diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..0486219 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,31 @@ +name: tests + +# Run the golden-master parity suite on every push and pull request. +on: + push: + branches: [main] + pull_request: + workflow_dispatch: # allow manual runs from the Actions tab + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # The model is stdlib-only; test across every supported interpreter. + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up uv (with Python ${{ matrix.python-version }}) + uses: astral-sh/setup-uv@v5 + with: + python-version: ${{ matrix.python-version }} + enable-cache: true + + # `uv run` creates/syncs the project environment (installing the package and the [dev] + # extra) and runs the command in it. It reuses an existing .venv without erroring. + - name: Run tests + run: uv run --python ${{ matrix.python-version }} --extra dev python -m pytest -q diff --git a/.gitignore b/.gitignore index 40ea563..40d0ead 100644 --- a/.gitignore +++ b/.gitignore @@ -12,21 +12,18 @@ main.pdf # Python __pycache__/ *.pyc +*.egg-info/ +.pytest_cache/ # OS .DS_Store # Figure previews rendered for review, not deliverables *-1.png -scripts/clean/figures/ # Working notes and superseded scripts, kept locally docs/ bearer-energy-research.md -scripts/agentic_ecal.py -scripts/agentic_ecal_v2.py -scripts/make_figures_current.py -scripts/fig_ecal_overlay.ipynb figures/make_figures_letters.py # Unrelated manuscripts that happen to live in this working directory @@ -52,5 +49,14 @@ Topologies_AAAI*.pdf # Infra-Bench lives in its own repository; this is a local working copy, not a vendored dependency. agent-infra/ -experimental_results/* -experiments_qwen3_8_27B/* \ No newline at end of file +# Experiment directories: data and per-experiment code are kept out of this repo. +experimental_results/ +experiments_qwen3_8_27B/ +experiments_qwen35_9B/ +experiments_fig12/ + +*.pdf +*.tex + +.claude +results/* \ No newline at end of file diff --git a/CHANGES-2026-09-01.md b/CHANGES-2026-09-01.md deleted file mode 100644 index d49ce3e..0000000 --- a/CHANGES-2026-09-01.md +++ /dev/null @@ -1,150 +0,0 @@ -# Session log, 1 September 2026 - -What changed in the manuscript today, why, and what a reader should be suspicious of. - -**Ground rules used throughout.** `main.tex` and `v1/` are frozen and were not touched — verified -with `git diff HEAD -- main.tex v1/` after every step. All work is in `v2/`. New material is green -(`\ROOF{}`), text flagged for removal is red (`\CUT{}`). - -**A LaTeX trap worth knowing.** A colour set inside a group does *not* survive a column break in -this two-column layout. Bare `\ROOF{}…\OLD{}` spans silently revert to black — or to blue, if the -surrounding region is inside a `\NEW` block. Coloured passages therefore re-assert the marker at -each paragraph, and at each clause where a paragraph is long enough to split. Existing green spans -in `v1/` have this latent bug; they happen not to break today. - ---- - -## At a glance - -| Section | What changed | Why | -|---|---|---| -| Structure | §IV split into IV/V; Amortization → IV-D; Placement moved ahead of Latency; "Scaling" paragraph → IV-B | Amortization consumes the case study's `E_W`, so it cannot precede it; the Scaling paragraph discusses Fig. 2 and Table II, which §IV-B introduces | -| Labels | `sec:results` was **duplicated** across both new sections | `\ref` was silently resolving to §V; one Introduction reference deleted, one repointed | -| Table I | `N` → `N_params`; added `N`, `β`, `γ`, `M`, `a_dec`, `c_0` | Table I defined `N` as parameters while the text used it for agent count — both senses appeared in one equation | -| §III-B / §IV-A | Peer law demoted to prose, slope `b` → `τ`; `a_dec`/`c_0` defined at the head of IV-A | `b` meant serving batch *and* slope in one subsection; `a_dec`/`c_0` appeared only in a caption and a legend, defined nowhere | -| §IV-C | New Fig. 3; old Fig. 4 removed; two paragraphs marked red | The paper referenced the case-study workflow six times without ever showing it | -| §V-B | New Model Placement subsection, Fig. 6, Table IV | Communication is a **memory** constraint: debate cuts hostable agents 5–14×, and KV width beats parameter count | -| §V-C | Table V → `table*`; batch-robustness sentence | It overflowed a column; hand-off volume is batch-independent, so the margin holds across b = 16–256 | -| §VI | New, scaffolded | Replaces the token denominator with task completion — the only way to locate a cost-optimal team size | - ---- - -## Structure - -§IV was split into two results sections, so the paper now runs I–VIII. Subsection order within -them changed twice: - -- **Amortization moved to §IV-D**, after "Energy of a Centralized RAG Workflow". It had drifted to - last. It cannot go earlier: `fig_amortization` computes `E_W` from the case study, and the - crossover is `G* = E_emb / E_W`, so placing it before the workflow subsection would make it - consume a quantity the reader has not met. -- **Model Placement moved ahead of Latency Budget.** Its opening sentence leaned on the latency - subsection as already-established and was rewritten. -- **A "Scaling" paragraph moved from §IV-C to §IV-B.** It discusses Fig. 2 and Table II, both of - which §IV-B introduces; it was sitting in the case-study subsection, two pages from its - referents. Its redundant `\textbf{Scaling.}` run-in was dropped, the subsection being so titled. - -Both new results sections initially carried the *same* `\label{sec:results}`, so `\ref` silently -resolved to §V. Now `sec:results-scale` (IV) and `sec:results-comms` (V). One reference in the -Introduction was deleted as asked; a second, on the amortization crossover, had been resolving to -the wrong section and now points at IV. - -## Notation - -- **`N` → `N_params` for parameter count**, 13 occurrences including Eqs. (4), (5), (8). Table I - defined `N` as parameters while §III-B and §IV already used it for agent count. Both senses - appear in the placement constraint, so the collision was live. -- **Table I gained `N` (team width), `β`, `γ`, `M`, and `a_dec, c_0`.** `β` and `γ` were used in - Eq. (5) and never defined — that predates this session. -- **The peer-law slope was renamed `b → τ`.** It collided with serving batch `b`, and both senses - appeared within one subsection. - -## Equations - -- **The peer law was demoted from a numbered equation to prose.** It is a two-parameter straight - line; the finding is the fit quality (R² ≥ 0.998 across 16 models), not the line. The paper drops - from 10 numbered equations to 9. -- **The placement memory constraint was kept as Eq. (9)** and *not* promoted to §III, because it - takes `p_in(k)` as an argument and that is now defined only in §V. -- **`a_dec` and `c_0` are now defined**, at the head of §IV-A where Fig. 1(b) lives. They appeared - only in a figure caption and a figure legend, defined nowhere. Eq. (5) is unchanged at your - direction; since its `2·N_params·β` implies `a_dec` ≈ 2.7× the measured value, the text reads the - equation as the mechanism setting the two terms rather than as a prediction of their size. - -## Figures and tables - -- **Fig. 3 (new, `fig_workflow`)** — the case-study workflow, which the paper referenced six times - without ever showing. Panels: schematic, prompt composition, energy per useful bit at b = 1/64/256, - hand-off energy per bearer. Panel (b) recomputes the composition exactly as `Workflow.run` does - and asserts the totals match. -- **Fig. 6 (new, `fig_model_placement`)** and **Table IV** — the measured communication law and the - memory-bound placement result. -- **Fig. 4 (`fig_comm`) panel (c)** — retitled from a colloquial label; distinct markers plus an - annotation, because the three text protocols coincide (latency is set by the propagation floor of - 11 messages, not by payload) and looked like a missing curve. -- **Table V (`tab_dimensioning`) is now `table*`** — it overflowed a single column. -- **The old Fig. 4 (`fig_components`) was removed from v2**, with its dedicated paragraph and a - dangling reference in the surviving "Component breakdown" text. -- **§VI (new) is scaffolded, not populated** — see below. - -## New analysis - -**The measured communication law.** 6.69M team-item records, 16 open-weight models, teams of 1–30 -at full mesh. Round 1 precedes any exchange and its prompt equals a single agent's, giving a -no-communication baseline inside the same campaign. Per-agent prefill is then linear in peers read, -`p_in = p0 + τk`, with R² ≥ 0.998 for all 16 models and τ = 79–126 tokens (median 103, against a -120-token budget). Output length barely moves, so communication is almost purely a prefill -phenomenon. Team prefill is therefore quadratic in N: 41× more prefill at N = 30 than the same -agents run in isolation. - -**Placement is bound by memory, not bandwidth.** A peer's message must stay resident in the -receiver's context for the whole forward pass, so debate inflates the KV footprint linearly in N. -Across every device/model pair, communication cuts hostable agents 5–14×. An 8 GB module cannot host -the 30-agent team it is debating with. KV width matters more than parameter count: OLMo-2-7B (MHA, -512 KB/token) holds 31 agents on an A100 where the *larger* Qwen2.5-7B (4 KV heads, 56 KB/token) -holds 347. - -**A structural claim, independently confirmed.** Prefilling a token costs ~496× more than moving it -on 5G. Pricing the measured peer messages the same way gives 613× at N = 30 — reproducing §III-E's -conclusion from real multi-agent traffic rather than from one synthetic workflow. - -## §VI — scaffolded, awaiting runs - -The metric divides by useful output *tokens*, which is a proxy: a workflow that fails its task still -emits tokens and still scores. The infrastructure benchmark replaces that denominator with task -completion, which lets the paper answer a question the token form cannot — how many agents to use. -Success saturates in N while energy does not (Eq. 7), so joules per *solved* task has an interior -minimum. - -Everything except the numbers is written. `make_figure_infra.py` reads -`results/infra_runs.csv` and regenerates the figure, table and **all nine macros**, so the prose -needs no edits when runs land. Committed placeholders render a red `??` or "pending benchmark runs". - -Statistical note built into the text: at 1–2 runs over 71 tasks the binomial CI is ±6–12 pp, so the -location of the minimum is provisional; the *shape* of the trade stands regardless. - ---- - -## Open items — read before submitting - -1. **§V-A quotes a superseded workflow.** Its 200/900/1600 hand-off tokens, 43.2 kbit, 7,720 - prefill, 1,600 output and 2.4×10³ J are internally consistent and hand-verifiable, but come from - a **planner + 2 workers + writer** topology priced on H100 at η = 0.05 with f = 16 — the removed - builder in `figures/make_figures_letters.py`. No code in the tree reproduces them; the current - chain topology gives 1,290 / 4,990 / 1,020. Table III is stale by ~2× for the same reason, and - `\caseEnergy`, `\handoffBits`, `\parityJpb`, `\parityFactor`, `\parityVolume`, `\crossoverG` all - rest on it. Audit finding C5 records the regime choice as a co-author decision; the docstring - claiming it "settles C5" overstates that. -2. **`f` = 16 in the manuscript vs `BITS_PER_TOKEN = 17.0` in the code.** The quoted vocabulary - ranges also disagree (32k–128k vs 128k–152k). -3. **`call_latency()` is unvalidated.** It disagrees with measured wall-clock by ≥2×, and whether it - over- or under-predicts depends on unresolved concurrency semantics in the sweeps (is - `wall_latency_s` the batch time or the sweep time?). Fig. 7's entire latency budget rests on it. -4. **b = 1 is an extrapolation** below the fitted range b ∈ [2,256]. Now stated in Fig. 3's caption; - Fig. 7(c)'s lowest curve has the same issue and no such note. -5. **`TODO - move to results`** still renders in the body text of all three manuscripts, and a - sentence fragment follows it. -6. **Eq. (8) is orphaned** under "Energy of Retrieval"; **`sec:workflow` is undefined** everywhere. -7. **Shared artifacts.** `fig_comm.pdf` and `tab_dimensioning.tex` were regenerated and are shared - with `v1`, so v1's *rendering* changes on its next build even though its `.tex` is untouched. -8. **`v2/infranet.png`** is committed but not yet wired into anything. diff --git a/CHANGES-2026-09-03.md b/CHANGES-2026-09-03.md deleted file mode 100644 index a8facec..0000000 --- a/CHANGES-2026-09-03.md +++ /dev/null @@ -1,149 +0,0 @@ -# Manuscript changes, 2–3 September 2026 - -For co-authors. Covers the work after `CHANGES-2026-09-01.md`. The manuscript itself is no longer -in this repository — the LaTeX sources live in a working copy outside it, and `figures/` and -`tables/` are regenerated by the scripts here rather than committed. - ---- - -## At a glance - -| Area | What changed | Why | -|---|---|---| -| §III-E | Gained a table of transport intensities `ε` | The OSI model charged every bit by `ε` without ever saying what `ε` is for a real link | -| §IV-A | `c_0` for Qwen2.5-7B corrected `0.00 → 0.02` | Came in with the `origin` merge; the "Qwen has no measurable floor" framing was wrong | -| §V-A | **New**: *Where a Team Fits: Accelerators, Models and Transports* | Placement was asserted, never evidenced | -| §V-B | **Rewritten**: *What a Hand-off Must Carry* | It previously answered the same question as §V-A and got the same answer | -| §VI | **New**: *Energy per Solved Task on an Infrastructure Benchmark* | First place in the paper where energy meets task success | -| Throughout | Every stale case-study macro is now unused | They came from a workflow topology no longer in the code | - -Figures 1–2 regenerated; Figures 6, 7 and 8 are new or rebuilt. The paper is 11 pages, 0 errors. - ---- - -## The stale numbers are gone - -`CHANGES-2026-09-01.md` opened its list of concerns with §V-A quoting a superseded workflow. That -is resolved, not by correcting the numbers but by the sections that used them being rewritten. All -six macros now have **zero uses**: - -`\caseEnergy` (2.4×10³ J), `\handoffBits` (43.2 kbit), `\caseEcal`, `\parityJpb`, `\parityVolume`, -`\parityFactor` (5.0×10⁴). - -The Conclusion's parity claim was restated from the current model: parity with compute would demand -a link some **3×10³** times costlier per bit than a mobile bearer, replacing the 5×10⁴ that derived -from the removed topology. The definitions remain in the preamble as dead code and should be -deleted once everyone has seen this. - -## §V-A — where a team fits - -Structure: heterogeneity of scale and topology → the sixteen models in four weight classes → the -eight orchestration architectures → results. - -Figure 7 has three panels at `N = 10`: - -- **(a)** how many of the 16 models load per accelerator. Weight restricts deployment only at the - far edge: 8 GB admits 2 models, 24 GB admits 10, 64 GB admits all 16 including the 4-bit 70B pair. -- **(b)** concurrent 10-agent **sessions** per accelerator, from 2 on an 8 GB module to 59 on an - A100. Within the 8–16 GB class alone this varies **16×** — GPT-OSS-20B and Qwen2.5-7B sustain 112 - and 87 sessions at 48 and 56 kB/token, while the *lighter* OLMo-2-7B sustains 7 at 512 kB/token. - Capacity is governed by key-value width, not by parameter count. -- **(c)** inter-agent edges against team size for the eight architectures. - -**The identity behind panel (c).** Each edge contributes one peer message to some agent's prompt, so -the summed prefill of a workflow is `N·p₀ + E·τ`. A DAG's *shape* therefore cannot affect cost; only -its edge count can. Six of the eight architectures require `E ≈ N−1` at every team size and -coincide exactly. Only Diamond (`≈1.5N`) and Cascading-Chain (`≈5N`) separate, and the dispersion is -bounded — 2.5× at N=5 rising only to 4.8× at N=30. - -`N = 10` was chosen as a mid-range operating point. At N=5 memory stops binding (every accelerator -holds hundreds of agents); at N=30 the far edge falls to zero sessions and reads as a contrived -corner. - -## §V-B — what a hand-off must carry - -The previous version swept placement across bearers and concluded communication was negligible, -which §V-A already establishes. It now asks the question §V-A does not: what each message contains. - -**The old figure was largely redundant.** Its three panels plotted volume, energy and latency, but -energy is volume times a bearer constant (log–log correlation **1.0000**) and latency is -propagation-dominated (0.9921). Two of three panels carried no information the first did not. The -rebuilt figure asks three different questions: - -- **(a)** what one session puts on the wire at K=3. Re-shipping the transcript costs 6.5× an - incremental hand-off; migrating the key-value cache costs **5.7×10⁵** times as much. -- **(b)** how that grows with loop depth. Re-shipping grows **quadratically** (30× from K=1 to 6) - while incremental and summary grow linearly (≈6×). The gap widens with depth rather than being a - constant factor. -- **(c)** the bearer intensity `ε*` at which transmission would equal the compute it accompanies. - -**The result.** Every text protocol keeps `ε* > 3×10⁻³` J/b at all depths — above a constrained -NB-IoT uplink, five orders above an optical backbone. Cache migration falls to `5.1×10⁻⁸` J/b, -leaving **5× of margin on a backbone and none on any mobile bearer**. Attention state is not -transportable; an agent's context is bound to the accelerator that built it, which is *why* §V-A's -capacity limit cannot be relieved by moving contexts elsewhere. That link is what makes the two -subsections one argument. - -The protocol spans 5.7×10⁵, against 4.8× for architecture and ~10⁵ for bearer: **what a hand-off -contains dominates where it is sent and how it is carried.** - -## §VI — energy per solved task - -Kubeply Infra-Bench `kubernetes-core` on ephemeral k3s clusters through the Harbor harness: 15 -tasks (5 easy / 5 medium / 5 hard), 60-minute deadline, 50-turn cap, binary reward on final cluster -state. Qwen3.5-9B, five runs per topology. - -| topology | score | kJ/run | **kJ per solved task** | -|---|---|---|---| -| Single | 48.0% | 18.1 | **2.5** | -| PS | 54.7% | 34.4 | 4.2 | -| Single+V | 49.3% | 37.5 | 5.1 | -| PS+V | 44.0% | 51.9 | 7.9 | -| I+PS+V | 48.0% | 75.3 | **10.5** | - -Energy spans 4.2× while score spans eleven points. Only proposer+synthesiser buys anything -(+6.7 pp for 1.9× energy). Adding a verifier *loses* 10.7 pp while costing more; the fullest -topology returns to the single agent's score at 4.2× its energy. - -The proposer sweep is sharper: N=1→5 raises energy per solved task **3.9 → 12.8 kJ** while the score -*falls* 53.3 → 46.7%. That is §III-B's super-linear token growth becoming super-linear energy with -no corresponding gain. - -**Accounting choice that matters:** prefill is charged on **uncached** input only, cached input -being served from the prefix cache. Cached input is 6–19 M tokens against 0.3–2.4 M uncached, so -charging it would change every number. Someone who knows the harness should confirm this. - ---- - -## Open items - -1. **`sec:workflow` is still undefined.** One reference remains, in the Introduction. It is the only - undefined reference left. -2. **§VI rests on 1–5 runs over 15 tasks.** Adjacent topologies are not separated; only the extremes - are robust. Timeouts count as failures and are not uniform across topologies, so part of the - elaborate designs' penalty is a deadline effect. Both caveats are stated in the section. -3. **Qwen3.5-9B has no measured calibration.** §VI prices its tokens with the Qwen2.5-7B - coefficients — ordering preserved, absolute scale not. -4. **`c_0 = 0.02` should be confirmed as a re-fit.** The upstream commit message was "Updated kv - floor to reproduce original figure", which describes tuning toward a figure rather than - regressing against data, and the paper now states it as measured. -5. **Edge counts in Fig. 7(c) are derived**, from each architecture's stated construction rather - than from separate measurement. -6. **The orchestration-architecture manuscript is an anonymised submission** whose distribution and - citation are restricted. `refs.bib` carries this in the entry's `note`. If it is our own - concurrent work, self-cite; otherwise Fig. 6 and §V-A cannot cite it as they stand. -7. **`f` = 16 in the manuscript vs `BITS_PER_TOKEN = 17.0` in the code**, unchanged from 1 September. -8. **The six dead macros** should be deleted from the preamble. - -## Reproducing the figures - -```bash -cd scripts/clean -python3 make_figures.py --figs validation scaling comm --out /figures -python3 make_figure_capacity.py /figures # Fig. 7 -python3 make_figure_infra.py /figures # Fig. 8 -``` - -`make_figure_topology.py` draws placement scenarios as network segments. It is not currently used -by the manuscript; it was explored before Fig. 7 took its present form and is kept because the -segment view may still be wanted. diff --git a/CHANGES-2026-09-08.md b/CHANGES-2026-09-08.md deleted file mode 100644 index 4bab758..0000000 --- a/CHANGES-2026-09-08.md +++ /dev/null @@ -1,230 +0,0 @@ -# Manuscript changes, 8 September 2026 - -For co-authors. Covers the work after `CHANGES-2026-09-03.md`. Today is Figure 6 only — the -capacity figure and its caption. `section5-provenance.md` was re-checked against the newly pulled -generators at the same time and seven of its findings closed. - -**The manuscript is edited on Overleaf, not here.** The `.tex` in this working copy is not the live -source and was deliberately left untouched. Every LaTeX block below is paste-ready for Overleaf; -the figure and script changes are in this repo and are already applied. - ---- - -## At a glance - -| Area | What changed | Why | -|---|---|---| -| Fig. 6(a) | Cells now read `n/N` and are coloured by **fraction**, not raw count | A "3" and a "7" both meant *all of them*; on a shared colour scale the small classes rendered fainter than the large one even when equally satisfied — the opposite of the truth | -| Fig. 6(b) | Colorbar labels decades only | `tick_params` defaults to `which="major"`, so the `LogNorm` minor labels kept the default font size and overprinted each other | -| Fig. 6 caption | Panel (a) described as a **fraction**; class populations stated | The caption said "Number of the $16$ … models"; no cell ever had 16 as its denominator | -| §V-B + Fig. 7 caption | `5.7\times10^{5}` → **`3.7\times10^{5}`**, both occurrences | The protocol span is KV / incremental = 370,086. The wrong value inverts the paragraph's own conclusion | -| §V audit | Seven rows closed, one reclassified | Fig. 6's generator arrived and computes edge counts independently of `dag.py`, which drives no figure in Section V | - -No plotted value changed in either panel. The figure says the same thing; it now says it legibly. - ---- - -## Fig. 6(a) — the denominator was invisible - -The four weight classes hold **3, 7, 3 and 3** of the 16 models. Every cell is either the whole -class or Orin's 2 of 3: - -``` - <8 GB 8-16 16-32 32-64 - orin 2/3 0/7 0/3 0/3 - l4 3/3 7/7 0/3 0/3 - rtx4090 3/3 7/7 0/3 0/3 - agx_orin 3/3 7/7 3/3 3/3 - a100 3/3 7/7 3/3 3/3 -``` - -The panel printed the numerator alone. Three problems compounded: - -1. the denominator changes per column and was stated nowhere, so "3" read as worse than "7"; -2. the colour scale was shared across columns with different maxima, so the `<8 GB` column rendered - permanently fainter than `8–16` even where both were fully satisfied; -3. the colorbar was labelled **"models (of 16)"** while running 0–7. - -Together these invited exactly the wrong reading — that fewer light models fit than medium ones. - -**In `make_figure_capacity.py`:** `grids()` now also returns the per-class population; `heat()` -takes a `denoms` argument and, when given, colours by the fraction `n/N` and labels each cell -`n/N`; the colorbar reads *"fraction of weight class"* over 0–1; and a `0` cell shows `0/7` rather -than `--`, since with a denominator present the zero is informative. Panel (b) passes no `denoms` -and is untouched. - -Every fully-satisfied class now renders the same colour whatever its size. The console table the -script prints carries the denominator too, so the figure and the log agree. - -## Fig. 6(b) — colorbar, not data - -The session-log entry worth keeping: **this was already in the version we pulled**, not introduced -by the panel-(a) work. Verified by rendering `git show HEAD:` of the file side by side. - -`cb.ax.tick_params(labelsize=5.5)` defaults to `which="major"`. Under `LogNorm` the *minor* labels — -`2×10⁰`, `3×10⁰`, `2×10¹`, `4×10¹` — therefore kept the default font size and overprinted one -another across the colorbar. Two lines fix it: - -```python -cb.ax.tick_params(labelsize=5.5, which="both") -if log: - cb.ax.yaxis.set_minor_formatter(matplotlib.ticker.NullFormatter()) -``` - -Guarded on `log`, so panel (a)'s linear 0–1 colorbar is unaffected. Only `10¹` remains. Cells, -colours and the `--` blanks are unchanged. - -## The caption - -Panel (a) no longer reports a number, so the first sentence had to change. Stating the 3/7/3/3 -populations is also what makes the `n/N` cells readable without the reader counting — the missing -denominator was the original defect. - -```latex -\caption{\ROOF{}Where an agent team fits, and how much traffic its architecture generates. - (a) Fraction of each weight class whose models load on each accelerator, cells labelled - $n/N$; the four classes hold $3$, $7$, $3$ and $3$ of the $16$ measured open-weight models. - (b) Median number of concurrent ten-agent sessions the same accelerator sustains - once each agent's resident context is included; an agent is a context, not a model instance, so - all agents on a device share one copy of the weights. Contexts are taken from the measured - single-exchange condition. (c) Inter-agent edges against team size for the eight orchestration - architectures of Fig.~\ref{fig:topologies}. Because the summed prefill of a workflow is - $N p_{0}+E\tau$, edge count alone determines cost; six of the eight architectures require - $E\approx N-1$ and coincide, and the dotted line marks the $N=10$ used in (a) and (b). Edge - counts follow from the construction each architecture is defined by rather than from separate - measurement.\OLD{}}\label{fig:capacity} -``` - -Only the panel-(a) sentence differs; every other claim in the caption was re-verified and holds. -The closing sentence — *"Edge counts follow from the construction … rather than from separate -measurement"* — is now **more** accurate than when it was written: `edges()` is a closed form per -architecture that reads no data at all. - -**Paste into Overleaf** — this repo's `.tex` is not the live source. - -## The protocol span — `5.7\times10^{5}` is wrong twice - -The number appears in §V-B and again in the Fig. 7 caption. At K=3 the three candidate ratios are - -| ratio | value | orders | -|---|---|---| -| KV / full context | 56,936 = **5.69×10⁴** | 4.76 | -| KV / incremental | 370,086 = **3.70×10⁵** | 5.57 | -| KV / summary | 238,765 = 2.39×10⁵ | 5.38 | - -and none of them is 5.7×10⁵. Nor is it a depth artifact: swept over K=1…8, KV/incremental runs -123k, 247k, 370k, 493k, 617k — stepping *over* 5.7×10⁵ between K=4 and K=5 without landing on it — -while KV/full-context asymptotes to ≈60k. - -**Where 3.7×10⁵ comes from.** It is not read from anywhere; it is the ratio of the two extreme -protocols, at the figure's own parameters (N=4 agents, K=3 rounds → 12 hand-offs, 250 tokens per -turn, f = 17 bit/token): - -``` -incremental (cheapest): 12 msgs x 250 tok x 17 bit = 51,000 bits -KV migration (dearest): 12 msgs x 131,072 B/tok x 1500 tok x 8 - = 18,874,368,000 bits - -span = 18,874,368,000 / 51,000 = 370,086 = 3.70 x 10^5 -``` - -The KV context is `N*K*p_out/2 = 1500` tokens, and 131,072 B/token is Llama-3.1-8B's cache entry. -Ordered cheapest first the four protocols are incremental 0.051, summary 0.079, full context -0.3315, KV migration 18,874 Mbit — so the span is KV over incremental, and full context is not an -extreme at all. - -**How it arose.** The mantissa 5.7 matches KV/full-context (5.69×10⁴) to three significant figures. -The exponent matches the *words* of the preceding sentence, "five orders of magnitude above the -transcript", which is 4.76 rounded up. The number is a splice of the correct mantissa with an -exponent imported from the rounded prose describing that same ratio; the "therefore" joining the -two sentences is the seam. - -**Why the fix is 3.7×10⁵ and not 5.7×10⁴.** Tempting as the mantissa evidence is, the smaller -value inverts the paragraph's own conclusion. The sentence ranks three spans and concludes that -"what a hand-off contains dominates both where it is sent and how it is carried", and the bearer -span is $10^{-3}/10^{-8} = 10^{5}$: - -- at **3.7×10⁵** → protocol > bearer (10⁵) > architecture (4.8×). The claim holds. -- at **5.7×10⁴** → protocol < bearer. The bearer would dominate the protocol by 1.8× and the - sentence would assert the opposite of its own numbers. - -Two further reasons agree: "spans" ordinarily means the range between extremes, which is KV over -incremental; and the caption's parallel construction — "costs $6.5\times$ an incremental hand-off, -and … times as much" — makes the incremental hand-off the comparator in both places. Same -substitution, both occurrences. - -**This is an inference about intent, not a derivation.** The published number matches neither -candidate, so somebody has to decide which comparison the sentence was making. Both readings are -internally coherent: - -| if the sentence meant | correct value | what else must change | -|---|---|---| -| the span of the protocol family (dearest / cheapest) | **3.70×10⁵** | nothing; the ranking and conclusion stand | -| the KV-vs-transcript gap the previous sentence describes | **5.69×10⁴** | the closing clause — the protocol would no longer beat the bearer | - -We took the first. An author who knows the sentence was always about the transcript comparison -should take the second and rewrite the conclusion, because at 5.69×10⁴ the bearer wins. - -**Optional, and recommended.** With the corrected value the closing clause rests on two very -different margins: the protocol beats the *architecture* by five orders (3.7×10⁵ / 4.8 ≈ 7.7×10⁴) -but the *bearer* by only 3.7×. "Dominates both" flattens them, and a referee who divides will -notice. Suggested replacement for the final clause: - -```latex -what a hand-off contains dominates the choice of graph by five orders of magnitude and outweighs -even the choice of bearer. -``` - -**Leave #35 alone.** "Five orders of magnitude above the transcript" is a *different* comparison — -KV against full context, 4.76 orders — and is correctly rounded. It verified clean. - -**Paste into Overleaf** — this repo's `.tex` is not the live source. - -## What the audit re-check closed - -`make_figure_capacity.py` supplies Fig. 6 and carries its own `edges()` model. The earlier audit -compared against `experimental_results/ringelmann/topology/dag.py`, which drives no figure in -Section V. Re-checked against the real source: - -| # | was | now | -|---|---|---| -| 22 | ⚠️ "8–9, not 8–10" | ✅ the six give {8, 9, 10} at N=10, as written | -| 24 | ❌ E = N(N−1)/2 | ✅ `sum(min(5,i))` = 5N−15, ≈5N to leading order | -| 25 | ❌ 2.5×→15.5×, growing | ✅ **2.50× → 4.82×**, and it plateaus | -| 31 | ⚠️ | ✅ `edges()` reads no data | -| 31b | ⚠️ misleading encoding | ✅ fixed above | -| 39 | ❌ inherited #24 | ✅ 4.82× | -| 51 | 🔍 stale generator | ✅ `fig_comm` now draws the captioned panels | - -Everything else re-ran unchanged: fleet 16 models at 5.74–50.29 GiB, KV 36–512 kB/token, sessions -112 / 87 / 7, the 16.0× within-class spread, traffic 0.3315 / 0.051 / 0.079 Mbit and 18.9 Gbit, and -every ε\* — 5.101×10⁻⁸ at K=6, a 5.10× margin over optical backbone, 19.6× against 5G. - -**One thing to be suspicious of.** There are now two Cascading-Chains. `edges()` caps refiners at -five preceding reports, matching the manuscript; `dag.py`'s `CascadingChainDAG` implements full -history (`parents = tuple(range(i))`). Nothing in Section V depends on the DAG version — no CSV -anywhere contains `cascading_chain` — so the two never meet in this paper. It deserves a footnote -so a later reader does not run the harness, get E = N(N−1)/2, and think Fig. 6(c) is wrong. - -## Open items - -1. **Three edits are pending on Overleaf** — the Fig. 6 caption, and the two - `5.7\times10^{5}` → `3.7\times10^{5}` substitutions in §V-B and the Fig. 7 caption. All are - drafted above, ready to paste. -2. **Output paths disagree.** `make_figure_capacity.py`, `make_figure_infra.py` and - `make_figure_topology.py` default to `v2/figures/`; `make_figures.py` and - `make_figure_workflow.py` default to `figures/`, which is what the manuscript includes. `v2/` - holds no `.tex`, so the capacity figure currently lands where the manuscript will not find it. -3. **Fig. 5 `topologies_v1.pdf` still has no generator.** `make_figure_topology.py` is a *different* - figure — three network topologies with a team placed on each — and writes `fig_topology.pdf`, - which the manuscript does not include. -4. **`fig_infra.pdf` cannot be rebuilt**: `results/infra_runs.csv` is absent. - -## Reproducing - -```bash -cd scripts/clean -python make_figure_capacity.py # writes v2/figures/fig_capacity.pdf, prints the n/N grid -``` - -The printed grid should match the block at the top of this note. diff --git a/CHANGES-2026-09-09.md b/CHANGES-2026-09-09.md deleted file mode 100644 index 062461b..0000000 --- a/CHANGES-2026-09-09.md +++ /dev/null @@ -1,111 +0,0 @@ -# Code changes — 9 September 2026 - -Figure-generation and model code only. The manuscript is maintained outside this repository in a -`vN/` working copy and is not tracked here. - -This commit also carries a small amount of uncommitted work from the 8 September session (the -`_paper_figures()` output helper and the `agentic_ecal.py` comment corrections), which had been -left in the working tree. - -## New files - -**`scripts/clean/roles.py`** — one functional taxonomy of agent roles, shared by every figure that -draws agents. The concrete role names differ between figures because they come from three -independent sources: the case study names its own agents, the orchestration architectures inherit -the names of the study that defined them, and the infrastructure benchmark uses its configuration -keys. Those names are left alone, for traceability to the artefacts, and the figures are unified by -functional **class** instead, carried by colour: - -| class | meaning | members | -|---|---|---| -| `generate` | produces a candidate or a plan | planner, proposal, proposer | -| `transform` | refines ONE line of work | analyst, writer, critic, refiner, verifier | -| `ground` | acquires state external to the workflow | retriever, inspector | -| `aggregate` | reduces COMPETING candidates | manager, synthesiser, duelist | - -The transform/aggregate line is drawn at *competing candidates*, not at fan-in: a cascading refiner -reads its own history and still refines one draft, whereas a duelist picking one of two rival -answers, and a synthesiser merging three, both reduce a candidate set. `role_class()` raises -`KeyError` on an unknown name deliberately, so a new role must be classified rather than silently -defaulted. - -`ground` has no counterpart in the orchestration taxonomy, which admits neither retrieval nor -tools. That absence is a result rather than an oversight. - -**`scripts/clean/fig_topologies.py`** — generator for `topologies_v1.pdf` (the eight orchestration -architectures), previously untracked. Now sources its palette from `roles.py`. - -**`scripts/clean/make_figure_overview.py`** — generator for `fig_overview.pdf`, the framing figure. -Rewritten from a two-layer form (workflow graph above a substrate, joined by projection lines) to a -single diagram with tier on the vertical axis and workflow order on the horizontal, so that -placement is position. The previous version was authored 7.1 in wide but included at -`\columnwidth`, which put its labels on the page below 3 pt; everything is now sized for the column -it is placed in. It computes and prints the prefill-to-transport ratio it annotates -(`crossing_ratio()`: 496x for a 2 700-token hand-off on a 5G bearer) rather than hard-coding it. - -## Modified - -**`agentic_ecal.py`** -- Removed equation numbers from comments and docstrings. They referred to the numbering of an - earlier manuscript revision and had drifted; the comments now name the quantity ("the two-rate - model", "the prefill rate") instead of a number that goes stale on every reorder. -- `four_agent_rag_workflow()` docstring no longer claims to settle audit finding C5, which - `docs/o2-audit-report.md` records as still open for the co-authors, and no longer quotes energies - that had gone stale twice. It points at `make_figure_workflow.py` for current values. -- `llm_call_energy()` docstring now states explicitly that it derives its rates from datasheet - quantities rather than from the measured `TWO_RATE` coefficients, since the two paths had become - easy to confuse once both were plotted. - -**`make_figures.py`** (Figs. 2, 3 and the communication figure) -- Added `datasheet_energy_per_query()` / `datasheet_curve()`: the same two-rate form with rates - **derived** from `N`, `beta`, `P`, `B_HBM`, `eta_pre` rather than fitted, so the figure can show - the gap between prediction and calibration. It splits by `n_calls` where the calibrated path does - not, because the derived decode rate carries a cache term evaluated at the call's mean context - and is therefore quadratic in it — applying it to per-query totals would price one enormous call - and inflate that term. -- Fig. 3 restyled: measurements as markers only so the overlays stay legible; panel (b) batches on - a blue ramp (orange withdrawn, since it denotes Llama elsewhere); solid = calibrated, - dashed = datasheet in both (a) and (b); composite `HandlerTuple` legend keys so a coloured - overlay is not represented by a single grey sample. -- Panel (c) reverted to measured-only. The overlay overshot badly at large batch (1.58 against a - measured 1.42 on Qwen, 1.45 against 1.17 on Llama); the overshoot is anchored at `N=2`, where the - model under-predicts by 46–63% at `b=256`, so plotting it displayed a localised residual as a - systematic slope error. The reason is recorded in the code. - -**`make_figure_workflow.py`** (Fig. 4) -- Agent boxes tinted by functional class via `roles.py`, with a class key between title and - schematic. -- Panel (d) now covers all three serving batches. `E_W` is 281.5 J at `b=64` and 238.1 J at - `b=256` — 18% apart, thinner than the line width on a seven-decade log axis — so the batched - regime is drawn as a band bounded by the two rather than as two overprinting lines. -- Schematic spacing: boxes 0.72 -> 0.84 tall with the two text lines at 0.68h/0.25h, and the axes - y-range trimmed to the drawn content. The axes previously ran `ylim(0, 4.05)` over content - spanning ~0.4–3.6, and that dead space read as a gap between the panel and the row beneath it. - -**`make_figure_infra.py`** (Fig. 9) -- Added `COMPOSITION` and `class_grid()`: each topology carries a fixed four-slot grid recording - which functional classes it instantiates, filled where present and outlined where absent. The - marks in this figure are per-topology rather than per-node, so composition is how the shared - class colours apply here. -- Bars and markers made neutral so the grid carries the colour signal, and panel (c)'s two series - recoloured off the class palette to avoid collision. - -**`make_figure_capacity.py`, `make_figure_topology.py`, `make_figures_placement.py`, -`make_figures.py`, `make_figure_infra.py`** — all now resolve their output directory through -`_paper_figures()`, which picks the highest-numbered `vN/` working copy that exists, instead of -hard-coding `v2/figures` or the repo-root `figures/`. The hard-coded defaults went stale on every -revision and silently wrote into a superseded copy. - -## Verification - -- `python3 agentic_ecal.py` self-tests pass. -- `make_figure_workflow.py`'s `sum(p_in) == run()['prefill_tokens']` assertion still holds. -- Fig. 2 regenerated: `R^2 = 0.9957 / 0.9962`, MAPE `10.8% / 11.9%`, `n = 378` per model. -- Fig. 3 regenerated: measured exponents `1.10 -> 1.42` (Qwen), `1.02 -> 1.17` (Llama). -- Fig. 8 regenerated: capacity `2` sessions on the 8 GB module to `59` on the A100. - -## Note - -`make_figure_capacity.py` uses `TEAM = 10`, and its panels are *concurrent sessions* at `N=10` and -*inter-agent edge count*. A manuscript caption describing them as agents at `N=30` and transmission -energy was corrected against this code, not the other way round. diff --git a/README.md b/README.md index 29a6270..f11a783 100644 --- a/README.md +++ b/README.md @@ -34,55 +34,118 @@ sender and receiver are charged. ## Layout +The model now lives at the repository root as a modular package, alongside the figure scripts, +the measured data, tests, and examples: + ``` -scripts/clean/ - agentic_ecal.py the model: Hardware/LLM descriptors, two-rate call model, Workflow, the metric - osi.py eCAL Eq. (3), the OSI-layer transmission model, for E_tx - placement.py bearers, deployment archetypes, and the generated placement table - make_figures.py the shared figures + tab_dimensioning - model_placement.py device tiers with MEMORY, 16-model fleet geometry, the measured peer law - fit_peer_law.py fits the peer law from ~/DATA/data_steiner_Aug26 (see below) +agentic_ecal_pkg/ the model, as a modular package (import agentic_ecal_pkg as ae) + descriptors.py Hardware, LLM descriptors + rates.py the two-rate call model: prefill/decode rates, call energy & latency + calib.py TwoRateCalib (measured coefficients) + components.py Tool, Retrieval, Step + workflow.py Workflow (.run() breakdown, .agentic_ecal() J/bit) + registries.py HW, LLMS, TWO_RATE presets (Python data) + builders.py four_agent_rag_workflow, tree_workflow, debate_workflow + validation.py literature-anchor calibration checks + cli.py `agentic-ecal` / `python -m agentic_ecal_pkg` (add --json) +examples/ runnable usage examples +tests/ pytest parity suite: package == frozen reference, bit-exact (~3000 cases) + agentic_ecal_reference.py the original single-file model, frozen as the golden master +figure_scripts/ every figure/analysis script (import agentic_ecal_pkg) + osi.py eCAL Eq. (3), the OSI-layer transmission model, for E_tx + placement.py bearers, deployment archetypes, and the generated placement table + model_placement.py device tiers with MEMORY, 16-model fleet geometry, the measured peer law + make_figures.py the shared figures + tab_dimensioning make_figures_placement.py fig_model_placement + tab_model_placement make_figure_workflow.py fig_workflow, the case-study schematic - make_figure_infra.py fig_infra + tab_infra + infra_macros (needs benchmark runs) - results/ the measured sweeps (A100, vLLM), plus the cached Steiner aggregate + make_figure_infra.py fig_infra + tab_infra + infra_macros (needs benchmark runs) + fit_peer_law.py fits the peer law from ~/DATA/data_steiner_Aug26 (see below) +results/ the measured sweeps (A100, vLLM), plus the cached Steiner aggregate + (not tracked in git; provided separately) ``` +`agentic_ecal_pkg` is a **behaviour-preserving** refactor of the original single-file model (now +frozen as `tests/agentic_ecal_reference.py`): it re-exports the identical public API +(`import agentic_ecal_pkg as ae` is a drop-in) and produces bit-for-bit identical numbers, which +the parity test suite enforces against that reference. + The manuscript no longer lives here. This repository holds the model, the measured data, and the scripts that generate the paper's figures and tables; the LaTeX sources are maintained separately. -Running the scripts recreates `figures/` and `tables/` from `scripts/clean/results/`. +Running the scripts recreates `figures/` and `tables/` from `results/`. + +## Setup (uv) + +From the repository root: + +```bash +uv venv +source .venv/bin/activate +uv pip install -r requirements.txt # numpy, pandas, matplotlib + pytest + +# optional: install the model as a package (enables the `agentic-ecal` CLI from anywhere) +uv pip install -e . +``` + +The model itself is **pure standard library**; numpy/pandas/matplotlib are only needed to +regenerate the figures, and pytest only to run the tests. + +## Use it + +```python +import agentic_ecal_pkg as ae + +wf = ae.four_agent_rag_workflow() # the paper's case study (Llama-3 8B on A100) +print(wf.run()["e_total"], "J") # 5240.26 J +print(wf.agentic_ecal(), "J/bit") # 0.3022 J/bit +``` + +```bash +python examples/basic_call.py # per-call / per-token energy across batches +python examples/case_study.py # the four-agent RAG breakdown +python examples/custom_workflow.py # build a Workflow from Steps by hand +python -m agentic_ecal_pkg # derived-vs-measured coefficients, reduction to eCAL at K=1 +python -m agentic_ecal_pkg --json # the same, machine-readable +``` + +## Tests (pytest) + +A golden-master / parity suite loads the frozen reference `tests/agentic_ecal_reference.py` and the new package side by +side and asserts bit-exact identical results across every public function, registry, workflow +builder, and the full `Workflow.run()` breakdown; `tests/test_paper_numbers.py` additionally pins +the anchors and case-study figures quoted in the manuscript. + +```bash +pytest # ~3000 parametrized cases +``` -## Reproduce +## Reproduce the figures ```bash -pip install -r requirements.txt -cd scripts/clean -python3 agentic_ecal.py # derived-vs-measured coefficients, reduction to eCAL at K=1 -python3 placement.py # tables/tab_placement.tex (generated but currently unused) -python3 make_figures.py # the shared figures + tab_dimensioning -python3 make_figures_placement.py # fig_model_placement + tab_model_placement -python3 make_figure_workflow.py # fig_workflow +python figure_scripts/make_figures.py # the shared figures + tab_dimensioning +python figure_scripts/make_figures_placement.py # fig_model_placement + tab_model_placement +python figure_scripts/make_figure_workflow.py # fig_workflow ``` -The scripts write into `figures/` and `tables/` at the repository root, creating them if absent. +The scripts write into `figures/` (or the newest `vN/figures/`) and `tables/` at the repository +root, creating them if absent. -`make_figures_placement.py` reads the cached aggregate -`scripts/clean/results/steiner_tokens_by_cell.csv`, which is committed, so it runs without the -raw data. To rebuild that aggregate from scratch you need `~/DATA/data_steiner_Aug26` (132 GB, -not in the repo) and `python3 fit_peer_law.py`; it summarises the `teams_*.csv` files in chunks, -which are ~20x smaller than the `agents_*.csv` ones. +The `results/` directory is **not tracked in git** (see `.gitignore`); obtain the measured CSVs +separately and place them there. `make_figures_placement.py` reads the cached aggregate +`results/steiner_tokens_by_cell.csv`, so it runs from that aggregate alone, without the raw data. +To rebuild that aggregate from scratch you need +`~/DATA/data_steiner_Aug26` (132 GB, not in the repo) and `python figure_scripts/fit_peer_law.py`; +it summarises the `teams_*.csv` files in chunks, which are ~20x smaller than the `agents_*.csv` ones. `make_figure_infra.py` waits on the infrastructure-benchmark runs. It expects -`scripts/clean/results/infra_runs.csv` with columns `topology, n_proposers, run_id, task_id, -difficulty, success, prefill_tokens, decode_tokens, wall_s`, and exits with that message if the -file is absent. Until then it emits placeholders that render a conspicuous red `??` or -"pending benchmark runs", so no placeholder can ship unnoticed. +`results/infra_runs.csv` with columns `topology, n_proposers, run_id, task_id, difficulty, +success, prefill_tokens, decode_tokens, wall_s`, and exits with that message if the file is +absent. Until then it emits placeholders that render a conspicuous red `??` or "pending benchmark +runs", so no placeholder can ship unnoticed. -`scripts/clean/results/infranet.png` is the infrastructure-benchmark result figure that motivates -that section. +`results/infranet.png` is the infrastructure-benchmark result figure that motivates that section. -`agentic_ecal.py` is stdlib-only. `make_figures.py` and `placement.py` need numpy, pandas and -matplotlib, and the measured CSVs in `scripts/clean/results/`. +`agentic_ecal_pkg` is stdlib-only. The figure scripts need numpy, pandas +and matplotlib, and the measured CSVs in `results/`. ## Status diff --git a/agentic_ecal_pkg/__init__.py b/agentic_ecal_pkg/__init__.py new file mode 100644 index 0000000..341022e --- /dev/null +++ b/agentic_ecal_pkg/__init__.py @@ -0,0 +1,46 @@ +"""agentic-eCAL: analytical two-rate energy model for agentic workflows of open-weight LLMs. + +This package is a modular, behaviour-preserving refactor of the single-file ``agentic_ecal.py``. +It extends the eCAL methodology (Chou et al., IEEE JSAC 2026) from a single AI-model lifecycle to +an *agentic workflow*: a directed graph of LLM calls, tool invocations, retrievals, and +inter-agent messages built on a pre-trained open-weight model. + +It contains the model only -- no plotting, no data loading, no pandas/matplotlib import -- so it +can be imported and tested on its own. Figures live in ``figure_scripts/``. + +The call model is TWO-RATE: prefill is compute-bound and priced at ``eta_pre``, decode is +memory-bandwidth-bound and amortises the weight read over the serving batch. All energies are in +Joules unless noted. + +The top-level namespace re-exports the full public API, so ``import agentic_ecal_pkg as ae`` is a +drop-in for the original ``import agentic_ecal as ae``. +""" + +from __future__ import annotations + +from .constants import BITS_PER_TOKEN, GWH_TO_J, MFU_LATENCY, MFU_THROUGHPUT +from .descriptors import Hardware, LLM +from .calib import TwoRateCalib +from .rates import ( + call_latency, + decode_rate, + embedding_flops, + joules_per_token, + llm_call_energy, + prefill_rate, +) +from .components import Retrieval, Step, Tool +from .workflow import Workflow +from .registries import HW, LLMS, TWO_RATE +from .builders import debate_workflow, four_agent_rag_workflow, tree_workflow +from .validation import validate_against_samsi, validate_current_gen, validate_two_rate + +__all__ = [ + "Hardware", "LLM", "Tool", "Retrieval", "Step", "Workflow", + "HW", "LLMS", "MFU_LATENCY", "MFU_THROUGHPUT", "BITS_PER_TOKEN", "GWH_TO_J", + "prefill_rate", "decode_rate", "llm_call_energy", "call_latency", + "joules_per_token", + "embedding_flops", "validate_two_rate", "TwoRateCalib", "TWO_RATE", + "four_agent_rag_workflow", "tree_workflow", "debate_workflow", + "validate_against_samsi", "validate_current_gen", +] diff --git a/agentic_ecal_pkg/__main__.py b/agentic_ecal_pkg/__main__.py new file mode 100644 index 0000000..d7b1945 --- /dev/null +++ b/agentic_ecal_pkg/__main__.py @@ -0,0 +1,8 @@ +"""Enable ``python -m agentic_ecal_pkg`` to run the self-consistency demo.""" + +from __future__ import annotations + +from .cli import main + +if __name__ == "__main__": + main() diff --git a/agentic_ecal_pkg/builders.py b/agentic_ecal_pkg/builders.py new file mode 100644 index 0000000..efe97e6 --- /dev/null +++ b/agentic_ecal_pkg/builders.py @@ -0,0 +1,135 @@ +"""Workflow builders: the paper's case study plus the tree and debate topologies.""" + +from __future__ import annotations + +from typing import List, Optional + +from .descriptors import Hardware, LLM +from .components import Step, Tool, Retrieval +from .workflow import Workflow +from .registries import HW, LLMS + + +def four_agent_rag_workflow(model: Optional[LLM] = None, hw: Optional[Hardware] = None, + rounds: int = 1) -> Workflow: + """Planner -> retriever -> analyst -> writer, carrying history, repeated ``rounds`` times. + + ``rounds=1`` is the single-pass case study; larger values re-run the same team over the + accumulated transcript, which is the depth axis of the composition figure. + + Single definition of the case study, so the component breakdown, the amortization curve and + the numbers quoted in the text cannot drift apart. Callers choose the model and hardware; + the token budget and topology are fixed. + + This is the only case-study workflow in the codebase. A competing planner-plus-two-workers + topology, which produced the 2.4e3 J and 43.2 kbit the manuscript once quoted, was removed; + audit finding C5 records that regime choice as still open for the co-authors rather than + settled here. Energies therefore depend on the operating point and are not quoted in this + docstring, where they went stale twice; run make_figure_workflow.py for current values. + + Args: + model (Optional[LLM]): The model to run on. Defaults to ``LLMS["llama3_8b"]``. + hw (Optional[Hardware]): The accelerator to run on. Defaults to ``HW["a100"]``. + rounds (int): Number of times to repeat the four-agent team over the accumulated + transcript. Defaults to ``1`` (the single-pass case study). + + Returns: + Workflow: The configured case-study workflow. + """ + ret = Retrieval(k_chunks=5, chunk_tokens=256) + tool = Tool(name="code", energy=2.0, obs_tokens=150) + steps = [ + Step(agent="planner", p_in_local=300, p_out=200), + Step(agent="retriever", p_in_local=120, p_out=120, retrieval=ret), + Step(agent="analyst", p_in_local=200, p_out=300, tool=tool), + Step(agent="writer", p_in_local=200, p_out=400), + ] * max(int(rounds), 1) + # round_size stays 1: the four agents run in sequence, so each reads its predecessors' + # output within the same pass. Grouping them into one parallel round would remove that. + # + # tx_energy_per_message = 0: this is the CENTRALIZED case study, four co-located agents, and + # the co-located column of the placement table is zero by construction. It was 0.5 J, a flat + # per-message constant, which is the wrong shape as well as the wrong value: osi.segment_energy + # and osi.endpoint_stack_energy are exactly linear in payload with zero intercept, so there is + # no fixed per-message cost for a constant to represent, and the three hand-offs here differ + # 3.85x in size (200/320/770 tokens). The value was also bearer-independent, where the correct + # figure spans 0.003 J (metro) to 17 J (NB-IoT) per message. Distribution is priced ON TOP of + # E_W by placement.py/osi.py -- which is what fig_workflow(d) and the placement section do -- + # so carrying a second, flat transmission model inside E_W double-counted the same hand-offs + # by a different method. debate_workflow already sets 0.0. + return Workflow(model=model or LLMS["llama3_8b"], hw=hw or HW["a100"], steps=steps, + carry_history=True, gamma_v=0.10, tx_energy_per_message=0.0) + + +def tree_workflow(model: LLM, hw: Hardware, fanout: int, depth: int, + p_out: int = 250, sys_tokens: int = 400, p_in_root: int = 300, + batch: float = 1.0, aggregate: bool = True) -> Workflow: + """A fan-out tree: a root delegates to ``fanout`` children, each of which may delegate again. + + Every node reads only its parent's output, so a node's prompt is bounded by one hand-off + however wide or deep the tree becomes. That is the structural difference from a chain, where + each step reads the whole accumulated transcript. With ``aggregate`` the root reads its + children's outputs back on the way up, which is the only place a prompt grows with fan-out. + + Args: + model (LLM): The model every node runs on. + hw (Hardware): The accelerator every node runs on. + fanout (int): Number of children each node delegates to. + depth (int): Number of delegation levels below the root. + p_out (int): Generated tokens per node. Defaults to ``250``. + sys_tokens (int): System prompt prepended to every call. Defaults to ``400``. + p_in_root (int): Local prompt tokens at the root. Defaults to ``300``. + batch (float): Serving batch ``b``. Defaults to ``1.0``. + aggregate (bool): If ``True``, the root reads its children's outputs back. Defaults to + ``True``. + + Returns: + Workflow: The configured fan-out tree workflow. + """ + steps: List[Step] = [Step(agent="root", p_in_local=p_in_root, p_out=p_out)] + nodes = 1 + for level in range(1, depth + 1): + nodes = fanout ** level + steps += [Step(agent=f"L{level}_{i}", p_in_local=p_out, p_out=p_out) + for i in range(nodes)] + if aggregate: + steps.append(Step(agent="root", p_in_local=nodes * p_out, p_out=p_out)) + return Workflow(model=model, hw=hw, steps=steps, sys_tokens=sys_tokens, + carry_history=False, gamma_v=0.10, tx_energy_per_message=0.5, + serving_batch=batch) + + +def debate_workflow(model: LLM, hw: Hardware, n_agents: int, rounds: int, p_out: float, + sys_tokens: int, history_rounds: Optional[int] = 1, + exclude_self: bool = True) -> Workflow: + """The measured multi-agent protocol: ``n_agents`` agents debating for ``rounds`` rounds. + + Each round is one pass over the team, and by default an agent entering round r reads the + outputs its n-1 peers produced in round r-1. Setting ``history_rounds=None`` retains the whole + transcript instead and ``carry_history=False`` retains none; the token-validation figure plots + all three against measurement. + + ``p_out`` is the generated tokens per call and ``sys_tokens`` the fixed prompt scaffold, both + taken from the measurements rather than assumed. + + Args: + model (LLM): The model every agent runs on. + hw (Hardware): The accelerator every agent runs on. + n_agents (int): Number of agents (team width, i.e. the round size). + rounds (int): Number of debate rounds. + p_out (float): Generated tokens per call (rounded to an int per step). + sys_tokens (int): Fixed prompt scaffold prepended to every call. + history_rounds (Optional[int]): Rounds of transcript retained; ``None`` retains all, + ``0`` retains none. Defaults to ``1`` (previous round only). + exclude_self (bool): Skip an agent's own output within the retained window. Defaults to + ``True``. + + Returns: + Workflow: The configured debate workflow. + """ + steps = [Step(agent=f"agent{i}", p_in_local=0, p_out=int(round(p_out))) + for _ in range(rounds) for i in range(n_agents)] + return Workflow(model=model, hw=hw, steps=steps, sys_tokens=sys_tokens, + carry_history=True, history_rounds=history_rounds, + round_size=n_agents, exclude_self=exclude_self, + gamma_v=0.10, tx_energy_per_message=0.0) diff --git a/agentic_ecal_pkg/calib.py b/agentic_ecal_pkg/calib.py new file mode 100644 index 0000000..b375b61 --- /dev/null +++ b/agentic_ecal_pkg/calib.py @@ -0,0 +1,52 @@ +"""Measured two-rate calibration coefficients (paper Eqs. 4-5, fitted counterpart).""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class TwoRateCalib: + """Two-rate coefficients (J/token) measured directly on the A100 sweeps. + + ``c_dec(b) = a_dec / b + kv_floor``. These are fitted, not derived: they are what a + least-squares fit of the two-rate call model to the measured per-query energies returns, averaged over + GPQA / GSM-Hard / MMLU-Hard under vLLM. They are the calibration counterpart of Eqs. (4)-(5), + and the validation figure uses them so that the figure tests the two-rate *form* independently + of the hardware derivation. + + Attributes: + name (str): Human-readable model name. + c_pre (float): Prefill coefficient, in Joules per prompt token. + a_dec (float): Decode weight-read term, in Joules (amortised over the batch). + kv_floor (float): Per-token KV-cache floor, in Joules, reached at large batch. Defaults + to ``0.0``. + """ + name: str + c_pre: float + a_dec: float + kv_floor: float = 0.0 + + def c_dec(self, batch: float) -> float: + """Decode rate at a given serving batch. + + Args: + batch (float): Serving batch size ``b`` (clamped to a minimum of 1). + + Returns: + float: ``a_dec / max(b, 1) + kv_floor``, in Joules per generated token. + """ + return self.a_dec / max(float(batch), 1.0) + self.kv_floor + + def energy(self, p_in: float, p_out: float, batch: float) -> float: + """Energy of one call under the two-rate model with measured coefficients. + + Args: + p_in (float): Prompt (prefill) tokens. + p_out (float): Generated (decode) tokens. + batch (float): Serving batch size ``b``. + + Returns: + float: ``c_pre * p_in + c_dec(batch) * p_out``, in Joules. + """ + return self.c_pre * p_in + self.c_dec(batch) * p_out diff --git a/agentic_ecal_pkg/cli.py b/agentic_ecal_pkg/cli.py new file mode 100644 index 0000000..648794b --- /dev/null +++ b/agentic_ecal_pkg/cli.py @@ -0,0 +1,83 @@ +"""Command-line entry point: the model's self-consistency demo, with an optional ``--json`` mode. + +The default (human) output is byte-for-byte identical to running the original +``agentic_ecal.py`` module directly, so it doubles as a reproduction check. +""" + +from __future__ import annotations + +import argparse +import json + +from .components import Step +from .registries import HW, LLMS +from .rates import llm_call_energy +from .validation import validate_two_rate, validate_against_samsi, validate_current_gen +from .workflow import Workflow + + +def _demo_numbers() -> dict: + """Compute the values the demo reports, so the human and ``--json`` paths agree. + + Returns: + dict: ``{"two_rate": ..., "anchors": ..., "reduce": ...}`` with the derived-vs-measured + coefficients, the literature-anchor implied batches, and the one-shot reduction check. + """ + m, hw = LLMS["llama3_8b"], HW["h100"] + wf = Workflow(model=m, hw=hw, steps=[Step(p_in_local=600, p_out=300)], + sys_tokens=0, carry_history=True, gamma_v=0.0, serving_batch=64) + direct = llm_call_energy(m, hw, 600, 300, 64) + one_shot = wf.run()["e_total"] + return { + "two_rate": validate_two_rate(), + "anchors": {**validate_against_samsi(), **validate_current_gen()}, + "reduce": { + "one_shot_J": one_shot, + "direct_J": direct, + "match": abs(one_shot - direct) < 1e-6, + "agentic_ecal_J_per_bit": wf.agentic_ecal(), + }, + } + + +def _print_human() -> None: + """Emit the original ``__main__`` output verbatim.""" + print("[two-rate] derived vs measured coefficients (A100, nothing fitted):") + for name, v in validate_two_rate().items(): + print(f" {name:<13} c_pre {v['c_pre'][0]:.4f} vs {v['c_pre'][1]} " + f"a_dec {v['a_dec'][0]} vs {v['a_dec'][1]}") + print("\n[anchors] what serving batch each literature figure implies:") + for k, v in {**validate_against_samsi(), **validate_current_gen()}.items(): + print(f" {k}: {v}") + + m, hw = LLMS["llama3_8b"], HW["h100"] + wf = Workflow(model=m, hw=hw, steps=[Step(p_in_local=600, p_out=300)], + sys_tokens=0, carry_history=True, gamma_v=0.0, serving_batch=64) + direct = llm_call_energy(m, hw, 600, 300, 64) + one_shot = wf.run()["e_total"] + print(f"\n[reduce] one-shot workflow {one_shot:.2f} J == direct call {direct:.2f} J: " + f"{abs(one_shot - direct) < 1e-6}") + print(f" agentic-eCAL (one-shot, b=64): {wf.agentic_ecal():.2e} J/bit") + + +def main(argv=None) -> None: + """Entry point for ``agentic-ecal`` / ``python -m agentic_ecal_pkg``. + + Args: + argv (Optional[list[str]]): Argument vector to parse. Defaults to ``None`` (uses + ``sys.argv``). + """ + parser = argparse.ArgumentParser( + prog="agentic-ecal", + description="Two-rate energy model for agentic LLM workflows (self-consistency demo).") + parser.add_argument("--json", action="store_true", + help="emit the demo values as machine-readable JSON instead of text") + args = parser.parse_args(argv) + if args.json: + print(json.dumps(_demo_numbers(), indent=2)) + else: + _print_human() + + +if __name__ == "__main__": + main() diff --git a/agentic_ecal_pkg/components.py b/agentic_ecal_pkg/components.py new file mode 100644 index 0000000..4662673 --- /dev/null +++ b/agentic_ecal_pkg/components.py @@ -0,0 +1,114 @@ +"""Workflow components: tools, retrieval, and a single step executed by an agent.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Optional + +from .descriptors import Hardware +from .rates import embedding_flops + + +@dataclass +class Tool: + """A non-LLM tool invocation (code exec, API, search). + + Attributes: + name (str): Human-readable tool name. + energy (float): Energy per call, in Joules. Defaults to ``0.0``. + obs_tokens (int): Tokens the tool result adds to the running context. Defaults to ``0``. + """ + name: str + energy: float = 0.0 + obs_tokens: int = 0 # tokens the tool result adds to the running context + + +@dataclass +class Retrieval: + """RAG retrieval step: query embedding, vector-index search, and injection of k chunks. + + The embedder is a distinct bi-encoder rather than the generator. The default values + correspond to a BERT-base-class encoder: bge-base-en-v1.5, e5-base and all-mpnet-base-v2 each + comprise approximately 109M parameters at 768 dimensions. Accordingly, ``dim`` bears no + relation to the generator's hidden size (4096 for Llama-3 8B, 3584 for Qwen2.5-7B), and + ``chunk_tokens`` remains within the 512-token input limit of such an encoder. + + The energy of retrieval itself is negligible: 0.199 J, or 0.013% of the case-study workflow. + The dominant cost is the ``added_context`` injected into each subsequent prompt. Those same + 1280 tokens incur approximately 300 J of prefill in the generator, some three orders of + magnitude above the retrieval term. ``k_chunks`` and ``chunk_tokens``, expressed in the + generator's tokenizer, are therefore the only parameters of material consequence: + substituting bge-large for MiniLM alters the workflow total by less than 0.05%, and + ``n_vectors`` is inert. + + Two simplifications follow from this insensitivity and are recorded rather than corrected. + The encoder is priced at the generator's accelerator and utilisation, which a model of 110M + parameters would not attain. Only the query embedding is charged; embedding the corpus is a + genuine cost, but one incurred offline and amortised across all queries. + + Attributes: + embed_params (float): Bi-encoder parameters (BERT-base class, approx. 109M). Defaults to + ``1.1e8``. + query_tokens (int): Tokens in the query to embed. Defaults to ``64``. + n_vectors (float): Index size; immaterial to the result. Defaults to ``1e6``. + dim (int): Bi-encoder embedding width (not the generator hidden size). Defaults to ``768``. + k_chunks (int): Number of retrieved chunks appended to the prompt. Defaults to ``5``. + chunk_tokens (int): Tokens per chunk, in the generator's tokenizer. Defaults to ``256``. + ann (bool): Approximate (sublinear) search if ``True``, else exhaustive. Both are + dominated by the embedding term. Defaults to ``True``. + """ + embed_params: float = 1.1e8 # bi-encoder parameters (BERT-base class, approx. 109M) + query_tokens: int = 64 + n_vectors: float = 1e6 # index size; immaterial to the result (see above) + dim: int = 768 # bi-encoder embedding width, not the generator hidden size + k_chunks: int = 5 # k_chunks * chunk_tokens is appended to the prompt, + chunk_tokens: int = 256 # expressed in the generator's tokenizer + ann: bool = True # approximate or exhaustive search; both are dominated + # by the embedding term + + def energy(self, hw: Hardware) -> float: + """Energy of the retrieval step (query embedding + index search). + + Args: + hw (Hardware): The accelerator the encoder is priced on. + + Returns: + float: Retrieval energy, in Joules (dominated by the embedding term). + """ + embed_flops = embedding_flops(self.embed_params, self.query_tokens) + if self.ann: + # HNSW-style: ~ dim * log2(n_vectors) distance ops, *2 FLOPs each + search_flops = 2.0 * self.dim * math.log2(max(self.n_vectors, 2.0)) + else: + search_flops = 2.0 * self.dim * self.n_vectors + return (embed_flops + search_flops) / hw.flops_eff * hw.power + + @property + def added_context(self) -> int: + """Tokens injected into the prompt by this retrieval. + + Returns: + int: ``k_chunks * chunk_tokens``. + """ + return self.k_chunks * self.chunk_tokens + + +@dataclass +class Step: + """One workflow step executed by an agent: an LLM call plus optional tool/retrieval. + + Attributes: + agent (str): Name of the agent executing this step. Defaults to ``"main"``. + p_in_local (int): *New* prompt tokens contributed at this step, excluding carried + history. Defaults to ``0``. + p_out (int): Generated (decode) tokens. Defaults to ``256``. + tool (Optional[Tool]): A tool invoked after the call, if any. Defaults to ``None``. + retrieval (Optional[Retrieval]): A retrieval performed before the call, if any. Defaults + to ``None``. + """ + agent: str = "main" + p_in_local: int = 0 # *new* prompt tokens contributed at this step (excl. carried history) + p_out: int = 256 # generated tokens + tool: Optional[Tool] = None + retrieval: Optional[Retrieval] = None diff --git a/agentic_ecal_pkg/constants.py b/agentic_ecal_pkg/constants.py new file mode 100644 index 0000000..09bef36 --- /dev/null +++ b/agentic_ecal_pkg/constants.py @@ -0,0 +1,26 @@ +"""Grounded numeric constants for the agentic-eCAL model. + +Every value here is literature-grounded; see ``docs/research-notes.md`` for the source of each. +All energies are in Joules unless noted. +""" + +from __future__ import annotations + +# --------------------------------------------------------------------------- +# Token -> bit conversion +# --------------------------------------------------------------------------- +# +# f is the information-theoretic capacity log2(|V|) of a token drawn from a vocabulary of size +# |V|: ~17 bits for the 128k-152k vocabularies of Llama-3 / Qwen-2.5. It is a pure denominator +# scale, so it cancels from every scaling exponent and sets only the absolute J/bit level. + +BITS_PER_TOKEN = 17.0 + +# MFU_LATENCY / MFU_THROUGHPUT are residues of the retired single-rate model. Nothing in the +# two-rate path reads them; they survive only through Hardware.mfu, which prices the retrieval +# term. The throughput value's old comment claimed 0.25 "reproduces ~0.39 J/token Llama-3 70B", +# which was a calibration claim about the model that no longer exists (see validate_current_gen). +MFU_LATENCY = 0.05 # single-stream / latency-bound decode +MFU_THROUGHPUT = 0.25 # throughput-optimized batched serving + +GWH_TO_J = 3.6e12 # 1 GWh = 3.6e12 J diff --git a/agentic_ecal_pkg/descriptors.py b/agentic_ecal_pkg/descriptors.py new file mode 100644 index 0000000..12a7452 --- /dev/null +++ b/agentic_ecal_pkg/descriptors.py @@ -0,0 +1,104 @@ +"""Hardware and model descriptors -- the leaf dataclasses the rest of the package builds on.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + + +# --------------------------------------------------------------------------- +# Hardware / model descriptors +# --------------------------------------------------------------------------- + +@dataclass +class Hardware: + """Accelerator descriptor for the two-rate energy model. + + ``mfu`` (model-FLOP utilization, achieved/peak) is the calibration knob. Single-stream, + latency-bound decode is memory-bound and reaches only ~5% MFU; throughput-optimized batched + serving reaches ~30-50%. We calibrate it against measured per-token energy (research-notes.md) + rather than asserting peak FLOP/s, because estimating from nameplate TDP overestimates measured + energy by up to 4.1x (ML.ENERGY, arXiv:2505.06371). + + Attributes: + name (str): Human-readable accelerator name. + flops_peak (float): Peak dense FLOP/s at the working precision (fp16/bf16). + power (float): Board power drawn while busy, in Watts. + mfu (float): Model-FLOP utilization (achieved/peak); prices the retrieval term. Defaults + to ``0.10`` (mid latency/throughput regime). + bandwidth (float): Peak HBM bandwidth ``B_HBM``, in byte/s. Required by the decode rate. + eta_pre (float): Prefill utilization; compute-bound, so near peak. Back-solving the fitted + ``c_pre`` of ``TWO_RATE`` gives 0.74 (Llama-3.1-8B) and 0.85 (Qwen2.5-7B) on the A100 + sweeps. Defaults to ``0.85``. + bw_efficiency (float): ``eps``, the sustained fraction of peak HBM bandwidth (model- + bandwidth utilization). Measured at ~55-72% at batch 1; degrades with batch. Defaults + to ``0.70``. + """ + name: str + flops_peak: float # peak dense FLOP/s at the working precision + power: float # board power drawn while busy [W] + mfu: float = 0.10 # default: mid latency/throughput regime + bandwidth: float = 0.0 # B_HBM, peak HBM bandwidth [byte/s] + eta_pre: float = 0.85 # prefill utilisation; compute-bound, so near peak. Back-solving + # the fitted c_pre of TWO_RATE gives 0.74 (Llama-3.1-8B) and + # 0.85 (Qwen2.5-7B) on the A100 sweeps. + bw_efficiency: float = 0.70 # eps, sustained fraction of peak HBM bandwidth. Reported MBU + # is 72% (pytorch.org/blog/accelerating-generative-ai-2) and + # 55-60% (databricks.com/blog/llm-inference-performance- + # engineering-best-practices), both at batch 1. See the + # two-rate comment block for why it degrades with batch. + + @property + def flops_eff(self) -> float: + """Effective (achieved) FLOP/s. + + Returns: + float: ``flops_peak * mfu``, the utilisation-derated throughput. + """ + return self.flops_peak * self.mfu + + +@dataclass +class LLM: + """Open-weight decoder-only LLM descriptor. + + Attributes: + name (str): Human-readable model name. + n_params (float): Total parameter count. + n_layers (int): Number of transformer blocks. + d_model (int): Hidden size. + e_pretrain (float): Embodied (pre-training) energy in Joules; ``0.0`` if unknown. + n_active (Optional[float]): Active (non-embedding) parameters per token. Defaults to + ``n_params`` for dense models; for Mixture-of-Experts, the routed/active subset. + d_kv (Optional[int]): KV width ``n_kv_heads * head_dim``. Defaults to ``d_model`` + (multi-head attention). + bytes_per_param (float): ``beta``, the element size of weights and cache in bytes + (bf16 = 2). Defaults to ``2.0``. + """ + name: str + n_params: float # total parameters + n_layers: int # transformer blocks + d_model: int # hidden size + e_pretrain: float = 0.0 # embodied (pre-training) energy [J], 0 if unknown + + # Active (non-embedding) parameters per token. For dense models == n_params; + # for Mixture-of-Experts this is the routed/active subset. + n_active: Optional[float] = None + d_kv: Optional[int] = None # n_kv_heads * head_dim; equals d_model under multi-head + bytes_per_param: float = 2.0 # beta, element size of weights and cache (bf16 = 2) + + def __post_init__(self) -> None: + if self.n_active is None: + self.n_active = self.n_params + if self.d_kv is None: + self.d_kv = self.d_model + + @property + def kv_bytes_per_token(self) -> float: + """KV-cache traffic coefficient ``gamma``. + + Returns: + float: KV-cache bytes read per decoded token, per token of context + (``2 * n_layers * d_kv * bytes_per_param``). + """ + return 2.0 * self.n_layers * self.d_kv * self.bytes_per_param diff --git a/agentic_ecal_pkg/rates.py b/agentic_ecal_pkg/rates.py new file mode 100644 index 0000000..9060bce --- /dev/null +++ b/agentic_ecal_pkg/rates.py @@ -0,0 +1,142 @@ +"""The two-rate call model (paper Eqs. 3-5): per-token energy rates, call energy, and latency. + +A call has two phases with different energy characteristics, so one rate cannot describe both. + + E_call(p_in, p_out; b) = c_pre * p_in + c_dec(b) * p_out (the two-rate model) + +PREFILL processes the whole prompt in one parallel pass. It is compute-bound at near-peak +utilisation and essentially batch-independent: + + c_pre = 2 N P / (eta_pre * Pi) (the prefill rate) + +DECODE emits one token at a time and is memory-bandwidth-bound. Each step streams the weight +bytes from HBM once for the whole batch, so that cost amortises over the b concurrently +decoding sequences, while every sequence re-reads its own cache and that term does not: + + c_dec(b) = P / (eps B_HBM) * ( N beta / b + gamma * cbar ) (the decode rate) + +eps is the fraction of peak HBM bandwidth a serving stack actually sustains, reported as +model-bandwidth utilisation, (achieved bandwidth) / (peak bandwidth). See ``Hardware.bw_efficiency`` +for the measured values and the batch-degradation caveats. +""" + +from __future__ import annotations + +from .descriptors import Hardware, LLM + + +def embedding_flops(n_params: float, n_tokens: float) -> float: + """Forward FLOPs of a small encoder, used for the retrieval term. + + Args: + n_params (float): Encoder parameter count ``N``. + n_tokens (float): Tokens embedded ``p_q``. + + Returns: + float: ``2 * N * p_q``, the forward FLOPs. + """ + return 2.0 * n_params * n_tokens + + +def prefill_rate(model: LLM, hw: Hardware) -> float: + """Prefill energy rate ``c_pre``: compute-bound and batch-independent. + + Args: + model (LLM): The model descriptor. + hw (Hardware): The accelerator descriptor. + + Returns: + float: Joules per prompt token, ``2 N P / (eta_pre * Pi)``. + """ + return 2.0 * model.n_active * hw.power / (hw.eta_pre * hw.flops_peak) + + +def decode_rate(model: LLM, hw: Hardware, batch: float = 1.0, context: float = 0.0) -> float: + """Decode energy rate ``c_dec(b)``: memory-bound, amortising over the batch. + + Args: + model (LLM): The model descriptor. + hw (Hardware): The accelerator descriptor (must have ``bandwidth`` set). + batch (float): Serving batch size ``b`` (clamped to a minimum of 1). Defaults to ``1.0``. + context (float): Mean context length the KV term is evaluated at. Defaults to ``0.0``. + + Returns: + float: Joules per generated token. + + Raises: + ValueError: If ``hw.bandwidth`` is not set (``<= 0``). + """ + if hw.bandwidth <= 0.0: + raise ValueError(f"{hw.name} has no HBM bandwidth set") + weights = model.n_active * model.bytes_per_param / max(float(batch), 1.0) + cache = model.kv_bytes_per_token * context + return hw.power * (weights + cache) / (hw.bandwidth * hw.bw_efficiency) + + +def call_latency(model: LLM, hw: Hardware, p_in: float, p_out: float, + batch: float = 1.0) -> float: + """Wall-clock time of one call, which is not energy divided by power. + + Energy per request divides by the serving batch, because the weight read of a decode step is + shared by every sequence in flight. Latency does not: the step occupies its full duration for + all of them. Prefill is one compute-bound pass; each decode step costs the weight read plus + the cache read of the whole batch. + + Args: + model (LLM): The model descriptor. + hw (Hardware): The accelerator descriptor. + p_in (float): Prompt (prefill) tokens. + p_out (float): Generated (decode) tokens. + batch (float): Serving batch size ``b`` (clamped to a minimum of 1). Defaults to ``1.0``. + + Returns: + float: Wall-clock time of the call, in seconds. + """ + b = max(float(batch), 1.0) + cbar = p_in + p_out / 2.0 + t_pre = 2.0 * model.n_active * p_in / (hw.eta_pre * hw.flops_peak) + step = (model.n_active * model.bytes_per_param + + b * model.kv_bytes_per_token * cbar) / (hw.bandwidth * hw.bw_efficiency) + return t_pre + p_out * step + + +def llm_call_energy(model: LLM, hw: Hardware, p_in: float, p_out: float, + batch: float = 1.0) -> float: + """Energy of one LLM call under the two-rate model, from datasheet quantities. + + The rates come from :func:`prefill_rate` / :func:`decode_rate` rather than from the measured + ``TWO_RATE`` coefficients, so this is a derivation rather than a calibration. The figures use + the measured path instead; see ``two_rate_energy_per_query`` in ``make_figures.py``. + + The cache term is evaluated at the call's mean context, ``p_in + p_out/2``, since the + transcript grows by one token per decode step. + + Args: + model (LLM): The model descriptor. + hw (Hardware): The accelerator descriptor. + p_in (float): Prompt (prefill) tokens. + p_out (float): Generated (decode) tokens. + batch (float): Serving batch size ``b``. Defaults to ``1.0``. + + Returns: + float: Energy of the call, in Joules. + """ + cbar = p_in + p_out / 2.0 + return prefill_rate(model, hw) * p_in + decode_rate(model, hw, batch, cbar) * p_out + + +def joules_per_token(model: LLM, hw: Hardware, p_in: float = 512, p_out: float = 512, + batch: float = 1.0) -> float: + """Energy per generated token for a representative call. + + Args: + model (LLM): The model descriptor. + hw (Hardware): The accelerator descriptor. + p_in (float): Prompt (prefill) tokens. Defaults to ``512``. + p_out (float): Generated (decode) tokens. Defaults to ``512``. + batch (float): Serving batch size ``b``. Defaults to ``1.0``. + + Returns: + float: ``llm_call_energy(...) / p_out``, in Joules per generated token. + """ + return llm_call_energy(model, hw, p_in, p_out, batch) / p_out diff --git a/agentic_ecal_pkg/registries.py b/agentic_ecal_pkg/registries.py new file mode 100644 index 0000000..3232c9b --- /dev/null +++ b/agentic_ecal_pkg/registries.py @@ -0,0 +1,67 @@ +"""Representative hardware, model, and measured-calibration presets. + +Kept as Python literals (not externalised YAML) so the numbers are bit-exact and the package +stays dependency-free. Specs/architectures are from vendor datasheets and official model cards; +see ``docs/research-notes.md`` for the source of each value. +""" + +from __future__ import annotations + +from .constants import GWH_TO_J, MFU_LATENCY +from .descriptors import Hardware, LLM +from .calib import TwoRateCalib + + +# Representative accelerators. +# H100 SXM bf16 dense ~989 TFLOP/s (with sparsity ~1979); board power ~700 W. +# A100 80GB bf16 dense ~312 TFLOP/s; board power ~400 W (SXM). +HW = { + "h100": Hardware("NVIDIA H100 SXM", flops_peak=989e12, power=700.0, mfu=MFU_LATENCY, + bandwidth=3.35e12), + # H100 NVL, the board TokenPowerBench actually ran on. Their released results record + # gpu_memory_total_mb = 95830 (94 GB, against the SXM's 80) and a per-GPU draw that caps at + # 394.5 W over all 185 runs, so the 700 W / 3.35 TB/s SXM entry above misprices every number + # taken from that source by ~2x. NVL: 94 GB HBM3, 3.9 TB/s, 400 W board cap. + "h100_nvl": Hardware("NVIDIA H100 NVL 94GB", flops_peak=989e12, power=400.0, + mfu=MFU_LATENCY, bandwidth=3.9e12), + "a100": Hardware( "NVIDIA A100 80GB", flops_peak=312e12, power=400.0, mfu=MFU_LATENCY, + bandwidth=2.039e12), +} + + +# Representative open-weight models. Architectures from official model cards; pre-training +# (embodied) energy = GPU-hours x 700 W from the Llama-3 model card (research-notes.md): +# 8B : 1.3M H100-h x 700 W = 0.91 GWh = 3.28e12 J +# 70B : 6.4M H100-h x 700 W = 4.48 GWh = 1.61e13 J +LLMS = { + # Llama-3 8B / Llama-3.1-8B-Instruct: 32 layers, d_model 4096. Also the measured model. + # GQA: 8 KV heads x head_dim 128 -> d_kv = 1024. + "llama3_8b": LLM("Llama-3 8B", n_params=8.03e9, n_layers=32, d_model=4096, + e_pretrain=0.91 * GWH_TO_J, d_kv=1024), + # Llama-3 70B: 80 layers, d_model 8192. + "llama3_70b": LLM("Llama-3 70B", n_params=70.6e9, n_layers=80, d_model=8192, + e_pretrain=4.48 * GWH_TO_J, d_kv=1024), + # LLaMA-65B: 80 layers, d_model 8192. Retained only to back validate_against_samsi(); it was + # dropped from Table II and from the amortization figure. No explicit d_kv, so d_kv = d_model + # = 8192 -- multi-head attention, correct for LLaMA-1 and 8x the 70B's GQA width. + # 65B : 1,022,362 A100-h x 400 W = 0.409 GWh = 1.47e12 J + # Same convention as the Llama-3 entries above (GPU-hours x TDP, no PUE), which is why this + # is below the 449 MWh the LLaMA paper reports for the same run. + "llama_65b": LLM("LLaMA 65B", n_params=65.2e9, n_layers=80, d_model=8192, + e_pretrain=0.409 * GWH_TO_J), + # Qwen2.5-7B-Instruct: 28 layers, d_model 3584 = 28 query heads x head_dim 128. GQA with + # 4 KV heads, which cuts KV-cache traffic 7x but NOT the attention FLOPs below (every query + # head still scores against the full key sequence), so d_model is the right width here. + # The second measured model; the notebook referenced this key but it was missing from the dict. + # GQA: 4 KV heads x 128 -> d_kv = 512, half of Llama-3.1-8B despite similar size. + "qwen2_5_7b": LLM("Qwen2.5-7B", n_params=7.615e9, n_layers=28, d_model=3584, + e_pretrain=0.0, d_kv=512), +} + + +# Both models keep a decode floor at large batch, where the weight read has amortised away and +# the per-token KV traffic is what is left. Llama's wider KV puts its floor ~3x above Qwen's. +TWO_RATE = { + "qwen2_5_7b": TwoRateCalib("Qwen2.5-7B", c_pre=0.023, a_dec=3.2, kv_floor=0.02), + "llama3_8b": TwoRateCalib("Llama-3.1-8B", c_pre=0.028, a_dec=3.3, kv_floor=0.06), +} diff --git a/agentic_ecal_pkg/validation.py b/agentic_ecal_pkg/validation.py new file mode 100644 index 0000000..6271a42 --- /dev/null +++ b/agentic_ecal_pkg/validation.py @@ -0,0 +1,108 @@ +"""Calibration checks against the measured anchors in the literature. + +``implied_batch`` is a package-internal helper (not part of the public API) used by the two +current-generation validators; it is the honest inverse of the two-rate model -- given a measured +J/output-token anchor, it reports the serving batch that anchor implies. +""" + +from __future__ import annotations + +from .descriptors import Hardware, LLM +from .registries import HW, LLMS +from .rates import prefill_rate, joules_per_token + + +def validate_two_rate() -> dict: + """Derived rates against the coefficients a two-rate fit obtains from the A100 sweeps. + + Nothing here is fitted: P is the board power, Pi and B_HBM are datasheet, eta_pre is the + near-peak prefill utilisation. Measured values are c_pre = 0.023 (Qwen) / 0.028 (Llama) + J per prompt token and a_dec = 3.2 / 3.3 J, from the vLLM sweeps on an A100. + + Returns: + dict: Mapping model name to ``{"c_pre": (derived, measured), "a_dec": (derived, + measured)}`` tuples. + """ + out = {} + for key, c_pre_meas, a_dec_meas in (("qwen2_5_7b", 0.023, 3.2), ("llama3_8b", 0.028, 3.3)): + m, hw = LLMS[key], HW["a100"] + a_dec = hw.power * m.n_active * m.bytes_per_param / (hw.bandwidth * hw.bw_efficiency) + out[m.name] = { + "c_pre": (round(prefill_rate(m, hw), 4), c_pre_meas), + "a_dec": (round(a_dec, 2), a_dec_meas), + } + return out + + +def implied_batch(model: LLM, hw: Hardware, j_per_token: float, + p_in: float = 64, p_out: float = 512) -> float: + """Serving batch at which the two-rate model reproduces a measured J/output-token anchor. + + A batch-aware call model should not be tuned to hit a literature number; it should say what + operating point that number corresponds to. Solving the two-rate model for b is the honest use of it. + + Args: + model (LLM): The model descriptor. + hw (Hardware): The accelerator descriptor. + j_per_token (float): The measured energy anchor, in Joules per output token. + p_in (float): Prompt tokens assumed for the anchor. Defaults to ``64``. + p_out (float): Output tokens assumed for the anchor. Defaults to ``512``. + + Returns: + float: The serving batch that reproduces the anchor, or ``inf`` when the residual is + non-positive (the anchor is at or below the ``b -> inf`` floor). + """ + cbar = p_in + p_out / 2.0 + fixed = prefill_rate(model, hw) * p_in / p_out \ + + hw.power * model.kv_bytes_per_token * cbar / (hw.bandwidth * hw.bw_efficiency) + weight_term = hw.power * model.n_active * model.bytes_per_param \ + / (hw.bandwidth * hw.bw_efficiency) + residual = j_per_token - fixed + return weight_term / residual if residual > 0 else float("inf") + + +def validate_against_samsi() -> dict: + """Samsi et al. (2023) measured 3-4 J/token for LLaMA-65B on A100s. + + Retained as a diagnostic only: the Table II row this used to back was DROPPED, because a 2023 + sweep billing 8-32 shards at 300 W-1 kW aggregate sat far below the bandwidth roofline the + decode rate assumes. See table2-row1-options.md. + + Under the decode rate the b = 1 limit is P N beta / B_HBM = 25.6 J/token, so that anchor cannot be a + single-stream measurement; it implies a serving batch of roughly 7-9. Tensor parallelism does + not change this, since sharding over n devices divides the bytes each reads by n while + multiplying board power by n, leaving P W / B invariant. + + Returns: + dict: The b=1 J/token figure and the serving batches implied by the 3 and 4 J/token + anchors. + """ + m, hw = LLMS["llama_65b"], HW["a100"] + return {"J/token at b=1": round(joules_per_token(m, hw, 64, 512, 1.0), 1), + "implied batch for 3 J/token": round(implied_batch(m, hw, 3.0), 1), + "implied batch for 4 J/token": round(implied_batch(m, hw, 4.0), 1)} + + +def validate_current_gen() -> dict: + """2026 anchors from TokenPowerBench, on the board it actually ran (H100 NVL). + + The 0.70 J/token used here for the 70B and the 0.39 it replaced are BOTH figure reads of that + paper, from different panels; 0.39 is not fabricated, contrary to an earlier note in + ``table2-row2-options.md``. 0.70 is corroborated by the paper's own text: it reports energy + per token rising 7.3x from Llama3-1B to 70B, and its Fig. 4 puts the 1B near 0.10 J/token. + Which panel 0.39 came from is not yet settled -- the engine comparison is the likely source, + since the same text reports TensorRT-LLM and vLLM cutting energy per token 25-40% below + Transformers, and 0.70 x 0.6 = 0.42. + + The 8B anchor below is not a figure read at all: it is the released per-run data, 0.084 + J/token on the active device at batch 32. + + Returns: + dict: The serving batches implied by the 70B (0.70 J/token) and 8B (0.084 J/token) + anchors on the H100 NVL. + """ + hw = HW["h100_nvl"] + return {"llama3_70b implied batch at 0.70 J/token (Fig. 3b)": + round(implied_batch(LLMS["llama3_70b"], hw, 0.70), 1), + "llama3_8b implied batch at 0.084 J/token (released runs, active GPU)": + round(implied_batch(LLMS["llama3_8b"], hw, 0.084, p_in=50, p_out=382), 1)} diff --git a/agentic_ecal_pkg/workflow.py b/agentic_ecal_pkg/workflow.py new file mode 100644 index 0000000..5234103 --- /dev/null +++ b/agentic_ecal_pkg/workflow.py @@ -0,0 +1,185 @@ +"""The ``Workflow`` aggregator: walk steps, price each call, return the energy breakdown.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import List, Optional + +from .constants import BITS_PER_TOKEN +from .descriptors import Hardware, LLM +from .components import Step +from .calib import TwoRateCalib +from .rates import prefill_rate, decode_rate, call_latency + + +@dataclass +class Workflow: + """An agentic workflow over an open-weight model. + + ``carry_history`` toggles whether each step's prompt accumulates the running transcript + (the dominant driver of super-linear token growth in agent loops). + + Three further fields describe *how much* transcript is retained, because all-or-nothing does + not describe measured agent protocols. Steps are grouped into rounds of ``round_size``; a step + sees the outputs of the ``history_rounds`` most recent *completed* rounds, and never those of + its own round (agents within a round act in parallel). + + ``round_size`` steps per round; 0 or 1 means every step is its own round. + ``history_rounds`` rounds of transcript retained. None retains all of them, which with + ``round_size`` = 1 is the original accumulating behaviour; 1 retains only + the previous round; 0 retains none, equivalent to carry_history = False. + ``exclude_self`` within the retained window, skip outputs the same agent produced. This is + the difference between an agent re-reading its own reasoning and a debate + participant reading only its N-1 peers. + + The defaults reproduce the original semantics exactly. + + Attributes: + model (LLM): The model every step runs on. + hw (Hardware): The accelerator every step runs on. + steps (List[Step]): The ordered steps of the workflow. Defaults to an empty list. + sys_tokens (int): System / tool-schema prompt prepended to every call. Defaults to ``400``. + carry_history (bool): Whether each step's prompt accumulates the running transcript. + Defaults to ``True``. + history_rounds (Optional[int]): Rounds of transcript retained; ``None`` retains every + completed round. Defaults to ``None``. + round_size (int): Steps per round (team width in a debate). Defaults to ``1``. + exclude_self (bool): Skip the agent's own output within the retained window. Defaults to + ``False``. + serving_batch (float): Decode batch ``b`` the decode rate amortises over. Defaults to + ``1.0``. + calib (Optional[TwoRateCalib]): Measured coefficients; ``None`` uses the derived Eqs. + (4)-(5). Defaults to ``None``. + gamma_v (float): Virtualization / orchestration overhead (eCAL's ``gamma_v``). Defaults + to ``0.10``. + tx_energy_per_message (float): Inter-agent transmission energy per hand-off, in Joules. + Defaults to ``0.0``. + """ + model: LLM + hw: Hardware + steps: List[Step] = field(default_factory=list) + sys_tokens: int = 400 # system / tool-schema prompt prepended to every call + carry_history: bool = True + history_rounds: Optional[int] = None # None = every completed round + round_size: int = 1 # steps per round (team width in a debate) + exclude_self: bool = False # skip the agent's own output within the window + serving_batch: float = 1.0 # b of the decode rate; decode amortises over it + calib: Optional["TwoRateCalib"] = None # measured coefficients; None uses Eqs. (4)-(5) + gamma_v: float = 0.10 # virtualization / orchestration overhead (eCAL's gamma_v) + tx_energy_per_message: float = 0.0 # inter-agent transmission energy [J] (OSI model) + + def run(self) -> dict: + """Compute the full energy breakdown of the workflow. + + Returns: + dict: A breakdown with keys (all energies in Joules, times in seconds, counts in + tokens): + + - ``e_llm``: total LLM-call energy. + - ``e_prefill``: prefill component of ``e_llm``. + - ``e_decode``: decode component of ``e_llm``. + - ``t_llm``: the request's share of device time. + - ``t_wall``: wall-clock time of the calls run back to back. + - ``e_tool``: total tool energy. + - ``e_retrieval``: total retrieval energy. + - ``e_transmission``: total inter-agent transmission energy. + - ``e_orchestration``: the ``gamma_v`` overhead on the core energy. + - ``e_total``: ``(1 + gamma_v)`` times the core energy. + - ``prefill_tokens``: total prompt tokens across all calls. + - ``decode_tokens``: total generated tokens across all calls. + - ``useful_output_tokens``: generated tokens counted as useful output. + """ + rs = max(self.round_size, 1) + transcript = [] # (round index, agent, tokens) for outputs and tool observations + e_llm = e_tool = e_ret = e_tx = 0.0 + e_prefill = e_decode = 0.0 # the two terms of the two-rate call model + t_llm = 0.0 # request's share of device time [s] + t_wall = 0.0 # wall-clock time of the calls, run back to back [s] + useful_out = 0 + prefill_tokens = decode_tokens = 0 + prev_agent = None + + for k, st in enumerate(self.steps): + rnd = k // rs + + # retrieval (RAG) before the call + added = 0 + if st.retrieval is not None: + e_ret += st.retrieval.energy(self.hw) + added += st.retrieval.added_context + + # carried transcript: completed rounds inside the retention window only + carried = 0 + if self.carry_history: + lo = 0 if self.history_rounds is None else rnd - self.history_rounds + carried = sum(tok for (r, ag, tok) in transcript + if lo <= r < rnd and not (self.exclude_self and ag == st.agent)) + + # prompt = system + carried history + new local prompt + retrieved context + p_in = self.sys_tokens + st.p_in_local + added + carried + + if self.calib is not None: + pre = self.calib.c_pre * p_in + dec = self.calib.c_dec(self.serving_batch) * st.p_out + else: + cbar = p_in + st.p_out / 2.0 + pre = prefill_rate(self.model, self.hw) * p_in + dec = decode_rate(self.model, self.hw, self.serving_batch, cbar) * st.p_out + e_prefill += pre + e_decode += dec + e_call = pre + dec + e_llm += e_call + t_llm += e_call / self.hw.power + t_wall += call_latency(self.model, self.hw, p_in, st.p_out, + self.serving_batch) + prefill_tokens += p_in + decode_tokens += st.p_out + useful_out += st.p_out + + # tool execution after the call + if st.tool is not None: + e_tool += st.tool.energy + transcript.append((rnd, st.agent, st.tool.obs_tokens)) + + # inter-agent hand-off transmission + if prev_agent is not None and st.agent != prev_agent: + e_tx += self.tx_energy_per_message + prev_agent = st.agent + + transcript.append((rnd, st.agent, st.p_out)) # joins the transcript + + e_core = e_llm + e_tool + e_ret + e_tx + e_total = (1.0 + self.gamma_v) * e_core + return { + "e_llm": e_llm, + "e_prefill": e_prefill, + "e_decode": e_decode, + "t_llm": t_llm, + "t_wall": t_wall, + "e_tool": e_tool, + "e_retrieval": e_ret, + "e_transmission": e_tx, + "e_orchestration": self.gamma_v * e_core, + "e_total": e_total, + "prefill_tokens": prefill_tokens, + "decode_tokens": decode_tokens, + "useful_output_tokens": useful_out, + } + + def agentic_ecal(self, bits_per_token: float = BITS_PER_TOKEN, + e_emb_amortized: float = 0.0) -> float: + """agentic-eCAL in J/bit: total workflow energy per useful output bit. + + Args: + bits_per_token (float): Information content per token. Defaults to + :data:`~agentic_ecal_pkg.constants.BITS_PER_TOKEN`. + e_emb_amortized (float): Share of the model's embodied (pre-training) energy + attributed to this single workflow invocation, in Joules. Defaults to ``0.0``. + + Returns: + float: ``(e_total + e_emb_amortized) / (bits_per_token * useful_output_tokens)``, in + Joules per bit; ``nan`` when there is no useful output. + """ + r = self.run() + useful_bits = bits_per_token * r["useful_output_tokens"] + return (r["e_total"] + e_emb_amortized) / useful_bits if useful_bits else float("nan") diff --git a/examples/basic_call.py b/examples/basic_call.py new file mode 100644 index 0000000..cee5c8f --- /dev/null +++ b/examples/basic_call.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Per-call and per-token energy for a preset model on preset hardware. + +Run: python examples/basic_call.py +""" + +import os +import sys + +# Make the package importable when running from a source checkout without installing it. +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import agentic_ecal_pkg as ae + + +def main() -> None: + model = ae.LLMS["llama3_8b"] + hw = ae.HW["a100"] + p_in, p_out = 512, 512 + + for batch in (1, 32, 256): + e = ae.llm_call_energy(model, hw, p_in, p_out, batch) + jpt = ae.joules_per_token(model, hw, p_in, p_out, batch) + t = ae.call_latency(model, hw, p_in, p_out, batch) + print(f"{model.name} on {hw.name}, batch={batch:>3}: " + f"{e:8.2f} J/call | {jpt:6.3f} J/token | {t:6.3f} s latency") + + +if __name__ == "__main__": + main() diff --git a/examples/case_study.py b/examples/case_study.py new file mode 100644 index 0000000..c06e125 --- /dev/null +++ b/examples/case_study.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""The paper's four-agent RAG case study: full energy breakdown and the agentic-eCAL metric. + +Run: python examples/case_study.py +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import agentic_ecal_pkg as ae + + +def main() -> None: + wf = ae.four_agent_rag_workflow() # Llama-3 8B on A100, single pass + r = wf.run() + + print("Four-agent RAG case study (planner -> retriever -> analyst -> writer)") + print(f" model / hardware : {wf.model.name} / {wf.hw.name}") + print(f" prefill tokens : {r['prefill_tokens']}") + print(f" decode tokens : {r['decode_tokens']}") + print() + for key in ("e_llm", "e_prefill", "e_decode", "e_tool", "e_retrieval", + "e_transmission", "e_orchestration", "e_total"): + print(f" {key:<16}: {r[key]:10.3f} J") + print() + print(f" agentic-eCAL : {wf.agentic_ecal():.4f} J/bit") + + +if __name__ == "__main__": + main() diff --git a/examples/custom_workflow.py b/examples/custom_workflow.py new file mode 100644 index 0000000..da46587 --- /dev/null +++ b/examples/custom_workflow.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Build a workflow by hand from Steps, a Tool, and a Retrieval, then price it. + +Run: python examples/custom_workflow.py +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import agentic_ecal_pkg as ae + + +def main() -> None: + steps = [ + ae.Step(agent="planner", p_in_local=300, p_out=200), + ae.Step(agent="searcher", p_in_local=120, p_out=150, + retrieval=ae.Retrieval(k_chunks=4, chunk_tokens=200)), + ae.Step(agent="coder", p_in_local=200, p_out=350, + tool=ae.Tool(name="python", energy=3.0, obs_tokens=180)), + ae.Step(agent="writer", p_in_local=150, p_out=400), + ] + wf = ae.Workflow( + model=ae.LLMS["qwen2_5_7b"], + hw=ae.HW["h100_nvl"], + steps=steps, + carry_history=True, + serving_batch=32, + calib=ae.TWO_RATE["qwen2_5_7b"], # use the measured coefficients + ) + r = wf.run() + print(f"Custom 4-step workflow on {wf.model.name} / {wf.hw.name} (batch {int(wf.serving_batch)})") + print(f" total energy : {r['e_total']:.2f} J") + print(f" useful output : {r['useful_output_tokens']} tokens") + print(f" agentic-eCAL : {wf.agentic_ecal():.4f} J/bit") + + # Compare with the derived (datasheet) rates instead of the measured calibration. + wf.calib = None + print(f" agentic-eCAL (derived rates): {wf.agentic_ecal():.4f} J/bit") + + +if __name__ == "__main__": + main() diff --git a/figure_scripts/_pkg_path.py b/figure_scripts/_pkg_path.py new file mode 100644 index 0000000..8f4d353 --- /dev/null +++ b/figure_scripts/_pkg_path.py @@ -0,0 +1,16 @@ +"""Put the repository root on ``sys.path`` so ``import agentic_ecal_pkg`` resolves. + +The figure scripts live in ``figure_scripts/`` while the package lives one level up at the +repository root in ``agentic_ecal_pkg/``. Importing this module (before importing the package) +adds the repo root to the import path. Import it for its side effect: + + import _pkg_path # noqa: F401 + import agentic_ecal_pkg as ae +""" + +import os +import sys + +_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if _ROOT not in sys.path: + sys.path.insert(0, _ROOT) diff --git a/scripts/clean/fig_topologies.py b/figure_scripts/fig_topologies.py similarity index 99% rename from scripts/clean/fig_topologies.py rename to figure_scripts/fig_topologies.py index 74d9bb9..c0625ae 100644 --- a/scripts/clean/fig_topologies.py +++ b/figure_scripts/fig_topologies.py @@ -13,7 +13,7 @@ def _paper_figures(): """Newest vN/figures alongside the repo; the manuscript is maintained outside it.""" - root = Path(__file__).resolve().parents[2] + root = Path(__file__).resolve().parents[1] vs = sorted((d for d in root.iterdir() if d.is_dir() and re.fullmatch(r"v\d+", d.name)), key=lambda d: int(d.name[1:])) diff --git a/scripts/clean/fit_peer_law.py b/figure_scripts/fit_peer_law.py similarity index 100% rename from scripts/clean/fit_peer_law.py rename to figure_scripts/fit_peer_law.py diff --git a/scripts/clean/make_figure_capacity.py b/figure_scripts/make_figure_capacity.py similarity index 97% rename from scripts/clean/make_figure_capacity.py rename to figure_scripts/make_figure_capacity.py index 0574b69..e348a5f 100644 --- a/scripts/clean/make_figure_capacity.py +++ b/figure_scripts/make_figure_capacity.py @@ -25,7 +25,8 @@ matplotlib.use("Agg") import matplotlib.pyplot as plt -import agentic_ecal as ae +import _pkg_path # noqa: F401 (puts repo root on sys.path) +import agentic_ecal_pkg as ae import model_placement as mp import placement as pl import osi @@ -34,7 +35,7 @@ def _paper_figures(default="figures"): """Newest vN/figures alongside the repo, or repo-root figures/ if none exists.""" - root = os.path.dirname(os.path.dirname(HERE)) + root = os.path.dirname(HERE) vs = sorted((d for d in os.listdir(root) if re.fullmatch(r"v\d+", d) and os.path.isdir(os.path.join(root, d))), key=lambda d: int(d[1:])) @@ -179,8 +180,8 @@ def main(out_dir=OUT_DEFAULT): ax[2].set_ylabel("inter-agent edges", fontsize=7) ax[2].set_title("(c) topology as a traffic lever", fontsize=8) ax[2].tick_params(labelsize=6.5); ax[2].grid(alpha=0.25, lw=0.4) - ax[2].annotate("six architectures\ncoincide at $N-1$", xy=(15, 14), xytext=(5.2, 30), - fontsize=5.2, color="0.4", + ax[2].annotate("six architectures\ncoincide at $N-1$", xy=(19, 18), xytext=(30, 7.6), + fontsize=5.2, color="0.4", ha="right", va="center", arrowprops=dict(arrowstyle="->", color="0.55", lw=0.7)) ax[2].legend(fontsize=5.2, frameon=False, loc="upper left") ax[2].axvline(TEAM, color="0.75", ls=":", lw=0.8, zorder=0) diff --git a/scripts/clean/make_figure_infra.py b/figure_scripts/make_figure_infra.py similarity index 98% rename from scripts/clean/make_figure_infra.py rename to figure_scripts/make_figure_infra.py index a7c5a7c..71e64b8 100644 --- a/scripts/clean/make_figure_infra.py +++ b/figure_scripts/make_figure_infra.py @@ -21,14 +21,15 @@ matplotlib.use("Agg") import matplotlib.pyplot as plt -import agentic_ecal as ae +import _pkg_path # noqa: F401 (puts repo root on sys.path) +import agentic_ecal_pkg as ae import roles HERE = os.path.dirname(os.path.abspath(__file__)) def _paper_figures(default="figures"): """Newest vN/figures alongside the repo, or repo-root figures/ if none exists.""" - root = os.path.dirname(os.path.dirname(HERE)) + root = os.path.dirname(HERE) vs = sorted((d for d in os.listdir(root) if re.fullmatch(r"v\d+", d) and os.path.isdir(os.path.join(root, d))), key=lambda d: int(d[1:])) diff --git a/scripts/clean/make_figure_overview.py b/figure_scripts/make_figure_overview.py similarity index 96% rename from scripts/clean/make_figure_overview.py rename to figure_scripts/make_figure_overview.py index bf44143..99c48de 100644 --- a/scripts/clean/make_figure_overview.py +++ b/figure_scripts/make_figure_overview.py @@ -10,8 +10,14 @@ def build_figure(out_pdf=None): if len(sys.argv) > 1: out_pdf = sys.argv[1] else: - base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - out_pdf = os.path.join(base_dir, "v6", "figures", "fig_overview.pdf") + import re + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + vs = sorted((d for d in os.listdir(base_dir) + if re.fullmatch(r"v\d+", d) + and os.path.isdir(os.path.join(base_dir, d, "figures"))), + key=lambda d: int(d[1:])) + sub = os.path.join(vs[-1], "figures") if vs else "figures" + out_pdf = os.path.join(base_dir, sub, "fig_overview.pdf") plt.rcParams.update({ "font.family": "DejaVu Sans", "font.size": 7, @@ -252,6 +258,7 @@ def build_figure(out_pdf=None): fontsize=3.8, color="#78350f", zorder=5) plt.subplots_adjust(left=0.005, right=0.995, top=0.995, bottom=0.005) + os.makedirs(os.path.dirname(out_pdf) or ".", exist_ok=True) fig.savefig(out_pdf, bbox_inches="tight", pad_inches=0.01) plt.close(fig) print("Generated:", out_pdf) diff --git a/scripts/clean/make_figure_topology.py b/figure_scripts/make_figure_topology.py similarity index 98% rename from scripts/clean/make_figure_topology.py rename to figure_scripts/make_figure_topology.py index 6270cbd..31ab1e6 100644 --- a/scripts/clean/make_figure_topology.py +++ b/figure_scripts/make_figure_topology.py @@ -28,7 +28,8 @@ import matplotlib.pyplot as plt from matplotlib.patches import FancyBboxPatch, Circle -import agentic_ecal as ae +import _pkg_path # noqa: F401 (puts repo root on sys.path) +import agentic_ecal_pkg as ae import model_placement as mp import placement as pl import osi @@ -37,7 +38,7 @@ def _paper_figures(default="figures"): """Newest vN/figures alongside the repo, or repo-root figures/ if none exists.""" - root = os.path.dirname(os.path.dirname(HERE)) + root = os.path.dirname(HERE) vs = sorted((d for d in os.listdir(root) if re.fullmatch(r"v\d+", d) and os.path.isdir(os.path.join(root, d))), key=lambda d: int(d[1:])) diff --git a/scripts/clean/make_figure_workflow.py b/figure_scripts/make_figure_workflow.py similarity index 98% rename from scripts/clean/make_figure_workflow.py rename to figure_scripts/make_figure_workflow.py index 5902d2e..57af9f5 100644 --- a/scripts/clean/make_figure_workflow.py +++ b/figure_scripts/make_figure_workflow.py @@ -21,7 +21,8 @@ import matplotlib.pyplot as plt from matplotlib.patches import FancyBboxPatch, FancyArrowPatch -import agentic_ecal as ae +import _pkg_path # noqa: F401 (puts repo root on sys.path) +import agentic_ecal_pkg as ae import roles import osi import placement as pl @@ -30,13 +31,13 @@ def _paper_figures(default="figures"): """Newest vN/figures alongside the repo, or repo-root figures/ if none exists.""" - root = os.path.dirname(os.path.dirname(HERE)) + root = os.path.dirname(HERE) vs = sorted((d for d in os.listdir(root) if re.fullmatch(r"v\d+", d) and os.path.isdir(os.path.join(root, d))), key=lambda d: int(d[1:])) return os.path.join(root, vs[-1] if vs else "", default) -ROOT = os.path.dirname(os.path.dirname(HERE)) +ROOT = os.path.dirname(HERE) FIGDIR = _paper_figures() # colours for the four prompt sources, in the order they are stacked @@ -57,7 +58,7 @@ def _paper_figures(default="figures"): # only the names were wrong. The two fibre rows are spelled out because that is where the # ambiguity was; the radio rows keep their short forms, which are unambiguous as they stand. BEARERS = ("backbone", "ftth", "5g", "edgecell") -TABLE3_NAME = {"backbone": "Optical backbone", "ftth": "Metro / fixed", +TABLE3_NAME = {"backbone": "optical backbone", "ftth": "metro / fixed", "5g": "5G", "edgecell": "edge cell"} diff --git a/scripts/clean/make_figures.py b/figure_scripts/make_figures.py similarity index 97% rename from scripts/clean/make_figures.py rename to figure_scripts/make_figures.py index c48ef18..1750c8a 100644 --- a/scripts/clean/make_figures.py +++ b/figure_scripts/make_figures.py @@ -26,7 +26,8 @@ from matplotlib.ticker import LogLocator, NullFormatter, ScalarFormatter sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import agentic_ecal as ae # noqa: E402 +import _pkg_path # noqa: F401 (puts repo root on sys.path) +import agentic_ecal_pkg as ae # noqa: E402 import placement as pl # noqa: E402 import osi # noqa: E402 @@ -35,7 +36,7 @@ def _paper_figures(default="figures"): """Newest vN/figures alongside the repo, or repo-root figures/ if none exists.""" - root = os.path.dirname(os.path.dirname(HERE)) + root = os.path.dirname(HERE) vs = sorted((d for d in os.listdir(root) if re.fullmatch(r"v\d+", d) and os.path.isdir(os.path.join(root, d))), key=lambda d: int(d[1:])) @@ -68,11 +69,16 @@ def _paper_figures(default="figures"): ETAS = (0.25, 0.30, 0.60, 0.70) ETA_GREYS = ("0.80", "0.62", "0.40", "0.10") # light -> dark with increasing eta -# Amortization: single-rate at one eta for ALL three curves, so the comparison is method-uniform. -AMORT_ETA = 0.05 +# Amortization: derived two-rate at one serving batch for both curves, so the comparison is +# method-uniform. (AMORT_ETA removed: a single-rate residue, read nowhere once fig_amortization +# moved to the two-rate path at CASE_BATCH.) +# +# LLaMA-65B was dropped. Qwen2.5-7B cannot replace it: it trained on 18T tokens but publishes no +# GPU-hours and no energy, so e_pretrain = 0 and its curve would collapse onto its own operational +# floor with no crossover. Both models below carry disclosed GPU-hours x TDP from the Llama-3 +# model card. AMORT_MODELS = ( ("Llama-3 8B", "llama3_8b", "#1f77b4", "-"), - ("LLaMA-65B", "llama_65b", "#2ca02c", "--"), ("Llama-3 70B", "llama3_70b", "#d62728", "-."), ) @@ -577,9 +583,10 @@ def fig_components(out_dir): def fig_amortization(out_dir): """Embodied cost amortises over served invocations G; crossover G* = E_emb / E_W. - Single-rate at one eta for all three curves: the 65B and 70B have no measured calibration of - their own, so a uniform method is the only comparable one. The crossover RATIOS between models - are invariant to hardware and eta alike, which is why the text quotes ratios, not absolutes. + Derived two-rate at one serving batch for all three curves: the 65B and 70B have no measured + calibration of their own, so a uniform method is the only comparable one. The crossover RATIOS + between models are invariant to hardware and batch alike, which is why the text quotes ratios, + not absolutes. """ hw = CASE_HW G = np.logspace(3, 12, 120) @@ -813,7 +820,7 @@ def write_dimensioning_table(out_dir): print(f" [dimension] {kind:<13} {bits/1e6:9.3g} Mb/session, {load/1e6:9.3g} Mb/s per " f"accelerator, {per_link[1]:,.1f} accelerators per 10 GbE ({binding}-bound)") - tex = f"""% Generated by scripts/clean/make_figures.py -- do not edit by hand. + tex = f"""% Generated by figure_scripts/make_figures.py -- do not edit by hand. \\begin{{table*}}[t] \\caption{{Site dimensioning for an agentic service. One accelerator serving at batch ${batch}$ completes ${sessions:.2f}$ sessions per second.}} diff --git a/scripts/clean/make_figures_placement.py b/figure_scripts/make_figures_placement.py similarity index 95% rename from scripts/clean/make_figures_placement.py rename to figure_scripts/make_figures_placement.py index e4f0cdd..00cdc8c 100644 --- a/scripts/clean/make_figures_placement.py +++ b/figure_scripts/make_figures_placement.py @@ -7,7 +7,7 @@ figures/fig_model_placement.pdf tables/tab_model_placement.tex -Measured basis: scripts/clean/results/steiner_tokens_by_cell.csv, summarised by +Measured basis: results/steiner_tokens_by_cell.csv, summarised by ``fit_peer_law.py`` from ~/DATA/data_steiner_Aug26 (6.69M team-item records, 16 open-weight models, team sizes 1..30 at full mesh, 3 debate rounds, peer_token_budget=120). Token counts only; task accuracy is not analysed. @@ -22,7 +22,8 @@ matplotlib.use("Agg") import matplotlib.pyplot as plt -import agentic_ecal as ae +import _pkg_path # noqa: F401 (puts repo root on sys.path) +import agentic_ecal_pkg as ae import osi import model_placement as mp @@ -30,14 +31,14 @@ def _paper_figures(default="figures"): """Newest vN/figures alongside the repo, or repo-root figures/ if none exists.""" - root = os.path.dirname(os.path.dirname(HERE)) + root = os.path.dirname(HERE) vs = sorted((d for d in os.listdir(root) if re.fullmatch(r"v\d+", d) and os.path.isdir(os.path.join(root, d))), key=lambda d: int(d[1:])) return os.path.join(root, vs[-1] if vs else "", default) -ROOT = os.path.dirname(os.path.dirname(HERE)) -RESULTS = os.path.join(HERE, "results", "steiner_tokens_by_cell.csv") +ROOT = os.path.dirname(HERE) +RESULTS = os.path.join(os.path.dirname(HERE), "results", "steiner_tokens_by_cell.csv") FIGDIR = _paper_figures() TABDIR = _paper_figures("tables") @@ -174,7 +175,7 @@ def table(d): rows.append(f"{m.name} & {w:.1f} & {kv:.0f} & " + " & ".join(cells) + " \\\\") body = "\n".join(rows) hdr = " & ".join(mp.DEVICES[t].label.replace("\n", " ") for t in TIERS) - tex = f"""% Generated by scripts/clean/make_figures_placement.py -- do not edit by hand. + tex = f"""% Generated by figure_scripts/make_figures_placement.py -- do not edit by hand. \\begin{{table*}}[t] \\ROOF{{}} \\caption{{Agents a device can host concurrently at the measured $N={TEAM}$ full-mesh debate diff --git a/scripts/clean/model_placement.py b/figure_scripts/model_placement.py similarity index 98% rename from scripts/clean/model_placement.py rename to figure_scripts/model_placement.py index 8cf944c..193b8c8 100644 --- a/scripts/clean/model_placement.py +++ b/figure_scripts/model_placement.py @@ -34,7 +34,8 @@ from dataclasses import dataclass from typing import Dict, Optional -import agentic_ecal as ae +import _pkg_path # noqa: F401 (puts repo root on sys.path) +import agentic_ecal_pkg as ae __all__ = ["Device", "DEVICES", "FLEET", "PeerLaw", "PEER_LAW", "weight_bytes", "kv_bytes_per_token", "resident_bytes", @@ -136,7 +137,7 @@ def p_in(self, k: float, communicating: bool = True) -> float: return self.a + self.b * k if communicating else self.solo -# Fitted on ~/DATA/data_steiner_Aug26; see scripts/clean/fit_peer_law.py. +# Fitted on ~/DATA/data_steiner_Aug26; see figure_scripts/fit_peer_law.py. PEER_LAW: Dict[str, PeerLaw] = { "gpt-oss-20b": PeerLaw(192, 287, 78.6, 172), "marin-8b": PeerLaw(541, 660, 90.5, 118), diff --git a/scripts/clean/osi.py b/figure_scripts/osi.py similarity index 100% rename from scripts/clean/osi.py rename to figure_scripts/osi.py diff --git a/scripts/clean/placement.py b/figure_scripts/placement.py similarity index 97% rename from scripts/clean/placement.py rename to figure_scripts/placement.py index a876988..57548c4 100644 --- a/scripts/clean/placement.py +++ b/figure_scripts/placement.py @@ -23,7 +23,8 @@ from dataclasses import dataclass from typing import Dict, Sequence, Tuple -import agentic_ecal as ae +import _pkg_path # noqa: F401 (puts repo root on sys.path) +import agentic_ecal_pkg as ae import osi __all__ = ["Bearer", "BEARERS", "ARCHETYPES", "PLACEMENTS", "PathCost", @@ -211,7 +212,7 @@ def latex_table(wf: ae.Workflow, path: str) -> str: f"{full.latency * 1e3:.0f} \\\\") body = "\n".join(rows) - tex = f"""% Generated by scripts/clean/placement.py -- do not edit by hand. + tex = f"""% Generated by figure_scripts/placement.py -- do not edit by hand. \\begin{{table*}}[t] \\caption{{Cost of distributing the four-agent workflow over composed network paths.}} \\label{{tab:placement}} @@ -245,7 +246,7 @@ def latex_table(wf: ae.Workflow, path: str) -> str: import os _hw = dataclasses.replace(ae.HW["h100"], mfu=ae.MFU_LATENCY) _wf = ae.four_agent_rag_workflow(hw=_hw) - _root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + _root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) _out = os.path.join(_root, "tables") os.makedirs(_out, exist_ok=True) latex_table(_wf, os.path.join(_out, "tab_placement.tex")) diff --git a/scripts/clean/roles.py b/figure_scripts/roles.py similarity index 100% rename from scripts/clean/roles.py rename to figure_scripts/roles.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..965c06f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,27 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "agentic-ecal" +version = "0.1.0" +description = "Analytical two-rate energy model for agentic workflows of open-weight LLMs." +readme = "README.md" +requires-python = ">=3.9" +# The model is pure-Python, standard library only. +dependencies = [] + +[project.optional-dependencies] +dev = ["pytest>=7"] + +[project.scripts] +agentic-ecal = "agentic_ecal_pkg.cli:main" + +[tool.setuptools] +# Ship only the package; the retained flat agentic_ecal.py (golden-master reference) and the +# figure_scripts/ are intentionally excluded from the distribution. +packages = ["agentic_ecal_pkg"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" diff --git a/requirements.txt b/requirements.txt index 59c405e..c993a32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,9 @@ -# The model (scripts/clean/agentic_ecal.py) is pure-Python, stdlib only. +# The model (scripts/clean/agentic_ecal_pkg/, and the legacy scripts/clean/agentic_ecal.py) is +# pure-Python, stdlib only. # These are needed to load the measured sweeps and regenerate the figures. numpy>=1.24 pandas>=2.0 matplotlib>=3.7 + +# Dev / test: the parity suite under scripts/clean/tests/. +pytest>=7 diff --git a/scripts/clean/results/agents_Llama-3.1-8B-Instruct.csv b/scripts/clean/results/agents_Llama-3.1-8B-Instruct.csv deleted file mode 100644 index bc10d25..0000000 --- a/scripts/clean/results/agents_Llama-3.1-8B-Instruct.csv +++ /dev/null @@ -1,271 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,2,2,3,0,0,32,1450.555,1982.3,801.5,155.84 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,2,5,3,0,0,32,3897.554,8242.0,2091.1,418.96 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,2,10,3,0,0,32,7022.765,26870.3,3658.6,755.7 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,2,20,3,0,0,32,15890.344,93735.0,7756.3,1710.9 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,2,30,3,0,0,32,23782.731,203830.3,10698.0,2560.53 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,2,2,3,0,0,32,2199.096,1895.4,1217.6,236.03 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,2,5,3,0,0,32,6360.022,8341.6,3492.1,683.15 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,2,10,3,0,0,32,12589.895,28532.8,6713.0,1353.02 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,2,20,3,0,0,32,25443.104,103955.0,12814.0,2735.99 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,2,30,3,0,0,32,40113.964,226872.2,19217.5,4310.03 -meta-llama/Llama-3.1-8B-Instruct,gpqa,2,2,3,0,0,32,2257.422,3236.6,1242.4,242.39 -meta-llama/Llama-3.1-8B-Instruct,gpqa,2,5,3,0,0,32,5688.528,11601.9,3097.5,610.86 -meta-llama/Llama-3.1-8B-Instruct,gpqa,2,10,3,0,0,32,11759.624,34969.2,6259.7,1263.66 -meta-llama/Llama-3.1-8B-Instruct,gpqa,2,20,3,0,0,32,21081.239,115927.0,10428.1,2267.5 -meta-llama/Llama-3.1-8B-Instruct,gpqa,2,30,3,0,0,32,31383.757,242669.4,14410.6,3375.6 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,2,2,3,0,1,32,1394.455,2018.6,769.8,149.96 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,2,5,3,0,1,32,4184.526,8155.6,2284.3,449.92 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,2,10,3,0,1,32,7560.695,26636.9,3972.6,813.82 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,2,20,3,0,1,32,16619.484,94243.8,8150.7,1790.29 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,2,30,3,0,1,32,26191.967,202177.6,11944.6,2820.34 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,2,2,3,0,1,32,2107.492,1917.3,1145.7,226.3 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,2,5,3,0,1,32,5798.544,8367.8,3180.0,623.01 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,2,10,3,0,1,32,11229.191,28669.9,5964.8,1206.92 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,2,20,3,0,1,32,21782.693,104238.1,10827.1,2344.05 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,2,30,3,0,1,32,34934.028,228262.5,16372.2,3756.6 -meta-llama/Llama-3.1-8B-Instruct,gpqa,2,2,3,0,1,32,2068.811,2627.1,1145.7,222.27 -meta-llama/Llama-3.1-8B-Instruct,gpqa,2,5,3,0,1,32,5082.786,10039.0,2773.9,546.19 -meta-llama/Llama-3.1-8B-Instruct,gpqa,2,10,3,0,1,32,10333.794,31650.0,5479.1,1111.17 -meta-llama/Llama-3.1-8B-Instruct,gpqa,2,20,3,0,1,32,19629.749,108785.6,9632.3,2113.29 -meta-llama/Llama-3.1-8B-Instruct,gpqa,2,30,3,0,1,32,31472.521,229765.3,14531.4,3385.7 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,2,2,3,0,2,32,1406.604,1980.7,771.4,151.22 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,2,5,3,0,2,32,3534.95,8122.7,1896.8,380.18 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,2,10,3,0,2,32,7360.227,26392.5,3863.7,792.28 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,2,20,3,0,2,32,14822.34,94221.9,7161.9,1597.82 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,2,30,3,0,2,32,23717.487,201471.6,10623.2,2554.37 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,2,2,3,0,2,32,1985.821,1867.2,1072.3,213.31 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,2,5,3,0,2,32,5232.19,8287.8,2817.5,562.39 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,2,10,3,0,2,32,10286.889,28098.1,5429.3,1106.16 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,2,20,3,0,2,32,23052.39,102768.1,11485.9,2480.07 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,2,30,3,0,2,32,35633.897,223392.2,16664.0,3832.63 -meta-llama/Llama-3.1-8B-Instruct,gpqa,2,2,3,0,2,32,2110.269,3147.9,1165.2,226.77 -meta-llama/Llama-3.1-8B-Instruct,gpqa,2,5,3,0,2,32,5031.233,11264.2,2746.8,540.64 -meta-llama/Llama-3.1-8B-Instruct,gpqa,2,10,3,0,2,32,9409.148,33408.4,4962.3,1012.08 -meta-llama/Llama-3.1-8B-Instruct,gpqa,2,20,3,0,2,32,18409.303,111274.4,9000.9,1982.51 -meta-llama/Llama-3.1-8B-Instruct,gpqa,2,30,3,0,2,32,30110.331,233069.1,13796.8,3241.08 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,4,2,3,0,0,32,823.532,1985.4,853.9,88.65 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,4,5,3,0,0,32,2103.91,8167.5,2116.6,226.47 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,4,10,3,0,0,32,4272.12,26807.9,3999.6,459.96 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,4,20,3,0,0,32,9156.008,93493.1,7826.0,987.45 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,4,30,3,0,0,32,14922.238,202424.0,11322.1,1609.17 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,4,2,3,0,0,32,1157.437,1899.2,1138.1,124.21 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,4,5,3,0,0,32,3291.528,8329.5,3477.0,353.55 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,4,10,3,0,0,32,6786.946,28621.9,6789.9,729.52 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,4,20,3,0,0,32,14472.625,104232.5,13136.5,1558.98 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,4,30,3,0,0,32,22716.755,227082.2,18620.7,2445.25 -meta-llama/Llama-3.1-8B-Instruct,gpqa,4,2,3,0,0,32,1231.157,3237.8,1287.8,132.21 -meta-llama/Llama-3.1-8B-Instruct,gpqa,4,5,3,0,0,32,3070.524,11611.2,3150.0,329.95 -meta-llama/Llama-3.1-8B-Instruct,gpqa,4,10,3,0,0,32,5997.636,34773.8,5935.5,645.16 -meta-llama/Llama-3.1-8B-Instruct,gpqa,4,20,3,0,0,32,12083.432,115895.1,10572.2,1301.77 -meta-llama/Llama-3.1-8B-Instruct,gpqa,4,30,3,0,0,32,19283.444,242719.0,15068.5,2076.83 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,4,2,3,0,1,32,795.821,2010.9,787.2,85.57 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,4,5,3,0,1,32,2167.707,8134.8,2232.9,233.16 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,4,10,3,0,1,32,4375.384,26873.4,4265.7,471.18 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,4,20,3,0,1,32,9108.003,94156.9,7773.0,982.0 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,4,30,3,0,1,32,14804.502,201031.0,11133.5,1595.96 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,4,2,3,0,1,32,1182.128,1926.7,1222.2,126.99 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,4,5,3,0,1,32,3087.553,8381.1,3178.4,331.99 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,4,10,3,0,1,32,6094.215,28663.4,5922.6,655.51 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,4,20,3,0,1,32,13318.545,104553.1,11872.0,1434.33 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,4,30,3,0,1,32,21262.75,227416.9,17020.7,2289.08 -meta-llama/Llama-3.1-8B-Instruct,gpqa,4,2,3,0,1,32,1054.86,2625.1,1112.2,113.65 -meta-llama/Llama-3.1-8B-Instruct,gpqa,4,5,3,0,1,32,2893.859,10065.7,3028.9,311.53 -meta-llama/Llama-3.1-8B-Instruct,gpqa,4,10,3,0,1,32,5443.602,31592.8,5366.4,586.43 -meta-llama/Llama-3.1-8B-Instruct,gpqa,4,20,3,0,1,32,10820.15,108440.0,9371.2,1166.92 -meta-llama/Llama-3.1-8B-Instruct,gpqa,4,30,3,0,1,32,18330.775,229273.2,14305.6,1974.31 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,4,2,3,0,2,32,689.064,1986.2,728.3,74.24 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,4,5,3,0,2,32,1806.58,8099.8,1852.3,194.81 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,4,10,3,0,2,32,3938.812,26428.5,3737.0,424.85 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,4,20,3,0,2,32,9061.688,93893.1,7746.9,977.61 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,4,30,3,0,2,32,14507.556,202182.2,10710.6,1564.42 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,4,2,3,0,2,32,1104.941,1861.0,1079.6,118.8 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,4,5,3,0,2,32,2935.69,8263.1,3006.1,315.94 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,4,10,3,0,2,32,5966.706,28176.9,5796.8,642.33 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,4,20,3,0,2,32,12425.398,102786.9,10943.5,1338.6 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,4,30,3,0,2,32,22225.648,223405.3,18093.8,2391.4 -meta-llama/Llama-3.1-8B-Instruct,gpqa,4,2,3,0,2,32,1115.097,3150.6,1146.7,120.0 -meta-llama/Llama-3.1-8B-Instruct,gpqa,4,5,3,0,2,32,2529.197,11265.2,2632.2,272.51 -meta-llama/Llama-3.1-8B-Instruct,gpqa,4,10,3,0,2,32,5271.418,33267.5,5183.2,568.03 -meta-llama/Llama-3.1-8B-Instruct,gpqa,4,20,3,0,2,32,10544.366,111153.8,9109.4,1136.9 -meta-llama/Llama-3.1-8B-Instruct,gpqa,4,30,3,0,2,32,17155.004,232516.9,13125.0,1848.26 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,8,2,3,0,0,32,463.83,1984.9,735.4,50.11 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,8,5,3,0,0,32,1279.281,8203.9,2142.8,137.91 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,8,10,3,0,0,32,2390.211,26821.3,3907.8,257.74 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,8,20,3,0,0,32,5551.22,94075.6,7456.3,599.06 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,8,30,3,0,0,32,9886.852,202554.3,11093.7,1065.26 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,8,2,3,0,0,32,746.564,1890.4,1153.4,80.2 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,8,5,3,0,0,32,1767.006,8352.2,3284.0,190.26 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,8,10,3,0,0,32,3722.621,28389.1,6627.7,400.55 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,8,20,3,0,0,32,8718.683,103931.9,13438.2,939.36 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,8,30,3,0,0,32,14653.89,226185.0,18939.3,1576.14 -meta-llama/Llama-3.1-8B-Instruct,gpqa,8,2,3,0,0,32,659.017,3235.3,1191.7,70.87 -meta-llama/Llama-3.1-8B-Instruct,gpqa,8,5,3,0,0,32,1692.662,11605.8,3133.4,182.16 -meta-llama/Llama-3.1-8B-Instruct,gpqa,8,10,3,0,0,32,3584.357,34999.2,6175.0,386.01 -meta-llama/Llama-3.1-8B-Instruct,gpqa,8,20,3,0,0,32,7093.487,115819.5,10103.8,765.37 -meta-llama/Llama-3.1-8B-Instruct,gpqa,8,30,3,0,0,32,12665.38,242743.5,15333.9,1363.18 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,8,2,3,0,1,32,484.575,2011.3,795.7,52.24 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,8,5,3,0,1,32,1217.503,8183.9,2212.2,131.26 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,8,10,3,0,1,32,2541.408,26780.6,4369.8,274.06 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,8,20,3,0,1,32,5712.718,94216.9,7925.7,616.67 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,8,30,3,0,1,32,9882.503,201858.8,11271.5,1065.41 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,8,2,3,0,1,32,688.996,1914.8,1154.0,74.06 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,8,5,3,0,1,32,1697.082,8364.7,3113.8,182.58 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,8,10,3,0,1,32,3366.475,28666.2,5752.4,362.59 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,8,20,3,0,1,32,8115.779,104592.5,11997.0,875.15 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,8,30,3,0,1,32,14261.228,227611.9,18210.1,1534.33 -meta-llama/Llama-3.1-8B-Instruct,gpqa,8,2,3,0,1,32,575.201,2616.5,1113.4,62.05 -meta-llama/Llama-3.1-8B-Instruct,gpqa,8,5,3,0,1,32,1454.764,10033.0,2812.4,156.75 -meta-llama/Llama-3.1-8B-Instruct,gpqa,8,10,3,0,1,32,3191.672,31673.8,5615.2,344.0 -meta-llama/Llama-3.1-8B-Instruct,gpqa,8,20,3,0,1,32,6772.336,108537.5,9556.7,730.81 -meta-llama/Llama-3.1-8B-Instruct,gpqa,8,30,3,0,1,32,11788.199,230515.3,13872.9,1268.94 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,8,2,3,0,2,32,419.006,1989.1,796.2,45.29 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,8,5,3,0,2,32,1133.953,8105.8,1855.5,122.22 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,8,10,3,0,2,32,2438.848,26366.9,4091.6,263.01 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,8,20,3,0,2,32,5222.719,93642.5,7024.5,564.17 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,8,30,3,0,2,32,9483.113,201905.7,10312.7,1022.23 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,8,2,3,0,2,32,647.3,1867.8,986.1,69.68 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,8,5,3,0,2,32,1602.826,8281.2,2828.3,172.56 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,8,10,3,0,2,32,3180.242,28027.8,5430.0,342.78 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,8,20,3,0,2,32,7791.375,102837.5,11463.9,839.83 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,8,30,3,0,2,32,13398.848,223605.9,16805.9,1441.89 -meta-llama/Llama-3.1-8B-Instruct,gpqa,8,2,3,0,2,32,665.99,3148.7,1103.0,71.79 -meta-llama/Llama-3.1-8B-Instruct,gpqa,8,5,3,0,2,32,1454.887,11293.0,2823.8,156.82 -meta-llama/Llama-3.1-8B-Instruct,gpqa,8,10,3,0,2,32,2933.362,33445.0,5138.6,316.26 -meta-llama/Llama-3.1-8B-Instruct,gpqa,8,20,3,0,2,32,6593.625,110976.9,9140.6,711.39 -meta-llama/Llama-3.1-8B-Instruct,gpqa,8,30,3,0,2,32,12072.295,233115.8,14465.2,1300.3 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,16,2,3,0,0,32,356.777,1984.8,735.9,39.45 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,16,5,3,0,0,32,908.221,8161.1,2283.0,97.8 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,16,10,3,0,0,32,1620.545,26877.8,4126.6,174.76 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,16,20,3,0,0,32,4144.423,94385.6,7587.4,446.32 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,16,30,3,0,0,32,7725.708,202749.3,11133.1,831.17 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,16,2,3,0,0,32,554.196,1892.6,1391.3,59.66 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,16,5,3,0,0,32,1203.022,8335.9,3624.9,129.42 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,16,10,3,0,0,32,2439.387,28442.5,6911.0,262.64 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,16,20,3,0,0,32,6026.855,103990.6,13236.7,648.58 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,16,30,3,0,0,32,10701.486,226130.6,18516.2,1150.07 -meta-llama/Llama-3.1-8B-Instruct,gpqa,16,2,3,0,0,32,431.304,3240.3,1094.5,46.56 -meta-llama/Llama-3.1-8B-Instruct,gpqa,16,5,3,0,0,32,983.622,11582.2,2880.3,105.92 -meta-llama/Llama-3.1-8B-Instruct,gpqa,16,10,3,0,0,32,2204.715,34939.2,6103.9,237.57 -meta-llama/Llama-3.1-8B-Instruct,gpqa,16,20,3,0,0,32,5138.317,115533.2,10306.5,553.45 -meta-llama/Llama-3.1-8B-Instruct,gpqa,16,30,3,0,0,32,9472.426,242792.0,14738.6,1018.65 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,16,2,3,0,1,32,374.212,2014.6,844.1,40.28 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,16,5,3,0,1,32,683.945,8178.8,2120.6,73.97 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,16,10,3,0,1,32,1696.979,26761.0,4275.4,183.18 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,16,20,3,0,1,32,4088.287,94334.4,7842.1,440.75 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,16,30,3,0,1,32,7647.183,201066.6,11154.1,822.96 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,16,2,3,0,1,32,495.161,1919.1,1143.6,53.28 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,16,5,3,0,1,32,1015.101,8388.1,2784.1,109.38 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,16,10,3,0,1,32,2252.962,28714.3,6066.3,242.72 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,16,20,3,0,1,32,5646.925,104151.2,11970.7,608.36 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,16,30,3,0,1,32,10204.827,227207.8,16958.4,1096.87 -meta-llama/Llama-3.1-8B-Instruct,gpqa,16,2,3,0,1,32,505.96,2611.8,1292.0,54.48 -meta-llama/Llama-3.1-8B-Instruct,gpqa,16,5,3,0,1,32,937.729,10060.2,2901.8,101.12 -meta-llama/Llama-3.1-8B-Instruct,gpqa,16,10,3,0,1,32,2223.179,31706.5,6044.2,239.57 -meta-llama/Llama-3.1-8B-Instruct,gpqa,16,20,3,0,1,32,4962.248,108305.6,9837.4,534.96 -meta-llama/Llama-3.1-8B-Instruct,gpqa,16,30,3,0,1,32,9020.261,228776.3,13871.5,970.0 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,16,2,3,0,2,32,277.625,1988.9,759.6,29.96 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,16,5,3,0,2,32,630.313,8125.9,1894.3,68.14 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,16,10,3,0,2,32,1473.301,26539.7,3679.3,159.01 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,16,20,3,0,2,32,4143.891,93622.5,7685.3,447.11 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,16,30,3,0,2,32,7677.659,201618.8,10594.8,826.15 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,16,2,3,0,2,32,449.139,1862.1,966.2,48.44 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,16,5,3,0,2,32,1025.802,8283.1,2862.0,110.47 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,16,10,3,0,2,32,2221.745,28072.8,5919.2,239.25 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,16,20,3,0,2,32,5316.104,102960.0,11011.7,573.06 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,16,30,3,0,2,32,10170.025,223993.1,16805.2,1093.34 -meta-llama/Llama-3.1-8B-Instruct,gpqa,16,2,3,0,2,32,543.6,3145.9,1055.4,58.59 -meta-llama/Llama-3.1-8B-Instruct,gpqa,16,5,3,0,2,32,888.17,11266.3,2659.7,95.7 -meta-llama/Llama-3.1-8B-Instruct,gpqa,16,10,3,0,2,32,1910.375,33444.1,4767.9,206.19 -meta-llama/Llama-3.1-8B-Instruct,gpqa,16,20,3,0,2,32,4486.58,110906.2,8505.8,483.94 -meta-llama/Llama-3.1-8B-Instruct,gpqa,16,30,3,0,2,32,8744.002,232340.6,13035.8,941.08 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,2,3,0,0,32,251.983,1975.8,701.3,28.06 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,3,0,0,32,592.026,8198.0,2220.2,63.8 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,10,3,0,0,32,1122.42,26941.3,3624.4,120.98 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,20,3,0,0,32,3403.991,93688.8,7698.3,365.76 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,30,3,0,0,32,7153.108,203595.0,11240.1,767.56 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,2,3,0,0,32,401.174,1899.8,1303.3,43.02 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,3,0,0,32,706.543,8375.5,3400.8,76.02 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,10,3,0,0,32,1580.547,28508.1,6962.0,170.01 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,20,3,0,0,32,4535.065,104386.2,13501.0,487.32 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,30,3,0,0,32,9084.253,226705.3,18413.9,974.99 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,2,3,0,0,32,379.029,3231.5,1242.7,40.75 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,3,0,0,32,692.931,11569.1,2869.2,74.76 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,10,3,0,0,32,1441.655,35002.0,5693.2,155.21 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,20,3,0,0,32,4278.646,115709.4,10728.2,459.71 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,30,3,0,0,32,8521.438,243355.6,14896.5,914.38 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,2,3,0,1,32,333.065,2020.0,808.8,35.77 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,3,0,1,32,581.712,8158.9,2025.1,62.71 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,10,3,0,1,32,1135.884,26895.0,4051.4,122.59 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,20,3,0,1,32,3342.296,94078.8,7422.4,359.44 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,30,3,0,1,32,7126.871,202100.7,11572.0,764.61 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,2,3,0,1,32,393.529,1914.5,1126.0,42.39 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,3,0,1,32,659.551,8380.9,3054.2,70.97 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,10,3,0,1,32,1498.253,28696.6,5932.3,161.1 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,20,3,0,1,32,4395.566,104648.1,11573.2,472.4 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,30,3,0,1,32,9002.416,227102.8,16857.1,966.1 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,2,3,0,1,32,419.72,2623.2,1523.6,45.07 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,3,0,1,32,676.175,10089.8,2928.6,72.9 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,10,3,0,1,32,1456.216,31651.9,5608.9,156.76 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,20,3,0,1,32,3953.714,108542.5,9508.2,425.18 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,30,3,0,1,32,8485.699,230790.0,14562.3,910.37 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,2,3,0,2,32,173.995,1999.1,717.3,18.91 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,3,0,2,32,579.964,8141.4,1959.8,62.45 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,10,3,0,2,32,1120.352,26428.4,3751.8,120.75 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,20,3,0,2,32,3389.63,94218.8,7283.9,364.44 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,30,3,0,2,32,6936.254,201126.6,10370.1,744.23 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,2,3,0,2,32,395.323,1863.8,1154.7,42.48 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,3,0,2,32,698.607,8261.6,3090.7,75.08 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,10,3,0,2,32,1403.804,28078.1,5407.3,150.94 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,20,3,0,2,32,4260.293,102298.1,11204.3,457.49 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,30,3,0,2,32,9087.095,222718.1,17677.4,974.95 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,2,3,0,2,32,404.586,3156.8,1036.2,43.55 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,3,0,2,32,635.404,11250.5,2738.3,68.3 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,10,3,0,2,32,1361.265,33317.5,4936.8,146.55 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,20,3,0,2,32,3843.59,110827.5,8762.2,413.01 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,30,3,0,2,32,8033.455,232317.1,13626.1,861.98 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,2,3,0,0,32,300.256,1991.5,771.1,33.29 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,5,3,0,0,32,543.689,8183.3,2185.2,58.49 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,10,3,0,0,32,1112.201,26872.8,3972.4,119.51 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,20,3,0,0,32,3469.99,94021.9,7706.3,372.39 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,30,3,0,0,32,7239.858,202036.8,11038.7,776.35 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,2,3,0,0,32,395.652,1887.6,1128.8,42.67 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,5,3,0,0,32,624.505,8335.2,3517.0,67.12 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,10,3,0,0,32,1419.541,28560.6,6995.3,152.44 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,20,3,0,0,32,4564.821,104481.9,12708.1,489.86 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,30,3,0,0,32,9862.753,226591.9,19309.5,1057.68 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,2,3,0,0,32,372.582,3229.8,1085.1,40.04 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,5,3,0,0,32,627.107,11602.0,3244.2,67.37 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,10,3,0,0,32,1418.238,34949.1,6170.7,152.24 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,20,3,0,0,32,4306.543,115436.4,10328.1,462.04 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,30,3,0,0,32,8904.987,243102.5,14552.4,954.92 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,2,3,0,1,32,388.568,2020.6,953.3,41.8 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,5,3,0,1,32,550.53,8130.6,2247.9,59.13 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,10,3,0,1,32,1067.972,26738.1,4105.2,114.95 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,20,3,0,1,32,3602.808,94402.0,8233.4,386.55 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,30,3,0,1,32,7399.304,200753.5,11851.6,793.54 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,2,3,0,1,32,327.863,1908.1,1017.7,35.19 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,5,3,0,1,32,609.952,8376.7,3196.2,65.58 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,10,3,0,1,32,1374.112,28673.1,6072.9,147.53 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,20,3,0,1,32,4534.758,104635.0,12079.0,486.72 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,30,3,0,1,32,9726.305,227962.5,17855.3,1043.05 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,2,3,0,1,32,410.018,2614.0,1330.2,44.11 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,5,3,0,1,32,604.987,10073.5,2936.6,65.01 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,10,3,0,1,32,1312.506,31707.9,5381.4,141.25 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,20,3,0,1,32,4195.251,108146.3,10003.5,449.87 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,30,3,0,1,32,8771.013,229277.9,14630.0,940.76 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,2,3,0,2,32,357.012,1985.8,782.9,38.36 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,5,3,0,2,32,478.895,8162.3,1923.4,51.57 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,10,3,0,2,32,1114.936,26322.8,3795.2,119.79 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,20,3,0,2,32,3505.813,94143.8,7578.9,376.14 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,30,3,0,2,32,7129.515,201500.7,10593.1,764.58 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,2,3,0,2,32,392.201,1867.6,1083.2,42.23 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,5,3,0,2,32,602.163,8254.1,3025.2,64.67 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,10,3,0,2,32,1308.724,28169.1,5485.6,140.58 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,20,3,0,2,32,4366.237,103030.0,11285.8,468.49 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,30,3,0,2,32,9679.89,222438.8,17950.8,1038.08 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,2,3,0,2,32,422.473,3154.5,1195.0,45.39 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,5,3,0,2,32,606.099,11243.9,2797.8,65.34 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,10,3,0,2,32,1241.656,33390.9,5057.8,133.38 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,20,3,0,2,32,3981.388,111071.2,8531.0,427.19 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,30,3,0,2,32,8549.431,232242.2,13566.0,916.76 diff --git a/scripts/clean/results/agents_Qwen2.5-7B-Instruct.csv b/scripts/clean/results/agents_Qwen2.5-7B-Instruct.csv deleted file mode 100644 index d1f7f3b..0000000 --- a/scripts/clean/results/agents_Qwen2.5-7B-Instruct.csv +++ /dev/null @@ -1,271 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,2,3,0,0,32,149.642,1839.0,496.0,16.22 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,5,3,0,0,32,393.4,7194.6,1296.0,42.91 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,10,3,0,0,32,926.007,23180.7,2605.4,100.55 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,20,3,0,0,32,2631.404,83051.9,5480.9,284.61 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,30,3,0,0,32,5302.021,182302.6,8604.6,571.93 -Qwen/Qwen2.5-7B-Instruct,gsmhard,16,2,3,0,0,32,381.053,1862.4,862.5,41.14 -Qwen/Qwen2.5-7B-Instruct,gsmhard,16,5,3,0,0,32,598.633,8104.4,2018.1,64.67 -Qwen/Qwen2.5-7B-Instruct,gsmhard,16,10,3,0,0,32,1434.323,27726.0,4118.0,154.58 -Qwen/Qwen2.5-7B-Instruct,gsmhard,16,20,3,0,0,32,3731.029,102066.3,8538.0,402.5 -Qwen/Qwen2.5-7B-Instruct,gsmhard,16,30,3,0,0,32,6871.406,223325.8,12162.9,740.26 -Qwen/Qwen2.5-7B-Instruct,gpqa,16,2,3,0,0,32,216.016,3091.1,642.4,23.31 -Qwen/Qwen2.5-7B-Instruct,gpqa,16,5,3,0,0,32,499.084,10794.2,1621.4,54.04 -Qwen/Qwen2.5-7B-Instruct,gpqa,16,10,3,0,0,32,1176.403,31710.1,3220.9,127.31 -Qwen/Qwen2.5-7B-Instruct,gpqa,16,20,3,0,0,32,3453.712,105845.2,7160.7,372.75 -Qwen/Qwen2.5-7B-Instruct,gpqa,16,30,3,0,0,32,6804.225,222669.8,10978.1,733.29 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,2,3,0,1,32,171.447,1862.2,526.1,18.51 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,5,3,0,1,32,414.655,7334.6,1386.8,45.1 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,10,3,0,1,32,994.375,23711.7,2833.9,107.6 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,20,3,0,1,32,2790.008,84941.5,5972.0,301.65 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,30,3,0,1,32,5295.581,183204.8,8871.9,571.07 -Qwen/Qwen2.5-7B-Instruct,gsmhard,16,2,3,0,1,32,214.582,1876.1,744.6,23.2 -Qwen/Qwen2.5-7B-Instruct,gsmhard,16,5,3,0,1,32,576.314,8207.6,2008.8,62.31 -Qwen/Qwen2.5-7B-Instruct,gsmhard,16,10,3,0,1,32,1376.628,27921.4,4145.9,148.7 -Qwen/Qwen2.5-7B-Instruct,gsmhard,16,20,3,0,1,32,3688.747,103019.9,8262.8,398.04 -Qwen/Qwen2.5-7B-Instruct,gsmhard,16,30,3,0,1,32,6921.918,223890.7,12462.2,745.7 -Qwen/Qwen2.5-7B-Instruct,gpqa,16,2,3,0,1,32,203.828,2475.4,615.1,22.18 -Qwen/Qwen2.5-7B-Instruct,gpqa,16,5,3,0,1,32,514.165,9076.6,1612.9,55.57 -Qwen/Qwen2.5-7B-Instruct,gpqa,16,10,3,0,1,32,1138.312,28151.6,3173.8,123.16 -Qwen/Qwen2.5-7B-Instruct,gpqa,16,20,3,0,1,32,3208.753,97101.3,6509.8,346.79 -Qwen/Qwen2.5-7B-Instruct,gpqa,16,30,3,0,1,32,6350.45,207381.6,9895.1,684.63 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,2,3,0,2,32,226.003,1841.4,525.5,24.46 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,5,3,0,2,32,390.108,7184.3,1303.4,42.42 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,10,3,0,2,32,944.577,23192.9,2627.2,102.38 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,20,3,0,2,32,2719.53,83211.3,5551.1,293.94 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,30,3,0,2,32,5307.226,179642.9,8661.5,572.85 -Qwen/Qwen2.5-7B-Instruct,gsmhard,16,2,3,0,2,32,209.161,1843.1,707.6,22.67 -Qwen/Qwen2.5-7B-Instruct,gsmhard,16,5,3,0,2,32,589.96,7962.9,1909.4,63.69 -Qwen/Qwen2.5-7B-Instruct,gsmhard,16,10,3,0,2,32,1346.37,27395.9,3862.2,145.37 -Qwen/Qwen2.5-7B-Instruct,gsmhard,16,20,3,0,2,32,3585.333,100805.1,7817.8,386.69 -Qwen/Qwen2.5-7B-Instruct,gsmhard,16,30,3,0,2,32,6801.305,219857.1,11474.0,732.62 -Qwen/Qwen2.5-7B-Instruct,gpqa,16,2,3,0,2,32,186.859,3002.7,538.9,20.24 -Qwen/Qwen2.5-7B-Instruct,gpqa,16,5,3,0,2,32,452.615,10329.6,1424.1,49.11 -Qwen/Qwen2.5-7B-Instruct,gpqa,16,10,3,0,2,32,1075.155,30230.1,2916.1,116.41 -Qwen/Qwen2.5-7B-Instruct,gpqa,16,20,3,0,2,32,3140.093,100662.2,6254.8,339.45 -Qwen/Qwen2.5-7B-Instruct,gpqa,16,30,3,0,2,32,6052.065,209778.9,9134.3,652.68 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,2,3,0,0,32,92.616,1844.2,501.6,10.07 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,3,0,0,32,231.773,7180.5,1295.5,25.47 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,10,3,0,0,32,623.53,23086.6,2677.3,68.14 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,20,3,0,0,32,2051.445,82780.8,5529.3,222.15 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,30,3,0,0,32,4740.719,182801.4,9016.7,510.89 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,2,3,0,0,32,205.693,1865.1,801.8,22.23 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,3,0,0,32,411.728,8120.3,2053.4,44.54 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,10,3,0,0,32,849.786,27598.1,4149.6,92.15 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,20,3,0,0,32,2825.839,101793.2,8226.0,304.03 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,30,3,0,0,32,5997.409,222539.3,12348.7,645.19 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,2,3,0,0,32,158.304,3094.1,642.8,17.05 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,3,0,0,32,309.365,10781.8,1613.3,33.61 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,10,3,0,0,32,802.28,31721.5,3374.6,87.28 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,20,3,0,0,32,2777.631,105903.8,7215.9,299.61 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,30,3,0,0,32,6067.006,223144.9,10920.2,652.88 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,2,3,0,1,32,99.925,1870.5,539.4,11.02 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,3,0,1,32,249.308,7284.6,1322.2,27.38 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,10,3,0,1,32,638.271,23916.8,2863.1,69.66 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,20,3,0,1,32,2159.195,85082.7,5850.4,233.31 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,30,3,0,1,32,4612.032,183230.9,9047.7,497.02 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,2,3,0,1,32,132.815,1881.5,733.6,14.27 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,3,0,1,32,323.779,8222.2,2052.5,35.24 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,10,3,0,1,32,840.937,27927.3,3979.1,91.45 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,20,3,0,1,32,2918.976,103232.3,8461.1,314.09 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,30,3,0,1,32,5926.879,223801.6,12421.6,637.2 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,2,3,0,1,32,149.726,2476.4,612.3,16.15 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,3,0,1,32,356.43,9068.3,1564.2,38.51 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,10,3,0,1,32,839.034,28122.2,3348.9,90.95 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,20,3,0,1,32,2563.938,96792.5,6572.4,276.62 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,30,3,0,1,32,5685.468,206768.6,10106.8,612.05 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,2,3,0,2,32,121.378,1839.8,533.4,13.22 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,3,0,2,32,249.421,7226.9,1304.1,27.24 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,10,3,0,2,32,628.974,23135.3,2580.0,68.65 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,20,3,0,2,32,2142.096,82987.5,5569.7,231.86 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,30,3,0,2,32,4634.071,179515.4,8310.7,499.37 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,2,3,0,2,32,191.14,1837.8,784.2,20.55 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,3,0,2,32,359.466,7984.8,1833.8,38.82 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,10,3,0,2,32,942.019,27446.3,3966.7,101.94 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,20,3,0,2,32,2888.037,100610.0,8113.0,310.6 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,30,3,0,2,32,5969.665,219770.8,11369.1,641.8 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,2,3,0,2,32,127.961,2997.4,544.4,13.8 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,3,0,2,32,268.981,10315.7,1431.8,29.35 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,10,3,0,2,32,715.539,30173.1,2912.9,77.75 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,20,3,0,2,32,2515.842,100705.2,6114.6,271.59 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,30,3,0,2,32,5437.033,208653.9,9092.7,585.09 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,2,3,0,0,32,102.846,1853.5,528.9,11.14 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,5,3,0,0,32,207.011,7180.7,1339.9,22.83 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,10,3,0,0,32,572.462,23256.9,2677.9,62.54 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,20,3,0,0,32,2196.869,83344.0,5794.2,237.7 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,30,3,0,0,32,4653.916,181447.5,8446.7,501.24 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,2,3,0,0,32,169.104,1865.2,806.9,18.3 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,5,3,0,0,32,387.247,8098.9,2025.1,41.82 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,10,3,0,0,32,874.24,27713.5,4082.8,94.31 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,20,3,0,0,32,3102.22,102130.7,8651.8,333.2 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,30,3,0,0,32,6356.211,222700.5,12214.7,682.15 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,2,3,0,0,32,126.15,3091.6,628.6,13.7 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,5,3,0,0,32,268.825,10785.2,1634.6,29.31 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,10,3,0,0,32,763.731,31757.5,3258.0,83.1 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,20,3,0,0,32,2967.409,106018.2,7214.2,319.76 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,30,3,0,0,32,6348.819,223214.1,10929.9,682.67 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,2,3,0,1,32,111.913,1873.3,544.2,12.13 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,5,3,0,1,32,190.505,7326.0,1319.4,20.79 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,10,3,0,1,32,578.943,23975.2,2888.9,63.45 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,20,3,0,1,32,2210.411,85181.7,5849.3,239.14 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,30,3,0,1,32,4795.438,183492.5,8922.4,517.18 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,2,3,0,1,32,122.09,1881.4,709.2,13.37 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,5,3,0,1,32,293.369,8212.2,2018.0,31.93 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,10,3,0,1,32,777.792,27998.3,4020.6,84.77 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,20,3,0,1,32,3081.692,103118.6,8394.8,331.55 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,30,3,0,1,32,6384.597,223682.6,12427.3,686.39 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,2,3,0,1,32,142.658,2466.1,583.7,15.45 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,5,3,0,1,32,236.248,9062.5,1531.0,25.73 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,10,3,0,1,32,741.46,28196.9,3125.9,80.57 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,20,3,0,1,32,2677.434,97076.4,6508.8,288.94 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,30,3,0,1,32,5900.445,207061.9,9868.2,635.24 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,2,3,0,2,32,107.909,1835.4,498.1,11.79 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,5,3,0,2,32,214.673,7189.9,1302.3,23.47 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,10,3,0,2,32,599.402,23342.5,2683.9,65.49 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,20,3,0,2,32,2234.285,84068.8,5750.7,241.59 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,30,3,0,2,32,4729.287,179030.7,8367.6,509.67 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,2,3,0,2,32,129.394,1833.7,713.5,14.09 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,5,3,0,2,32,346.864,7971.6,1910.1,37.64 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,10,3,0,2,32,920.713,27427.2,3947.1,99.31 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,20,3,0,2,32,3032.957,101000.1,7949.1,325.85 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,30,3,0,2,32,6413.673,219862.8,11504.3,689.19 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,2,3,0,2,32,175.31,2996.4,571.6,18.94 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,5,3,0,2,32,217.953,10287.3,1409.2,23.93 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,10,3,0,2,32,675.26,30205.3,2936.5,73.58 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,20,3,0,2,32,2598.812,100919.7,6132.8,280.6 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,30,3,0,2,32,5634.144,208506.5,9197.9,606.58 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,2,2,3,0,0,32,870.283,1836.3,497.9,93.47 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,2,5,3,0,0,32,2305.798,7121.8,1304.5,248.25 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,2,10,3,0,0,32,4979.869,23284.8,2737.4,536.28 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,2,20,3,0,0,32,10770.484,82872.1,5552.5,1161.29 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,2,30,3,0,0,32,17626.015,181183.2,8531.7,1900.63 -Qwen/Qwen2.5-7B-Instruct,gsmhard,2,2,3,0,0,32,1339.812,1863.4,771.6,143.92 -Qwen/Qwen2.5-7B-Instruct,gsmhard,2,5,3,0,0,32,3524.015,8090.8,2016.7,378.59 -Qwen/Qwen2.5-7B-Instruct,gsmhard,2,10,3,0,0,32,7564.352,27810.7,4201.2,813.85 -Qwen/Qwen2.5-7B-Instruct,gsmhard,2,20,3,0,0,32,15659.043,102065.0,8213.6,1687.68 -Qwen/Qwen2.5-7B-Instruct,gsmhard,2,30,3,0,0,32,24348.868,223225.4,12007.1,2623.41 -Qwen/Qwen2.5-7B-Instruct,gpqa,2,2,3,0,0,32,1098.426,3089.7,623.4,118.14 -Qwen/Qwen2.5-7B-Instruct,gpqa,2,5,3,0,0,32,2874.155,10754.7,1620.6,309.05 -Qwen/Qwen2.5-7B-Instruct,gpqa,2,10,3,0,0,32,6082.925,31689.2,3326.6,654.87 -Qwen/Qwen2.5-7B-Instruct,gpqa,2,20,3,0,0,32,14238.935,105757.1,7328.1,1535.54 -Qwen/Qwen2.5-7B-Instruct,gpqa,2,30,3,0,0,32,22895.175,223440.2,11034.3,2467.97 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,2,2,3,0,1,32,935.895,1868.2,537.9,100.59 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,2,5,3,0,1,32,2313.39,7307.6,1308.1,248.77 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,2,10,3,0,1,32,5072.811,23701.0,2794.3,545.91 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,2,20,3,0,1,32,11492.669,85007.0,5956.7,1239.31 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,2,30,3,0,1,32,18404.627,183943.4,8943.9,1984.58 -Qwen/Qwen2.5-7B-Instruct,gsmhard,2,2,3,0,1,32,1254.114,1877.7,725.8,134.82 -Qwen/Qwen2.5-7B-Instruct,gsmhard,2,5,3,0,1,32,3598.548,8190.9,2053.6,386.82 -Qwen/Qwen2.5-7B-Instruct,gsmhard,2,10,3,0,1,32,7250.601,27897.7,4025.2,780.25 -Qwen/Qwen2.5-7B-Instruct,gsmhard,2,20,3,0,1,32,15911.028,103031.1,8346.2,1714.65 -Qwen/Qwen2.5-7B-Instruct,gsmhard,2,30,3,0,1,32,25059.668,224252.6,12396.8,2699.17 -Qwen/Qwen2.5-7B-Instruct,gpqa,2,2,3,0,1,32,1062.603,2467.8,606.8,114.12 -Qwen/Qwen2.5-7B-Instruct,gpqa,2,5,3,0,1,32,2879.758,9064.5,1630.7,309.47 -Qwen/Qwen2.5-7B-Instruct,gpqa,2,10,3,0,1,32,5849.097,28141.6,3202.8,629.78 -Qwen/Qwen2.5-7B-Instruct,gpqa,2,20,3,0,1,32,13172.545,97358.8,6751.0,1420.57 -Qwen/Qwen2.5-7B-Instruct,gpqa,2,30,3,0,1,32,20650.914,207516.5,9835.0,2227.17 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,2,2,3,0,2,32,861.519,1829.1,489.9,92.5 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,2,5,3,0,2,32,2344.403,7202.2,1325.1,252.19 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,2,10,3,0,2,32,4946.794,23287.2,2706.4,532.61 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,2,20,3,0,2,32,10893.295,83787.6,5574.4,1175.6 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,2,30,3,0,2,32,17436.798,180390.1,8329.4,1880.8 -Qwen/Qwen2.5-7B-Instruct,gsmhard,2,2,3,0,2,32,1270.878,1829.4,719.7,136.44 -Qwen/Qwen2.5-7B-Instruct,gsmhard,2,5,3,0,2,32,3246.37,7963.0,1847.0,349.02 -Qwen/Qwen2.5-7B-Instruct,gsmhard,2,10,3,0,2,32,7224.162,27367.5,3995.6,777.55 -Qwen/Qwen2.5-7B-Instruct,gsmhard,2,20,3,0,2,32,15560.119,100690.6,8119.8,1677.16 -Qwen/Qwen2.5-7B-Instruct,gsmhard,2,30,3,0,2,32,23461.862,219781.9,11423.8,2528.46 -Qwen/Qwen2.5-7B-Instruct,gpqa,2,2,3,0,2,32,946.166,2998.4,536.0,101.75 -Qwen/Qwen2.5-7B-Instruct,gpqa,2,5,3,0,2,32,2568.319,10276.8,1442.4,276.01 -Qwen/Qwen2.5-7B-Instruct,gpqa,2,10,3,0,2,32,5386.285,30341.2,2934.8,579.87 -Qwen/Qwen2.5-7B-Instruct,gpqa,2,20,3,0,2,32,11943.623,100162.7,6063.8,1288.54 -Qwen/Qwen2.5-7B-Instruct,gpqa,2,30,3,0,2,32,19618.869,209844.5,9277.1,2116.21 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,4,2,3,0,0,32,461.311,1839.4,504.3,49.93 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,4,5,3,0,0,32,1184.791,7129.5,1290.5,127.8 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,4,10,3,0,0,32,2663.021,23364.8,2740.5,287.25 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,4,20,3,0,0,32,6044.166,83020.8,5527.9,653.11 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,4,30,3,0,0,32,10439.382,180977.1,8567.0,1126.41 -Qwen/Qwen2.5-7B-Instruct,gsmhard,4,2,3,0,0,32,716.016,1865.4,805.8,77.01 -Qwen/Qwen2.5-7B-Instruct,gsmhard,4,5,3,0,0,32,1835.886,8100.5,2019.1,197.85 -Qwen/Qwen2.5-7B-Instruct,gsmhard,4,10,3,0,0,32,3877.131,27705.9,4052.6,418.11 -Qwen/Qwen2.5-7B-Instruct,gsmhard,4,20,3,0,0,32,8773.372,102031.3,8285.8,946.38 -Qwen/Qwen2.5-7B-Instruct,gsmhard,4,30,3,0,0,32,14481.947,222912.4,12344.7,1560.46 -Qwen/Qwen2.5-7B-Instruct,gpqa,4,2,3,0,0,32,566.494,3077.2,608.8,60.91 -Qwen/Qwen2.5-7B-Instruct,gpqa,4,5,3,0,0,32,1719.878,10765.5,1801.6,185.44 -Qwen/Qwen2.5-7B-Instruct,gpqa,4,10,3,0,0,32,3255.339,31702.2,3304.8,351.41 -Qwen/Qwen2.5-7B-Instruct,gpqa,4,20,3,0,0,32,7955.588,106008.5,7235.8,858.57 -Qwen/Qwen2.5-7B-Instruct,gpqa,4,30,3,0,0,32,13545.444,223664.0,11057.0,1460.17 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,4,2,3,0,1,32,483.662,1872.3,531.4,52.11 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,4,5,3,0,1,32,1225.076,7349.0,1336.9,132.15 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,4,10,3,0,1,32,2677.669,23769.7,2774.4,288.98 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,4,20,3,0,1,32,6316.796,84752.7,5816.5,682.07 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,4,30,3,0,1,32,10899.787,184822.0,9031.4,1176.29 -Qwen/Qwen2.5-7B-Instruct,gsmhard,4,2,3,0,1,32,645.291,1880.7,727.5,69.48 -Qwen/Qwen2.5-7B-Instruct,gsmhard,4,5,3,0,1,32,1903.201,8214.8,2095.5,205.13 -Qwen/Qwen2.5-7B-Instruct,gsmhard,4,10,3,0,1,32,3838.209,27852.9,4013.7,414.11 -Qwen/Qwen2.5-7B-Instruct,gsmhard,4,20,3,0,1,32,8929.513,103132.9,8402.9,963.05 -Qwen/Qwen2.5-7B-Instruct,gsmhard,4,30,3,0,1,32,14280.981,223726.6,12176.3,1538.68 -Qwen/Qwen2.5-7B-Instruct,gpqa,4,2,3,0,1,32,546.111,2472.9,591.8,58.9 -Qwen/Qwen2.5-7B-Instruct,gpqa,4,5,3,0,1,32,1431.243,9046.0,1550.8,154.54 -Qwen/Qwen2.5-7B-Instruct,gpqa,4,10,3,0,1,32,3068.64,28076.3,3131.0,331.32 -Qwen/Qwen2.5-7B-Instruct,gpqa,4,20,3,0,1,32,7315.562,96733.3,6616.1,789.88 -Qwen/Qwen2.5-7B-Instruct,gpqa,4,30,3,0,1,32,12426.071,207331.1,9991.1,1340.54 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,4,2,3,0,2,32,478.236,1833.8,519.1,51.43 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,4,5,3,0,2,32,1210.904,7212.7,1309.8,130.69 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,4,10,3,0,2,32,2691.418,23403.8,2757.1,290.67 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,4,20,3,0,2,32,6236.359,83681.3,5649.7,673.9 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,4,30,3,0,2,32,10499.023,181020.9,8501.5,1133.26 -Qwen/Qwen2.5-7B-Instruct,gsmhard,4,2,3,0,2,32,821.003,1835.3,828.7,88.38 -Qwen/Qwen2.5-7B-Instruct,gsmhard,4,5,3,0,2,32,1743.327,7994.2,1913.8,187.89 -Qwen/Qwen2.5-7B-Instruct,gsmhard,4,10,3,0,2,32,3728.32,27366.9,3851.8,402.08 -Qwen/Qwen2.5-7B-Instruct,gsmhard,4,20,3,0,2,32,8650.098,100814.5,8088.9,933.05 -Qwen/Qwen2.5-7B-Instruct,gsmhard,4,30,3,0,2,32,14003.406,219843.8,11646.1,1509.51 -Qwen/Qwen2.5-7B-Instruct,gpqa,4,2,3,0,2,32,498.522,2995.2,529.2,53.76 -Qwen/Qwen2.5-7B-Instruct,gpqa,4,5,3,0,2,32,1337.266,10314.9,1447.5,144.23 -Qwen/Qwen2.5-7B-Instruct,gpqa,4,10,3,0,2,32,2905.899,30373.4,2941.9,313.77 -Qwen/Qwen2.5-7B-Instruct,gpqa,4,20,3,0,2,32,6923.573,100152.9,6191.7,747.66 -Qwen/Qwen2.5-7B-Instruct,gpqa,4,30,3,0,2,32,11815.239,209909.2,9349.1,1274.4 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,8,2,3,0,0,32,254.876,1843.6,517.2,27.57 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,8,5,3,0,0,32,653.14,7161.4,1305.4,70.71 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,8,10,3,0,0,32,1501.141,23326.6,2724.7,162.14 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,8,20,3,0,0,32,3744.354,83530.8,5571.0,404.33 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,8,30,3,0,0,32,6850.308,180740.0,8551.8,739.48 -Qwen/Qwen2.5-7B-Instruct,gsmhard,8,2,3,0,0,32,405.422,1863.3,822.3,43.67 -Qwen/Qwen2.5-7B-Instruct,gsmhard,8,5,3,0,0,32,962.228,8105.2,1999.0,103.82 -Qwen/Qwen2.5-7B-Instruct,gsmhard,8,10,3,0,0,32,2192.599,27760.6,4085.6,236.44 -Qwen/Qwen2.5-7B-Instruct,gsmhard,8,20,3,0,0,32,5381.997,102219.4,8481.0,580.41 -Qwen/Qwen2.5-7B-Instruct,gsmhard,8,30,3,0,0,32,9156.734,222323.6,12238.4,986.53 -Qwen/Qwen2.5-7B-Instruct,gpqa,8,2,3,0,0,32,338.436,3099.1,640.0,36.47 -Qwen/Qwen2.5-7B-Instruct,gpqa,8,5,3,0,0,32,845.954,10826.2,1674.2,91.32 -Qwen/Qwen2.5-7B-Instruct,gpqa,8,10,3,0,0,32,1864.666,31766.7,3323.4,201.28 -Qwen/Qwen2.5-7B-Instruct,gpqa,8,20,3,0,0,32,4890.998,106026.4,7251.4,527.96 -Qwen/Qwen2.5-7B-Instruct,gpqa,8,30,3,0,0,32,8830.903,223146.8,10802.0,952.15 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,8,2,3,0,1,32,278.267,1877.2,558.5,30.13 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,8,5,3,0,1,32,666.389,7335.1,1337.5,72.05 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,8,10,3,0,1,32,1574.409,23842.3,2893.1,170.09 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,8,20,3,0,1,32,3882.454,85064.6,5820.4,419.28 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,8,30,3,0,1,32,6956.033,183877.5,8792.3,750.44 -Qwen/Qwen2.5-7B-Instruct,gsmhard,8,2,3,0,1,32,361.52,1874.8,748.9,39.13 -Qwen/Qwen2.5-7B-Instruct,gsmhard,8,5,3,0,1,32,1006.977,8177.8,2070.3,108.62 -Qwen/Qwen2.5-7B-Instruct,gsmhard,8,10,3,0,1,32,2151.071,27870.8,4033.9,231.94 -Qwen/Qwen2.5-7B-Instruct,gsmhard,8,20,3,0,1,32,5350.823,103148.5,8382.5,577.16 -Qwen/Qwen2.5-7B-Instruct,gsmhard,8,30,3,0,1,32,9265.379,223465.9,12578.6,998.69 -Qwen/Qwen2.5-7B-Instruct,gpqa,8,2,3,0,1,32,287.512,2472.4,581.3,31.08 -Qwen/Qwen2.5-7B-Instruct,gpqa,8,5,3,0,1,32,782.195,9041.7,1534.9,84.6 -Qwen/Qwen2.5-7B-Instruct,gpqa,8,10,3,0,1,32,1733.488,28021.2,3084.0,187.53 -Qwen/Qwen2.5-7B-Instruct,gpqa,8,20,3,0,1,32,4536.569,96859.5,6631.7,489.93 -Qwen/Qwen2.5-7B-Instruct,gpqa,8,30,3,0,1,32,8196.069,205996.0,9962.2,883.94 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,8,2,3,0,2,32,242.416,1828.6,497.3,26.33 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,8,5,3,0,2,32,665.824,7221.7,1340.0,72.13 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,8,10,3,0,2,32,1496.384,23278.2,2664.5,161.93 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,8,20,3,0,2,32,3844.279,83558.2,5631.2,415.41 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,8,30,3,0,2,32,6874.877,180599.1,8408.9,741.98 -Qwen/Qwen2.5-7B-Instruct,gsmhard,8,2,3,0,2,32,420.218,1837.2,770.3,45.22 -Qwen/Qwen2.5-7B-Instruct,gsmhard,8,5,3,0,2,32,963.997,7988.9,1897.0,104.03 -Qwen/Qwen2.5-7B-Instruct,gsmhard,8,10,3,0,2,32,2107.386,27335.3,3876.4,227.33 -Qwen/Qwen2.5-7B-Instruct,gsmhard,8,20,3,0,2,32,5177.129,100728.2,7928.2,558.26 -Qwen/Qwen2.5-7B-Instruct,gsmhard,8,30,3,0,2,32,8958.259,220158.0,11327.6,965.23 -Qwen/Qwen2.5-7B-Instruct,gpqa,8,2,3,0,2,32,347.648,2992.9,553.9,37.53 -Qwen/Qwen2.5-7B-Instruct,gpqa,8,5,3,0,2,32,728.626,10271.4,1447.5,78.81 -Qwen/Qwen2.5-7B-Instruct,gpqa,8,10,3,0,2,32,1654.605,30238.5,2913.9,178.91 -Qwen/Qwen2.5-7B-Instruct,gpqa,8,20,3,0,2,32,4333.876,100292.1,6209.6,468.08 -Qwen/Qwen2.5-7B-Instruct,gpqa,8,30,3,0,2,32,7777.758,209057.0,9171.3,839.12 diff --git a/scripts/clean/results/depth_carry_N1_Llama-3.1-8B-Instruct.csv b/scripts/clean/results/depth_carry_N1_Llama-3.1-8B-Instruct.csv deleted file mode 100644 index 937ca07..0000000 --- a/scripts/clean/results/depth_carry_N1_Llama-3.1-8B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,1,1,0,32,37.583,177.8,98.1,4.84 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,1,1,0,32,116.314,143.8,153.2,12.5 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,1,1,0,32,123.835,367.8,161.8,13.33 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,2,1,0,32,147.706,479.3,263.1,15.8 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,2,1,0,32,236.178,432.6,361.7,25.37 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,2,1,0,32,204.299,877.3,342.6,22.03 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,3,1,0,32,223.699,895.5,403.8,24.11 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,3,1,0,32,356.936,833.7,580.7,38.36 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,3,1,0,32,322.261,1498.1,569.6,34.76 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,4,1,0,32,286.245,1422.7,551.7,30.93 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,4,1,0,32,480.802,1351.8,820.2,51.58 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,4,1,0,32,495.734,2251.2,850.9,53.26 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,5,1,0,32,579.28,2086.0,843.4,62.39 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,5,1,0,32,605.302,1991.2,986.8,64.96 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,5,1,0,32,633.6,3115.3,1242.8,68.23 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,6,1,0,32,658.051,2870.4,1103.2,70.69 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,6,1,0,32,735.448,2747.9,1245.4,79.02 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,6,1,0,32,631.982,4115.1,1317.8,67.82 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,1,1,1,32,27.684,179.7,94.0,2.99 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,1,1,1,32,119.076,147.2,226.7,12.82 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,1,1,1,32,119.17,268.3,141.5,12.83 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,2,1,1,32,232.753,482.5,268.6,25.12 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,2,1,1,32,236.008,439.5,346.1,25.32 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,2,1,1,32,245.362,673.3,481.1,26.43 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,3,1,1,32,118.194,892.8,346.8,12.66 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,3,1,1,32,356.046,844.6,600.5,38.4 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,3,1,1,32,364.864,1196.4,742.9,39.32 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,4,1,1,32,473.163,1415.6,634.3,50.8 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,4,1,1,32,417.089,1366.2,703.7,44.8 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,4,1,1,32,499.294,1837.0,1169.9,53.57 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,5,1,1,32,467.289,2055.6,711.6,50.1 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,5,1,1,32,298.903,2005.7,778.0,32.15 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,5,1,1,32,611.945,2603.0,1216.0,65.68 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,6,1,1,32,492.317,2820.3,849.3,52.8 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,6,1,1,32,576.322,2769.8,1078.8,62.09 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,6,1,1,32,761.155,3485.9,1791.2,81.88 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,1,1,2,32,28.831,176.9,86.3,3.15 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,1,1,2,32,119.144,143.3,206.5,12.77 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,1,1,2,32,122.789,360.2,134.5,13.25 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,2,1,2,32,142.42,472.6,233.3,15.34 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,2,1,2,32,236.197,426.9,338.9,25.35 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,2,1,2,32,245.515,848.9,333.1,26.38 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,3,1,2,32,172.222,873.5,370.1,18.7 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,3,1,2,32,354.51,814.0,518.0,38.08 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,3,1,2,32,316.066,1454.2,531.8,33.98 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,4,1,2,32,190.633,1398.6,494.2,20.44 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,4,1,2,32,446.115,1335.7,721.6,47.88 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,4,1,2,32,456.956,2180.3,777.8,49.16 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,5,1,2,32,277.481,2031.6,638.8,29.93 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,5,1,2,32,432.772,1941.4,808.6,46.73 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,5,1,2,32,632.323,3023.2,1097.5,67.9 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,6,1,2,32,652.248,2776.8,993.0,70.26 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,6,1,2,32,567.741,2673.8,957.8,61.12 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,6,1,2,32,664.849,3989.4,1195.8,71.46 diff --git a/scripts/clean/results/depth_carry_N1_Q256_Llama-3.1-8B-Instruct.csv b/scripts/clean/results/depth_carry_N1_Q256_Llama-3.1-8B-Instruct.csv deleted file mode 100644 index 8b8e789..0000000 --- a/scripts/clean/results/depth_carry_N1_Q256_Llama-3.1-8B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,1,1,0,256,19.305,182.2,97.1,16.62 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,1,1,0,256,20.867,147.0,172.6,17.95 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,1,1,0,198,27.968,300.4,136.7,18.59 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,2,1,0,256,40.962,486.9,267.4,35.35 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,2,1,0,256,43.076,436.5,351.8,37.1 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,2,1,0,198,58.824,738.3,389.0,39.16 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,3,1,0,256,69.079,903.1,434.5,59.48 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,3,1,0,256,71.394,838.0,531.0,61.51 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,3,1,0,198,100.545,1293.5,655.4,66.94 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,4,1,0,256,96.503,1431.3,585.3,83.28 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,4,1,0,256,101.002,1359.7,733.2,87.22 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,4,1,0,198,138.453,1970.7,914.4,92.07 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,5,1,0,256,126.728,2076.6,731.4,109.28 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,5,1,0,256,135.746,2000.4,968.9,117.02 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,5,1,0,198,181.793,2762.4,1201.6,121.04 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,6,1,0,256,160.254,2842.8,892.1,138.0 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,6,1,0,256,168.047,2760.0,1147.2,144.93 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,6,1,0,198,223.85,3683.2,1421.3,148.93 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,1,1,1,256,19.541,183.0,96.2,16.83 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,1,1,1,256,20.463,147.8,160.1,17.65 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,1,1,1,198,28.236,300.4,131.2,18.84 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,2,1,1,256,40.724,487.5,247.5,35.05 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,2,1,1,256,42.829,438.0,336.8,37.14 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,2,1,1,198,60.282,738.8,410.9,40.14 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,3,1,1,256,67.918,899.9,396.4,58.45 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,3,1,1,256,73.267,838.3,554.4,63.27 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,3,1,1,198,99.999,1294.2,651.8,66.49 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,4,1,1,256,95.164,1423.5,546.1,82.01 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,4,1,1,256,100.686,1356.9,718.0,86.71 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,4,1,1,198,139.364,1971.4,923.9,92.84 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,5,1,1,256,116.479,2074.9,702.0,100.72 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,5,1,1,256,131.385,1998.1,887.9,113.24 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,5,1,1,198,181.419,2771.3,1181.0,120.69 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,6,1,1,256,160.405,2829.5,886.7,138.26 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,6,1,1,256,167.51,2758.5,1109.8,144.13 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,6,1,1,198,224.316,3692.1,1425.7,149.27 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,1,1,2,256,19.818,180.2,100.8,17.07 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,1,1,2,256,20.76,146.6,160.7,18.02 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,1,1,2,198,28.28,300.4,133.7,18.88 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,2,1,2,256,40.597,480.5,251.1,35.02 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,2,1,2,256,43.24,434.7,354.5,37.27 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,2,1,2,198,58.609,738.2,380.9,39.28 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,3,1,2,256,68.654,889.8,416.5,59.12 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,3,1,2,256,72.276,834.5,543.7,62.27 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,3,1,2,198,101.358,1292.7,674.7,67.45 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,4,1,2,256,97.253,1415.6,600.4,83.83 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,4,1,2,256,100.076,1354.9,715.8,86.2 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,4,1,2,198,139.7,1967.6,916.5,93.1 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,5,1,2,256,117.429,2057.1,721.5,101.34 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,5,1,2,256,131.732,1990.8,908.5,113.5 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,5,1,2,198,178.775,2766.0,1147.0,118.83 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,6,1,2,256,163.577,2816.4,953.1,140.72 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,6,1,2,256,165.813,2755.5,1089.0,142.69 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,6,1,2,198,225.083,3685.3,1425.3,149.81 diff --git a/scripts/clean/results/depth_carry_N1_Q256_Qwen2.5-7B-Instruct.csv b/scripts/clean/results/depth_carry_N1_Q256_Qwen2.5-7B-Instruct.csv deleted file mode 100644 index 2628833..0000000 --- a/scripts/clean/results/depth_carry_N1_Q256_Qwen2.5-7B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,1,1,0,256,6.949,181.1,61.5,5.99 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,1,1,0,256,8.607,146.7,108.4,7.73 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,1,1,0,198,11.752,299.5,76.5,8.09 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,2,1,0,256,16.525,465.2,146.2,14.81 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,2,1,0,256,29.138,431.6,249.5,25.42 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,2,1,0,198,25.392,717.2,184.1,17.26 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,3,1,0,256,22.287,840.3,240.4,20.4 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,3,1,0,256,33.269,827.6,381.6,29.46 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,3,1,0,198,42.094,1238.3,309.8,28.44 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,4,1,0,256,30.697,1309.4,330.8,27.46 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,4,1,0,256,58.988,1347.2,526.6,51.36 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,4,1,0,198,49.54,1860.2,412.6,33.85 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,5,1,0,256,44.622,1879.9,430.5,39.88 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,5,1,0,256,71.723,1985.7,656.3,62.85 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,5,1,0,198,79.873,2595.2,523.0,53.9 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,6,1,0,256,64.204,2541.1,521.7,56.77 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,6,1,0,256,94.917,2740.1,780.7,82.74 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,6,1,0,198,96.982,3434.2,627.1,65.58 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,1,1,1,256,7.071,181.6,62.0,6.33 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,1,1,1,256,8.988,148.0,107.8,8.02 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,1,1,1,198,9.944,299.5,77.8,6.99 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,2,1,1,256,19.69,466.7,150.7,17.46 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,2,1,1,256,18.311,432.7,242.1,16.11 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,2,1,1,198,24.514,716.0,189.3,16.71 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,3,1,1,256,22.372,841.9,243.7,19.91 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,3,1,1,256,36.206,829.1,381.3,31.58 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,3,1,1,198,34.071,1233.3,302.8,23.27 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,4,1,1,256,46.203,1310.5,337.7,40.31 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,4,1,1,256,50.349,1345.3,516.9,44.17 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,4,1,1,198,58.3,1856.3,421.1,39.39 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,5,1,1,256,62.156,1881.6,436.9,54.37 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,5,1,1,256,83.2,1971.7,658.1,71.91 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,5,1,1,198,81.064,2590.3,544.8,54.9 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,6,1,1,256,72.328,2543.7,516.8,63.57 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,6,1,1,256,97.09,2725.7,796.0,84.74 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,6,1,1,198,95.963,3427.7,655.9,64.99 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,1,1,2,256,7.157,179.1,60.9,6.63 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,1,1,2,256,8.506,146.4,105.0,7.62 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,1,1,2,198,9.065,299.5,76.0,6.25 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,2,1,2,256,15.877,460.7,145.2,14.31 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,2,1,2,256,27.243,428.3,239.1,23.96 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,2,1,2,198,18.822,716.2,181.8,12.86 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,3,1,2,256,25.136,827.7,235.4,22.46 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,3,1,2,256,46.419,822.3,379.3,40.43 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,3,1,2,198,38.635,1229.7,296.8,26.22 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,4,1,2,256,33.902,1290.4,321.4,30.22 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,4,1,2,256,49.96,1330.5,497.3,43.64 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,4,1,2,198,45.074,1856.6,410.4,30.6 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,5,1,2,256,47.826,1851.4,419.8,42.63 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,5,1,2,256,67.178,1962.9,634.8,59.09 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,5,1,2,198,72.662,2590.0,524.2,49.29 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,6,1,2,256,65.701,2503.6,505.7,58.25 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,6,1,2,256,95.064,2711.2,766.2,83.08 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,6,1,2,198,119.384,3433.9,673.5,80.05 diff --git a/scripts/clean/results/depth_carry_N1_Qwen2.5-7B-Instruct.csv b/scripts/clean/results/depth_carry_N1_Qwen2.5-7B-Instruct.csv deleted file mode 100644 index e89c1f0..0000000 --- a/scripts/clean/results/depth_carry_N1_Qwen2.5-7B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,1,1,0,32,20.386,176.3,62.2,2.18 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,1,1,0,32,108.808,143.3,130.5,11.67 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,1,1,0,32,25.13,366.7,80.9,2.72 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,2,1,0,32,61.428,456.9,162.0,6.7 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,2,1,0,32,62.185,422.8,248.4,6.75 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,2,1,0,32,53.761,854.4,185.3,5.82 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,3,1,0,32,99.532,828.8,263.1,10.78 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,3,1,0,32,168.627,815.3,407.9,18.15 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,3,1,0,32,77.652,1444.8,300.5,8.31 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,4,1,0,32,100.044,1294.6,334.7,10.84 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,4,1,0,32,170.777,1324.9,526.6,18.46 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,4,1,0,32,114.319,2139.1,406.5,12.46 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,5,1,0,32,181.319,1864.7,464.2,19.53 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,5,1,0,32,291.326,1947.4,677.9,31.41 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,5,1,0,32,153.014,2967.3,560.9,16.49 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,6,1,0,32,230.821,2513.5,569.4,24.97 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,6,1,0,32,246.631,2691.9,789.8,26.56 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,6,1,0,32,195.785,3874.1,645.3,21.04 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,1,1,1,32,24.263,177.6,69.5,2.54 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,1,1,1,32,26.956,147.3,108.1,2.92 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,1,1,1,32,27.119,267.5,75.8,2.92 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,2,1,1,32,54.961,463.6,160.8,5.85 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,2,1,1,32,54.862,433.2,235.6,5.9 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,2,1,1,32,68.068,645.2,169.0,7.42 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,3,1,1,32,76.237,833.5,249.5,8.22 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,3,1,1,32,105.507,831.5,379.1,11.34 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,3,1,1,32,101.777,1120.8,296.5,10.97 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,4,1,1,32,108.253,1325.5,377.7,11.71 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,4,1,1,32,122.278,1355.7,507.1,13.25 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,4,1,1,32,166.097,1699.7,413.5,17.91 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,5,1,1,32,143.487,1878.8,451.0,15.67 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,5,1,1,32,246.622,1979.2,666.8,26.55 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,5,1,1,32,185.507,2367.2,517.6,20.08 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,6,1,1,32,185.315,2535.3,547.2,19.9 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,6,1,1,32,193.915,2750.1,780.7,20.89 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,6,1,1,32,206.356,3147.0,589.6,22.4 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,1,1,2,32,21.33,175.8,61.2,2.4 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,1,1,2,32,30.854,142.5,98.2,3.37 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,1,1,2,32,19.443,361.4,61.4,2.11 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,2,1,2,32,51.436,453.7,147.2,5.68 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,2,1,2,32,60.799,416.5,224.8,6.61 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,2,1,2,32,40.086,829.2,150.4,4.36 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,3,1,2,32,91.616,818.1,246.4,9.82 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,3,1,2,32,91.599,798.1,341.1,9.96 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,3,1,2,32,59.201,1386.9,242.6,6.44 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,4,1,2,32,110.054,1279.6,333.5,12.0 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,4,1,2,32,114.95,1300.2,450.1,12.32 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,4,1,2,32,89.066,2053.5,351.0,9.69 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,5,1,2,32,154.166,1834.5,437.2,16.67 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,5,1,2,32,223.955,1927.0,627.8,24.21 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,5,1,2,32,105.384,2825.6,442.8,11.35 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,6,1,2,32,191.097,2509.9,543.9,20.68 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,6,1,2,32,202.989,2659.9,699.5,21.88 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,6,1,2,32,175.052,3699.3,558.3,18.69 diff --git a/scripts/clean/results/depth_carry_N1_apcOFF_Llama-3.1-8B-Instruct.csv b/scripts/clean/results/depth_carry_N1_apcOFF_Llama-3.1-8B-Instruct.csv deleted file mode 100644 index 9ab3cc3..0000000 --- a/scripts/clean/results/depth_carry_N1_apcOFF_Llama-3.1-8B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,1,1,0,32,117.438,177.8,122.2,12.54 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,1,1,0,32,119.059,143.8,213.3,12.81 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,1,1,0,32,124.665,367.8,172.9,13.43 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,2,1,0,32,152.489,481.8,265.2,16.39 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,2,1,0,32,187.147,430.2,323.2,20.08 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,2,1,0,32,186.556,876.2,357.2,20.08 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,3,1,0,32,277.77,896.7,420.5,29.74 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,3,1,0,32,314.727,831.0,583.7,33.88 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,3,1,0,32,389.269,1504.0,600.2,41.81 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,4,1,0,32,403.845,1421.6,575.2,43.37 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,4,1,0,32,444.651,1351.0,800.0,47.92 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,4,1,0,32,526.548,2247.2,796.8,56.49 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,5,1,0,32,468.498,2079.8,761.5,50.34 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,5,1,0,32,574.922,1991.3,916.6,61.72 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,5,1,0,32,674.841,3113.7,1119.3,72.27 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,6,1,0,32,532.489,2819.2,852.7,57.18 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,6,1,0,32,736.062,2774.2,1269.9,79.04 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,6,1,0,32,816.986,4103.8,1335.0,87.88 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,1,1,1,32,117.093,179.7,122.2,12.6 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,1,1,1,32,118.013,147.2,166.4,12.67 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,1,1,1,32,119.784,268.3,145.5,12.92 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,2,1,1,32,172.384,483.1,247.8,18.58 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,2,1,1,32,238.871,439.4,335.0,25.69 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,2,1,1,32,250.29,672.1,452.9,26.91 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,3,1,1,32,274.797,891.0,432.8,29.65 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,3,1,1,32,366.592,842.8,584.6,39.33 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,3,1,1,32,382.565,1191.9,715.8,41.08 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,4,1,1,32,344.781,1410.2,564.8,37.07 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,4,1,1,32,421.765,1369.5,696.1,45.39 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,4,1,1,32,519.359,1833.2,994.4,55.6 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,5,1,1,32,342.67,2059.8,688.4,36.95 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,5,1,1,32,554.246,2015.4,891.0,59.53 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,5,1,1,32,657.424,2596.8,1249.2,70.45 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,6,1,1,32,374.271,2819.0,808.4,40.34 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,6,1,1,32,637.808,2770.4,1058.2,68.41 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,6,1,1,32,798.264,3476.9,1510.0,85.68 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,1,1,2,32,26.835,176.9,87.6,2.88 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,1,1,2,32,117.223,143.3,153.8,12.59 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,1,1,2,32,74.071,360.2,117.5,7.98 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,2,1,2,32,107.406,474.0,235.9,11.48 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,2,1,2,32,238.099,425.3,319.8,25.6 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,2,1,2,32,203.531,848.1,304.9,21.89 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,3,1,2,32,189.815,880.1,385.1,20.4 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,3,1,2,32,365.669,814.8,569.0,39.27 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,3,1,2,32,374.132,1447.7,598.3,40.17 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,4,1,2,32,317.532,1401.8,541.7,34.07 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,4,1,2,32,492.729,1320.6,731.8,52.94 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,4,1,2,32,477.795,2166.4,764.9,51.2 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,5,1,2,32,367.092,2041.9,681.8,39.47 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,5,1,2,32,568.532,1944.0,882.2,61.15 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,5,1,2,32,586.348,3015.9,948.5,63.01 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,6,1,2,32,453.99,2803.1,835.9,48.78 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,6,1,2,32,756.204,2686.5,1066.1,81.59 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,6,1,2,32,735.06,3966.9,1138.6,78.96 diff --git a/scripts/clean/results/depth_carry_N1_apcOFF_Qwen2.5-7B-Instruct.csv b/scripts/clean/results/depth_carry_N1_apcOFF_Qwen2.5-7B-Instruct.csv deleted file mode 100644 index 605091f..0000000 --- a/scripts/clean/results/depth_carry_N1_apcOFF_Qwen2.5-7B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,1,1,0,32,22.425,176.3,62.0,2.33 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,1,1,0,32,22.328,143.3,101.5,2.47 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,1,1,0,32,33.117,366.7,81.9,4.53 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,2,1,0,32,53.843,456.0,151.2,5.82 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,2,1,0,32,71.065,422.1,239.1,7.62 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,2,1,0,32,63.67,853.2,190.4,6.8 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,3,1,0,32,82.849,825.7,242.9,8.89 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,3,1,0,32,180.814,812.0,399.1,19.48 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,3,1,0,32,93.319,1439.5,301.0,10.02 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,4,1,0,32,159.911,1294.9,360.4,17.28 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,4,1,0,32,241.148,1319.7,549.5,25.98 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,4,1,0,32,145.366,2139.1,418.1,15.63 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,5,1,0,32,213.546,1870.5,468.3,22.86 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,5,1,0,32,300.155,1946.1,685.6,32.2 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,5,1,0,32,202.681,2938.9,538.9,21.62 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,6,1,0,32,268.06,2521.0,571.1,28.77 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,6,1,0,32,233.969,2685.7,751.8,25.29 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,6,1,0,32,248.133,3852.0,647.2,26.61 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,1,1,1,32,23.352,177.6,68.8,2.59 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,1,1,1,32,33.65,147.3,109.1,3.71 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,1,1,1,32,29.063,267.5,70.8,3.11 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,2,1,1,32,57.251,461.9,165.4,6.17 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,2,1,1,32,75.04,433.0,243.0,8.09 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,2,1,1,32,72.808,649.4,171.8,7.8 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,3,1,1,32,97.49,835.1,262.3,10.6 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,3,1,1,32,103.541,832.7,376.3,11.14 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,3,1,1,32,113.197,1123.0,276.5,12.25 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,4,1,1,32,124.339,1315.5,359.6,13.52 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,4,1,1,32,154.45,1350.3,528.2,16.79 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,4,1,1,32,160.694,1699.2,381.7,17.18 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,5,1,1,32,165.188,1884.4,455.9,17.64 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,5,1,1,32,193.531,1989.7,656.4,20.8 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,5,1,1,32,197.68,2379.7,479.0,21.28 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,6,1,1,32,207.223,2551.5,557.4,22.34 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,6,1,1,32,229.568,2754.2,785.8,24.63 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,6,1,1,32,297.457,3156.1,607.6,31.76 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,1,1,2,32,24.102,175.8,64.8,2.67 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,1,1,2,32,31.911,142.5,96.3,3.44 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,1,1,2,32,21.297,361.4,61.8,2.3 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,2,1,2,32,56.008,453.5,147.5,6.12 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,2,1,2,32,63.531,416.1,211.4,6.78 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,2,1,2,32,49.5,828.3,152.5,5.34 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,3,1,2,32,92.686,818.3,243.7,9.96 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,3,1,2,32,106.73,807.1,342.1,11.46 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,3,1,2,32,79.292,1391.1,246.1,8.7 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,4,1,2,32,128.792,1277.4,324.7,13.9 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,4,1,2,32,234.753,1299.0,492.8,25.2 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,4,1,2,32,112.788,2056.6,344.9,12.21 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,5,1,2,32,171.174,1836.6,426.8,18.37 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,5,1,2,32,177.754,1927.0,573.9,19.17 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,5,1,2,32,146.654,2829.6,443.3,15.7 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,6,1,2,32,219.801,2511.9,544.4,23.52 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,6,1,2,32,211.959,2659.0,705.8,22.89 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,6,1,2,32,185.053,3711.2,548.4,19.84 diff --git a/scripts/clean/results/depth_carry_N1_apcON_Llama-3.1-8B-Instruct.csv b/scripts/clean/results/depth_carry_N1_apcON_Llama-3.1-8B-Instruct.csv deleted file mode 100644 index 3d92d5e..0000000 --- a/scripts/clean/results/depth_carry_N1_apcON_Llama-3.1-8B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,1,1,0,32,32.635,177.8,96.3,3.48 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,1,1,0,32,117.163,143.8,167.1,12.56 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,1,1,0,32,123.063,367.8,161.8,13.29 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,2,1,0,32,158.906,481.2,274.2,17.09 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,2,1,0,32,237.973,430.7,415.6,25.66 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,2,1,0,32,249.856,876.4,456.6,26.82 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,3,1,0,32,267.213,898.1,452.1,28.7 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,3,1,0,32,356.774,833.5,565.1,38.34 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,3,1,0,32,370.511,1498.2,680.0,39.9 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,4,1,0,32,387.752,1421.8,629.7,41.72 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,4,1,0,32,481.194,1355.1,879.3,51.76 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,4,1,0,32,423.419,2249.0,958.7,45.5 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,5,1,0,32,457.078,2077.0,795.7,49.29 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,5,1,0,32,485.353,1986.5,963.9,52.11 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,5,1,0,32,628.1,3125.1,1150.3,67.46 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,6,1,0,32,492.452,2857.9,909.3,52.81 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,6,1,0,32,595.776,2745.0,1103.1,64.04 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,6,1,0,32,729.281,4104.4,1405.8,78.48 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,1,1,1,32,28.616,179.7,93.9,3.08 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,1,1,1,32,120.134,147.2,231.3,12.83 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,1,1,1,32,119.772,268.3,146.2,12.85 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,2,1,1,32,64.144,483.6,209.0,6.91 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,2,1,1,32,237.336,439.2,369.5,25.48 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,2,1,1,32,241.002,673.5,391.8,25.85 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,3,1,1,32,132.467,893.7,363.8,14.39 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,3,1,1,32,290.843,841.1,540.4,31.32 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,3,1,1,32,362.706,1194.1,664.2,39.0 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,4,1,1,32,186.121,1419.1,501.4,20.07 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,4,1,1,32,320.35,1365.2,635.3,34.34 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,4,1,1,32,491.171,1836.7,958.0,52.77 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,5,1,1,32,306.212,2060.7,664.1,32.94 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,5,1,1,32,448.448,2015.2,906.4,48.31 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,5,1,1,32,621.633,2608.1,1283.2,66.88 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,6,1,1,32,428.315,2817.0,902.8,46.22 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,6,1,1,32,446.552,2784.3,1002.7,47.92 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,6,1,1,32,755.688,3492.3,1600.9,81.4 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,1,1,2,32,39.36,176.9,90.2,4.23 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,1,1,2,32,117.212,143.3,154.4,12.54 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,1,1,2,32,123.711,360.2,146.1,13.28 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,2,1,2,32,141.275,472.3,233.5,15.31 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,2,1,2,32,234.604,426.1,327.5,25.32 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,2,1,2,32,248.043,850.1,361.2,26.63 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,3,1,2,32,233.433,871.0,401.5,25.18 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,3,1,2,32,279.239,818.2,473.0,29.95 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,3,1,2,32,362.707,1452.4,536.7,39.19 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,4,1,2,32,261.167,1398.3,522.9,28.01 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,4,1,2,32,422.358,1321.8,626.6,45.4 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,4,1,2,32,475.527,2182.7,791.7,51.1 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,5,1,2,32,332.039,2031.0,658.9,35.69 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,5,1,2,32,467.468,1953.5,820.9,50.06 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,5,1,2,32,626.848,3032.4,1050.8,67.4 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,6,1,2,32,396.407,2772.2,819.1,42.67 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,6,1,2,32,635.507,2674.5,1000.0,68.21 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,6,1,2,32,686.526,3971.7,1266.2,73.74 diff --git a/scripts/clean/results/depth_carry_N1_apcON_Qwen2.5-7B-Instruct.csv b/scripts/clean/results/depth_carry_N1_apcON_Qwen2.5-7B-Instruct.csv deleted file mode 100644 index e84dccf..0000000 --- a/scripts/clean/results/depth_carry_N1_apcON_Qwen2.5-7B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,1,1,0,32,21.429,176.3,61.1,2.31 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,1,1,0,32,22.185,143.3,103.8,2.41 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,1,1,0,32,24.062,366.7,83.0,2.58 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,2,1,0,32,57.742,455.9,161.9,6.21 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,2,1,0,32,133.892,422.6,267.5,14.47 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,2,1,0,32,57.929,856.8,193.1,6.29 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,3,1,0,32,101.069,828.3,260.8,10.92 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,3,1,0,32,104.007,814.5,382.2,11.18 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,3,1,0,32,74.442,1449.5,304.1,8.0 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,4,1,0,32,92.58,1292.0,332.8,10.01 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,4,1,0,32,129.759,1320.0,502.0,14.02 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,4,1,0,32,118.319,2150.2,418.2,12.69 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,5,1,0,32,145.974,1866.5,447.8,15.85 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,5,1,0,32,212.315,1956.2,655.4,22.78 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,5,1,0,32,148.003,2962.4,533.1,16.01 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,6,1,0,32,146.691,2502.0,514.5,15.88 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,6,1,0,32,216.326,2688.5,780.5,23.19 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,6,1,0,32,170.166,3886.7,665.7,18.26 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,1,1,1,32,23.311,177.6,68.9,2.53 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,1,1,1,32,29.967,147.3,107.8,3.23 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,1,1,1,32,27.075,267.5,73.5,2.91 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,2,1,1,32,54.234,463.0,162.1,5.79 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,2,1,1,32,61.479,435.1,247.2,6.61 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,2,1,1,32,68.142,646.6,167.1,7.38 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,3,1,1,32,74.738,845.1,267.7,8.09 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,3,1,1,32,96.392,835.8,376.6,10.38 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,3,1,1,32,96.932,1125.0,277.5,10.35 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,4,1,1,32,98.875,1315.8,353.6,10.73 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,4,1,1,32,133.712,1357.9,527.2,14.44 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,4,1,1,32,155.117,1694.4,381.5,16.68 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,5,1,1,32,127.722,1875.9,452.3,13.8 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,5,1,1,32,163.34,1999.1,650.2,17.81 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,5,1,1,32,157.287,2376.5,473.8,16.89 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,6,1,1,32,156.08,2567.6,560.8,16.71 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,6,1,1,32,207.8,2761.6,806.2,23.68 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,6,1,1,32,226.392,3171.6,611.8,24.26 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,1,1,2,32,23.225,175.8,62.3,2.55 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,1,1,2,32,21.385,142.5,95.3,2.31 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,1,1,2,32,19.583,361.4,61.4,2.09 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,2,1,2,32,52.491,453.6,148.8,5.64 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,2,1,2,32,72.081,418.5,223.8,7.78 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,2,1,2,32,73.338,831.2,167.8,8.09 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,3,1,2,32,80.767,815.9,240.0,8.8 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,3,1,2,32,92.309,803.0,347.5,9.88 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,3,1,2,32,58.497,1397.7,253.4,6.25 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,4,1,2,32,124.932,1281.8,342.3,13.44 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,4,1,2,32,120.18,1303.7,465.0,12.88 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,4,1,2,32,81.657,2062.1,349.4,8.77 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,5,1,2,32,141.866,1842.0,433.5,15.37 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,5,1,2,32,160.727,1924.2,589.3,17.19 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,5,1,2,32,106.397,2845.6,458.6,11.57 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,6,1,2,32,174.516,2493.5,527.1,18.79 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,6,1,2,32,289.176,2661.7,752.7,31.3 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,6,1,2,32,151.953,3721.0,556.1,16.21 diff --git a/scripts/clean/results/depth_carry_N5_Llama-3.1-8B-Instruct.csv b/scripts/clean/results/depth_carry_N5_Llama-3.1-8B-Instruct.csv deleted file mode 100644 index 8b08d5a..0000000 --- a/scripts/clean/results/depth_carry_N5_Llama-3.1-8B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,1,1,0,32,146.85,888.8,504.9,15.87 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,1,1,0,32,165.708,719.1,910.7,17.87 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,1,1,0,32,159.657,1838.8,772.7,17.21 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,2,1,0,32,357.734,4289.8,1423.1,38.47 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,2,1,0,32,449.074,4478.1,2352.7,48.31 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,2,1,0,32,428.374,6564.8,1841.4,46.3 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,3,1,0,32,638.774,10509.7,2051.2,68.88 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,3,1,0,32,791.874,11324.2,3352.1,85.24 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,3,1,0,32,786.168,14398.1,2974.9,84.77 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,4,1,0,32,1026.067,19737.9,2778.9,110.32 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,4,1,0,32,1212.773,21182.7,4406.1,130.48 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,4,1,0,32,1175.281,25165.3,3757.0,126.48 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,5,1,0,32,1375.003,31894.2,3588.8,147.84 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,5,1,0,32,1734.718,34113.3,6004.2,186.36 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,5,1,0,32,1581.03,38984.7,4478.6,170.08 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,6,1,0,32,1871.146,46867.5,4468.9,201.14 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,6,1,0,32,2165.177,49930.3,6996.5,232.57 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,6,1,0,32,2099.968,56229.2,5535.8,226.02 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,1,1,1,32,139.267,898.3,511.7,15.06 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,1,1,1,32,166.644,735.9,869.7,17.93 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,1,1,1,32,142.857,1341.6,641.3,15.46 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,2,1,1,32,289.401,4224.4,1368.1,31.32 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,2,1,1,32,434.145,4487.2,2096.7,46.77 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,2,1,1,32,430.922,5496.4,1915.6,46.31 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,3,1,1,32,633.603,10463.6,2202.0,68.17 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,3,1,1,32,775.591,11250.0,3091.9,83.67 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,3,1,1,32,722.746,12788.1,2748.8,77.77 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,4,1,1,32,1016.929,19566.2,2928.3,109.38 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,4,1,1,32,1188.184,21149.4,4296.1,127.74 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,4,1,1,32,1160.776,22990.8,3709.9,125.09 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,5,1,1,32,1205.726,31620.3,3240.8,129.82 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,5,1,1,32,1600.191,33919.5,4857.9,171.91 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,5,1,1,32,1615.525,36456.2,4633.5,173.59 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,6,1,1,32,1892.613,46828.4,4434.6,203.42 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,6,1,1,32,2079.446,49658.0,6019.9,223.57 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,6,1,1,32,2048.12,52705.4,5309.2,220.18 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,1,1,2,32,128.194,884.5,477.7,13.81 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,1,1,2,32,160.727,716.6,727.3,17.32 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,1,1,2,32,139.981,1801.2,570.7,15.08 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,2,1,2,32,276.611,4195.3,1191.9,29.79 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,2,1,2,32,423.292,4420.9,1972.0,45.85 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,2,1,2,32,422.47,6323.0,1745.6,45.51 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,3,1,2,32,560.573,10427.5,1914.5,60.52 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,3,1,2,32,780.314,11121.1,3036.8,84.0 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,3,1,2,32,745.437,13852.3,2599.7,80.26 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,4,1,2,32,991.028,19499.2,2554.8,106.88 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,4,1,2,32,1227.975,20786.1,4373.2,131.96 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,4,1,2,32,1161.725,24428.6,3392.6,124.94 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,5,1,2,32,1279.933,31744.1,3236.8,137.96 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,5,1,2,32,1613.173,33591.7,4762.2,173.39 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,5,1,2,32,1558.378,37821.6,4654.3,167.66 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,6,1,2,32,1560.242,46458.4,3763.0,168.13 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,6,1,2,32,2006.829,49487.3,5662.9,215.57 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,6,1,2,32,2049.548,54431.9,5550.8,220.58 diff --git a/scripts/clean/results/depth_carry_N5_Qwen2.5-7B-Instruct.csv b/scripts/clean/results/depth_carry_N5_Qwen2.5-7B-Instruct.csv deleted file mode 100644 index 6d8d7d6..0000000 --- a/scripts/clean/results/depth_carry_N5_Qwen2.5-7B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,1,1,0,32,31.404,881.6,301.3,3.48 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,1,1,0,32,55.228,716.7,524.3,6.16 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,1,1,0,32,48.775,1833.4,392.8,5.51 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,2,1,0,32,133.191,3713.1,778.6,14.64 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,2,1,0,32,186.922,4295.2,1269.7,20.39 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,2,1,0,32,204.29,6075.2,1025.5,22.27 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,3,1,0,32,245.573,8964.1,1293.7,26.78 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,3,1,0,32,370.867,10854.9,1997.3,40.29 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,3,1,0,32,341.799,13080.1,1608.0,37.1 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,4,1,0,32,420.106,16819.2,1830.8,45.82 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,4,1,0,32,572.473,20507.0,2640.7,62.29 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,4,1,0,32,591.221,22999.9,2255.7,64.13 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,5,1,0,32,637.2,27645.7,2423.8,69.18 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,5,1,0,32,893.309,33189.7,3382.3,96.64 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,5,1,0,32,855.513,35774.4,2915.0,92.57 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,6,1,0,32,815.942,40236.6,2799.3,88.8 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,6,1,0,32,1137.001,48894.7,4092.3,122.84 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,6,1,0,32,1076.677,51443.1,3549.5,116.49 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,1,1,1,32,36.554,888.1,347.1,3.99 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,1,1,1,32,125.226,736.7,568.5,13.53 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,1,1,1,32,123.518,1337.3,390.4,13.32 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,2,1,1,32,139.732,3891.3,825.6,15.29 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,2,1,1,32,236.365,4373.0,1332.4,25.8 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,2,1,1,32,162.925,4917.3,942.9,17.71 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,3,1,1,32,268.811,9349.6,1359.3,29.29 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,3,1,1,32,367.823,11009.5,2057.6,40.05 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,3,1,1,32,394.969,11127.8,1525.9,42.82 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,4,1,1,32,427.103,17354.5,1831.3,46.33 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,4,1,1,32,601.585,20688.8,2806.0,65.18 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,4,1,1,32,712.349,20163.9,2245.9,76.84 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,5,1,1,32,632.079,27426.2,2302.8,68.7 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,5,1,1,32,864.236,33423.9,3576.2,93.54 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,5,1,1,32,902.798,32095.8,2888.0,97.42 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,6,1,1,32,784.449,40490.7,2844.1,85.53 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,6,1,1,32,1029.284,49088.7,4245.4,111.36 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,6,1,1,32,1127.821,46855.6,3421.5,121.99 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,1,1,2,32,44.705,879.1,318.8,4.95 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,1,1,2,32,139.236,712.3,523.3,15.06 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,1,1,2,32,46.327,1807.0,326.6,5.12 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,2,1,2,32,137.558,3718.4,783.7,15.17 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,2,1,2,32,249.458,4239.1,1235.7,27.15 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,2,1,2,32,149.379,5699.9,851.4,16.34 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,3,1,2,32,266.618,9020.8,1308.4,29.05 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,3,1,2,32,489.943,10723.1,1976.6,52.87 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,3,1,2,32,284.639,12189.6,1392.9,31.11 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,4,1,2,32,467.5,16913.3,1797.1,50.71 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,4,1,2,32,676.051,20263.4,2563.2,73.02 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,4,1,2,32,523.524,21484.7,1998.2,56.97 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,5,1,2,32,677.807,27493.4,2361.3,73.52 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,5,1,2,32,840.327,32455.5,3207.0,90.89 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,5,1,2,32,787.301,33851.8,2567.0,85.16 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,6,1,2,32,874.286,40751.3,2873.7,94.62 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,6,1,2,32,1191.06,48119.1,4112.4,128.71 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,6,1,2,32,952.233,48453.7,3089.4,103.1 diff --git a/scripts/clean/results/depth_free_N1_Llama-3.1-8B-Instruct.csv b/scripts/clean/results/depth_free_N1_Llama-3.1-8B-Instruct.csv deleted file mode 100644 index 5c16ac1..0000000 --- a/scripts/clean/results/depth_free_N1_Llama-3.1-8B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,1,0,0,32,29.722,177.8,92.1,3.24 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,1,0,0,32,118.272,143.8,194.8,12.65 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,1,0,0,32,123.724,367.8,161.8,13.27 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,2,0,0,32,147.699,478.8,272.2,15.93 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,2,0,0,32,237.872,432.8,398.6,25.55 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,2,0,0,32,247.508,875.8,433.7,26.67 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,3,0,0,32,263.125,803.8,446.7,28.2 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,3,0,0,32,352.333,721.9,568.4,37.8 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,3,0,0,32,367.267,1394.0,689.6,39.45 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,4,0,0,32,400.703,1128.5,629.6,43.1 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,4,0,0,32,469.761,1011.7,719.7,50.35 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,4,0,0,32,403.372,1910.2,766.9,43.32 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,5,0,0,32,472.542,1457.2,770.8,50.76 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,5,0,0,32,594.211,1293.8,1102.7,63.96 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,5,0,0,32,532.769,2428.3,1107.3,57.25 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,6,0,0,32,540.93,1783.3,939.8,58.27 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,6,0,0,32,709.178,1574.8,1243.9,76.21 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,6,0,0,32,728.756,2945.2,1382.3,78.32 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,1,0,1,32,26.894,179.7,94.0,2.99 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,1,0,1,32,117.298,147.2,172.6,12.59 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,1,0,1,32,119.108,268.3,141.4,12.81 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,2,0,1,32,166.873,484.3,245.2,18.05 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,2,0,1,32,237.341,438.5,420.4,25.53 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,2,0,1,32,240.762,673.3,399.1,25.89 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,3,0,1,32,226.417,800.6,424.0,24.34 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,3,0,1,32,281.455,731.2,525.9,30.3 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,3,0,1,32,364.748,1091.2,819.2,39.13 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,4,0,1,32,392.938,1123.0,638.9,42.23 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,4,0,1,32,318.791,1021.2,656.6,34.16 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,4,0,1,32,491.334,1512.7,1190.2,52.79 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,5,0,1,32,376.922,1438.8,699.1,40.54 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,5,0,1,32,509.471,1314.6,894.1,54.82 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,5,0,1,32,611.306,1932.5,1491.3,65.74 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,6,0,1,32,430.553,1763.0,819.3,46.38 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,6,0,1,32,612.817,1600.7,1075.4,65.9 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,6,0,1,32,739.453,2346.2,1888.3,79.4 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,1,0,2,32,29.492,176.9,86.3,3.16 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,1,0,2,32,76.772,143.3,137.0,8.24 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,1,0,2,32,123.889,360.2,134.5,13.22 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,2,0,2,32,71.557,472.2,220.7,7.69 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,2,0,2,32,159.117,426.6,306.8,17.06 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,2,0,2,32,186.304,849.1,324.2,20.04 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,3,0,2,32,87.145,793.0,329.1,9.36 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,3,0,2,32,206.805,708.2,450.8,22.38 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,3,0,2,32,309.746,1358.0,506.2,33.36 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,4,0,2,32,375.578,1114.3,542.6,40.24 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,4,0,2,32,312.59,988.8,548.9,33.85 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,4,0,2,32,485.698,1869.4,788.1,52.18 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,5,0,2,32,308.605,1434.9,644.3,33.31 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,5,0,2,32,582.807,1276.4,803.0,62.46 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,5,0,2,32,542.061,2377.0,1044.2,58.33 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,1,6,0,2,32,379.561,1748.5,786.6,40.87 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,1,6,0,2,32,470.813,1556.7,875.5,50.52 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,1,6,0,2,32,737.751,2883.7,1291.2,79.34 diff --git a/scripts/clean/results/depth_free_N1_Q256_Llama-3.1-8B-Instruct.csv b/scripts/clean/results/depth_free_N1_Q256_Llama-3.1-8B-Instruct.csv deleted file mode 100644 index 0fcd838..0000000 --- a/scripts/clean/results/depth_free_N1_Q256_Llama-3.1-8B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,1,0,0,256,19.657,182.2,101.2,16.92 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,1,0,0,256,20.661,147.0,167.3,17.73 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,1,0,0,198,28.19,300.4,135.3,18.78 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,2,0,0,256,40.235,487.0,245.7,34.68 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,2,0,0,256,44.773,436.2,385.7,38.54 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,2,0,0,198,58.108,738.2,381.2,38.67 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,3,0,0,256,65.942,812.2,416.9,56.81 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,3,0,0,256,68.642,727.4,548.6,59.07 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,3,0,0,198,97.313,1190.0,676.8,64.73 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,4,0,0,256,87.834,1138.8,583.3,75.72 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,4,0,0,256,93.128,1018.3,781.1,80.22 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,4,0,0,198,127.601,1640.7,917.2,84.85 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,5,0,0,256,108.565,1463.3,726.2,93.61 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,5,0,0,256,113.241,1306.3,926.7,97.53 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,5,0,0,198,161.444,2091.3,1218.8,107.48 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,6,0,0,256,129.327,1788.9,863.8,111.62 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,6,0,0,256,134.724,1596.0,1132.0,116.2 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,6,0,0,198,194.858,2538.7,1535.5,129.63 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,1,0,1,256,19.865,183.0,101.1,17.17 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,1,0,1,256,20.45,147.8,160.1,17.62 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,1,0,1,198,28.214,300.4,134.8,18.81 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,2,0,1,256,40.661,487.9,249.2,35.1 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,2,0,1,256,44.052,438.2,372.5,38.18 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,2,0,1,198,60.485,739.2,419.8,40.29 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,3,0,1,256,66.31,811.6,414.6,57.06 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,3,0,1,256,68.835,727.4,543.8,59.44 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,3,0,1,198,96.467,1190.7,655.0,64.17 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,4,0,1,256,86.748,1134.2,559.6,74.78 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,4,0,1,256,90.585,1016.2,724.5,78.02 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,4,0,1,198,130.933,1641.4,977.7,87.19 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,5,0,1,256,107.931,1461.1,711.3,93.02 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,5,0,1,256,111.115,1306.4,881.3,95.75 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,5,0,1,198,158.264,2091.3,1152.2,105.28 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,6,0,1,256,129.902,1785.7,885.0,111.98 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,6,0,1,256,132.624,1595.0,1081.3,114.24 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,6,0,1,198,191.93,2543.3,1451.6,127.76 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,1,0,2,256,12.431,180.2,86.8,10.71 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,1,0,2,256,21.157,146.6,172.4,18.4 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,1,0,2,198,28.411,300.4,135.9,18.97 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,2,0,2,256,32.495,481.3,245.7,28.27 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,2,0,2,256,43.012,434.1,347.0,37.02 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,2,0,2,198,58.453,738.7,382.2,39.13 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,3,0,2,256,56.75,803.0,422.6,49.15 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,3,0,2,256,67.946,725.3,519.6,58.48 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,3,0,2,198,97.119,1188.3,650.2,64.57 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,4,0,2,256,79.752,1125.5,588.9,69.04 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,4,0,2,256,90.093,1014.4,712.7,77.84 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,4,0,2,198,129.05,1638.2,938.3,85.92 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,5,0,2,256,108.863,1452.6,728.5,93.75 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,5,0,2,256,109.957,1303.6,851.2,94.85 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,5,0,2,198,163.169,2089.9,1245.1,108.49 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,256,1,6,0,2,256,130.266,1773.3,897.1,112.25 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,256,1,6,0,2,256,132.363,1593.5,1087.2,114.05 -meta-llama/Llama-3.1-8B-Instruct,gpqa,256,1,6,0,2,198,192.257,2541.5,1464.8,128.05 diff --git a/scripts/clean/results/depth_free_N1_Q256_Qwen2.5-7B-Instruct.csv b/scripts/clean/results/depth_free_N1_Q256_Qwen2.5-7B-Instruct.csv deleted file mode 100644 index 156be90..0000000 --- a/scripts/clean/results/depth_free_N1_Q256_Qwen2.5-7B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,1,0,0,256,6.833,181.1,59.8,5.91 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,1,0,0,256,9.257,146.7,110.6,8.18 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,1,0,0,198,11.82,299.5,78.7,8.1 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,2,0,0,256,16.651,465.2,147.9,14.89 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,2,0,0,256,30.321,431.2,249.5,26.31 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,2,0,0,198,25.704,717.4,184.5,17.49 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,3,0,0,256,20.678,768.8,238.6,18.78 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,3,0,0,256,43.873,720.9,382.1,38.35 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,3,0,0,198,32.689,1150.5,303.3,22.24 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,4,0,0,256,29.298,1076.3,336.6,26.51 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,4,0,0,256,54.554,1012.2,514.9,47.62 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,4,0,0,198,45.082,1587.9,419.9,30.65 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,5,0,0,256,37.692,1382.2,432.4,33.91 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,5,0,0,256,53.701,1302.6,646.9,47.31 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,5,0,0,198,63.267,2020.1,530.9,43.0 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,6,0,0,256,47.328,1691.7,528.0,42.29 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,6,0,0,256,66.941,1594.5,796.7,59.08 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,6,0,0,198,71.984,2457.4,639.7,49.01 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,1,0,1,256,6.233,181.6,61.1,5.39 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,1,0,1,256,17.642,148.0,111.1,15.49 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,1,0,1,198,9.065,299.5,77.2,6.25 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,2,0,1,256,19.329,466.9,149.9,17.15 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,2,0,1,256,27.271,432.6,245.0,23.77 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,2,0,1,198,20.947,715.6,186.3,14.24 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,3,0,1,256,27.712,770.7,245.1,24.56 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,3,0,1,256,30.712,724.2,380.3,26.85 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,3,0,1,198,41.063,1147.8,302.9,27.91 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,4,0,1,256,38.713,1076.3,337.4,33.86 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,4,0,1,256,40.78,1015.6,512.7,36.18 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,4,0,1,198,55.494,1583.6,427.6,37.46 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,5,0,1,256,62.79,1382.5,432.3,54.57 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,5,0,1,256,64.605,1308.2,661.9,56.19 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,5,0,1,198,73.376,2018.7,536.2,49.54 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,6,0,1,256,56.81,1693.1,525.4,50.07 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,6,0,1,256,82.198,1599.4,806.4,71.82 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,6,0,1,198,68.013,2452.5,636.7,46.35 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,1,0,2,256,6.313,179.1,59.9,5.56 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,1,0,2,256,8.705,146.4,110.1,7.72 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,1,0,2,198,8.145,299.5,74.7,5.63 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,2,0,2,256,15.875,459.6,142.2,14.2 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,2,0,2,256,20.092,427.6,237.2,17.88 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,2,0,2,198,21.007,716.0,182.8,14.46 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,3,0,2,256,23.965,758.9,231.3,21.27 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,3,0,2,256,29.37,717.0,368.3,25.97 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,3,0,2,198,31.302,1148.7,296.7,21.5 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,4,0,2,256,37.812,1063.0,329.2,33.4 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,4,0,2,256,48.963,1005.6,509.5,42.77 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,4,0,2,198,48.884,1585.1,429.8,33.04 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,5,0,2,256,42.221,1364.9,413.5,37.39 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,5,0,2,256,55.58,1296.7,634.9,48.79 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,5,0,2,198,63.667,2021.2,538.2,42.97 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,1,6,0,2,256,58.379,1669.4,510.4,51.2 -Qwen/Qwen2.5-7B-Instruct,gsmhard,256,1,6,0,2,256,64.91,1583.0,760.4,57.06 -Qwen/Qwen2.5-7B-Instruct,gpqa,256,1,6,0,2,198,85.655,2462.8,685.3,57.85 diff --git a/scripts/clean/results/depth_free_N1_Qwen2.5-7B-Instruct.csv b/scripts/clean/results/depth_free_N1_Qwen2.5-7B-Instruct.csv deleted file mode 100644 index e7975ed..0000000 --- a/scripts/clean/results/depth_free_N1_Qwen2.5-7B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,1,0,0,32,20.508,176.3,62.5,2.2 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,1,0,0,32,21.325,143.3,98.7,2.28 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,1,0,0,32,25.086,366.7,83.1,2.71 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,2,0,0,32,51.101,455.8,153.1,5.54 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,2,0,0,32,70.571,421.8,250.0,7.65 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,2,0,0,32,54.71,856.0,189.1,6.02 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,3,0,0,32,101.346,756.5,275.3,11.03 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,3,0,0,32,168.421,705.3,397.8,18.09 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,3,0,0,32,70.373,1356.8,298.0,7.81 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,4,0,0,32,144.307,1062.1,398.7,15.6 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,4,0,0,32,275.557,995.1,555.6,29.67 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,4,0,0,32,111.061,1866.4,422.7,11.96 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,5,0,0,32,200.528,1364.5,514.6,21.71 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,5,0,0,32,161.389,1280.9,632.7,17.37 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,5,0,0,32,109.266,2363.7,502.2,11.93 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,6,0,0,32,235.086,1670.7,625.5,25.29 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,6,0,0,32,204.953,1565.0,743.8,22.2 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,6,0,0,32,150.374,2874.1,628.7,16.1 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,1,0,1,32,22.194,177.6,68.1,2.46 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,1,0,1,32,27.952,147.3,108.2,2.97 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,1,0,1,32,26.898,267.5,75.8,2.94 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,2,0,1,32,52.165,462.4,159.6,5.75 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,2,0,1,32,61.44,433.8,242.4,6.58 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,2,0,1,32,69.061,647.3,181.2,7.45 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,3,0,1,32,74.508,763.4,254.5,8.02 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,3,0,1,32,95.168,725.3,378.9,10.22 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,3,0,1,32,107.32,1043.6,286.1,11.56 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,4,0,1,32,109.955,1074.1,379.2,11.85 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,4,0,1,32,191.743,1020.0,514.7,20.59 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,4,0,1,32,160.42,1436.7,392.1,17.28 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,5,0,1,32,123.313,1379.8,456.1,13.28 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,5,0,1,32,165.697,1308.5,647.5,17.86 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,5,0,1,32,204.03,1839.7,491.7,22.15 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,6,0,1,32,147.431,1671.8,531.4,16.06 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,6,0,1,32,198.425,1601.5,786.7,21.55 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,6,0,1,32,247.17,2236.5,609.8,26.92 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,1,0,2,32,23.178,175.8,62.0,2.55 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,1,0,2,32,31.644,142.5,99.7,3.4 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,1,0,2,32,19.312,361.4,62.2,2.12 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,2,0,2,32,55.889,453.9,149.7,6.02 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,2,0,2,32,68.958,415.6,215.8,7.45 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,2,0,2,32,40.76,830.8,152.8,4.4 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,3,0,2,32,111.874,750.7,252.2,12.1 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,3,0,2,32,174.989,699.6,364.8,18.73 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,3,0,2,32,57.12,1319.5,248.3,6.13 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,4,0,2,32,147.314,1050.3,345.2,15.93 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,4,0,2,32,143.632,983.9,465.9,15.41 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,4,0,2,32,71.131,1807.1,329.8,7.78 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,5,0,2,32,217.89,1351.9,453.8,23.39 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,5,0,2,32,176.088,1269.0,577.4,19.11 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,5,0,2,32,90.535,2292.9,421.0,9.91 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,1,6,0,2,32,297.424,1652.8,561.5,31.96 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,1,6,0,2,32,210.822,1553.6,716.1,22.75 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,1,6,0,2,32,110.36,2784.0,506.2,12.0 diff --git a/scripts/clean/results/depth_free_N5_Llama-3.1-8B-Instruct.csv b/scripts/clean/results/depth_free_N5_Llama-3.1-8B-Instruct.csv deleted file mode 100644 index 30fb1e5..0000000 --- a/scripts/clean/results/depth_free_N5_Llama-3.1-8B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,1,0,0,32,147.036,888.8,511.0,15.85 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,1,0,0,32,165.778,719.1,907.8,17.84 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,1,0,0,32,167.285,1838.8,847.3,18.06 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,2,0,0,32,361.528,4301.7,1285.8,38.94 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,2,0,0,32,445.911,4475.5,2346.7,47.97 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,2,0,0,32,445.145,6554.2,1977.8,48.14 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,3,0,0,32,599.07,8156.7,2275.8,64.46 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,3,0,0,32,730.742,8358.1,3613.9,78.71 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,3,0,0,32,709.198,11583.6,3120.9,76.43 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,4,0,0,32,802.853,12152.8,3097.2,86.36 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,4,0,0,32,983.013,12225.3,4782.5,105.78 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,4,0,0,32,960.569,16604.1,4343.5,103.54 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,5,0,0,32,975.311,16161.9,3815.8,105.18 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,5,0,0,32,1231.222,16084.4,5793.1,132.57 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,5,0,0,32,1267.637,21699.1,5866.0,136.52 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,6,0,0,32,1318.907,19965.6,4829.1,141.92 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,6,0,0,32,1442.363,19967.5,6630.3,155.19 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,6,0,0,32,1526.473,26684.3,6951.6,164.37 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,1,0,1,32,138.352,898.3,487.2,14.94 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,1,0,1,32,165.635,735.9,882.5,17.82 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,1,0,1,32,146.867,1341.6,725.1,15.9 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,2,0,1,32,370.724,4226.7,1391.5,39.89 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,2,0,1,32,444.393,4476.7,2173.2,47.86 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,2,0,1,32,413.311,5521.9,1840.6,44.6 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,3,0,1,32,593.68,8168.8,2248.8,63.93 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,3,0,1,32,679.848,8367.2,2936.6,73.24 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,3,0,1,32,691.666,10037.8,2986.2,74.47 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,4,0,1,32,785.313,12067.0,2631.1,84.6 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,4,0,1,32,920.804,12219.2,4089.4,99.23 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,4,0,1,32,941.261,14580.8,4181.3,101.31 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,5,0,1,32,1099.882,16088.8,4115.3,118.43 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,5,0,1,32,1164.272,16144.5,5169.7,125.63 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,5,0,1,32,1195.886,19126.0,5025.4,128.74 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,6,0,1,32,1329.327,20155.6,5054.0,143.09 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,6,0,1,32,1415.694,20082.0,6169.8,152.65 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,6,0,1,32,1434.164,23640.9,6219.1,154.53 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,1,0,2,32,145.807,884.5,483.9,15.79 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,1,0,2,32,143.99,716.6,699.6,15.54 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,1,0,2,32,77.099,1801.2,545.1,8.36 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,2,0,2,32,363.413,4190.8,1271.1,39.25 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,2,0,2,32,438.483,4416.1,2021.3,47.26 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,2,0,2,32,349.842,6308.4,1682.3,37.93 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,3,0,2,32,382.66,8103.1,1858.8,41.33 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,3,0,2,32,693.628,8268.9,2893.2,74.73 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,3,0,2,32,617.099,11233.6,2624.0,66.66 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,4,0,2,32,745.962,12072.1,2886.2,80.45 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,4,0,2,32,938.011,12097.3,3840.0,100.81 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,4,0,2,32,896.578,16195.8,3846.1,96.67 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,5,0,2,32,926.281,16034.8,3318.5,99.76 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,5,0,2,32,1208.574,16016.2,5016.1,130.45 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,5,0,2,32,1180.905,21061.1,5023.9,127.52 -meta-llama/Llama-3.1-8B-Instruct,mmlu_hard,64,5,6,0,2,32,1269.622,20007.6,4238.5,136.81 -meta-llama/Llama-3.1-8B-Instruct,gsmhard,64,5,6,0,2,32,1447.161,19800.0,5775.6,156.1 -meta-llama/Llama-3.1-8B-Instruct,gpqa,64,5,6,0,2,32,1446.153,26053.8,5617.3,155.75 diff --git a/scripts/clean/results/depth_free_N5_Qwen2.5-7B-Instruct.csv b/scripts/clean/results/depth_free_N5_Qwen2.5-7B-Instruct.csv deleted file mode 100644 index 5d2c7c9..0000000 --- a/scripts/clean/results/depth_free_N5_Qwen2.5-7B-Instruct.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,1,0,0,32,32.167,881.6,300.8,3.56 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,1,0,0,32,53.577,716.7,525.2,5.95 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,1,0,0,32,48.92,1833.4,390.4,5.51 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,2,0,0,32,135.401,3731.9,793.3,14.87 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,2,0,0,32,191.1,4290.2,1277.9,20.82 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,2,0,0,32,188.59,6048.7,1019.8,20.55 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,3,0,0,32,216.435,7100.9,1241.7,23.82 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,3,0,0,32,309.291,8102.2,1972.5,33.83 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,3,0,0,32,273.413,10747.8,1592.1,30.03 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,4,0,0,32,351.663,10680.6,1881.5,38.35 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,4,0,0,32,394.084,11939.2,2675.8,43.11 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,4,0,0,32,421.923,15559.6,2236.0,45.93 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,5,0,0,32,526.118,14346.4,2488.8,57.36 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,5,0,0,32,522.426,15733.2,3471.8,57.23 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,5,0,0,32,622.081,20473.0,3018.6,67.61 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,6,0,0,32,553.296,18126.5,2914.5,60.27 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,6,0,0,32,785.816,19470.6,4073.3,85.33 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,6,0,0,32,716.434,25271.9,3617.8,77.82 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,1,0,1,32,39.059,888.1,348.5,4.34 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,1,0,1,32,55.567,736.7,539.0,6.21 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,1,0,1,32,50.71,1337.3,364.0,5.63 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,2,0,1,32,138.369,3893.2,845.1,15.29 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,2,0,1,32,204.608,4386.6,1320.1,22.38 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,2,0,1,32,240.484,4910.0,1008.0,26.1 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,3,0,1,32,230.522,7321.7,1364.5,25.11 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,3,0,1,32,312.182,8193.9,2076.9,34.0 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,3,0,1,32,365.283,9058.8,1578.4,39.55 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,4,0,1,32,326.017,10929.0,1893.0,35.78 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,4,0,1,32,418.588,12059.0,2734.5,45.71 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,4,0,1,32,480.52,13330.5,2286.8,52.42 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,5,0,1,32,439.416,14480.5,2424.3,48.01 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,5,0,1,32,600.554,15895.2,3531.7,65.24 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,5,0,1,32,513.678,17692.0,2888.9,56.03 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,6,0,1,32,541.513,18308.8,3026.4,59.15 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,6,0,1,32,694.966,19676.2,4129.6,75.53 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,6,0,1,32,676.839,21887.2,3354.7,73.44 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,1,0,2,32,39.142,879.1,314.3,4.39 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,1,0,2,32,138.352,712.3,530.1,15.05 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,1,0,2,32,46.343,1807.0,330.0,5.04 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,2,0,2,32,140.212,3728.0,792.4,15.45 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,2,0,2,32,330.842,4219.5,1287.9,35.63 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,2,0,2,32,151.264,5706.7,862.3,16.6 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,3,0,2,32,238.207,7220.5,1326.0,26.08 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,3,0,2,32,300.405,7979.1,1863.1,32.79 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,3,0,2,32,257.813,10298.8,1451.4,28.24 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,4,0,2,32,352.526,10745.5,1821.3,38.6 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,4,0,2,32,550.209,11756.8,2579.8,59.67 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,4,0,2,32,363.916,14990.8,1956.9,39.63 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,5,0,2,32,450.487,14512.0,2449.7,49.23 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,5,0,2,32,643.853,15505.7,3266.2,69.78 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,5,0,2,32,479.819,19762.0,2629.5,52.43 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,64,5,6,0,2,32,611.793,18051.8,2948.9,66.66 -Qwen/Qwen2.5-7B-Instruct,gsmhard,64,5,6,0,2,32,793.468,19266.4,4055.0,86.03 -Qwen/Qwen2.5-7B-Instruct,gpqa,64,5,6,0,2,32,660.507,24574.9,3158.1,71.66 diff --git a/scripts/clean/results/infranet.png b/scripts/clean/results/infranet.png deleted file mode 100644 index 6c55f99..0000000 Binary files a/scripts/clean/results/infranet.png and /dev/null differ diff --git a/scripts/clean/results/probe.csv b/scripts/clean/results/probe.csv deleted file mode 100644 index c5be842..0000000 --- a/scripts/clean/results/probe.csv +++ /dev/null @@ -1,3 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,2,3,0,0,32,163.504,1843.1,506.0,17.7 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,256,2,3,0,0,32,103.805,1839.6,515.4,11.31 diff --git a/scripts/clean/results/qwen3_depth_carry_thinkOFF.csv b/scripts/clean/results/qwen3_depth_carry_thinkOFF.csv deleted file mode 100644 index a198c14..0000000 --- a/scripts/clean/results/qwen3_depth_carry_thinkOFF.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,enable_thinking,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,per_query_think_tok,per_query_answer_tok,accuracy,trunc_frac,wall_latency_s -Qwen/Qwen3-8B,mmlu_hard,64,1,1,1,0,0,32,465.575,159.3,235.1,1.0,234.1,0.5938,0.031,49.86 -Qwen/Qwen3-8B,gsmhard,64,1,1,1,0,0,32,492.63,126.3,497.8,0.9,496.9,0.1562,0.094,52.78 -Qwen/Qwen3-8B,gpqa,64,1,1,1,0,0,32,44.79,349.7,105.3,1.0,104.3,0.3438,0.0,4.87 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,1,0,0,32,129.777,440.6,232.6,2.0,230.6,0.5938,0.0,13.91 -Qwen/Qwen3-8B,gsmhard,64,1,2,1,0,0,32,547.468,393.6,556.7,1.9,554.8,0.375,0.031,58.69 -Qwen/Qwen3-8B,gpqa,64,1,2,1,0,0,32,98.167,831.7,249.2,2.0,247.2,0.3125,0.0,10.65 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,1,0,0,32,594.746,825.0,475.1,3.0,472.1,0.5938,0.01,63.72 -Qwen/Qwen3-8B,gsmhard,64,1,3,1,0,0,32,608.065,775.9,609.5,3.0,606.5,0.375,0.01,65.22 -Qwen/Qwen3-8B,gpqa,64,1,3,1,0,0,32,135.216,1434.8,395.5,3.0,392.5,0.3125,0.0,14.74 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,1,0,0,32,601.794,1318.7,580.4,4.0,576.5,0.625,0.008,64.47 -Qwen/Qwen3-8B,gsmhard,64,1,4,1,0,0,32,686.69,1286.5,1039.9,3.9,1036.0,0.3438,0.023,73.6 -Qwen/Qwen3-8B,gpqa,64,1,4,1,0,0,32,155.157,2137.3,494.8,4.0,490.8,0.3438,0.0,16.86 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,1,0,0,32,322.897,1923.4,585.8,5.0,580.8,0.625,0.0,34.75 -Qwen/Qwen3-8B,gsmhard,64,1,5,1,0,0,32,672.408,1902.7,1111.4,4.9,1106.5,0.3438,0.019,72.29 -Qwen/Qwen3-8B,gpqa,64,1,5,1,0,0,32,209.836,2975.5,639.2,5.0,634.2,0.3125,0.0,22.8 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,1,0,0,32,342.661,2609.7,678.8,6.0,672.8,0.5938,0.0,36.74 -Qwen/Qwen3-8B,gsmhard,64,1,6,1,0,0,32,759.132,2644.6,1173.1,6.0,1167.2,0.3438,0.01,81.51 -Qwen/Qwen3-8B,gpqa,64,1,6,1,0,0,32,270.102,3923.4,767.1,6.0,761.1,0.3125,0.0,29.11 -Qwen/Qwen3-8B,mmlu_hard,64,1,1,1,0,1,32,19.299,160.6,79.8,1.0,78.8,0.6875,0.0,2.13 -Qwen/Qwen3-8B,gsmhard,64,1,1,1,0,1,32,42.079,130.3,154.2,1.0,153.2,0.1875,0.0,4.52 -Qwen/Qwen3-8B,gpqa,64,1,1,1,0,1,32,25.906,250.5,85.7,1.0,84.7,0.25,0.0,2.83 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,1,0,1,32,485.453,441.7,326.9,2.0,324.9,0.6875,0.016,52.02 -Qwen/Qwen3-8B,gsmhard,64,1,2,1,0,1,32,112.325,403.2,328.1,2.0,326.1,0.3125,0.0,12.12 -Qwen/Qwen3-8B,gpqa,64,1,2,1,0,1,32,88.096,628.1,229.3,2.0,227.3,0.25,0.0,9.51 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,1,0,1,32,555.429,829.8,460.1,3.0,457.2,0.7188,0.01,59.6 -Qwen/Qwen3-8B,gsmhard,64,1,3,1,0,1,32,157.758,790.6,473.4,3.0,470.4,0.3125,0.0,17.05 -Qwen/Qwen3-8B,gpqa,64,1,3,1,0,1,32,149.152,1111.8,372.4,3.0,369.4,0.25,0.0,16.18 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,1,0,1,32,197.883,1330.0,464.0,4.0,460.0,0.6875,0.0,21.17 -Qwen/Qwen3-8B,gsmhard,64,1,4,1,0,1,32,187.575,1302.9,616.8,4.0,612.8,0.2812,0.0,20.34 -Qwen/Qwen3-8B,gpqa,64,1,4,1,0,1,32,156.779,1707.8,468.0,4.0,464.0,0.25,0.0,16.98 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,1,0,1,32,198.311,1952.4,576.8,5.0,571.8,0.6875,0.0,21.47 -Qwen/Qwen3-8B,gsmhard,64,1,5,1,0,1,32,244.07,1934.1,759.8,5.0,754.8,0.25,0.0,26.33 -Qwen/Qwen3-8B,gpqa,64,1,5,1,0,1,32,190.904,2406.7,563.2,5.0,558.2,0.25,0.0,20.54 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,1,0,1,32,200.927,2647.0,637.5,6.0,631.5,0.7188,0.0,21.85 -Qwen/Qwen3-8B,gsmhard,64,1,6,1,0,1,32,290.431,2684.3,886.9,6.0,880.9,0.3438,0.0,31.25 -Qwen/Qwen3-8B,gpqa,64,1,6,1,0,1,32,226.751,3226.2,666.1,6.0,660.1,0.25,0.0,24.38 -Qwen/Qwen3-8B,mmlu_hard,64,1,1,1,0,2,32,27.029,158.8,80.7,1.0,79.7,0.5,0.0,2.92 -Qwen/Qwen3-8B,gsmhard,64,1,1,1,0,2,32,480.324,125.5,378.1,1.0,377.1,0.1562,0.062,51.47 -Qwen/Qwen3-8B,gpqa,64,1,1,1,0,2,32,42.034,344.4,93.1,1.0,92.1,0.3438,0.0,4.58 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,1,0,2,32,71.595,431.9,185.3,2.0,183.3,0.5625,0.0,7.64 -Qwen/Qwen3-8B,gsmhard,64,1,2,1,0,2,32,526.231,391.5,524.8,2.0,522.8,0.3125,0.031,56.38 -Qwen/Qwen3-8B,gpqa,64,1,2,1,0,2,32,96.888,815.0,224.2,2.0,222.2,0.3125,0.0,10.47 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,1,0,2,32,88.494,810.1,296.6,3.0,293.6,0.5625,0.0,9.64 -Qwen/Qwen3-8B,gsmhard,64,1,3,1,0,2,32,557.064,772.1,641.9,3.0,639.0,0.2812,0.021,59.83 -Qwen/Qwen3-8B,gpqa,64,1,3,1,0,2,32,217.646,1395.1,381.8,3.0,378.8,0.3438,0.0,23.28 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,1,0,2,32,158.177,1277.8,406.1,4.0,402.1,0.5938,0.0,17.21 -Qwen/Qwen3-8B,gsmhard,64,1,4,1,0,2,32,596.558,1271.0,773.0,4.0,769.0,0.3438,0.016,64.0 -Qwen/Qwen3-8B,gpqa,64,1,4,1,0,2,32,181.916,2095.8,486.7,4.0,482.7,0.3438,0.0,19.59 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,1,0,2,32,148.781,1867.2,490.4,5.0,485.4,0.5938,0.0,16.12 -Qwen/Qwen3-8B,gsmhard,64,1,5,1,0,2,32,664.318,1883.3,1142.2,4.9,1137.3,0.375,0.025,71.44 -Qwen/Qwen3-8B,gpqa,64,1,5,1,0,2,32,200.883,2907.0,590.1,5.1,585.0,0.3125,0.0,21.7 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,1,0,2,32,181.226,2544.7,588.7,6.0,582.7,0.5938,0.0,19.4 -Qwen/Qwen3-8B,gsmhard,64,1,6,1,0,2,32,693.415,2614.1,949.5,6.0,943.5,0.3438,0.005,74.4 -Qwen/Qwen3-8B,gpqa,64,1,6,1,0,2,32,300.265,3833.3,714.2,6.0,708.2,0.3125,0.0,32.32 diff --git a/scripts/clean/results/qwen3_depth_carry_thinkON.csv b/scripts/clean/results/qwen3_depth_carry_thinkON.csv deleted file mode 100644 index f173132..0000000 --- a/scripts/clean/results/qwen3_depth_carry_thinkON.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,enable_thinking,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,per_query_think_tok,per_query_answer_tok,accuracy,trunc_frac,wall_latency_s -Qwen/Qwen3-8B,mmlu_hard,64,1,1,1,1,0,32,546.962,155.3,1382.0,1313.6,68.4,0.8125,0.062,60.99 -Qwen/Qwen3-8B,gsmhard,64,1,1,1,1,0,32,518.479,122.3,1276.2,1188.2,88.0,0.7812,0.094,55.58 -Qwen/Qwen3-8B,gpqa,64,1,1,1,1,0,32,733.408,345.7,2968.5,2917.6,50.9,0.375,0.438,78.53 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,1,1,0,32,1022.507,426.0,2208.5,2053.1,155.3,0.8438,0.078,109.52 -Qwen/Qwen3-8B,gsmhard,64,1,2,1,1,0,32,999.582,381.1,2063.5,1849.4,214.2,0.8125,0.047,107.1 -Qwen/Qwen3-8B,gpqa,64,1,2,1,1,0,32,1462.705,791.7,5991.1,5865.4,125.7,0.3438,0.422,156.62 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,1,1,0,32,1554.054,776.8,3497.2,3268.1,229.1,0.8125,0.094,166.61 -Qwen/Qwen3-8B,gsmhard,64,1,3,1,1,0,32,1515.122,730.8,3022.7,2740.3,282.3,0.7812,0.062,162.41 -Qwen/Qwen3-8B,gpqa,64,1,3,1,1,0,32,2171.479,1316.1,8441.8,8243.2,198.6,0.4062,0.385,232.57 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,1,1,0,32,2034.782,1270.8,4153.7,3812.9,340.8,0.875,0.039,218.1 -Qwen/Qwen3-8B,gsmhard,64,1,4,1,1,0,32,2026.118,1199.8,3995.9,3605.6,390.3,0.8125,0.062,217.16 -Qwen/Qwen3-8B,gpqa,64,1,4,1,1,0,32,2834.934,1851.4,11020.0,10782.7,237.3,0.4688,0.453,303.59 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,1,1,0,32,2578.44,1796.9,5211.6,4804.8,406.8,0.9062,0.062,276.35 -Qwen/Qwen3-8B,gsmhard,64,1,5,1,1,0,32,2523.051,1791.1,4807.5,4312.2,495.2,0.8125,0.044,270.59 -Qwen/Qwen3-8B,gpqa,64,1,5,1,1,0,32,3636.494,2524.4,13719.7,13421.4,298.2,0.375,0.444,389.74 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,1,1,0,32,3032.246,2461.5,5590.4,5084.1,506.3,0.9062,0.047,324.98 -Qwen/Qwen3-8B,gsmhard,64,1,6,1,1,0,32,3120.228,2399.8,6555.1,5977.6,577.5,0.7812,0.094,334.52 -Qwen/Qwen3-8B,gpqa,64,1,6,1,1,0,32,4238.671,3299.5,16388.2,15977.8,410.4,0.4375,0.417,454.01 -Qwen/Qwen3-8B,mmlu_hard,64,1,1,1,1,1,32,496.71,156.6,977.2,904.9,72.3,0.8438,0.031,53.25 -Qwen/Qwen3-8B,gsmhard,64,1,1,1,1,1,32,496.404,126.3,1093.2,985.8,107.4,0.7188,0.031,53.19 -Qwen/Qwen3-8B,gpqa,64,1,1,1,1,1,32,690.861,246.5,3175.0,3137.0,38.0,0.2812,0.562,74.0 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,1,1,1,32,873.342,436.9,1996.0,1810.0,186.0,0.875,0.0,93.55 -Qwen/Qwen3-8B,gsmhard,64,1,2,1,1,1,32,1015.131,385.5,2112.8,1896.3,216.5,0.7188,0.062,108.82 -Qwen/Qwen3-8B,gpqa,64,1,2,1,1,1,32,1390.795,584.9,6462.2,6291.9,170.4,0.3438,0.547,148.95 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,1,1,1,32,757.873,815.4,2256.3,1986.4,269.9,0.9062,0.0,81.29 -Qwen/Qwen3-8B,gsmhard,64,1,3,1,1,1,32,1517.696,743.1,3119.1,2800.7,318.4,0.6875,0.073,162.76 -Qwen/Qwen3-8B,gpqa,64,1,3,1,1,1,32,2065.702,963.2,9401.3,9261.6,139.7,0.3125,0.562,221.18 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,1,1,1,32,1093.307,1275.9,2918.5,2568.0,350.5,0.9062,0.008,117.16 -Qwen/Qwen3-8B,gsmhard,64,1,4,1,1,1,32,2076.95,1232.2,4509.2,4054.7,454.4,0.7188,0.07,222.54 -Qwen/Qwen3-8B,gpqa,64,1,4,1,1,1,32,2691.907,1409.8,11706.1,11480.2,225.9,0.4062,0.523,288.3 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,1,1,1,32,1375.195,1874.0,3476.1,3018.5,457.6,0.9062,0.006,147.46 -Qwen/Qwen3-8B,gsmhard,64,1,5,1,1,1,32,2559.543,1805.4,5503.1,4977.4,525.7,0.7188,0.05,274.59 -Qwen/Qwen3-8B,gpqa,64,1,5,1,1,1,32,3383.08,1932.2,14625.2,14363.8,261.4,0.4062,0.531,362.47 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,1,1,1,32,1569.174,2598.8,4052.8,3498.7,554.2,0.9062,0.0,168.24 -Qwen/Qwen3-8B,gsmhard,64,1,6,1,1,1,32,3134.443,2506.2,6692.8,6036.1,656.8,0.6875,0.073,335.89 -Qwen/Qwen3-8B,gpqa,64,1,6,1,1,1,32,4045.473,2554.3,17253.6,16920.4,333.2,0.3438,0.5,433.07 -Qwen/Qwen3-8B,mmlu_hard,64,1,1,1,1,2,32,500.639,154.8,1125.9,1053.0,73.0,0.9062,0.031,53.66 -Qwen/Qwen3-8B,gsmhard,64,1,1,1,1,2,32,512.262,121.5,1104.5,1018.2,86.3,0.8438,0.062,54.82 -Qwen/Qwen3-8B,gpqa,64,1,1,1,1,2,32,705.272,340.4,2703.2,2648.3,54.9,0.4375,0.375,75.59 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,1,1,2,32,1006.348,420.2,2026.0,1871.0,155.0,0.9062,0.047,107.78 -Qwen/Qwen3-8B,gsmhard,64,1,2,1,1,2,32,1034.309,367.9,2229.7,2037.8,192.0,0.875,0.062,110.91 -Qwen/Qwen3-8B,gpqa,64,1,2,1,1,2,32,1436.425,774.5,5361.3,5241.2,120.1,0.4062,0.406,153.98 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,1,1,2,32,1489.69,792.8,2826.7,2575.6,251.1,0.9062,0.031,159.6 -Qwen/Qwen3-8B,gsmhard,64,1,3,1,1,2,32,1527.181,708.0,2923.0,2656.8,266.2,0.875,0.083,163.53 -Qwen/Qwen3-8B,gpqa,64,1,3,1,1,2,32,2095.717,1300.0,7454.2,7267.2,187.1,0.4375,0.365,224.4 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,1,1,2,32,1964.013,1263.8,3369.9,3029.8,340.1,0.875,0.031,210.51 -Qwen/Qwen3-8B,gsmhard,64,1,4,1,1,2,32,2040.733,1170.7,3876.8,3507.3,369.5,0.875,0.07,218.73 -Qwen/Qwen3-8B,gpqa,64,1,4,1,1,2,32,2690.007,1884.1,9508.1,9249.3,258.8,0.4375,0.359,288.13 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,1,1,2,32,2457.383,1832.1,4070.8,3642.7,428.1,0.875,0.031,263.32 -Qwen/Qwen3-8B,gsmhard,64,1,5,1,1,2,32,2530.37,1706.6,4503.3,4060.6,442.7,0.8438,0.062,271.11 -Qwen/Qwen3-8B,gpqa,64,1,5,1,1,2,32,3420.009,2544.0,12058.2,11736.0,322.2,0.4375,0.388,366.45 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,1,1,2,32,2957.198,2467.7,4825.0,4306.5,518.5,0.875,0.031,316.74 -Qwen/Qwen3-8B,gsmhard,64,1,6,1,1,2,32,3045.823,2375.1,5379.1,4819.8,559.2,0.875,0.052,326.74 -Qwen/Qwen3-8B,gpqa,64,1,6,1,1,2,32,4051.661,3268.1,14058.0,13649.8,408.2,0.4688,0.365,433.9 diff --git a/scripts/clean/results/qwen3_depth_free_thinkOFF.csv b/scripts/clean/results/qwen3_depth_free_thinkOFF.csv deleted file mode 100644 index 6e3777d..0000000 --- a/scripts/clean/results/qwen3_depth_free_thinkOFF.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,enable_thinking,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,per_query_think_tok,per_query_answer_tok,accuracy,trunc_frac,wall_latency_s -Qwen/Qwen3-8B,mmlu_hard,64,1,1,0,0,0,32,86.454,159.3,114.7,1.0,113.7,0.5938,0.0,9.24 -Qwen/Qwen3-8B,gsmhard,64,1,1,0,0,0,32,491.965,126.3,497.8,0.9,496.9,0.1562,0.094,52.73 -Qwen/Qwen3-8B,gpqa,64,1,1,0,0,0,32,45.429,349.7,105.3,1.0,104.3,0.3438,0.0,4.86 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,0,0,0,32,122.518,438.5,229.9,2.0,227.9,0.5625,0.0,13.21 -Qwen/Qwen3-8B,gsmhard,64,1,2,0,0,0,32,539.351,393.2,667.2,1.9,665.2,0.375,0.047,57.9 -Qwen/Qwen3-8B,gpqa,64,1,2,0,0,0,32,85.781,832.4,250.9,2.0,248.9,0.3438,0.0,9.26 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,0,0,0,32,236.663,735.1,378.2,3.0,375.2,0.5938,0.0,25.35 -Qwen/Qwen3-8B,gsmhard,64,1,3,0,0,0,32,609.208,667.5,745.8,2.9,742.9,0.375,0.021,65.49 -Qwen/Qwen3-8B,gpqa,64,1,3,0,0,0,32,126.834,1328.8,395.5,3.0,392.5,0.3125,0.0,13.73 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,0,0,0,32,1039.34,1037.1,729.1,3.9,725.2,0.625,0.016,111.29 -Qwen/Qwen3-8B,gsmhard,64,1,4,0,0,0,32,658.74,940.7,1036.7,3.9,1032.8,0.375,0.023,70.77 -Qwen/Qwen3-8B,gpqa,64,1,4,0,0,0,32,198.402,1828.1,550.0,4.0,546.0,0.3438,0.0,21.45 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,0,0,0,32,656.93,1333.5,722.2,5.0,717.2,0.6562,0.006,70.43 -Qwen/Qwen3-8B,gsmhard,64,1,5,0,0,0,32,732.477,1213.4,1094.8,4.9,1089.8,0.375,0.013,78.64 -Qwen/Qwen3-8B,gpqa,64,1,5,0,0,0,32,304.259,2322.0,744.7,5.0,739.8,0.3125,0.0,32.83 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,0,0,0,32,790.753,1635.1,888.7,6.0,882.7,0.5938,0.005,85.04 -Qwen/Qwen3-8B,gsmhard,64,1,6,0,0,0,32,1165.722,1490.1,1190.8,5.9,1184.9,0.375,0.01,124.91 -Qwen/Qwen3-8B,gpqa,64,1,6,0,0,0,32,370.637,2808.1,863.2,6.0,857.3,0.3438,0.0,39.89 -Qwen/Qwen3-8B,mmlu_hard,64,1,1,0,0,1,32,24.908,160.6,82.5,1.0,81.5,0.6562,0.0,2.73 -Qwen/Qwen3-8B,gsmhard,64,1,1,0,0,1,32,45.72,130.3,153.7,1.0,152.7,0.1875,0.0,4.96 -Qwen/Qwen3-8B,gpqa,64,1,1,0,0,1,32,24.258,250.5,84.8,1.0,83.8,0.25,0.0,2.63 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,0,0,1,32,484.393,441.6,315.8,2.0,313.8,0.6875,0.016,52.0 -Qwen/Qwen3-8B,gsmhard,64,1,2,0,0,1,32,108.273,402.6,324.2,2.0,322.2,0.3438,0.0,11.7 -Qwen/Qwen3-8B,gpqa,64,1,2,0,0,1,32,85.68,625.1,229.6,2.0,227.6,0.25,0.0,9.21 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,0,0,1,32,525.517,744.4,450.9,3.0,447.9,0.6875,0.01,56.43 -Qwen/Qwen3-8B,gsmhard,64,1,3,0,0,1,32,185.441,679.7,511.7,3.0,508.7,0.3438,0.0,19.96 -Qwen/Qwen3-8B,gpqa,64,1,3,0,0,1,32,185.87,1020.2,381.1,3.0,378.1,0.25,0.0,19.94 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,0,0,1,32,136.172,1050.0,457.0,4.0,453.0,0.6875,0.0,14.86 -Qwen/Qwen3-8B,gsmhard,64,1,4,0,0,1,32,250.669,957.0,672.2,4.0,668.2,0.375,0.0,26.97 -Qwen/Qwen3-8B,gpqa,64,1,4,0,0,1,32,178.929,1414.7,507.5,4.0,503.5,0.25,0.0,19.31 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,0,0,1,32,137.741,1352.4,548.1,5.0,543.1,0.6875,0.0,14.9 -Qwen/Qwen3-8B,gsmhard,64,1,5,0,0,1,32,281.442,1233.6,789.6,5.0,784.6,0.4062,0.0,30.29 -Qwen/Qwen3-8B,gpqa,64,1,5,0,0,1,32,247.433,1807.1,638.6,5.0,633.6,0.25,0.0,26.65 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,0,0,1,32,644.504,1655.8,833.8,6.0,827.8,0.7188,0.005,69.15 -Qwen/Qwen3-8B,gsmhard,64,1,6,0,0,1,32,359.105,1509.8,993.2,6.0,987.2,0.375,0.0,38.62 -Qwen/Qwen3-8B,gpqa,64,1,6,0,0,1,32,697.267,2199.8,871.5,6.0,865.6,0.25,0.005,74.85 -Qwen/Qwen3-8B,mmlu_hard,64,1,1,0,0,2,32,26.62,158.8,84.8,1.0,83.8,0.5,0.0,2.93 -Qwen/Qwen3-8B,gsmhard,64,1,1,0,0,2,32,480.555,125.5,377.5,0.9,376.5,0.1562,0.062,51.46 -Qwen/Qwen3-8B,gpqa,64,1,1,0,0,2,32,42.696,344.4,93.1,1.0,92.1,0.3438,0.0,4.6 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,0,0,2,32,46.306,433.0,182.8,2.0,180.8,0.5625,0.0,5.03 -Qwen/Qwen3-8B,gsmhard,64,1,2,0,0,2,32,524.517,391.3,506.9,1.9,504.9,0.3125,0.031,56.3 -Qwen/Qwen3-8B,gpqa,64,1,2,0,0,2,32,101.765,815.7,232.5,2.0,230.5,0.3125,0.0,10.87 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,0,0,2,32,129.656,726.0,325.4,3.0,322.4,0.625,0.0,13.91 -Qwen/Qwen3-8B,gsmhard,64,1,3,0,0,2,32,574.63,661.3,666.0,2.9,663.1,0.3125,0.021,61.67 -Qwen/Qwen3-8B,gpqa,64,1,3,0,0,2,32,115.284,1303.5,349.1,3.0,346.1,0.3125,0.0,12.63 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,0,0,2,32,154.617,1024.0,425.2,4.0,421.2,0.5625,0.0,16.77 -Qwen/Qwen3-8B,gsmhard,64,1,4,0,0,2,32,595.368,931.2,767.7,3.9,763.7,0.375,0.016,64.0 -Qwen/Qwen3-8B,gpqa,64,1,4,0,0,2,32,168.962,1790.3,465.7,4.0,461.7,0.3125,0.0,18.33 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,0,0,2,32,245.441,1312.8,546.2,5.0,541.2,0.625,0.0,26.42 -Qwen/Qwen3-8B,gsmhard,64,1,5,0,0,2,32,628.303,1200.2,918.6,5.0,913.6,0.4062,0.013,67.65 -Qwen/Qwen3-8B,gpqa,64,1,5,0,0,2,32,223.736,2285.7,612.8,5.0,607.8,0.3438,0.0,24.18 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,0,0,2,32,266.294,1614.0,682.3,6.0,676.3,0.5938,0.0,28.62 -Qwen/Qwen3-8B,gsmhard,64,1,6,0,0,2,32,670.534,1468.8,918.9,6.0,912.9,0.4062,0.005,71.96 -Qwen/Qwen3-8B,gpqa,64,1,6,0,0,2,32,477.817,2766.2,812.6,6.0,806.6,0.3125,0.0,51.45 diff --git a/scripts/clean/results/qwen3_depth_free_thinkON.csv b/scripts/clean/results/qwen3_depth_free_thinkON.csv deleted file mode 100644 index 3f40560..0000000 --- a/scripts/clean/results/qwen3_depth_free_thinkON.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,enable_thinking,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,per_query_think_tok,per_query_answer_tok,accuracy,trunc_frac,wall_latency_s -Qwen/Qwen3-8B,mmlu_hard,64,1,1,0,1,0,32,516.973,155.3,1303.4,1231.6,71.8,0.875,0.062,55.39 -Qwen/Qwen3-8B,gsmhard,64,1,1,0,1,0,32,519.52,122.3,1276.2,1188.2,88.0,0.7812,0.094,55.66 -Qwen/Qwen3-8B,gpqa,64,1,1,0,1,0,32,734.303,345.7,2968.5,2917.6,50.9,0.375,0.438,78.63 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,0,1,0,32,1031.853,428.3,2479.2,2315.2,164.0,0.8438,0.062,110.63 -Qwen/Qwen3-8B,gsmhard,64,1,2,0,1,0,32,1023.101,373.9,2255.0,2050.9,204.1,0.7812,0.094,109.68 -Qwen/Qwen3-8B,gpqa,64,1,2,0,1,0,32,1468.427,790.6,5865.7,5742.2,123.5,0.4062,0.406,157.35 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,0,1,0,32,1536.25,705.6,3281.7,3041.8,239.9,0.875,0.073,164.73 -Qwen/Qwen3-8B,gsmhard,64,1,3,0,1,0,32,1539.994,633.5,3377.3,3041.7,335.7,0.8125,0.073,165.02 -Qwen/Qwen3-8B,gpqa,64,1,3,0,1,0,32,2208.359,1227.5,8848.2,8661.3,186.9,0.375,0.438,236.64 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,0,1,0,32,2034.009,998.0,4367.4,4015.7,351.7,0.875,0.039,217.98 -Qwen/Qwen3-8B,gsmhard,64,1,4,0,1,0,32,2031.191,886.9,4339.5,3955.0,384.6,0.7812,0.07,217.8 -Qwen/Qwen3-8B,gpqa,64,1,4,0,1,0,32,2974.266,1673.3,11600.2,11373.5,226.7,0.3125,0.492,318.69 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,0,1,0,32,2548.621,1272.0,5357.7,4934.5,423.2,0.9062,0.056,273.18 -Qwen/Qwen3-8B,gsmhard,64,1,5,0,1,0,32,2547.285,1140.7,5394.7,4909.1,485.7,0.8125,0.075,272.99 -Qwen/Qwen3-8B,gpqa,64,1,5,0,1,0,32,3726.056,2107.4,14511.1,14239.1,271.9,0.2812,0.506,399.1 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,0,1,0,32,3093.192,1545.3,6707.8,6207.4,500.4,0.875,0.078,331.8 -Qwen/Qwen3-8B,gsmhard,64,1,6,0,1,0,32,3025.228,1396.8,6180.8,5603.8,576.9,0.7812,0.062,324.24 -Qwen/Qwen3-8B,gpqa,64,1,6,0,1,0,32,4430.4,2544.9,17143.8,16817.5,326.3,0.3438,0.5,474.71 -Qwen/Qwen3-8B,mmlu_hard,64,1,1,0,1,1,32,340.719,156.6,992.5,906.9,85.6,0.9062,0.0,36.58 -Qwen/Qwen3-8B,gsmhard,64,1,1,0,1,1,32,477.277,126.3,1143.5,1034.8,108.7,0.7188,0.0,51.21 -Qwen/Qwen3-8B,gpqa,64,1,1,0,1,1,32,689.602,246.5,3142.8,3100.4,42.5,0.2812,0.531,73.86 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,0,1,1,32,813.19,433.7,1768.5,1597.6,170.9,0.9062,0.0,87.29 -Qwen/Qwen3-8B,gsmhard,64,1,2,0,1,1,32,1025.731,385.3,2276.1,2065.9,210.2,0.7188,0.062,109.93 -Qwen/Qwen3-8B,gpqa,64,1,2,0,1,1,32,1362.7,573.2,6157.7,6070.3,87.4,0.2812,0.547,145.92 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,0,1,1,32,839.419,722.7,2292.2,2025.6,266.6,0.875,0.0,90.0 -Qwen/Qwen3-8B,gsmhard,64,1,3,0,1,1,32,1537.593,645.3,3335.6,3018.5,317.1,0.7188,0.083,164.92 -Qwen/Qwen3-8B,gpqa,64,1,3,0,1,1,32,1992.668,939.5,8560.5,8365.0,195.5,0.3438,0.427,213.53 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,0,1,1,32,1206.382,1012.2,3001.1,2635.8,365.3,0.9062,0.008,129.42 -Qwen/Qwen3-8B,gsmhard,64,1,4,0,1,1,32,2044.021,916.2,4353.0,3900.1,452.9,0.7188,0.062,219.2 -Qwen/Qwen3-8B,gpqa,64,1,4,0,1,1,32,2704.915,1263.6,11942.1,11715.3,226.8,0.3438,0.5,289.7 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,0,1,1,32,1644.79,1316.9,3778.3,3310.8,467.5,0.9062,0.0,176.31 -Qwen/Qwen3-8B,gsmhard,64,1,5,0,1,1,32,2555.129,1182.4,5449.2,4876.7,572.5,0.7188,0.062,273.86 -Qwen/Qwen3-8B,gpqa,64,1,5,0,1,1,32,3396.306,1596.2,15169.8,14902.3,267.4,0.3125,0.544,363.85 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,0,1,1,32,2183.201,1601.2,4839.6,4263.3,576.2,0.9062,0.005,234.02 -Qwen/Qwen3-8B,gsmhard,64,1,6,0,1,1,32,2823.03,1461.5,6052.8,5320.1,732.7,0.75,0.026,302.5 -Qwen/Qwen3-8B,gpqa,64,1,6,0,1,1,32,4070.874,1938.3,17977.4,17655.5,321.9,0.3438,0.536,436.09 -Qwen/Qwen3-8B,mmlu_hard,64,1,1,0,1,2,32,447.573,154.8,1011.3,939.0,72.3,0.9062,0.0,47.99 -Qwen/Qwen3-8B,gsmhard,64,1,1,0,1,2,32,513.03,121.5,1104.5,1018.2,86.3,0.8438,0.062,54.9 -Qwen/Qwen3-8B,gpqa,64,1,1,0,1,2,32,708.119,340.4,2724.0,2671.3,52.7,0.4375,0.406,75.86 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,0,1,2,32,994.99,425.6,1987.3,1823.6,163.7,0.875,0.031,106.56 -Qwen/Qwen3-8B,gsmhard,64,1,2,0,1,2,32,1008.468,368.7,2038.2,1840.5,197.7,0.875,0.047,108.04 -Qwen/Qwen3-8B,gpqa,64,1,2,0,1,2,32,1365.303,784.4,4961.2,4830.8,130.4,0.4688,0.312,146.24 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,0,1,2,32,1324.06,707.9,2893.2,2639.6,253.6,0.9062,0.021,142.03 -Qwen/Qwen3-8B,gsmhard,64,1,3,0,1,2,32,1523.896,614.4,2928.6,2672.7,255.9,0.875,0.094,163.36 -Qwen/Qwen3-8B,gpqa,64,1,3,0,1,2,32,2110.917,1218.7,7865.2,7679.7,185.5,0.4375,0.406,226.06 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,0,1,2,32,1965.415,991.8,3507.0,3176.5,330.6,0.875,0.031,210.61 -Qwen/Qwen3-8B,gsmhard,64,1,4,0,1,2,32,2020.886,867.6,3748.1,3382.6,365.5,0.875,0.062,216.53 -Qwen/Qwen3-8B,gpqa,64,1,4,0,1,2,32,2833.925,1665.3,10532.6,10282.3,250.3,0.4062,0.414,303.7 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,0,1,2,32,2119.971,1295.1,4245.6,3797.0,448.6,0.9375,0.013,227.22 -Qwen/Qwen3-8B,gsmhard,64,1,5,0,1,2,32,2509.021,1123.5,4510.1,4022.1,488.0,0.875,0.05,269.02 -Qwen/Qwen3-8B,gpqa,64,1,5,0,1,2,32,3375.736,2112.3,12384.5,12058.7,325.8,0.4688,0.394,361.73 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,0,1,2,32,1687.904,1578.4,4777.7,4233.7,544.0,0.9375,0.0,181.1 -Qwen/Qwen3-8B,gsmhard,64,1,6,0,1,2,32,3026.538,1373.7,5542.3,4985.8,556.6,0.875,0.062,324.24 -Qwen/Qwen3-8B,gpqa,64,1,6,0,1,2,32,4050.898,2577.2,14732.4,14313.6,418.8,0.4375,0.354,433.94 diff --git a/scripts/clean/results/qwen3_depth_selfcorrect_thinkOFF.csv b/scripts/clean/results/qwen3_depth_selfcorrect_thinkOFF.csv deleted file mode 100644 index 300076a..0000000 --- a/scripts/clean/results/qwen3_depth_selfcorrect_thinkOFF.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,enable_thinking,self_correct,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,per_query_think_tok,per_query_answer_tok,accuracy,trunc_frac,wall_latency_s -Qwen/Qwen3-8B,mmlu_hard,64,1,1,0,0,1,0,32,464.885,159.3,237.8,1.0,236.9,0.5938,0.031,49.86 -Qwen/Qwen3-8B,gsmhard,64,1,1,0,0,1,0,32,492.443,126.3,497.8,0.9,496.9,0.1562,0.094,52.77 -Qwen/Qwen3-8B,gpqa,64,1,1,0,0,1,0,32,44.853,349.7,105.3,1.0,104.3,0.3438,0.0,4.88 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,0,0,1,0,32,520.997,450.8,395.9,2.0,394.0,0.625,0.016,55.81 -Qwen/Qwen3-8B,gsmhard,64,1,2,0,0,1,0,32,570.114,401.3,513.1,2.0,511.1,0.4688,0.016,61.2 -Qwen/Qwen3-8B,gpqa,64,1,2,0,0,1,0,32,146.603,846.6,384.3,2.0,382.3,0.3125,0.0,15.78 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,0,0,1,0,32,616.518,769.6,600.2,3.0,597.2,0.6562,0.01,66.13 -Qwen/Qwen3-8B,gsmhard,64,1,3,0,0,1,0,32,716.074,683.0,1062.8,2.9,1059.9,0.4688,0.031,76.81 -Qwen/Qwen3-8B,gpqa,64,1,3,0,0,1,0,32,677.518,1359.7,760.1,3.0,757.1,0.375,0.01,72.66 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,0,0,1,0,32,722.142,1088.4,969.5,3.9,965.6,0.625,0.016,77.53 -Qwen/Qwen3-8B,gsmhard,64,1,4,0,0,1,0,32,823.451,963.9,1347.7,3.9,1343.8,0.5312,0.023,88.18 -Qwen/Qwen3-8B,gpqa,64,1,4,0,0,1,0,32,520.113,1869.3,1051.8,4.0,1047.8,0.3125,0.0,55.95 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,0,0,1,0,32,1961.8,1409.5,1520.9,4.9,1516.0,0.6562,0.031,210.31 -Qwen/Qwen3-8B,gsmhard,64,1,5,0,0,1,0,32,1130.111,1243.3,1508.2,5.0,1503.2,0.5938,0.013,121.34 -Qwen/Qwen3-8B,gpqa,64,1,5,0,0,1,0,32,864.015,2387.5,1350.2,4.9,1345.3,0.2812,0.006,92.8 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,0,0,1,0,32,1790.242,1731.8,1674.2,5.9,1668.2,0.5938,0.021,192.01 -Qwen/Qwen3-8B,gsmhard,64,1,6,0,0,1,0,32,1311.136,1526.7,1944.2,5.9,1938.3,0.5625,0.016,140.74 -Qwen/Qwen3-8B,gpqa,64,1,6,0,0,1,0,32,961.101,2903.1,1656.2,6.0,1650.2,0.3438,0.005,103.32 -Qwen/Qwen3-8B,mmlu_hard,64,1,1,0,0,1,1,32,30.744,160.6,87.2,1.0,86.2,0.7188,0.0,3.34 -Qwen/Qwen3-8B,gsmhard,64,1,1,0,0,1,1,32,47.558,130.3,159.2,1.0,158.2,0.2188,0.0,5.14 -Qwen/Qwen3-8B,gpqa,64,1,1,0,0,1,1,32,23.97,250.5,84.8,1.0,83.8,0.25,0.0,2.64 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,0,0,1,1,32,75.748,453.3,239.2,2.0,237.2,0.6875,0.0,8.22 -Qwen/Qwen3-8B,gsmhard,64,1,2,0,0,1,1,32,138.476,411.6,400.8,2.0,398.8,0.5312,0.0,14.87 -Qwen/Qwen3-8B,gpqa,64,1,2,0,0,1,1,32,136.504,636.1,301.4,2.0,299.4,0.3125,0.0,14.78 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,0,0,1,1,32,560.761,779.3,586.5,3.0,583.5,0.6875,0.01,60.07 -Qwen/Qwen3-8B,gsmhard,64,1,3,0,0,1,1,32,235.822,697.6,640.3,3.0,637.3,0.6562,0.0,25.49 -Qwen/Qwen3-8B,gpqa,64,1,3,0,0,1,1,32,178.711,1054.0,584.1,3.0,581.1,0.3125,0.0,19.3 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,0,0,1,1,32,228.132,1100.3,626.0,4.0,622.0,0.7188,0.0,24.5 -Qwen/Qwen3-8B,gsmhard,64,1,4,0,0,1,1,32,249.702,982.7,817.9,4.0,813.9,0.5625,0.0,26.86 -Qwen/Qwen3-8B,gpqa,64,1,4,0,0,1,1,32,401.899,1464.9,937.6,4.0,933.6,0.2812,0.0,43.36 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,0,0,1,1,32,726.885,1425.9,937.0,5.0,932.0,0.75,0.006,78.03 -Qwen/Qwen3-8B,gsmhard,64,1,5,0,0,1,1,32,357.643,1268.6,1035.8,5.0,1030.8,0.625,0.0,38.4 -Qwen/Qwen3-8B,gpqa,64,1,5,0,0,1,1,32,573.153,1878.6,1176.7,5.0,1171.7,0.2812,0.0,61.35 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,0,0,1,1,32,340.893,1756.3,1024.1,6.0,1018.1,0.6875,0.0,36.92 -Qwen/Qwen3-8B,gsmhard,64,1,6,0,0,1,1,32,814.131,1554.4,1419.6,6.0,1413.6,0.6562,0.005,87.35 -Qwen/Qwen3-8B,gpqa,64,1,6,0,0,1,1,32,604.378,2293.2,1418.3,6.0,1412.3,0.3125,0.0,65.03 -Qwen/Qwen3-8B,mmlu_hard,64,1,1,0,0,1,2,32,23.933,158.8,80.7,1.0,79.7,0.5,0.0,2.64 -Qwen/Qwen3-8B,gsmhard,64,1,1,0,0,1,2,32,480.475,125.5,388.2,0.9,387.3,0.1562,0.062,51.49 -Qwen/Qwen3-8B,gpqa,64,1,1,0,0,1,2,32,42.752,344.4,93.1,1.0,92.1,0.3438,0.0,4.58 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,0,0,1,2,32,112.104,443.7,276.7,2.0,274.7,0.625,0.0,12.09 -Qwen/Qwen3-8B,gsmhard,64,1,2,0,0,1,2,32,949.001,399.6,715.1,1.9,713.2,0.4062,0.047,101.68 -Qwen/Qwen3-8B,gpqa,64,1,2,0,0,1,2,32,504.995,826.3,432.7,2.0,430.7,0.3125,0.016,54.13 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,0,0,1,2,32,201.704,762.5,485.7,3.0,482.7,0.5625,0.0,21.66 -Qwen/Qwen3-8B,gsmhard,64,1,3,0,0,1,2,32,647.327,679.9,776.6,3.0,773.6,0.4062,0.021,69.55 -Qwen/Qwen3-8B,gpqa,64,1,3,0,0,1,2,32,314.759,1333.7,598.3,3.0,595.2,0.3438,0.0,33.77 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,0,0,1,2,32,342.726,1082.8,695.4,4.0,691.4,0.5938,0.0,36.83 -Qwen/Qwen3-8B,gsmhard,64,1,4,0,0,1,2,32,794.153,960.6,1037.9,4.0,1034.0,0.4688,0.016,85.01 -Qwen/Qwen3-8B,gpqa,64,1,4,0,0,1,2,32,801.437,1843.0,1017.9,4.0,1013.9,0.3438,0.008,85.96 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,0,0,1,2,32,603.06,1401.8,969.8,5.0,964.8,0.5938,0.0,64.87 -Qwen/Qwen3-8B,gsmhard,64,1,5,0,0,1,2,32,795.459,1241.8,1178.1,5.0,1173.2,0.4375,0.013,85.33 -Qwen/Qwen3-8B,gpqa,64,1,5,0,0,1,2,32,617.553,2351.9,1185.8,5.0,1180.8,0.3438,0.0,66.23 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,0,0,1,2,32,591.021,1721.1,1176.6,6.0,1170.6,0.625,0.0,63.6 -Qwen/Qwen3-8B,gsmhard,64,1,6,0,0,1,2,32,1294.632,1524.5,1583.5,5.9,1577.6,0.4688,0.016,139.09 -Qwen/Qwen3-8B,gpqa,64,1,6,0,0,1,2,32,669.825,2861.2,1422.2,6.0,1416.2,0.3125,0.0,71.88 diff --git a/scripts/clean/results/qwen3_depth_selfcorrect_thinkON.csv b/scripts/clean/results/qwen3_depth_selfcorrect_thinkON.csv deleted file mode 100644 index f9a38de..0000000 --- a/scripts/clean/results/qwen3_depth_selfcorrect_thinkON.csv +++ /dev/null @@ -1,55 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,enable_thinking,self_correct,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,per_query_think_tok,per_query_answer_tok,accuracy,trunc_frac,wall_latency_s -Qwen/Qwen3-8B,mmlu_hard,64,1,1,0,1,1,0,32,516.511,155.3,1242.5,1174.1,68.4,0.8438,0.062,55.33 -Qwen/Qwen3-8B,gsmhard,64,1,1,0,1,1,0,32,517.799,122.3,1276.2,1188.2,88.0,0.7812,0.094,55.55 -Qwen/Qwen3-8B,gpqa,64,1,1,0,1,1,0,32,732.712,345.7,2968.5,2917.6,50.9,0.375,0.438,78.47 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,0,1,1,0,32,1023.344,436.4,2374.7,2200.2,174.4,0.875,0.047,109.71 -Qwen/Qwen3-8B,gsmhard,64,1,2,0,1,1,0,32,1036.506,386.8,2557.8,2357.1,200.7,0.7812,0.062,111.02 -Qwen/Qwen3-8B,gpqa,64,1,2,0,1,1,0,32,1437.612,804.2,5998.1,5857.6,140.4,0.4375,0.438,154.02 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,0,1,1,0,32,1537.483,737.2,3602.2,3330.7,271.5,0.9062,0.062,164.99 -Qwen/Qwen3-8B,gsmhard,64,1,3,0,1,1,0,32,1552.34,648.4,3816.2,3503.7,312.5,0.8125,0.073,166.35 -Qwen/Qwen3-8B,gpqa,64,1,3,0,1,1,0,32,2231.053,1250.7,9131.5,8946.6,185.0,0.3438,0.5,239.08 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,0,1,1,0,32,2092.066,1043.3,5133.5,4741.3,392.2,0.875,0.07,224.29 -Qwen/Qwen3-8B,gsmhard,64,1,4,0,1,1,0,32,2083.169,910.9,5301.1,4897.5,403.6,0.8125,0.086,223.25 -Qwen/Qwen3-8B,gpqa,64,1,4,0,1,1,0,32,3000.651,1717.2,12055.7,11766.7,289.0,0.3438,0.492,321.39 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,0,1,1,0,32,2560.875,1345.9,5761.1,5269.9,491.2,0.875,0.05,274.63 -Qwen/Qwen3-8B,gsmhard,64,1,5,0,1,1,0,32,2644.569,1172.8,6949.0,6446.5,502.5,0.8125,0.106,283.39 -Qwen/Qwen3-8B,gpqa,64,1,5,0,1,1,0,32,3792.638,2167.2,15183.6,14842.1,341.5,0.3438,0.5,406.5 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,0,1,1,0,32,3124.939,1651.9,7318.0,6719.4,598.6,0.875,0.062,335.09 -Qwen/Qwen3-8B,gsmhard,64,1,6,0,1,1,0,32,3133.002,1439.3,7782.8,7165.6,617.2,0.8125,0.089,335.81 -Qwen/Qwen3-8B,gpqa,64,1,6,0,1,1,0,32,4650.291,2603.5,18867.7,18455.9,411.8,0.3438,0.531,498.38 -Qwen/Qwen3-8B,mmlu_hard,64,1,1,0,1,1,1,32,504.756,156.6,992.0,920.5,71.5,0.875,0.062,54.09 -Qwen/Qwen3-8B,gsmhard,64,1,1,0,1,1,1,32,516.801,126.3,1255.6,1140.9,114.6,0.7188,0.062,55.38 -Qwen/Qwen3-8B,gpqa,64,1,1,0,1,1,1,32,688.443,246.5,3145.9,3103.7,42.2,0.2812,0.531,73.79 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,0,1,1,1,32,775.32,445.4,1995.8,1808.6,187.2,0.9062,0.0,83.21 -Qwen/Qwen3-8B,gsmhard,64,1,2,0,1,1,1,32,1035.029,400.5,2576.7,2343.2,233.4,0.75,0.062,111.01 -Qwen/Qwen3-8B,gpqa,64,1,2,0,1,1,1,32,1377.937,584.0,6383.3,6298.2,85.1,0.25,0.594,147.63 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,0,1,1,1,32,1473.474,750.8,2952.1,2646.2,305.9,0.875,0.021,157.91 -Qwen/Qwen3-8B,gsmhard,64,1,3,0,1,1,1,32,1599.671,663.9,4327.9,3985.0,342.9,0.75,0.073,171.45 -Qwen/Qwen3-8B,gpqa,64,1,3,0,1,1,1,32,2036.873,956.3,9094.9,8905.1,189.8,0.2812,0.5,218.13 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,0,1,1,1,32,1927.675,1062.2,3757.9,3347.8,410.2,0.875,0.023,206.62 -Qwen/Qwen3-8B,gsmhard,64,1,4,0,1,1,1,32,2164.213,918.3,6219.4,5795.8,423.7,0.7188,0.125,232.01 -Qwen/Qwen3-8B,gpqa,64,1,4,0,1,1,1,32,2733.75,1293.3,12304.8,12077.6,227.2,0.2812,0.555,292.88 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,0,1,1,1,32,1505.374,1382.8,4316.2,3784.7,531.5,0.9062,0.0,161.34 -Qwen/Qwen3-8B,gsmhard,64,1,5,0,1,1,1,32,2696.862,1205.9,7629.0,7072.6,556.4,0.7188,0.081,289.13 -Qwen/Qwen3-8B,gpqa,64,1,5,0,1,1,1,32,3432.521,1639.0,15583.8,15308.8,274.9,0.2812,0.562,367.87 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,0,1,1,1,32,2043.553,1695.9,5111.0,4472.3,638.7,0.9062,0.005,219.03 -Qwen/Qwen3-8B,gsmhard,64,1,6,0,1,1,1,32,3192.273,1487.1,8585.7,7883.2,702.4,0.7188,0.078,342.03 -Qwen/Qwen3-8B,gpqa,64,1,6,0,1,1,1,32,4099.717,1987.6,18361.8,18018.0,343.8,0.2812,0.562,439.23 -Qwen/Qwen3-8B,mmlu_hard,64,1,1,0,1,1,2,32,501.764,154.8,1153.8,1079.9,73.9,0.9062,0.031,53.83 -Qwen/Qwen3-8B,gsmhard,64,1,1,0,1,1,2,32,512.042,121.5,1104.5,1018.2,86.3,0.8438,0.062,54.86 -Qwen/Qwen3-8B,gpqa,64,1,1,0,1,1,2,32,705.377,340.4,2703.2,2648.3,54.9,0.4375,0.375,75.59 -Qwen/Qwen3-8B,mmlu_hard,64,1,2,0,1,1,2,32,1013.317,437.8,2193.6,2016.7,176.9,0.8438,0.031,108.65 -Qwen/Qwen3-8B,gsmhard,64,1,2,0,1,1,2,32,1037.431,382.3,2460.6,2256.9,203.7,0.8125,0.062,111.15 -Qwen/Qwen3-8B,gpqa,64,1,2,0,1,1,2,32,1434.921,789.9,5517.1,5396.9,120.2,0.3438,0.422,153.68 -Qwen/Qwen3-8B,mmlu_hard,64,1,3,0,1,1,2,32,1481.137,737.2,2930.5,2644.5,286.0,0.9062,0.031,158.73 -Qwen/Qwen3-8B,gsmhard,64,1,3,0,1,1,2,32,1563.714,646.7,3771.3,3477.4,293.9,0.8125,0.073,167.56 -Qwen/Qwen3-8B,gpqa,64,1,3,0,1,1,2,32,2104.669,1256.2,7906.6,7690.8,215.8,0.375,0.375,225.53 -Qwen/Qwen3-8B,mmlu_hard,64,1,4,0,1,1,2,32,1991.549,1040.4,4022.4,3633.7,388.7,0.9375,0.031,213.46 -Qwen/Qwen3-8B,gsmhard,64,1,4,0,1,1,2,32,2088.3,909.8,5073.4,4665.8,407.7,0.8125,0.062,223.8 -Qwen/Qwen3-8B,gpqa,64,1,4,0,1,1,2,32,2882.607,1699.9,10813.4,10526.6,286.8,0.4062,0.438,309.01 -Qwen/Qwen3-8B,mmlu_hard,64,1,5,0,1,1,2,32,2250.746,1363.9,4899.9,4381.0,518.9,0.9062,0.019,241.33 -Qwen/Qwen3-8B,gsmhard,64,1,5,0,1,1,2,32,2600.169,1176.0,6227.1,5711.0,516.1,0.8125,0.062,278.77 -Qwen/Qwen3-8B,gpqa,64,1,5,0,1,1,2,32,3625.401,2154.7,13731.2,13341.4,389.8,0.4375,0.444,388.47 -Qwen/Qwen3-8B,mmlu_hard,64,1,6,0,1,1,2,32,2971.752,1654.2,5915.8,5308.6,607.1,0.875,0.031,318.42 -Qwen/Qwen3-8B,gsmhard,64,1,6,0,1,1,2,32,3111.776,1426.8,7236.0,6631.9,604.1,0.8438,0.073,333.67 -Qwen/Qwen3-8B,gpqa,64,1,6,0,1,1,2,32,4413.838,2595.8,16564.8,16132.2,432.7,0.4062,0.464,472.89 diff --git a/scripts/clean/results/qwen3_smoke.csv b/scripts/clean/results/qwen3_smoke.csv deleted file mode 100644 index d788902..0000000 --- a/scripts/clean/results/qwen3_smoke.csv +++ /dev/null @@ -1,2 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,enable_thinking,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,per_query_think_tok,per_query_answer_tok,accuracy,trunc_frac,wall_latency_s -Qwen/Qwen3-8B,mmlu_hard,64,1,1,1,1,0,2,4543.498,154.0,1505.5,1426.5,79.0,1.0,0.0,30.37 diff --git a/scripts/clean/results/smoke.csv b/scripts/clean/results/smoke.csv deleted file mode 100644 index 7265200..0000000 --- a/scripts/clean/results/smoke.csv +++ /dev/null @@ -1,3 +0,0 @@ -model,task,decode_batch,team_size,rounds,carry_history,run_id,n_queries,per_query_energy_j,per_query_in_tok,per_query_out_tok,wall_latency_s -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,2,2,0,0,8,126.051,1023.5,314.2,3.42 -Qwen/Qwen2.5-7B-Instruct,mmlu_hard,16,5,2,0,0,8,244.458,3497.5,741.8,6.61 diff --git a/scripts/clean/results/steiner_tokens_by_cell.csv b/scripts/clean/results/steiner_tokens_by_cell.csv deleted file mode 100644 index ae3efd2..0000000 --- a/scripts/clean/results/steiner_tokens_by_cell.csv +++ /dev/null @@ -1,3251 +0,0 @@ -model_short,task,condition,team_size,comm_k,comm_round,n,p_sum,o_sum,p_mean,o_mean -gpt-oss-20b,arc,comm_true_k1,2,1,1,3495,1268124,711850,362.8394849785408,203.6766809728183 -gpt-oss-20b,arc,comm_true_k1,2,1,2,3495,2164100,1673174,619.1988555078684,478.7336194563662 -gpt-oss-20b,arc,comm_true_k1,2,1,3,3495,2240912,1530385,641.1765379113018,437.87839771101574 -gpt-oss-20b,arc,comm_true_k14,15,14,1,3495,9510930,3895258,2721.296137339056,1114.5230329041487 -gpt-oss-20b,arc,comm_true_k14,15,14,2,3495,51901073,3668703,14850.092417739628,1049.7004291845494 -gpt-oss-20b,arc,comm_true_k14,15,14,3,3495,59987771,3695683,17163.882975679542,1057.4200286123032 -gpt-oss-20b,arc,comm_true_k19,20,19,1,3495,12681240,5234238,3628.3948497854076,1497.6360515021458 -gpt-oss-20b,arc,comm_true_k19,20,19,2,3495,87395810,4920758,25005.954220314736,1407.9422031473534 -gpt-oss-20b,arc,comm_true_k19,20,19,3,3495,102404105,4936316,29300.173104434907,1412.393705293276 -gpt-oss-20b,arc,comm_true_k2,3,2,1,3495,1902186,773890,544.2592274678111,221.4277539341917 -gpt-oss-20b,arc,comm_true_k2,3,2,2,3495,3801048,864705,1087.567381974249,247.41201716738198 -gpt-oss-20b,arc,comm_true_k2,3,2,3,3495,4069754,879729,1164.4503576537911,251.71072961373392 -gpt-oss-20b,arc,comm_true_k24,25,24,1,3495,15851550,6543722,4535.49356223176,1872.3095851216024 -gpt-oss-20b,arc,comm_true_k24,25,24,2,3495,132215790,6207108,37829.98283261803,1775.9965665236052 -gpt-oss-20b,arc,comm_true_k24,25,24,3,3495,156550000,6251742,44792.56080114449,1788.767381974249 -gpt-oss-20b,arc,comm_true_k29,30,29,1,3495,19021860,7892076,5442.592274678112,2258.104721030043 -gpt-oss-20b,arc,comm_true_k29,30,29,2,3495,185952352,7587989,53205.250929899856,2171.098426323319 -gpt-oss-20b,arc,comm_true_k29,30,29,3,3495,224431045,7594746,64214.89127324749,2173.0317596566524 -gpt-oss-20b,arc,comm_true_k4,5,4,1,3495,3170310,1290105,907.0987124463519,369.12875536480686 -gpt-oss-20b,arc,comm_true_k4,5,4,2,3495,8163256,1237795,2335.69556509299,354.16165951359085 -gpt-oss-20b,arc,comm_true_k4,5,4,3,3495,8983930,1230757,2570.509298998569,352.14792560801146 -gpt-oss-20b,arc,comm_true_k6,7,6,1,3495,4438434,1798166,1269.9381974248927,514.4967095851216 -gpt-oss-20b,arc,comm_true_k6,7,6,2,3495,13993552,1722021,4003.877539341917,492.70987124463517 -gpt-oss-20b,arc,comm_true_k6,7,6,3,3495,15644294,1711038,4476.1928469241775,489.5673819742489 -gpt-oss-20b,arc,comm_true_k9,10,9,1,3495,6340620,2600063,1814.1974248927038,743.9379113018598 -gpt-oss-20b,arc,comm_true_k9,10,9,2,3495,25465257,2474064,7286.196566523605,707.8866952789699 -gpt-oss-20b,arc,comm_true_k9,10,9,3,3495,29037995,2446557,8308.439198855507,700.0163090128756 -gpt-oss-20b,arc,ensemble,1,0,0,3495,634062,252581,181.4197424892704,72.2692417739628 -gpt-oss-20b,arc,parity_decode_N5,1,0,0,3495,696972,563540,199.4197424892704,161.24177396280402 -gpt-oss-20b,dat,comm_true_k1,2,1,1,4500,1989000,4967746,442.0,1103.9435555555556 -gpt-oss-20b,dat,comm_true_k1,2,1,2,4500,3474926,6270286,772.2057777777778,1393.396888888889 -gpt-oss-20b,dat,comm_true_k1,2,1,3,4500,3787270,6594720,841.6155555555556,1465.4933333333333 -gpt-oss-20b,dat,comm_true_k14,15,14,1,4500,14917500,7651285,3315.0,1700.2855555555554 -gpt-oss-20b,dat,comm_true_k14,15,14,2,4500,105869130,10502998,23526.47333333333,2333.9995555555556 -gpt-oss-20b,dat,comm_true_k14,15,14,3,4500,126237048,12430839,28052.677333333333,2762.4086666666667 -gpt-oss-20b,dat,comm_true_k19,20,19,1,4500,19890000,8780550,4420.0,1951.2333333333333 -gpt-oss-20b,dat,comm_true_k19,20,19,2,4500,182093900,13487216,40465.311111111114,2997.159111111111 -gpt-oss-20b,dat,comm_true_k19,20,19,3,4500,215598632,16387702,47910.80711111111,3641.7115555555556 -gpt-oss-20b,dat,comm_true_k2,3,2,1,4500,2983500,1050907,663.0,233.5348888888889 -gpt-oss-20b,dat,comm_true_k2,3,2,2,4500,6944415,3634003,1543.2033333333334,807.5562222222222 -gpt-oss-20b,dat,comm_true_k2,3,2,3,4500,7588360,5009194,1686.3022222222223,1113.1542222222222 -gpt-oss-20b,dat,comm_true_k24,25,24,1,4500,24862500,10937918,5525.0,2430.6484444444445 -gpt-oss-20b,dat,comm_true_k24,25,24,2,4500,278108725,16450509,61801.938888888886,3655.6686666666665 -gpt-oss-20b,dat,comm_true_k24,25,24,3,4500,327387946,18867516,72752.87688888889,4192.781333333333 -gpt-oss-20b,dat,comm_true_k29,30,29,1,4500,29835000,14649208,6630.0,3255.3795555555557 -gpt-oss-20b,dat,comm_true_k29,30,29,2,4500,392060580,19182136,87124.57333333333,4262.696888888889 -gpt-oss-20b,dat,comm_true_k29,30,29,3,4500,466805481,22790921,103734.55133333334,5064.649111111111 -gpt-oss-20b,dat,comm_true_k4,5,4,1,4500,4972500,1752267,1105.0,389.3926666666667 -gpt-oss-20b,dat,comm_true_k4,5,4,2,4500,15599005,4481990,3466.4455555555555,995.9977777777777 -gpt-oss-20b,dat,comm_true_k4,5,4,3,4500,17565566,6440992,3903.4591111111113,1431.3315555555555 -gpt-oss-20b,dat,comm_true_k6,7,6,1,4500,6961500,2448567,1547.0,544.126 -gpt-oss-20b,dat,comm_true_k6,7,6,2,4500,27468049,5416477,6104.010888888889,1203.6615555555557 -gpt-oss-20b,dat,comm_true_k6,7,6,3,4500,31225178,7531308,6938.928444444445,1673.624 -gpt-oss-20b,dat,comm_true_k9,10,9,1,4500,9945000,4267717,2210.0,948.3815555555556 -gpt-oss-20b,dat,comm_true_k9,10,9,2,4500,51019670,7249226,11337.704444444444,1610.939111111111 -gpt-oss-20b,dat,comm_true_k9,10,9,3,4500,59010850,9977193,13113.522222222222,2217.154 -gpt-oss-20b,dat,ensemble,1,0,0,4500,994500,351009,221.0,78.002 -gpt-oss-20b,dat,parity_decode_N5,1,0,0,4500,1084500,1130103,241.0,251.134 -gpt-oss-20b,eqbench,comm_true_k1,2,1,1,96,67116,194297,699.125,2023.9270833333333 -gpt-oss-20b,eqbench,comm_true_k1,2,1,2,96,101196,191043,1054.125,1990.03125 -gpt-oss-20b,eqbench,comm_true_k1,2,1,3,96,120328,191439,1253.4166666666667,1994.15625 -gpt-oss-20b,eqbench,comm_true_k14,15,14,1,96,503370,1457702,5243.4375,15184.395833333334 -gpt-oss-20b,eqbench,comm_true_k14,15,14,2,96,3210698,1474114,33444.770833333336,15355.354166666666 -gpt-oss-20b,eqbench,comm_true_k14,15,14,3,96,3255695,1474388,33913.489583333336,15358.208333333334 -gpt-oss-20b,eqbench,comm_true_k19,20,19,1,96,671160,1945015,6991.25,20260.572916666668 -gpt-oss-20b,eqbench,comm_true_k19,20,19,2,96,5459002,1965742,56864.604166666664,20476.479166666668 -gpt-oss-20b,eqbench,comm_true_k19,20,19,3,96,5539047,1965637,57698.40625,20475.385416666668 -gpt-oss-20b,eqbench,comm_true_k2,3,2,1,96,100674,291957,1048.6875,3041.21875 -gpt-oss-20b,eqbench,comm_true_k2,3,2,2,96,218377,294912,2274.7604166666665,3072.0 -gpt-oss-20b,eqbench,comm_true_k2,3,2,3,96,219489,294912,2286.34375,3072.0 -gpt-oss-20b,eqbench,comm_true_k24,25,24,1,96,838950,2440462,8739.0625,25421.479166666668 -gpt-oss-20b,eqbench,comm_true_k24,25,24,2,96,8342108,2457216,86896.95833333333,25596.0 -gpt-oss-20b,eqbench,comm_true_k24,25,24,3,96,8422061,2457529,87729.80208333333,25599.260416666668 -gpt-oss-20b,eqbench,comm_true_k29,30,29,1,96,1006740,2922745,10486.875,30445.260416666668 -gpt-oss-20b,eqbench,comm_true_k29,30,29,2,96,11784194,2948242,122752.02083333333,30710.854166666668 -gpt-oss-20b,eqbench,comm_true_k29,30,29,3,96,11904148,2947702,124001.54166666667,30705.229166666668 -gpt-oss-20b,eqbench,comm_true_k4,5,4,1,96,167790,487326,1747.8125,5076.3125 -gpt-oss-20b,eqbench,comm_true_k4,5,4,2,96,481631,491520,5016.989583333333,5120.0 -gpt-oss-20b,eqbench,comm_true_k4,5,4,3,96,485704,491520,5059.416666666667,5120.0 -gpt-oss-20b,eqbench,comm_true_k6,7,6,1,96,234906,682199,2446.9375,7106.239583333333 -gpt-oss-20b,eqbench,comm_true_k6,7,6,2,96,843705,688008,8788.59375,7166.75 -gpt-oss-20b,eqbench,comm_true_k6,7,6,3,96,847830,687936,8831.5625,7166.0 -gpt-oss-20b,eqbench,comm_true_k9,10,9,1,96,335580,974151,3495.625,10147.40625 -gpt-oss-20b,eqbench,comm_true_k9,10,9,2,96,1557458,982930,16223.520833333334,10238.854166666666 -gpt-oss-20b,eqbench,comm_true_k9,10,9,3,96,1570830,983040,16362.8125,10240.0 -gpt-oss-20b,eqbench,ensemble,1,0,0,96,33558,98258,349.5625,1023.5208333333334 -gpt-oss-20b,eqbench,parity_decode_N5,1,0,0,96,35574,321300,370.5625,3346.875 -gpt-oss-20b,fermi,comm_true_k1,2,1,1,1587,531720,789233,335.04725897920605,497.31127914303715 -gpt-oss-20b,fermi,comm_true_k1,2,1,2,1587,1176964,1248552,741.6282293635791,786.7372400756144 -gpt-oss-20b,fermi,comm_true_k1,2,1,3,1587,1223002,1201275,770.6376811594203,756.9470699432892 -gpt-oss-20b,fermi,comm_true_k14,15,14,1,1587,3987900,5300805,2512.8544423440453,3340.141776937618 -gpt-oss-20b,fermi,comm_true_k14,15,14,2,1587,39399482,4805790,24826.390674228103,3028.2230623818527 -gpt-oss-20b,fermi,comm_true_k14,15,14,3,1587,45101205,4475178,28419.15879017013,2819.897920604915 -gpt-oss-20b,fermi,comm_true_k19,20,19,1,1587,5317200,7076742,3350.4725897920607,4459.194706994329 -gpt-oss-20b,fermi,comm_true_k19,20,19,2,1587,68226836,6272709,42991.07498424701,3952.557655954631 -gpt-oss-20b,fermi,comm_true_k19,20,19,3,1587,78403866,5806795,49403.82230623819,3658.9760554505356 -gpt-oss-20b,fermi,comm_true_k2,3,2,1,1587,797580,1058223,502.5708884688091,666.8071833648394 -gpt-oss-20b,fermi,comm_true_k2,3,2,2,1587,2256306,1254043,1421.7429111531192,790.1972274732199 -gpt-oss-20b,fermi,comm_true_k2,3,2,3,1587,2503691,1128208,1577.6250787649653,710.9061121613106 -gpt-oss-20b,fermi,comm_true_k24,25,24,1,1587,6880500,9045236,4335.538752362949,5699.581600504096 -gpt-oss-20b,fermi,comm_true_k24,25,24,2,1587,106460456,7655092,67082.83301827348,4823.624448645242 -gpt-oss-20b,fermi,comm_true_k24,25,24,3,1587,118458253,7170914,74642.88153749213,4518.53434152489 -gpt-oss-20b,fermi,comm_true_k29,30,29,1,1587,9213660,12166742,5805.708884688091,7666.504095778198 -gpt-oss-20b,fermi,comm_true_k29,30,29,2,1587,159182824,8981454,100304.23692501575,5659.391304347826 -gpt-oss-20b,fermi,comm_true_k29,30,29,3,1587,161651579,8547096,101859.84814114682,5385.693761814745 -gpt-oss-20b,fermi,comm_true_k4,5,4,1,1587,1329300,1765116,837.6181474480152,1112.234404536862 -gpt-oss-20b,fermi,comm_true_k4,5,4,2,1587,5330186,1830894,3358.655324511657,1153.6824196597354 -gpt-oss-20b,fermi,comm_true_k4,5,4,3,1587,5996657,1649177,3778.6118462507875,1039.1789540012603 -gpt-oss-20b,fermi,comm_true_k6,7,6,1,1587,1861020,2458066,1172.6654064272211,1548.8758664146187 -gpt-oss-20b,fermi,comm_true_k6,7,6,2,1587,9649964,2379499,6080.632640201638,1499.36925015753 -gpt-oss-20b,fermi,comm_true_k6,7,6,3,1587,10934608,2188961,6890.112161310649,1379.3074984247007 -gpt-oss-20b,fermi,comm_true_k9,10,9,1,1587,2658600,3528349,1675.2362948960304,2223.2822936357907 -gpt-oss-20b,fermi,comm_true_k9,10,9,2,1587,18471056,3247408,11638.976685570258,2046.2558286074354 -gpt-oss-20b,fermi,comm_true_k9,10,9,3,1587,21032349,3006438,13252.897920604915,1894.4158790170131 -gpt-oss-20b,fermi,ensemble,1,0,0,1587,265860,352719,167.52362948960302,222.25519848771268 -gpt-oss-20b,fermi,parity_decode_N5,1,0,0,1587,306064,801764,192.85696282293637,505.20730938878387 -gpt-oss-20b,gsm8k,comm_true_k1,2,1,1,3957,1378878,1060298,348.46550416982564,267.9550164265858 -gpt-oss-20b,gsm8k,comm_true_k1,2,1,2,3957,2683130,1165469,678.0717715440991,294.5334849633561 -gpt-oss-20b,gsm8k,comm_true_k1,2,1,3,3957,2769848,1089490,699.9868587313622,275.33232246651505 -gpt-oss-20b,gsm8k,comm_true_k14,15,14,1,3957,10341585,8172444,2613.491281273692,2065.3131159969676 -gpt-oss-20b,gsm8k,comm_true_k14,15,14,2,3957,78351566,5496718,19800.749557745767,1389.1124589335354 -gpt-oss-20b,gsm8k,comm_true_k14,15,14,3,3957,84334618,5255345,21312.76674248168,1328.1134698003539 -gpt-oss-20b,gsm8k,comm_true_k19,20,19,1,3957,13788780,10880813,3484.6550416982564,2749.763204447814 -gpt-oss-20b,gsm8k,comm_true_k19,20,19,2,3957,134044608,7362616,33875.31159969674,1860.6560525650746 -gpt-oss-20b,gsm8k,comm_true_k19,20,19,3,3957,145241526,7048150,36704.95981804397,1781.1852413444528 -gpt-oss-20b,gsm8k,comm_true_k2,3,2,1,3957,2068317,1616117,522.6982562547385,408.4197624462977 -gpt-oss-20b,gsm8k,comm_true_k2,3,2,2,3957,4993703,1182711,1261.9921657821583,298.89082638362396 -gpt-oss-20b,gsm8k,comm_true_k2,3,2,3,3957,5213483,1161648,1317.5342431134698,293.56785443517816 -gpt-oss-20b,gsm8k,comm_true_k24,25,24,1,3957,17235975,13599090,4355.818802122821,3436.7172100075813 -gpt-oss-20b,gsm8k,comm_true_k24,25,24,2,3957,204266348,9244236,51621.518321961084,2336.172858225929 -gpt-oss-20b,gsm8k,comm_true_k24,25,24,3,3957,222480828,8845622,56224.62168309325,2235.4364417487996 -gpt-oss-20b,gsm8k,comm_true_k29,30,29,1,3957,20683170,16273247,5226.982562547384,4112.521354561536 -gpt-oss-20b,gsm8k,comm_true_k29,30,29,2,3957,289667097,11075582,73203.71417740712,2798.984584281021 -gpt-oss-20b,gsm8k,comm_true_k29,30,29,3,3957,315448657,10622889,79719.14505938842,2684.581501137225 -gpt-oss-20b,gsm8k,comm_true_k4,5,4,1,3957,3447195,2707982,871.1637604245641,684.3522870861764 -gpt-oss-20b,gsm8k,comm_true_k4,5,4,2,3957,11282760,1891557,2851.341925701289,478.0280515542077 -gpt-oss-20b,gsm8k,comm_true_k4,5,4,3,3957,11966844,1778630,3024.221379833207,449.4895122567602 -gpt-oss-20b,gsm8k,comm_true_k6,7,6,1,3957,4826073,3814754,1219.6292645943897,964.0520596411423 -gpt-oss-20b,gsm8k,comm_true_k6,7,6,2,3957,19989031,2610940,5051.562041950973,659.8281526408896 -gpt-oss-20b,gsm8k,comm_true_k6,7,6,3,3957,21308054,2460341,5384.901187768512,621.7692696487238 -gpt-oss-20b,gsm8k,comm_true_k9,10,9,1,3957,6894390,5416167,1742.3275208491282,1368.7558756633814 -gpt-oss-20b,gsm8k,comm_true_k9,10,9,2,3957,37370715,3681209,9444.20394238059,930.3030073287845 -gpt-oss-20b,gsm8k,comm_true_k9,10,9,3,3957,40063844,3512554,10124.802628253728,887.6810715188274 -gpt-oss-20b,gsm8k,ensemble,1,0,0,3957,689439,529297,174.23275208491282,133.7621935809957 -gpt-oss-20b,gsm8k,parity_decode_N5,1,0,0,3957,748794,937339,189.23275208491282,236.88122314885013 -gpt-oss-20b,gsmhard,comm_true_k1,2,1,1,3051,1075284,1374224,352.43657817109147,450.41756801048837 -gpt-oss-20b,gsmhard,comm_true_k1,2,1,2,3051,2227100,1402443,729.9573910193379,459.6666666666667 -gpt-oss-20b,gsmhard,comm_true_k1,2,1,3,3051,2297876,1305474,753.155031137332,427.8839724680433 -gpt-oss-20b,gsmhard,comm_true_k14,15,14,1,3051,8064630,10419315,2643.2743362831857,3415.049164208456 -gpt-oss-20b,gsmhard,comm_true_k14,15,14,2,3051,68375103,6848423,22410.71878072763,2244.6486397902327 -gpt-oss-20b,gsmhard,comm_true_k14,15,14,3,3051,74079706,6106259,24280.467387741723,2001.3959357587676 -gpt-oss-20b,gsmhard,comm_true_k19,20,19,1,3051,10752840,13842616,3524.3657817109142,4537.075057358244 -gpt-oss-20b,gsmhard,comm_true_k19,20,19,2,3051,117520883,9021406,38518.80793182563,2956.868567682727 -gpt-oss-20b,gsmhard,comm_true_k19,20,19,3,3051,128159983,8040611,42005.894133071124,2635.4018354637824 -gpt-oss-20b,gsmhard,comm_true_k2,3,2,1,3051,1612926,2088250,528.6548672566372,684.4477220583416 -gpt-oss-20b,gsmhard,comm_true_k2,3,2,2,3051,4185842,1607196,1371.9573910193378,526.7767944936087 -gpt-oss-20b,gsmhard,comm_true_k2,3,2,3,3051,4390612,1514225,1439.0730907899049,496.304490331039 -gpt-oss-20b,gsmhard,comm_true_k24,25,24,1,3051,13441050,17314143,4405.457227138643,5674.907571288102 -gpt-oss-20b,gsmhard,comm_true_k24,25,24,2,3051,179976058,11090211,58989.202884300226,3634.9429695181907 -gpt-oss-20b,gsmhard,comm_true_k24,25,24,3,3051,196086138,9950490,64269.46509341199,3261.3864306784662 -gpt-oss-20b,gsmhard,comm_true_k29,30,29,1,3051,16129260,20754611,5286.548672566371,6802.560144215012 -gpt-oss-20b,gsmhard,comm_true_k29,30,29,2,3051,255531304,13266773,83753.29531301213,4348.335955424451 -gpt-oss-20b,gsmhard,comm_true_k29,30,29,3,3051,278453646,11837862,91266.35398230089,3879.9941002949854 -gpt-oss-20b,gsmhard,comm_true_k4,5,4,1,3051,2688210,3459277,881.0914454277286,1133.8174369059325 -gpt-oss-20b,gsmhard,comm_true_k4,5,4,2,3051,9614708,2454462,3151.330055719436,804.4778761061947 -gpt-oss-20b,gsmhard,comm_true_k4,5,4,3,3051,10208515,2212117,3345.957063257948,725.0465421173386 -gpt-oss-20b,gsmhard,comm_true_k6,7,6,1,3051,3763494,4851137,1233.52802359882,1590.0154047853164 -gpt-oss-20b,gsmhard,comm_true_k6,7,6,2,3051,17171243,3343693,5628.070468698787,1095.9334644378891 -gpt-oss-20b,gsmhard,comm_true_k6,7,6,3,3051,18360055,2985012,6017.717141920682,978.3716814159292 -gpt-oss-20b,gsmhard,comm_true_k9,10,9,1,3051,5376420,6946130,1762.1828908554571,2276.6732218944608 -gpt-oss-20b,gsmhard,comm_true_k9,10,9,2,3051,32464591,4603481,10640.639462471321,1508.8433300557194 -gpt-oss-20b,gsmhard,comm_true_k9,10,9,3,3051,34952715,4146328,11456.150442477876,1359.0062274664044 -gpt-oss-20b,gsmhard,ensemble,1,0,0,3051,537642,689192,176.21828908554573,225.89052769583742 -gpt-oss-20b,gsmhard,parity_decode_N5,1,0,0,3051,583407,1277749,191.21828908554573,418.7967879383809 -gpt-oss-20b,math500,comm_true_k1,2,1,1,1500,562992,1027376,375.328,684.9173333333333 -gpt-oss-20b,math500,comm_true_k1,2,1,2,1500,1153404,990009,768.936,660.006 -gpt-oss-20b,math500,comm_true_k1,2,1,3,1500,1194392,969538,796.2613333333334,646.3586666666666 -gpt-oss-20b,math500,comm_true_k14,15,14,1,1500,4222440,7728595,2814.96,5152.3966666666665 -gpt-oss-20b,math500,comm_true_k14,15,14,2,1500,34511174,4790400,23007.449333333334,3193.6 -gpt-oss-20b,math500,comm_true_k14,15,14,3,1500,38356495,4250496,25570.996666666666,2833.664 -gpt-oss-20b,math500,comm_true_k19,20,19,1,1500,5629920,10323614,3753.28,6882.409333333333 -gpt-oss-20b,math500,comm_true_k19,20,19,2,1500,59290460,6155363,39526.973333333335,4103.575333333333 -gpt-oss-20b,math500,comm_true_k19,20,19,3,1500,66413209,5460462,44275.47266666667,3640.308 -gpt-oss-20b,math500,comm_true_k2,3,2,1,1500,844488,1549927,562.992,1033.2846666666667 -gpt-oss-20b,math500,comm_true_k2,3,2,2,1500,2135166,1203450,1423.444,802.3 -gpt-oss-20b,math500,comm_true_k2,3,2,3,1500,2248234,1173147,1498.8226666666667,782.098 -gpt-oss-20b,math500,comm_true_k24,25,24,1,1500,7037400,12887061,4691.6,8591.374 -gpt-oss-20b,math500,comm_true_k24,25,24,2,1500,90588325,7554704,60392.21666666667,5036.469333333333 -gpt-oss-20b,math500,comm_true_k24,25,24,3,1500,101933731,6784771,67955.82066666667,4523.180666666667 -gpt-oss-20b,math500,comm_true_k29,30,29,1,1500,8444880,15464619,5629.92,10309.746 -gpt-oss-20b,math500,comm_true_k29,30,29,2,1500,128447426,9012157,85631.61733333333,6008.104666666667 -gpt-oss-20b,math500,comm_true_k29,30,29,3,1500,144718137,8075567,96478.758,5383.711333333334 -gpt-oss-20b,math500,comm_true_k4,5,4,1,1500,1407480,2578123,938.32,1718.7486666666666 -gpt-oss-20b,math500,comm_true_k4,5,4,2,1500,4879879,1814750,3253.2526666666668,1209.8333333333333 -gpt-oss-20b,math500,comm_true_k4,5,4,3,1500,5247312,1685964,3498.208,1123.976 -gpt-oss-20b,math500,comm_true_k6,7,6,1,1500,1970472,3621009,1313.648,2414.006 -gpt-oss-20b,math500,comm_true_k6,7,6,2,1500,8673583,2408637,5782.388666666667,1605.758 -gpt-oss-20b,math500,comm_true_k6,7,6,3,1500,9442232,2237915,6294.821333333333,1491.9433333333334 -gpt-oss-20b,math500,comm_true_k9,10,9,1,1500,2814960,5167123,1876.64,3444.748666666667 -gpt-oss-20b,math500,comm_true_k9,10,9,2,1500,16406703,3297974,10937.802,2198.6493333333333 -gpt-oss-20b,math500,comm_true_k9,10,9,3,1500,17979914,3030684,11986.609333333334,2020.456 -gpt-oss-20b,math500,ensemble,1,0,0,1500,281496,516291,187.664,344.194 -gpt-oss-20b,math500,parity_decode_N5,1,0,0,1500,303996,1152235,202.664,768.1566666666666 -gpt-oss-20b,mmlu_hard,comm_true_k1,2,1,1,2173,923884,901412,425.1652093879429,414.8237459733088 -gpt-oss-20b,mmlu_hard,comm_true_k1,2,1,2,2172,1605780,1199142,739.3093922651934,552.0911602209944 -gpt-oss-20b,mmlu_hard,comm_true_k1,2,1,3,2172,1744098,1163726,802.9917127071823,535.7854511970534 -gpt-oss-20b,mmlu_hard,comm_true_k14,15,14,1,2172,6926580,5376343,3189.033149171271,2475.2960405156537 -gpt-oss-20b,mmlu_hard,comm_true_k14,15,14,2,2172,40784288,4136178,18777.29650092081,1904.3176795580112 -gpt-oss-20b,mmlu_hard,comm_true_k14,15,14,3,2172,50124778,3714784,23077.706261510128,1710.305709023941 -gpt-oss-20b,mmlu_hard,comm_true_k19,20,19,1,2172,9235440,7162676,4252.044198895028,3297.7329650092083 -gpt-oss-20b,mmlu_hard,comm_true_k19,20,19,2,2172,69286440,5516419,31899.834254143647,2539.7877532228363 -gpt-oss-20b,mmlu_hard,comm_true_k19,20,19,3,2172,86542553,4937594,39844.6376611418,2273.293738489871 -gpt-oss-20b,mmlu_hard,comm_true_k2,3,2,1,2172,1385316,1069214,637.8066298342542,492.27163904235726 -gpt-oss-20b,mmlu_hard,comm_true_k2,3,2,2,2172,2869259,1114161,1321.0216390423573,512.9654696132596 -gpt-oss-20b,mmlu_hard,comm_true_k2,3,2,3,2172,3198390,1019495,1472.5552486187846,469.3807550644567 -gpt-oss-20b,mmlu_hard,comm_true_k24,25,24,1,2172,11544300,8994524,5315.055248618784,4141.125230202579 -gpt-oss-20b,mmlu_hard,comm_true_k24,25,24,2,2172,104998461,6779141,48341.83287292818,3121.151473296501 -gpt-oss-20b,mmlu_hard,comm_true_k24,25,24,3,2172,131964742,6115847,60757.24769797422,2815.7674953959486 -gpt-oss-20b,mmlu_hard,comm_true_k29,30,29,1,2172,13853160,10788794,6378.066298342542,4967.216390423573 -gpt-oss-20b,mmlu_hard,comm_true_k29,30,29,2,2172,147943800,8106330,68114.08839779005,3732.196132596685 -gpt-oss-20b,mmlu_hard,comm_true_k29,30,29,3,2172,187696885,7375684,86416.61372007367,3395.8029465930017 -gpt-oss-20b,mmlu_hard,comm_true_k4,5,4,1,2172,2308860,1772534,1063.011049723757,816.0837937384898 -gpt-oss-20b,mmlu_hard,comm_true_k4,5,4,2,2172,6250685,1515424,2877.847605893186,697.7090239410682 -gpt-oss-20b,mmlu_hard,comm_true_k4,5,4,3,2172,7214141,1347040,3321.4277163904235,620.1841620626151 -gpt-oss-20b,mmlu_hard,comm_true_k6,7,6,1,2172,3232404,2472247,1488.2154696132598,1138.2352670349908 -gpt-oss-20b,mmlu_hard,comm_true_k6,7,6,2,2172,10815099,2016155,4979.327348066298,928.2481583793739 -gpt-oss-20b,mmlu_hard,comm_true_k6,7,6,3,2172,12775687,1781326,5881.992173112339,820.1316758747698 -gpt-oss-20b,mmlu_hard,comm_true_k9,10,9,1,2172,4617720,3578594,2126.022099447514,1647.6031307550645 -gpt-oss-20b,mmlu_hard,comm_true_k9,10,9,2,2172,19863723,2791511,9145.360497237569,1285.22605893186 -gpt-oss-20b,mmlu_hard,comm_true_k9,10,9,3,2172,23920550,2493844,11013.144567219153,1148.1786372007366 -gpt-oss-20b,mmlu_hard,ensemble,1,0,0,2172,461772,353057,212.6022099447514,162.54926335174954 -gpt-oss-20b,mmlu_hard,parity_decode_N5,1,0,0,2172,500868,754986,230.6022099447514,347.59944751381215 -llama3.1-8b,arc,comm_true_k1,2,1,1,3495,1061058,433454,303.5931330472103,124.02117310443491 -llama3.1-8b,arc,comm_true_k1,2,1,2,3495,2278936,717549,652.0560801144492,205.30729613733905 -llama3.1-8b,arc,comm_true_k1,2,1,3,3495,2740020,757852,783.9828326180258,216.83891273247497 -llama3.1-8b,arc,comm_true_k14,15,14,1,3495,7957935,3219693,2276.9484978540772,921.2283261802575 -llama3.1-8b,arc,comm_true_k14,15,14,2,3495,68261432,5243163,19531.167954220316,1500.1896995708155 -llama3.1-8b,arc,comm_true_k14,15,14,3,3495,92554375,5514335,26481.938483547925,1577.7782546494993 -llama3.1-8b,arc,comm_true_k19,20,19,1,3495,10610580,4294013,3035.931330472103,1228.6160228898427 -llama3.1-8b,arc,comm_true_k19,20,19,2,3495,117283935,6736790,33557.63519313305,1927.5507868383404 -llama3.1-8b,arc,comm_true_k19,20,19,3,3495,157772413,7115698,45142.32131616595,2035.9650929899858 -llama3.1-8b,arc,comm_true_k2,3,2,1,3495,1591587,644138,455.3896995708154,184.3027181688126 -llama3.1-8b,arc,comm_true_k2,3,2,2,3495,4202520,1087617,1202.4377682403433,311.1922746781116 -llama3.1-8b,arc,comm_true_k2,3,2,3,3495,5280668,1151468,1510.9207439198856,329.4615164520744 -llama3.1-8b,arc,comm_true_k24,25,24,1,3495,13263225,5371650,3794.9141630901286,1536.9527896995708 -llama3.1-8b,arc,comm_true_k24,25,24,2,3495,179327979,8197769,51309.865236051504,2345.570529327611 -llama3.1-8b,arc,comm_true_k24,25,24,3,3495,239005859,8842171,68385.08125894134,2529.94878397711 -llama3.1-8b,arc,comm_true_k29,30,29,1,3495,15915870,6451614,4553.8969957081545,1845.955364806867 -llama3.1-8b,arc,comm_true_k29,30,29,2,3495,254841441,9732662,72916.00600858369,2784.7387696709584 -llama3.1-8b,arc,comm_true_k29,30,29,3,3495,338593117,10420289,96879.2895565093,2981.4846924177396 -llama3.1-8b,arc,comm_true_k4,5,4,1,3495,2652645,1075431,758.9828326180258,307.70557939914164 -llama3.1-8b,arc,comm_true_k4,5,4,2,3495,9644838,1796783,2759.6103004291845,514.1010014306152 -llama3.1-8b,arc,comm_true_k4,5,4,3,3495,12498680,1958685,3576.1602288984263,560.4248927038626 -llama3.1-8b,arc,comm_true_k6,7,6,1,3495,3713703,1503170,1062.575965665236,430.09155937052935 -llama3.1-8b,arc,comm_true_k6,7,6,2,3495,17163357,2511333,4910.831759656652,718.5502145922746 -llama3.1-8b,arc,comm_true_k6,7,6,3,3495,22748512,2737347,6508.873247496424,783.218025751073 -llama3.1-8b,arc,comm_true_k9,10,9,1,3495,5305290,2148916,1517.9656652360516,614.8543633762517 -llama3.1-8b,arc,comm_true_k9,10,9,2,3495,32365187,3509366,9260.425464949929,1004.110443490701 -llama3.1-8b,arc,comm_true_k9,10,9,3,3495,43471209,3814983,12438.11416309013,1091.5545064377682 -llama3.1-8b,arc,ensemble,1,0,0,3495,530529,215331,151.79656652360515,61.61115879828326 -llama3.1-8b,arc,parity_decode_N5,1,0,0,3495,593439,808979,169.79656652360515,231.4675250357654 -llama3.1-8b,dat,comm_true_k1,2,1,1,4500,1719000,1002226,382.0,222.7168888888889 -llama3.1-8b,dat,comm_true_k1,2,1,2,4500,4175852,1093865,927.9671111111111,243.0811111111111 -llama3.1-8b,dat,comm_true_k1,2,1,3,4500,4187374,1081952,930.5275555555555,240.43377777777778 -llama3.1-8b,dat,comm_true_k14,15,14,1,4500,12892500,7499251,2865.0,1666.5002222222222 -llama3.1-8b,dat,comm_true_k14,15,14,2,4500,140111565,8139032,31135.903333333332,1808.6737777777778 -llama3.1-8b,dat,comm_true_k14,15,14,3,4500,140815524,8000276,31292.338666666667,1777.8391111111112 -llama3.1-8b,dat,comm_true_k19,20,19,1,4500,17190000,10004974,3820.0,2223.3275555555556 -llama3.1-8b,dat,comm_true_k19,20,19,2,4500,242596721,10394940,53910.38244444445,2309.9866666666667 -llama3.1-8b,dat,comm_true_k19,20,19,3,4500,243767169,10370021,54170.482,2304.449111111111 -llama3.1-8b,dat,comm_true_k2,3,2,1,4500,2578500,1490958,573.0,331.324 -llama3.1-8b,dat,comm_true_k2,3,2,2,4500,7934331,1665693,1763.1846666666668,370.154 -llama3.1-8b,dat,comm_true_k2,3,2,3,4500,7965972,1624991,1770.216,361.10911111111113 -llama3.1-8b,dat,comm_true_k24,25,24,1,4500,21487500,12506007,4775.0,2779.112666666667 -llama3.1-8b,dat,comm_true_k24,25,24,2,4500,372969960,12620749,82882.21333333333,2804.610888888889 -llama3.1-8b,dat,comm_true_k24,25,24,3,4500,374642201,12789640,83253.82244444445,2842.1422222222222 -llama3.1-8b,dat,comm_true_k29,30,29,1,4500,25785000,15011946,5730.0,3335.988 -llama3.1-8b,dat,comm_true_k29,30,29,2,4500,531250347,14707759,118055.63266666667,3268.390888888889 -llama3.1-8b,dat,comm_true_k29,30,29,3,4500,533142997,14791307,118476.22155555556,3286.9571111111113 -llama3.1-8b,dat,comm_true_k4,5,4,1,4500,4297500,2488970,955.0,553.1044444444444 -llama3.1-8b,dat,comm_true_k4,5,4,2,4500,18806232,2833601,4179.162666666667,629.6891111111111 -llama3.1-8b,dat,comm_true_k4,5,4,3,4500,18889848,2705665,4197.744,601.2588888888889 -llama3.1-8b,dat,comm_true_k6,7,6,1,4501,6017837,3494194,1337.0,776.3150411019774 -llama3.1-8b,dat,comm_true_k6,7,6,2,4501,34143981,3953461,7585.865585425461,878.3516996223061 -llama3.1-8b,dat,comm_true_k6,7,6,3,4500,34303584,3755568,7623.018666666667,834.5706666666666 -llama3.1-8b,dat,comm_true_k9,10,9,1,4500,8595000,5007422,1910.0,1112.7604444444444 -llama3.1-8b,dat,comm_true_k9,10,9,2,4500,65512230,5613107,14558.273333333333,1247.3571111111112 -llama3.1-8b,dat,comm_true_k9,10,9,3,4500,65840299,5381136,14631.177555555556,1195.808 -llama3.1-8b,dat,ensemble,1,0,0,4500,859500,499550,191.0,111.0111111111111 -llama3.1-8b,dat,parity_decode_N5,1,0,0,4500,949500,1499146,211.0,333.14355555555557 -llama3.1-8b,eqbench,comm_true_k1,2,1,1,96,61794,189828,643.6875,1977.375 -llama3.1-8b,eqbench,comm_true_k1,2,1,2,96,117024,161127,1219.0,1678.40625 -llama3.1-8b,eqbench,comm_true_k1,2,1,3,96,117034,165659,1219.1041666666667,1725.6145833333333 -llama3.1-8b,eqbench,comm_true_k14,15,14,1,96,463455,1420846,4827.65625,14800.479166666666 -llama3.1-8b,eqbench,comm_true_k14,15,14,2,96,3214894,1036082,33488.479166666664,10792.520833333334 -llama3.1-8b,eqbench,comm_true_k14,15,14,3,96,3215131,1038526,33490.947916666664,10817.979166666666 -llama3.1-8b,eqbench,comm_true_k19,20,19,1,96,617940,1895116,6436.875,19740.791666666668 -llama3.1-8b,eqbench,comm_true_k19,20,19,2,96,5485250,1372558,57138.020833333336,14297.479166666666 -llama3.1-8b,eqbench,comm_true_k19,20,19,3,96,5485423,1352713,57139.822916666664,14090.760416666666 -llama3.1-8b,eqbench,comm_true_k2,3,2,1,96,92691,284085,965.53125,2959.21875 -llama3.1-8b,eqbench,comm_true_k2,3,2,2,96,211505,234330,2203.1770833333335,2440.9375 -llama3.1-8b,eqbench,comm_true_k2,3,2,3,96,211507,235981,2203.1979166666665,2458.1354166666665 -llama3.1-8b,eqbench,comm_true_k24,25,24,1,96,772425,2370984,8046.09375,24697.75 -llama3.1-8b,eqbench,comm_true_k24,25,24,2,96,8354459,1666553,87025.61458333333,17359.927083333332 -llama3.1-8b,eqbench,comm_true_k24,25,24,3,96,8354796,1666578,87029.125,17360.1875 -llama3.1-8b,eqbench,comm_true_k29,30,29,1,96,926910,2840437,9655.3125,29587.885416666668 -llama3.1-8b,eqbench,comm_true_k29,30,29,2,96,11823488,2013544,123161.33333333333,20974.416666666668 -llama3.1-8b,eqbench,comm_true_k29,30,29,3,96,11823452,2010707,123160.95833333333,20944.864583333332 -llama3.1-8b,eqbench,comm_true_k4,5,4,1,96,154485,473458,1609.21875,4931.854166666667 -llama3.1-8b,eqbench,comm_true_k4,5,4,2,96,472351,374926,4920.322916666667,3905.4791666666665 -llama3.1-8b,eqbench,comm_true_k4,5,4,3,96,472325,374482,4920.052083333333,3900.8541666666665 -llama3.1-8b,eqbench,comm_true_k6,7,6,1,96,216279,664216,2252.90625,6918.916666666667 -llama3.1-8b,eqbench,comm_true_k6,7,6,2,96,829097,513829,8636.427083333334,5352.385416666667 -llama3.1-8b,eqbench,comm_true_k6,7,6,3,96,829189,505745,8637.385416666666,5268.177083333333 -llama3.1-8b,eqbench,comm_true_k9,10,9,1,96,308970,949252,3218.4375,9888.041666666666 -llama3.1-8b,eqbench,comm_true_k9,10,9,2,96,1543813,718239,16081.385416666666,7481.65625 -llama3.1-8b,eqbench,comm_true_k9,10,9,3,96,1543972,703351,16083.041666666666,7326.572916666667 -llama3.1-8b,eqbench,ensemble,1,0,0,96,30897,95151,321.84375,991.15625 -llama3.1-8b,eqbench,parity_decode_N5,1,0,0,96,32913,114915,342.84375,1197.03125 -llama3.1-8b,fermi,comm_true_k1,2,1,1,1587,521578,978800,328.6565847511027,616.7611846250787 -llama3.1-8b,fermi,comm_true_k1,2,1,2,1587,1325306,899380,835.1014492753624,566.7170762444864 -llama3.1-8b,fermi,comm_true_k1,2,1,3,1587,1322554,690855,833.3673597983617,435.32136105860116 -llama3.1-8b,fermi,comm_true_k14,15,14,1,1587,3911835,7338198,2464.9243856332705,4623.943289224952 -llama3.1-8b,fermi,comm_true_k14,15,14,2,1587,47954192,6930313,30216.882167611846,4366.926906112161 -llama3.1-8b,fermi,comm_true_k14,15,14,3,1587,45456082,4361714,28642.773787019534,2748.402016383113 -llama3.1-8b,fermi,comm_true_k19,20,19,1,1587,5215780,9779189,3286.565847511027,6162.059861373661 -llama3.1-8b,fermi,comm_true_k19,20,19,2,1587,83382363,9206730,52540.87145557656,5801.342155009452 -llama3.1-8b,fermi,comm_true_k19,20,19,3,1587,78511454,5431091,49471.61562696913,3422.2375551354758 -llama3.1-8b,fermi,comm_true_k2,3,2,1,1587,782367,1462254,492.98487712665406,921.3950850661626 -llama3.1-8b,fermi,comm_true_k2,3,2,2,1587,2572818,1418857,1621.1833648393194,894.0497794580971 -llama3.1-8b,fermi,comm_true_k2,3,2,3,1587,2567901,1081503,1618.085066162571,681.4763705103969 -llama3.1-8b,fermi,comm_true_k24,25,24,1,1587,6519725,12198170,4108.207309388784,7686.3074984247005 -llama3.1-8b,fermi,comm_true_k24,25,24,2,1587,128505847,11394466,80974.06868304977,7179.877756773787 -llama3.1-8b,fermi,comm_true_k24,25,24,3,1587,120470796,6680401,75911.0245746692,4209.4524259609325 -llama3.1-8b,fermi,comm_true_k29,30,29,1,1587,7823670,14607919,4929.848771266541,9204.737870195337 -llama3.1-8b,fermi,comm_true_k29,30,29,2,1587,183517791,13509965,115638.17958412098,8512.895400126024 -llama3.1-8b,fermi,comm_true_k29,30,29,3,1587,170237742,7817311,107270.15879017013,4925.84183994959 -llama3.1-8b,fermi,comm_true_k4,5,4,1,1587,1303945,2427766,821.6414618777568,1529.783238815375 -llama3.1-8b,fermi,comm_true_k4,5,4,2,1587,6237962,2340172,3930.6628859483303,1474.588531821046 -llama3.1-8b,fermi,comm_true_k4,5,4,3,1587,6140623,1798600,3869.3276622558287,1133.3333333333333 -llama3.1-8b,fermi,comm_true_k6,7,6,1,1587,1825523,3403083,1150.2980466288595,2144.3497164461246 -llama3.1-8b,fermi,comm_true_k6,7,6,2,1587,11452024,3505073,7216.146187775677,2208.615626969124 -llama3.1-8b,fermi,comm_true_k6,7,6,3,1587,11274444,2547168,7104.2495274102075,1605.0207939508507 -llama3.1-8b,fermi,comm_true_k9,10,9,1,1587,2607890,4868608,1643.2829237555136,3067.8059231253937 -llama3.1-8b,fermi,comm_true_k9,10,9,2,1587,22216466,4873775,13999.033396345305,3071.061751732829 -llama3.1-8b,fermi,comm_true_k9,10,9,3,1587,21448550,3312341,13515.154379332072,2087.171392564587 -llama3.1-8b,fermi,ensemble,1,0,0,1587,260789,491623,164.32829237555134,309.7813484562067 -llama3.1-8b,fermi,parity_decode_N5,1,0,0,1587,258144,3684341,162.66162570888469,2321.575929426591 -llama3.1-8b,gsm8k,comm_true_k1,2,1,1,3957,1153038,1231257,291.3919636087945,311.1592115238817 -llama3.1-8b,gsm8k,comm_true_k1,2,1,2,3957,3118518,1338355,788.1015921152389,338.22466515036643 -llama3.1-8b,gsm8k,comm_true_k1,2,1,3,3957,3145752,1281489,794.9840788476118,323.85367702805155 -llama3.1-8b,gsm8k,comm_true_k14,15,14,1,3957,8647785,9235567,2185.4397270659592,2333.9820571139753 -llama3.1-8b,gsm8k,comm_true_k14,15,14,2,3957,111052635,11188500,28064.855951478392,2827.5208491281273 -llama3.1-8b,gsm8k,comm_true_k14,15,14,3,3957,116854775,9679275,29531.15365175638,2446.1144806671723 -llama3.1-8b,gsm8k,comm_true_k19,20,19,1,3957,11530380,12285855,2913.9196360879455,3104.840788476118 -llama3.1-8b,gsm8k,comm_true_k19,20,19,2,3957,192988830,14717778,48771.50113722517,3719.428354814253 -llama3.1-8b,gsm8k,comm_true_k19,20,19,3,3957,202658118,12559168,51215.09173616376,3173.911549153399 -llama3.1-8b,gsm8k,comm_true_k2,3,2,1,3957,1729557,1824601,437.0879454131918,461.10715188273946 -llama3.1-8b,gsm8k,comm_true_k2,3,2,2,3957,6026286,2170900,1522.9431387414709,548.6226939600707 -llama3.1-8b,gsm8k,comm_true_k2,3,2,3,3957,6214367,2099910,1570.4743492544858,530.6823351023503 -llama3.1-8b,gsm8k,comm_true_k24,25,24,1,3957,14412975,15419587,3642.3995451099318,3896.7872125347485 -llama3.1-8b,gsm8k,comm_true_k24,25,24,2,3957,297503447,18223460,75184.09072529695,4605.3727571392465 -llama3.1-8b,gsm8k,comm_true_k24,25,24,3,3957,311445669,15484015,78707.52312357847,3913.069244377053 -llama3.1-8b,gsm8k,comm_true_k29,30,29,1,3957,17295570,18472641,4370.8794541319185,4668.344958301744 -llama3.1-8b,gsm8k,comm_true_k29,30,29,2,3957,424322702,22033290,107233.43492544857,5568.180439727066 -llama3.1-8b,gsm8k,comm_true_k29,30,29,3,3957,441757693,18289636,111639.54839524893,4622.096537781147 -llama3.1-8b,gsm8k,comm_true_k4,5,4,1,3957,2882595,3063889,728.4799090219864,774.2959312610564 -llama3.1-8b,gsm8k,comm_true_k4,5,4,2,3957,14534967,3765796,3673.228961334344,951.6795552186 -llama3.1-8b,gsm8k,comm_true_k4,5,4,3,3957,15217603,3501317,3845.742481678039,884.8412939095274 -llama3.1-8b,gsm8k,comm_true_k6,7,6,1,3958,4036732,4321817,1019.8918645780698,1091.9194037392622 -llama3.1-8b,gsm8k,comm_true_k6,7,6,2,3957,26650641,5391081,6735.062168309325,1362.4162244124336 -llama3.1-8b,gsm8k,comm_true_k6,7,6,3,3957,28035659,4887498,7085.0793530452365,1235.1523881728583 -llama3.1-8b,gsm8k,comm_true_k9,10,9,1,3957,5765190,6070510,1456.9598180439727,1534.119282284559 -llama3.1-8b,gsm8k,comm_true_k9,10,9,2,3957,51536771,7523596,13024.202931513773,1901.3383876674247 -llama3.1-8b,gsm8k,comm_true_k9,10,9,3,3957,54410157,6892803,13750.355572403336,1741.926459438969 -llama3.1-8b,gsm8k,ensemble,1,0,0,3957,576519,606995,145.69598180439726,153.39777609299975 -llama3.1-8b,gsm8k,parity_decode_N5,1,0,0,3957,635874,8798788,160.69598180439726,2223.6007076067726 -llama3.1-8b,gsmhard,comm_true_k1,2,1,1,3051,901290,1034758,295.4080629301868,339.1537200917732 -llama3.1-8b,gsmhard,comm_true_k1,2,1,2,3051,2441142,1263563,800.1120943952802,414.147164863979 -llama3.1-8b,gsmhard,comm_true_k1,2,1,3,3051,2479494,1142284,812.6823992133727,374.39659128154705 -llama3.1-8b,gsmhard,comm_true_k14,15,14,1,3051,6759675,7685768,2215.560471976401,2519.098000655523 -llama3.1-8b,gsmhard,comm_true_k14,15,14,2,3051,87261955,10957599,28601.09963946247,3591.477876106195 -llama3.1-8b,gsmhard,comm_true_k14,15,14,3,3051,91261127,9094485,29911.873811864963,2980.8210422812194 -llama3.1-8b,gsmhard,comm_true_k19,20,19,1,3051,9012900,10306061,2954.080629301868,3377.9288757784334 -llama3.1-8b,gsmhard,comm_true_k19,20,19,2,3051,151887787,14274542,49782.9521468371,4678.643723369387 -llama3.1-8b,gsmhard,comm_true_k19,20,19,3,3051,158151781,11644003,51836.04752540151,3816.4546050475255 -llama3.1-8b,gsmhard,comm_true_k2,3,2,1,3051,1351935,1519452,443.1120943952802,498.01769911504425 -llama3.1-8b,gsmhard,comm_true_k2,3,2,2,3051,4728084,2092025,1549.6833824975417,685.6850213044903 -llama3.1-8b,gsmhard,comm_true_k2,3,2,3,3051,4881669,1935919,1600.0226155358898,634.5195018026876 -llama3.1-8b,gsmhard,comm_true_k24,25,24,1,3051,11266125,12823164,3692.6007866273353,4202.938053097345 -llama3.1-8b,gsmhard,comm_true_k24,25,24,2,3051,233826505,17539618,76639.30022943298,5748.809570632579 -llama3.1-8b,gsmhard,comm_true_k24,25,24,3,3051,243384673,14247564,79772.0986561783,4669.801376597837 -llama3.1-8b,gsmhard,comm_true_k29,30,29,1,3051,13519350,15350976,4431.120943952802,5031.457227138643 -llama3.1-8b,gsmhard,comm_true_k29,30,29,2,3051,333704675,21243490,109375.50803015404,6962.795804654212 -llama3.1-8b,gsmhard,comm_true_k29,30,29,3,3051,345812787,17001908,113344.0796460177,5572.568993772534 -llama3.1-8b,gsmhard,comm_true_k4,5,4,1,3051,2253225,2569783,738.520157325467,842.2756473287446 -llama3.1-8b,gsmhard,comm_true_k4,5,4,2,3051,11406373,3707306,3738.5686660111437,1215.1117666338905 -llama3.1-8b,gsmhard,comm_true_k4,5,4,3,3051,11907070,3310604,3902.677810553917,1085.0881678138314 -llama3.1-8b,gsmhard,comm_true_k6,7,6,1,3051,3154515,3570218,1033.928220255654,1170.1796132415602 -llama3.1-8b,gsmhard,comm_true_k6,7,6,2,3051,20921548,5350645,6857.2756473287445,1753.734841035726 -llama3.1-8b,gsmhard,comm_true_k6,7,6,3,3051,21909891,4629848,7181.216322517207,1517.485414618158 -llama3.1-8b,gsmhard,comm_true_k9,10,9,1,3051,4506450,5093040,1477.040314650934,1669.3018682399213 -llama3.1-8b,gsmhard,comm_true_k9,10,9,2,3051,40506438,7553053,13276.446411012783,2475.599147820387 -llama3.1-8b,gsmhard,comm_true_k9,10,9,3,3051,42424569,6498518,13905.13569321534,2129.9632907243526 -llama3.1-8b,gsmhard,ensemble,1,0,0,3051,450645,512775,147.7040314650934,168.06784660766962 -llama3.1-8b,gsmhard,parity_decode_N5,1,0,0,3051,496410,8123891,162.7040314650934,2662.697803998689 -llama3.1-8b,math500,comm_true_k1,2,1,1,1500,476274,694028,317.516,462.68533333333335 -llama3.1-8b,math500,comm_true_k1,2,1,2,1500,1252110,1096149,834.74,730.766 -llama3.1-8b,math500,comm_true_k1,2,1,3,1500,1266336,1002683,844.224,668.4553333333333 -llama3.1-8b,math500,comm_true_k14,15,14,1,1500,3572055,5147343,2381.37,3431.562 -llama3.1-8b,math500,comm_true_k14,15,14,2,1500,44116070,8139934,29410.713333333333,5426.622666666667 -llama3.1-8b,math500,comm_true_k14,15,14,3,1500,45217674,6754317,30145.116,4502.878 -llama3.1-8b,math500,comm_true_k19,20,19,1,1500,4762740,6872055,3175.16,4581.37 -llama3.1-8b,math500,comm_true_k19,20,19,2,1500,76688308,10215863,51125.53866666667,6810.575333333333 -llama3.1-8b,math500,comm_true_k19,20,19,3,1500,78229470,8552260,52152.98,5701.506666666667 -llama3.1-8b,math500,comm_true_k2,3,2,1,1500,714411,1030774,476.274,687.1826666666667 -llama3.1-8b,math500,comm_true_k2,3,2,2,1500,2409774,1760629,1606.516,1173.7526666666668 -llama3.1-8b,math500,comm_true_k2,3,2,3,1500,2461236,1609131,1640.824,1072.754 -llama3.1-8b,math500,comm_true_k24,25,24,1,1500,5953425,8572504,3968.95,5715.002666666666 -llama3.1-8b,math500,comm_true_k24,25,24,2,1500,118145170,12446156,78763.44666666667,8297.437333333333 -llama3.1-8b,math500,comm_true_k24,25,24,3,1500,120312189,10221478,80208.126,6814.318666666667 -llama3.1-8b,math500,comm_true_k29,30,29,1,1500,7144110,10247605,4762.74,6831.736666666667 -llama3.1-8b,math500,comm_true_k29,30,29,2,1500,168491576,14853424,112327.71733333333,9902.282666666666 -llama3.1-8b,math500,comm_true_k29,30,29,3,1500,170916438,11873753,113944.292,7915.835333333333 -llama3.1-8b,math500,comm_true_k4,5,4,1,1500,1190685,1719333,793.79,1146.222 -llama3.1-8b,math500,comm_true_k4,5,4,2,1500,5804146,2918687,3869.4306666666666,1945.7913333333333 -llama3.1-8b,math500,comm_true_k4,5,4,3,1500,5955373,2650445,3970.248666666667,1766.9633333333334 -llama3.1-8b,math500,comm_true_k6,7,6,1,1501,1667855,2385097,1111.1625582944703,1589.0053297801467 -llama3.1-8b,math500,comm_true_k6,7,6,2,1500,10614946,4148734,7076.630666666667,2765.8226666666665 -llama3.1-8b,math500,comm_true_k6,7,6,3,1500,10924345,3661463,7282.8966666666665,2440.9753333333333 -llama3.1-8b,math500,comm_true_k9,10,9,1,1500,2381370,3404955,1587.58,2269.97 -llama3.1-8b,math500,comm_true_k9,10,9,2,1500,20523802,5657151,13682.534666666666,3771.434 -llama3.1-8b,math500,comm_true_k9,10,9,3,1500,21102934,4943307,14068.622666666666,3295.538 -llama3.1-8b,math500,ensemble,1,0,0,1500,238137,340118,158.758,226.74533333333332 -llama3.1-8b,math500,parity_decode_N5,1,0,0,1500,260637,4093075,173.758,2728.7166666666667 -llama3.1-8b,mmlu_hard,comm_true_k1,2,1,1,2172,792840,412991,365.02762430939225,190.14318600368324 -llama3.1-8b,mmlu_hard,comm_true_k1,2,1,2,2172,1692820,689735,779.3830570902394,317.55755064456724 -llama3.1-8b,mmlu_hard,comm_true_k1,2,1,3,2172,1897782,679848,873.7486187845304,313.0055248618784 -llama3.1-8b,mmlu_hard,comm_true_k14,15,14,1,2173,5949315,3134405,2737.834790612057,1442.4321214910262 -llama3.1-8b,mmlu_hard,comm_true_k14,15,14,2,2172,51293830,4694029,23615.9438305709,2161.155156537753 -llama3.1-8b,mmlu_hard,comm_true_k14,15,14,3,2172,61934395,4385574,28514.914825046042,2019.1408839779006 -llama3.1-8b,mmlu_hard,comm_true_k19,20,19,1,2172,7928400,4205667,3650.276243093923,1936.3107734806629 -llama3.1-8b,mmlu_hard,comm_true_k19,20,19,2,2172,88337472,5965995,40671.02762430939,2746.774861878453 -llama3.1-8b,mmlu_hard,comm_true_k19,20,19,3,2172,106152657,5747282,48873.23066298343,2646.078268876611 -llama3.1-8b,mmlu_hard,comm_true_k2,3,2,1,2172,1189260,632656,547.5414364640884,291.2780847145488 -llama3.1-8b,mmlu_hard,comm_true_k2,3,2,2,2172,3127561,1056932,1439.945211786372,486.6169429097606 -llama3.1-8b,mmlu_hard,comm_true_k2,3,2,3,2172,3625400,1033620,1669.1528545119706,475.8839779005525 -llama3.1-8b,mmlu_hard,comm_true_k24,25,24,1,2172,9910500,5231298,4562.845303867403,2408.516574585635 -llama3.1-8b,mmlu_hard,comm_true_k24,25,24,2,2172,135128019,7243675,62213.636740331494,3335.0253222836095 -llama3.1-8b,mmlu_hard,comm_true_k24,25,24,3,2172,161746921,6919137,74469.1164825046,3185.6063535911603 -llama3.1-8b,mmlu_hard,comm_true_k29,30,29,1,2172,11892600,6278116,5475.414364640884,2890.476979742173 -llama3.1-8b,mmlu_hard,comm_true_k29,30,29,2,2172,192035552,8608605,88414.15837937385,3963.446132596685 -llama3.1-8b,mmlu_hard,comm_true_k29,30,29,3,2172,228946879,8197846,105408.32366482505,3774.330570902394 -llama3.1-8b,mmlu_hard,comm_true_k4,5,4,1,2172,1982100,1048692,912.5690607734807,482.8232044198895 -llama3.1-8b,mmlu_hard,comm_true_k4,5,4,2,2172,7205767,1687964,3317.5722836095765,777.147329650092 -llama3.1-8b,mmlu_hard,comm_true_k4,5,4,3,2172,8526311,1678936,3925.557550644567,772.9907918968693 -llama3.1-8b,mmlu_hard,comm_true_k6,7,6,1,2172,2774940,1457569,1277.596685082873,671.0722836095764 -llama3.1-8b,mmlu_hard,comm_true_k6,7,6,2,2172,12867200,2355911,5924.125230202579,1084.6735727440148 -llama3.1-8b,mmlu_hard,comm_true_k6,7,6,3,2172,15386936,2297905,7084.22467771639,1057.9673112338858 -llama3.1-8b,mmlu_hard,comm_true_k9,10,9,1,2172,3964200,2091266,1825.1381215469614,962.829650092081 -llama3.1-8b,mmlu_hard,comm_true_k9,10,9,2,2172,24311056,3265861,11192.935543278085,1503.6192449355433 -llama3.1-8b,mmlu_hard,comm_true_k9,10,9,3,2172,29267997,3148299,13475.136740331493,1449.493093922652 -llama3.1-8b,mmlu_hard,ensemble,1,0,0,2172,396420,204940,182.51381215469613,94.35543278084714 -llama3.1-8b,mmlu_hard,parity_decode_N5,1,0,0,2172,435516,1600629,200.51381215469613,736.9378453038674 -llama3.3-70b-awq,gsm8k,comm_true_k1,2,1,1,1319,384346,384826,291.3919636087945,291.75587566338135 -llama3.3-70b-awq,gsm8k,comm_true_k1,2,1,2,1319,946730,381092,717.7634571645186,288.92494313874147 -llama3.3-70b-awq,gsm8k,comm_true_k1,2,1,3,1319,1058316,326394,802.3623957543593,247.45564821834722 -llama3.3-70b-awq,gsm8k,comm_true_k14,15,14,1,1319,2882595,2854000,2185.4397270659592,2163.7604245640637 -llama3.3-70b-awq,gsm8k,comm_true_k14,15,14,2,1319,31820209,2918667,24124.49507202426,2212.7877179681577 -llama3.3-70b-awq,gsm8k,comm_true_k14,15,14,3,1319,38737716,2466978,29369.00379075057,1870.3396512509478 -llama3.3-70b-awq,gsm8k,comm_true_k19,20,19,1,1319,3843460,3795816,2913.9196360879455,2877.79833206975 -llama3.3-70b-awq,gsm8k,comm_true_k19,20,19,2,1319,55178866,3878294,41833.86353297953,2940.3290371493554 -llama3.3-70b-awq,gsm8k,comm_true_k19,20,19,3,1319,67188804,3360557,50939.19939347991,2547.806671721001 -llama3.3-70b-awq,gsm8k,comm_true_k2,3,2,1,1319,576519,562119,437.0879454131918,426.17058377558755 -llama3.3-70b-awq,gsm8k,comm_true_k2,3,2,2,1319,1798684,597000,1363.6724791508718,452.6156178923427 -llama3.3-70b-awq,gsm8k,comm_true_k2,3,2,3,1319,2051592,507142,1555.4147081122062,384.48976497346473 -llama3.3-70b-awq,gsm8k,comm_true_k24,25,24,1,716,2608900,2404569,3643.7150837988825,3358.336592178771 -llama3.3-70b-awq,gsm8k,comm_true_k24,25,24,2,716,45565260,2510223,63638.6312849162,3505.8980446927376 -llama3.3-70b-awq,gsm8k,comm_true_k24,25,24,3,715,56034931,2202456,78370.53286713286,3080.358041958042 -llama3.3-70b-awq,gsm8k,comm_true_k29,30,29,1,300,1305780,1306714,4352.6,4355.713333333333 -llama3.3-70b-awq,gsm8k,comm_true_k29,30,29,2,300,27359723,1275318,91199.07666666666,4251.06 -llama3.3-70b-awq,gsm8k,comm_true_k29,30,29,3,300,33582931,1111640,111943.10333333333,3705.4666666666667 -llama3.3-70b-awq,gsm8k,comm_true_k4,5,4,1,1319,960865,929433,728.4799090219864,704.6497346474601 -llama3.3-70b-awq,gsm8k,comm_true_k4,5,4,2,1319,4266427,969953,3234.5921152388173,735.3699772554966 -llama3.3-70b-awq,gsm8k,comm_true_k4,5,4,3,1319,4997060,830671,3788.521607278241,629.773313115997 -llama3.3-70b-awq,gsm8k,comm_true_k6,7,6,1,1319,1345211,1340745,1019.8718726307809,1016.4859742228962 -llama3.3-70b-awq,gsm8k,comm_true_k6,7,6,2,1319,7752679,1374990,5877.69446550417,1042.4488248673238 -llama3.3-70b-awq,gsm8k,comm_true_k6,7,6,3,1319,9189236,1168239,6966.820318423048,885.7005307050796 -llama3.3-70b-awq,gsm8k,comm_true_k9,10,9,1,1319,1921730,1915548,1456.9598180439727,1452.27293404094 -llama3.3-70b-awq,gsm8k,comm_true_k9,10,9,2,1319,14895408,1930221,11292.955269143291,1463.3972706595905 -llama3.3-70b-awq,gsm8k,comm_true_k9,10,9,3,1319,17858520,1635440,13539.438968915845,1239.9090219863533 -llama3.3-70b-awq,gsm8k,ensemble,1,0,0,1319,192173,189283,145.69598180439726,143.5049279757392 -llama3.3-70b-awq,gsm8k,parity_decode_N5,1,0,0,48,7763,50401,161.72916666666666,1050.0208333333333 -marin-8b,arc,comm_true_k1,2,1,1,3495,3668328,290062,1049.5931330472104,82.9934191702432 -marin-8b,arc,comm_true_k1,2,1,2,3495,4611318,462829,1319.4042918454936,132.42603719599427 -marin-8b,arc,comm_true_k1,2,1,3,3495,4934164,528736,1411.7779685264663,151.28354792560802 -marin-8b,arc,comm_true_k14,15,14,1,3495,27512460,2180632,7871.948497854078,623.9290414878398 -marin-8b,arc,comm_true_k14,15,14,2,3495,72404583,3377111,20716.618884120173,966.2692417739628 -marin-8b,arc,comm_true_k14,15,14,3,3495,88957012,4430828,25452.65007153076,1267.7619456366238 -marin-8b,arc,comm_true_k19,20,19,1,3495,36683280,2910093,10495.931330472104,832.644635193133 -marin-8b,arc,comm_true_k19,20,19,2,3495,116006982,4089950,33192.269527897,1170.2288984263232 -marin-8b,arc,comm_true_k19,20,19,3,3495,138047171,5631951,39498.47525035765,1611.4309012875538 -marin-8b,arc,comm_true_k2,3,2,1,3495,5502492,437050,1574.3896995708155,125.05007153075823 -marin-8b,arc,comm_true_k2,3,2,2,3495,7504476,699371,2147.203433476395,200.10615164520743 -marin-8b,arc,comm_true_k2,3,2,3,3495,8235625,808216,2356.4020028612304,231.24921316165953 -marin-8b,arc,comm_true_k24,25,24,1,3495,45854100,3644612,13119.914163090129,1042.8074391988555 -marin-8b,arc,comm_true_k24,25,24,2,3495,169352540,4901240,48455.66237482117,1402.3576537911301 -marin-8b,arc,comm_true_k24,25,24,3,3495,198551438,6949239,56810.13962804006,1988.337339055794 -marin-8b,arc,comm_true_k29,30,29,1,3495,55024920,4362705,15743.896995708155,1248.2703862660944 -marin-8b,arc,comm_true_k29,30,29,2,3495,232188209,5719630,66434.39456366237,1636.5178826895565 -marin-8b,arc,comm_true_k29,30,29,3,3495,270095710,8026014,77280.60371959943,2296.4274678111587 -marin-8b,arc,comm_true_k4,5,4,1,3495,9170820,727063,2623.982832618026,208.02947067238912 -marin-8b,arc,comm_true_k4,5,4,2,3495,14441428,1190350,4132.025178826895,340.5865522174535 -marin-8b,arc,comm_true_k4,5,4,3,3495,16602697,1399858,4750.414020028612,400.5316165951359 -marin-8b,arc,comm_true_k6,7,6,1,3496,12842718,1016240,3673.546338672769,290.6864988558352 -marin-8b,arc,comm_true_k6,7,6,2,3496,22929126,1665631,6558.674485125858,476.4390732265446 -marin-8b,arc,comm_true_k6,7,6,3,3495,27133829,2038281,7763.613447782546,583.1991416309013 -marin-8b,arc,comm_true_k9,10,9,1,3495,18341640,1457889,5247.965665236052,417.1356223175966 -marin-8b,arc,comm_true_k9,10,9,2,3495,38591477,2313523,11041.9104434907,661.952217453505 -marin-8b,arc,comm_true_k9,10,9,3,3495,46611699,2901911,13336.68068669528,830.3035765379113 -marin-8b,arc,ensemble,1,0,0,3495,1834164,146544,524.7965665236052,41.929613733905576 -marin-8b,arc,parity_decode_N5,1,0,0,3495,1897074,528556,542.7965665236052,151.23204577968525 -marin-8b,dat,comm_true_k1,2,1,1,4500,5076000,664750,1128.0,147.72222222222223 -marin-8b,dat,comm_true_k1,2,1,2,4500,7190986,806775,1597.9968888888889,179.28333333333333 -marin-8b,dat,comm_true_k1,2,1,3,4500,7350996,866494,1633.5546666666667,192.55422222222222 -marin-8b,dat,comm_true_k14,15,14,1,4500,38070000,4998365,8460.0,1110.7477777777779 -marin-8b,dat,comm_true_k14,15,14,2,4500,146218696,6029930,32493.043555555556,1339.9844444444445 -marin-8b,dat,comm_true_k14,15,14,3,4500,158377204,6499747,35194.934222222226,1444.388222222222 -marin-8b,dat,comm_true_k19,20,19,1,4500,50760000,6661124,11280.0,1480.2497777777778 -marin-8b,dat,comm_true_k19,20,19,2,4500,242169740,7886157,53815.497777777775,1752.4793333333334 -marin-8b,dat,comm_true_k19,20,19,3,4500,262589097,8489417,58353.132666666665,1886.537111111111 -marin-8b,dat,comm_true_k2,3,2,1,4500,7614000,999313,1692.0,222.06955555555555 -marin-8b,dat,comm_true_k2,3,2,2,4500,12203856,1283124,2711.968,285.13866666666667 -marin-8b,dat,comm_true_k2,3,2,3,4500,12662662,1449265,2813.9248888888887,322.0588888888889 -marin-8b,dat,comm_true_k24,25,24,1,3364,47432400,6225593,14100.0,1850.6519024970273 -marin-8b,dat,comm_true_k24,25,24,2,3363,270478611,7509465,80427.77609277431,2232.9661016949153 -marin-8b,dat,comm_true_k24,25,24,3,3363,298001569,7999252,88611.82545346417,2378.6060065417782 -marin-8b,dat,comm_true_k29,30,29,1,3000,50760000,6656503,16920.0,2218.8343333333332 -marin-8b,dat,comm_true_k29,30,29,2,3000,336671886,8101018,112223.962,2700.3393333333333 -marin-8b,dat,comm_true_k29,30,29,3,3000,374354242,265048,124784.74733333333,88.34933333333333 -marin-8b,dat,comm_true_k4,5,4,1,4500,12690000,1664902,2820.0,369.9782222222222 -marin-8b,dat,comm_true_k4,5,4,2,4500,25080572,2080908,5573.460444444445,462.424 -marin-8b,dat,comm_true_k4,5,4,3,4500,26291292,2330623,5842.509333333333,517.9162222222222 -marin-8b,dat,comm_true_k6,7,6,1,4500,17766000,2336197,3948.0,519.1548888888889 -marin-8b,dat,comm_true_k6,7,6,2,4500,41745627,2904048,9276.806,645.344 -marin-8b,dat,comm_true_k6,7,6,3,4500,44352708,3217868,9856.157333333333,715.0817777777778 -marin-8b,dat,comm_true_k9,10,9,1,4501,25385640,3326022,5640.0,738.9517884914463 -marin-8b,dat,comm_true_k9,10,9,2,4501,73780147,3930693,16391.945567651634,873.2932681626305 -marin-8b,dat,comm_true_k9,10,9,3,4500,78273551,4278450,17394.122444444445,950.7666666666667 -marin-8b,dat,ensemble,1,0,0,4500,2538000,333564,564.0,74.12533333333333 -marin-8b,dat,parity_decode_N5,1,0,0,3100,1810400,448687,584.0,144.73774193548388 -marin-8b,eqbench,comm_true_k1,2,1,1,96,133410,175924,1389.6875,1832.5416666666667 -marin-8b,eqbench,comm_true_k1,2,1,2,96,186114,159378,1938.6875,1660.1875 -marin-8b,eqbench,comm_true_k1,2,1,3,96,187486,157030,1952.9791666666667,1635.7291666666667 -marin-8b,eqbench,comm_true_k14,15,14,1,64,667050,856586,10422.65625,13384.15625 -marin-8b,eqbench,comm_true_k14,15,14,2,64,2395742,769126,37433.46875,12017.59375 -marin-8b,eqbench,comm_true_k14,15,14,3,64,2489836,738394,38903.6875,11537.40625 -marin-8b,eqbench,comm_true_k19,20,19,1,64,889400,1150549,13896.875,17977.328125 -marin-8b,eqbench,comm_true_k19,20,19,2,64,3951697,886448,61745.265625,13850.75 -marin-8b,eqbench,comm_true_k19,20,19,3,64,4077955,886287,63718.046875,13848.234375 -marin-8b,eqbench,comm_true_k2,3,2,1,96,200115,262354,2084.53125,2732.8541666666665 -marin-8b,eqbench,comm_true_k2,3,2,2,96,313093,241658,3261.3854166666665,2517.2708333333335 -marin-8b,eqbench,comm_true_k2,3,2,3,96,316598,232053,3297.8958333333335,2417.21875 -marin-8b,eqbench,comm_true_k24,25,24,1,64,1111750,1429788,17371.09375,22340.4375 -marin-8b,eqbench,comm_true_k24,25,24,2,64,5857093,480808,91517.078125,7512.625 -marin-8b,eqbench,comm_true_k24,25,24,3,64,5788364,515751,90443.1875,8058.609375 -marin-8b,eqbench,comm_true_k29,30,29,1,64,1334100,1724741,20845.3125,26949.078125 -marin-8b,eqbench,comm_true_k29,30,29,2,64,8160237,130473,127503.703125,2038.640625 -marin-8b,eqbench,comm_true_k29,30,29,3,64,3174760,1444859,49605.625,22575.921875 -marin-8b,eqbench,comm_true_k4,5,4,1,96,333525,428117,3474.21875,4459.552083333333 -marin-8b,eqbench,comm_true_k4,5,4,2,96,632222,394275,6585.645833333333,4107.03125 -marin-8b,eqbench,comm_true_k4,5,4,3,96,645973,381989,6728.885416666667,3979.0520833333335 -marin-8b,eqbench,comm_true_k6,7,6,1,64,311290,398885,4863.90625,6232.578125 -marin-8b,eqbench,comm_true_k6,7,6,2,64,695988,358728,10874.8125,5605.125 -marin-8b,eqbench,comm_true_k6,7,6,3,64,713041,348966,11141.265625,5452.59375 -marin-8b,eqbench,comm_true_k9,10,9,1,64,444700,581052,6948.4375,9078.9375 -marin-8b,eqbench,comm_true_k9,10,9,2,64,1223438,499136,19116.21875,7799.0 -marin-8b,eqbench,comm_true_k9,10,9,3,64,1251831,485155,19559.859375,7580.546875 -marin-8b,eqbench,ensemble,1,0,0,96,66705,84618,694.84375,881.4375 -marin-8b,eqbench,parity_decode_N5,1,0,0,96,68721,159395,715.84375,1660.3645833333333 -marin-8b,fermi,comm_true_k1,2,1,1,1587,1874760,1305892,1181.3232514177694,822.8683049779459 -marin-8b,fermi,comm_true_k1,2,1,2,1587,2668502,883564,1681.4757403906742,556.7511027095148 -marin-8b,fermi,comm_true_k1,2,1,3,1587,2652626,686229,1671.4719596723378,432.406427221172 -marin-8b,fermi,comm_true_k14,15,14,1,1587,14060700,9881306,8859.92438563327,6226.405797101449 -marin-8b,fermi,comm_true_k14,15,14,2,1587,58328455,6550731,36753.909892879645,4127.744801512287 -marin-8b,fermi,comm_true_k14,15,14,3,1587,57323165,4499270,36120.45683679899,2835.0787649653435 -marin-8b,fermi,comm_true_k19,20,19,1,1587,18747600,13144980,11813.232514177695,8282.911153119092 -marin-8b,fermi,comm_true_k19,20,19,2,1587,97464676,10597079,61414.41461877757,6677.428481411468 -marin-8b,fermi,comm_true_k19,20,19,3,1587,96736601,7493554,60955.640201638314,4721.836168872086 -marin-8b,fermi,comm_true_k2,3,2,1,1587,2812140,1997976,1771.984877126654,1258.9640831758034 -marin-8b,fermi,comm_true_k2,3,2,2,1587,4592727,1433205,2893.96786389414,903.0907372400757 -marin-8b,fermi,comm_true_k2,3,2,3,1587,4566131,1044313,2877.209199747952,658.0422180214241 -marin-8b,fermi,comm_true_k24,25,24,1,1587,23434500,16548238,14766.540642722117,10427.371140516698 -marin-8b,fermi,comm_true_k24,25,24,2,1587,146383357,9041443,92239.04032766225,5697.191556395715 -marin-8b,fermi,comm_true_k24,25,24,3,1587,144353587,6953627,90960.04221802142,4381.617517328293 -marin-8b,fermi,comm_true_k29,30,29,1,1587,28121400,19773853,17719.84877126654,12459.894770006302 -marin-8b,fermi,comm_true_k29,30,29,2,1587,205016362,89503,129184.8531821046,56.39760554505356 -marin-8b,fermi,comm_true_k29,30,29,3,1587,53290170,12382444,33579.187145557655,7802.422180214241 -marin-8b,fermi,comm_true_k4,5,4,1,1587,4686900,3262360,2953.3081285444237,2055.6773787019533 -marin-8b,fermi,comm_true_k4,5,4,2,1587,9614815,2738225,6058.484562066793,1725.4095778197857 -marin-8b,fermi,comm_true_k4,5,4,3,1587,9590692,2068852,6043.284183994959,1303.6244486452426 -marin-8b,fermi,comm_true_k6,7,6,1,1587,6561660,4586197,4134.631379962193,2889.8531821046 -marin-8b,fermi,comm_true_k6,7,6,2,1587,16216331,3502955,10218.229993698802,2207.2810333963453 -marin-8b,fermi,comm_true_k6,7,6,3,1587,16130190,2613694,10163.950850661626,1646.940138626339 -marin-8b,fermi,comm_true_k9,10,9,1,1587,9373800,6543721,5906.616257088847,4123.327662255829 -marin-8b,fermi,comm_true_k9,10,9,2,1587,29060367,5171895,18311.510396975424,3258.913043478261 -marin-8b,fermi,comm_true_k9,10,9,3,1587,28864665,3693883,18188.194706994327,2327.588531821046 -marin-8b,fermi,ensemble,1,0,0,1587,937380,654075,590.6616257088847,412.14555765595463 -marin-8b,fermi,parity_decode_N5,1,0,0,1587,888183,1085055,559.6616257088847,683.7145557655955 -marin-8b,gsm8k,comm_true_k1,2,1,1,3957,4104960,846046,1037.3919636087946,213.80995703816023 -marin-8b,gsm8k,comm_true_k1,2,1,2,3957,5719460,865013,1445.4030831437958,218.60323477381854 -marin-8b,gsm8k,comm_true_k1,2,1,3,3957,5936680,853248,1500.2982057113975,215.6300227445034 -marin-8b,gsm8k,comm_true_k14,15,14,1,3957,30787200,6484682,7780.439727065959,1638.787465251453 -marin-8b,gsm8k,comm_true_k14,15,14,2,3957,113646529,7615342,28720.37629517311,1924.5241344452868 -marin-8b,gsm8k,comm_true_k14,15,14,3,3957,126340972,9189161,31928.474096537782,2322.254485721506 -marin-8b,gsm8k,comm_true_k19,20,19,1,3958,41061020,8631170,10374.183931278423,2180.689742294088 -marin-8b,gsm8k,comm_true_k19,20,19,2,3958,187915951,9649191,47477.50151591713,2437.8956543708946 -marin-8b,gsm8k,comm_true_k19,20,19,3,3957,208331371,11901726,52648.8175385393,3007.764973464746 -marin-8b,gsm8k,comm_true_k2,3,2,1,3957,6157440,1296059,1556.0879454131918,327.53575941369724 -marin-8b,gsm8k,comm_true_k2,3,2,2,3957,9667032,1368111,2443.0204700530703,345.7445034116755 -marin-8b,gsm8k,comm_true_k2,3,2,3,3957,10229028,1350074,2585.046247156937,341.18625221127115 -marin-8b,gsm8k,comm_true_k24,25,24,1,3957,51312000,10689900,12967.399545109933,2701.516300227445 -marin-8b,gsm8k,comm_true_k24,25,24,2,3957,280110099,11862699,70788.50113722518,2997.9021986353296 -marin-8b,gsm8k,comm_true_k24,25,24,3,3957,308970978,14090151,78082.1273692191,3560.816527672479 -marin-8b,gsm8k,comm_true_k29,30,29,1,3957,61574400,12966350,15560.879454131918,3276.8132423553197 -marin-8b,gsm8k,comm_true_k29,30,29,2,3957,390710320,11076049,98739.02451352034,2799.1026029820573 -marin-8b,gsm8k,comm_true_k29,30,29,3,3957,396864536,13798276,100294.297700278,3487.0548395248925 -marin-8b,gsm8k,comm_true_k4,5,4,1,3957,10262400,2156436,2593.4799090219863,544.96739954511 -marin-8b,gsm8k,comm_true_k4,5,4,2,3957,19743682,2340663,4989.558251200404,591.524639878696 -marin-8b,gsm8k,comm_true_k4,5,4,3,3957,21368879,2279182,5400.272681324235,575.9873641647713 -marin-8b,gsm8k,comm_true_k6,7,6,1,3957,14367360,3037324,3630.8718726307807,767.5825120040435 -marin-8b,gsm8k,comm_true_k6,7,6,2,3957,32729786,3444102,8271.363659337881,870.3821076573162 -marin-8b,gsm8k,comm_true_k6,7,6,3,3957,36195861,3388327,9147.298711144807,856.2868334596917 -marin-8b,gsm8k,comm_true_k9,10,9,1,3957,20524800,4289242,5186.9598180439725,1083.9631033611322 -marin-8b,gsm8k,comm_true_k9,10,9,2,3957,57598457,4862702,14556.092241597169,1228.8860247662371 -marin-8b,gsm8k,comm_true_k9,10,9,3,3957,64459974,4928682,16290.11220621683,1245.560272934041 -marin-8b,gsm8k,ensemble,1,0,0,3957,2052480,432198,518.6959818043973,109.22365428354814 -marin-8b,gsm8k,parity_decode_N5,1,0,0,3957,2111835,1691954,533.6959818043973,427.5850391710892 -marin-8b,gsmhard,comm_true_k1,2,1,1,3051,3177336,649673,1041.4080629301868,212.93772533595543 -marin-8b,gsmhard,comm_true_k1,2,1,2,3051,4473404,756567,1466.209111766634,247.97345132743362 -marin-8b,gsmhard,comm_true_k1,2,1,3,3051,4642730,751820,1521.7076368403802,246.41756801048837 -marin-8b,gsmhard,comm_true_k14,15,14,1,3051,23830020,5055733,7810.5604719764015,1657.0740740740741 -marin-8b,gsmhard,comm_true_k14,15,14,2,3051,90633569,6117144,29706.184529662405,2004.9636184857425 -marin-8b,gsmhard,comm_true_k14,15,14,3,3051,98728030,6869077,32359.236315961978,2251.418223533268 -marin-8b,gsmhard,comm_true_k19,20,19,1,3052,31783340,6734989,10413.938401048494,2206.7460681520315 -marin-8b,gsmhard,comm_true_k19,20,19,2,3051,150087140,8060256,49192.76958374304,2641.8407079646017 -marin-8b,gsmhard,comm_true_k19,20,19,3,3051,163255018,9315228,53508.691576532285,3053.1720747295967 -marin-8b,gsmhard,comm_true_k2,3,2,1,3051,4766004,1029938,1562.1120943952803,337.5739101933792 -marin-8b,gsmhard,comm_true_k2,3,2,2,3051,7592529,1232110,2488.5378564405114,403.8380858734841 -marin-8b,gsmhard,comm_true_k2,3,2,3,3051,8027047,1190760,2630.956079973779,390.2851524090462 -marin-8b,gsmhard,comm_true_k24,25,24,1,3051,39716700,8388197,13017.600786627336,2749.327105866929 -marin-8b,gsmhard,comm_true_k24,25,24,2,3051,223962575,9396272,73406.28482464765,3079.735168797116 -marin-8b,gsmhard,comm_true_k24,25,24,3,3051,242988032,10765720,79642.09505080302,3528.587348410357 -marin-8b,gsmhard,comm_true_k29,30,29,1,3051,47660040,10059845,15621.120943952803,3297.2287774500164 -marin-8b,gsmhard,comm_true_k29,30,29,2,3051,312588835,8089458,102454.5509668961,2651.411996066863 -marin-8b,gsmhard,comm_true_k29,30,29,3,3051,297837526,11817184,97619.64142903966,3873.216650278597 -marin-8b,gsmhard,comm_true_k4,5,4,1,3051,7943340,1671064,2603.520157325467,547.7102589314978 -marin-8b,gsmhard,comm_true_k4,5,4,2,3051,15577964,2115119,5105.855129465749,693.2543428384137 -marin-8b,gsmhard,comm_true_k4,5,4,3,3051,16790662,2053956,5503.330711242215,673.2074729596853 -marin-8b,gsmhard,comm_true_k6,7,6,1,3051,11120676,2324694,3644.9282202556537,761.944936086529 -marin-8b,gsmhard,comm_true_k6,7,6,2,3051,25880752,2998679,8482.711242215668,982.8511963290724 -marin-8b,gsmhard,comm_true_k6,7,6,3,3051,28349554,2875130,9291.88921665028,942.3566043920026 -marin-8b,gsmhard,comm_true_k9,10,9,1,3051,15886680,3298687,5207.040314650934,1081.182235332678 -marin-8b,gsmhard,comm_true_k9,10,9,2,3051,45722036,4148505,14985.918059652573,1359.7197640117995 -marin-8b,gsmhard,comm_true_k9,10,9,3,3051,50333938,4122317,16497.521468371026,1351.1363487381186 -marin-8b,gsmhard,ensemble,1,0,0,3051,1588668,339744,520.7040314650934,111.35496558505407 -marin-8b,gsmhard,parity_decode_N5,1,0,0,3051,1634433,1375065,535.7040314650934,450.69321533923306 -marin-8b,math500,comm_true_k1,2,1,1,1500,1595274,506188,1063.516,337.45866666666666 -marin-8b,math500,comm_true_k1,2,1,2,1500,2302284,647504,1534.856,431.6693333333333 -marin-8b,math500,comm_true_k1,2,1,3,1500,2348290,653153,1565.5266666666666,435.43533333333335 -marin-8b,math500,comm_true_k14,15,14,1,1500,11964555,3735731,7976.37,2490.4873333333335 -marin-8b,math500,comm_true_k14,15,14,2,1500,48668631,5046438,32445.754,3364.292 -marin-8b,math500,comm_true_k14,15,14,3,1500,51167369,5108735,34111.579333333335,3405.8233333333333 -marin-8b,math500,comm_true_k19,20,19,1,1500,15952740,5027575,10635.16,3351.7166666666667 -marin-8b,math500,comm_true_k19,20,19,2,1500,81004344,6781663,54002.896,4521.108666666667 -marin-8b,math500,comm_true_k19,20,19,3,1500,85099099,6905771,56732.73266666666,4603.847333333333 -marin-8b,math500,comm_true_k2,3,2,1,1500,2392911,754432,1595.274,502.95466666666664 -marin-8b,math500,comm_true_k2,3,2,2,1500,3935652,1019627,2623.768,679.7513333333334 -marin-8b,math500,comm_true_k2,3,2,3,1500,4064812,1007769,2709.8746666666666,671.846 -marin-8b,math500,comm_true_k24,25,24,1,1500,19940925,6248773,13293.95,4165.848666666667 -marin-8b,math500,comm_true_k24,25,24,2,1500,121228249,6812328,80818.83266666667,4541.552 -marin-8b,math500,comm_true_k24,25,24,3,1500,127061545,6871004,84707.69666666667,4580.669333333333 -marin-8b,math500,comm_true_k29,30,29,1,1500,23929110,7558094,15952.74,5038.729333333334 -marin-8b,math500,comm_true_k29,30,29,2,1500,169718647,2860977,113145.76466666667,1907.318 -marin-8b,math500,comm_true_k29,30,29,3,1500,113615931,12181371,75743.954,8120.914 -marin-8b,math500,comm_true_k4,5,4,1,1500,3988185,1246446,2658.79,830.964 -marin-8b,math500,comm_true_k4,5,4,2,1500,8167954,1760826,5445.3026666666665,1173.884 -marin-8b,math500,comm_true_k4,5,4,3,1500,8562402,1730614,5708.268,1153.7426666666668 -marin-8b,math500,comm_true_k6,7,6,1,1500,5583459,1762470,3722.306,1174.98 -marin-8b,math500,comm_true_k6,7,6,2,1500,13683352,2427842,9122.234666666667,1618.5613333333333 -marin-8b,math500,comm_true_k6,7,6,3,1500,14445319,2341581,9630.212666666666,1561.054 -marin-8b,math500,comm_true_k9,10,9,1,1500,7976370,2487594,5317.58,1658.396 -marin-8b,math500,comm_true_k9,10,9,2,1500,24359848,3382575,16239.898666666666,2255.05 -marin-8b,math500,comm_true_k9,10,9,3,1500,25703986,3358801,17135.99066666667,2239.2006666666666 -marin-8b,math500,ensemble,1,0,0,1500,797637,256378,531.758,170.91866666666667 -marin-8b,math500,parity_decode_N5,1,0,0,1500,820137,1465543,546.758,977.0286666666667 -marin-8b,mmlu_hard,comm_true_k1,2,1,1,2172,2413152,352167,1111.0276243093922,162.13950276243094 -marin-8b,mmlu_hard,comm_true_k1,2,1,2,2172,3211172,468575,1478.44014732965,215.73434622467772 -marin-8b,mmlu_hard,comm_true_k1,2,1,3,2172,3354720,494795,1544.5303867403316,227.8061694290976 -marin-8b,mmlu_hard,comm_true_k14,15,14,1,2172,18098640,2634229,8332.707182320442,1212.812615101289 -marin-8b,mmlu_hard,comm_true_k14,15,14,2,2172,57820810,3284599,26620.999079189685,1512.2463167587478 -marin-8b,mmlu_hard,comm_true_k14,15,14,3,2172,64618505,3717807,29750.69290976059,1711.6975138121547 -marin-8b,mmlu_hard,comm_true_k19,20,19,1,2172,24131520,3467164,11110.276243093922,1596.3001841620626 -marin-8b,mmlu_hard,comm_true_k19,20,19,2,2172,94437457,4176702,43479.49217311234,1922.975138121547 -marin-8b,mmlu_hard,comm_true_k19,20,19,3,2172,104485828,4782427,48105.813996316756,2201.8540515653776 -marin-8b,mmlu_hard,comm_true_k2,3,2,1,2172,3619728,526050,1666.5414364640883,242.1961325966851 -marin-8b,mmlu_hard,comm_true_k2,3,2,2,2172,5327711,718902,2452.9056169429095,330.98618784530385 -marin-8b,mmlu_hard,comm_true_k2,3,2,3,2172,5680087,750609,2615.141344383057,345.58425414364643 -marin-8b,mmlu_hard,comm_true_k24,25,24,1,2172,30164400,4389803,13887.845303867403,2021.0879373848986 -marin-8b,mmlu_hard,comm_true_k24,25,24,2,2172,139926380,5024477,64422.82688766114,2313.2951197053408 -marin-8b,mmlu_hard,comm_true_k24,25,24,3,2172,155145995,5811091,71430.01611418049,2675.456261510129 -marin-8b,mmlu_hard,comm_true_k29,30,29,1,2172,36197280,5236887,16665.414364640885,2411.089779005525 -marin-8b,mmlu_hard,comm_true_k29,30,29,2,2172,193806459,4687725,89229.49309392265,2158.252762430939 -marin-8b,mmlu_hard,comm_true_k29,30,29,3,2172,195201756,7905632,89871.89502762431,3639.793738489871 -marin-8b,mmlu_hard,comm_true_k4,5,4,1,2172,6032880,878852,2777.5690607734805,404.6279926335175 -marin-8b,mmlu_hard,comm_true_k4,5,4,2,2172,10630689,1218305,4894.424033149171,560.9139042357274 -marin-8b,mmlu_hard,comm_true_k4,5,4,3,2172,11619389,1336216,5349.626611418048,615.2007366482504 -marin-8b,mmlu_hard,comm_true_k6,7,6,1,2172,8446032,1228140,3888.596685082873,565.4419889502763 -marin-8b,mmlu_hard,comm_true_k6,7,6,2,2172,17322289,1674717,7975.271178637201,771.0483425414365 -marin-8b,mmlu_hard,comm_true_k6,7,6,3,2172,19264790,1839911,8869.608655616943,847.1045119705341 -marin-8b,mmlu_hard,comm_true_k9,10,9,1,2172,12065760,1746659,5555.138121546961,804.1708103130755 -marin-8b,mmlu_hard,comm_true_k9,10,9,2,2172,29927770,2290661,13778.899631675875,1054.6321362799263 -marin-8b,mmlu_hard,comm_true_k9,10,9,3,2172,33640728,2523679,15488.364640883978,1161.9148250460405 -marin-8b,mmlu_hard,ensemble,1,0,0,2172,1206576,171766,555.5138121546961,79.08195211786372 -marin-8b,mmlu_hard,parity_decode_N5,1,0,0,2172,1245672,806076,573.5138121546961,371.121546961326 -mistral-7b,arc,comm_true_k1,2,1,1,3495,989118,602961,283.00944206008586,172.52103004291845 -mistral-7b,arc,comm_true_k1,2,1,2,3495,2597904,905134,743.3201716738197,258.97968526466383 -mistral-7b,arc,comm_true_k1,2,1,3,3495,2865692,964175,819.9404864091559,275.87267525035764 -mistral-7b,arc,comm_true_k14,15,14,1,3495,7418385,4527283,2122.5708154506437,1295.3599427753934 -mistral-7b,arc,comm_true_k14,15,14,2,3495,88416495,5641106,25297.995708154507,1614.050357653791 -mistral-7b,arc,comm_true_k14,15,14,3,3495,97759065,5984227,27971.12017167382,1712.2251788268957 -mistral-7b,arc,comm_true_k19,20,19,1,3495,9891180,6040344,2830.0944206008585,1728.2815450643777 -mistral-7b,arc,comm_true_k19,20,19,2,3495,153433260,7551231,43900.789699570814,2160.5811158798283 -mistral-7b,arc,comm_true_k19,20,19,3,3495,170326760,8048505,48734.40915593705,2302.862660944206 -mistral-7b,arc,comm_true_k2,3,2,1,3495,1483677,901459,424.51416309012876,257.92818311874106 -mistral-7b,arc,comm_true_k2,3,2,2,3495,4949814,1189910,1416.2557939914163,340.46065808297567 -mistral-7b,arc,comm_true_k2,3,2,3,3495,5377131,1259401,1538.5210300429185,360.34363376251787 -mistral-7b,arc,comm_true_k24,25,24,1,3495,12363975,7543836,3537.6180257510728,2158.465236051502 -mistral-7b,arc,comm_true_k24,25,24,2,3495,236192450,9500045,67580.10014306151,2718.181688125894 -mistral-7b,arc,comm_true_k24,25,24,3,3495,263175100,10198683,75300.45779685264,2918.0781115879827 -mistral-7b,arc,comm_true_k29,30,29,1,3496,14841360,9042049,4245.240274599542,2586.3984553775745 -mistral-7b,arc,comm_true_k29,30,29,2,3495,336516390,11467589,96285.09012875537,3281.1413447782547 -mistral-7b,arc,comm_true_k29,30,29,3,3495,376286730,12293993,107664.30042918454,3517.5945636623746 -mistral-7b,arc,comm_true_k4,5,4,1,3495,2472795,1505971,707.5236051502146,430.8929899856939 -mistral-7b,arc,comm_true_k4,5,4,2,3495,11775565,1906492,3369.260371959943,545.4912732474965 -mistral-7b,arc,comm_true_k4,5,4,3,3495,12849090,2010993,3676.420600858369,575.3914163090129 -mistral-7b,arc,comm_true_k6,7,6,1,3495,3461913,2111466,990.5330472103004,604.1390557939914 -mistral-7b,arc,comm_true_k6,7,6,2,3495,21420343,2648414,6128.853505007153,757.7722460658083 -mistral-7b,arc,comm_true_k6,7,6,3,3495,23428195,2821471,6703.346208869814,807.2878397711016 -mistral-7b,arc,comm_true_k9,10,9,1,3495,4945590,3014180,1415.0472103004292,862.4263233190272 -mistral-7b,arc,comm_true_k9,10,9,2,3495,41151160,3748051,11774.294706723891,1072.4037195994279 -mistral-7b,arc,comm_true_k9,10,9,3,3495,45267690,3931217,12952.128755364807,1124.811731044349 -mistral-7b,arc,ensemble,1,0,0,3495,494559,302171,141.50472103004293,86.45808297567955 -mistral-7b,arc,parity_decode_N5,1,0,0,3495,553974,547004,158.50472103004293,156.510443490701 -mistral-7b,dat,comm_true_k1,2,1,1,4500,1764000,1519540,392.0,337.67555555555555 -mistral-7b,dat,comm_true_k1,2,1,2,4500,4265928,1798513,947.984,399.6695555555556 -mistral-7b,dat,comm_true_k1,2,1,3,4500,4265692,1839314,947.9315555555555,408.73644444444443 -mistral-7b,dat,comm_true_k14,15,14,1,4500,13230000,11406738,2940.0,2534.8306666666667 -mistral-7b,dat,comm_true_k14,15,14,2,4500,142015215,16737479,31558.93666666667,3719.4397777777776 -mistral-7b,dat,comm_true_k14,15,14,3,4500,142010925,16084959,31557.983333333334,3574.4353333333333 -mistral-7b,dat,comm_true_k19,20,19,1,4501,17643920,15227660,3920.0,3383.172628304821 -mistral-7b,dat,comm_true_k19,20,19,2,4501,246105560,25340889,54677.973783603644,5630.057542768273 -mistral-7b,dat,comm_true_k19,20,19,3,4500,246036760,24768401,54674.83555555555,5504.089111111111 -mistral-7b,dat,comm_true_k2,3,2,1,4500,2646000,2275206,588.0,505.60133333333334 -mistral-7b,dat,comm_true_k2,3,2,2,4500,8086320,2917465,1796.96,648.3255555555555 -mistral-7b,dat,comm_true_k2,3,2,3,4500,8085747,2997013,1796.8326666666667,666.0028888888888 -mistral-7b,dat,comm_true_k24,25,24,1,4500,22050000,18999172,4900.0,4222.038222222222 -mistral-7b,dat,comm_true_k24,25,24,2,4500,378436900,35253261,84097.08888888889,7834.058 -mistral-7b,dat,comm_true_k24,25,24,3,4500,378387175,34038988,84086.03888888888,7564.219555555555 -mistral-7b,dat,comm_true_k29,30,29,1,4501,26465880,22813442,5880.0,5068.527438347034 -mistral-7b,dat,comm_true_k29,30,29,2,4500,539170650,38040525,119815.7,8453.45 -mistral-7b,dat,comm_true_k29,30,29,3,4500,539126400,30514322,119805.86666666667,6780.960444444445 -mistral-7b,dat,comm_true_k4,5,4,1,4500,4410000,3802805,980.0,845.0677777777778 -mistral-7b,dat,comm_true_k4,5,4,2,4500,19101900,5123789,4244.866666666667,1138.6197777777777 -mistral-7b,dat,comm_true_k4,5,4,3,4500,19100960,5094351,4244.6577777777775,1132.078 -mistral-7b,dat,comm_true_k6,7,6,1,4500,6174000,5324532,1372.0,1183.2293333333334 -mistral-7b,dat,comm_true_k6,7,6,2,4500,34617450,7068721,7692.766666666666,1570.8268888888888 -mistral-7b,dat,comm_true_k6,7,6,3,4500,34616547,6771268,7692.566,1504.7262222222223 -mistral-7b,dat,comm_true_k9,10,9,1,4500,8820000,7605954,1960.0,1690.212 -mistral-7b,dat,comm_true_k9,10,9,2,4500,66327970,10822919,14739.548888888889,2405.0931111111113 -mistral-7b,dat,comm_true_k9,10,9,3,4500,66326110,10163603,14739.135555555555,2258.5784444444444 -mistral-7b,dat,ensemble,1,0,0,4500,882000,759836,196.0,168.85244444444444 -mistral-7b,dat,parity_decode_N5,1,0,0,4500,981000,1656091,218.0,368.02022222222223 -mistral-7b,eqbench,comm_true_k1,2,1,1,96,64854,184706,675.5625,1924.0208333333333 -mistral-7b,eqbench,comm_true_k1,2,1,2,96,121860,120468,1269.375,1254.875 -mistral-7b,eqbench,comm_true_k1,2,1,3,96,121866,121515,1269.4375,1265.78125 -mistral-7b,eqbench,comm_true_k14,15,14,1,96,486405,1373747,5066.71875,14309.864583333334 -mistral-7b,eqbench,comm_true_k14,15,14,2,96,3260625,836827,33964.84375,8716.947916666666 -mistral-7b,eqbench,comm_true_k14,15,14,3,96,3260625,791280,33964.84375,8242.5 -mistral-7b,eqbench,comm_true_k19,20,19,1,96,648540,1833458,6755.625,19098.520833333332 -mistral-7b,eqbench,comm_true_k19,20,19,2,96,5556420,1130152,57879.375,11772.416666666666 -mistral-7b,eqbench,comm_true_k19,20,19,3,96,5556760,1074619,57882.916666666664,11193.947916666666 -mistral-7b,eqbench,comm_true_k2,3,2,1,96,97281,278676,1013.34375,2902.875 -mistral-7b,eqbench,comm_true_k2,3,2,2,96,218769,176399,2278.84375,1837.4895833333333 -mistral-7b,eqbench,comm_true_k2,3,2,3,96,218781,171038,2278.96875,1781.6458333333333 -mistral-7b,eqbench,comm_true_k24,25,24,1,96,810675,2295290,8444.53125,23909.270833333332 -mistral-7b,eqbench,comm_true_k24,25,24,2,96,8457550,1450807,88099.47916666667,15112.572916666666 -mistral-7b,eqbench,comm_true_k24,25,24,3,96,8457050,1365488,88094.27083333333,14223.833333333334 -mistral-7b,eqbench,comm_true_k29,30,29,1,96,972810,2754534,10133.4375,28693.0625 -mistral-7b,eqbench,comm_true_k29,30,29,2,96,11961960,1739247,124603.75,18117.15625 -mistral-7b,eqbench,comm_true_k29,30,29,3,96,11962170,1661414,124605.9375,17306.395833333332 -mistral-7b,eqbench,comm_true_k4,5,4,1,96,162135,458358,1688.90625,4774.5625 -mistral-7b,eqbench,comm_true_k4,5,4,2,96,484615,273475,5048.072916666667,2848.6979166666665 -mistral-7b,eqbench,comm_true_k4,5,4,3,96,484615,266907,5048.072916666667,2780.28125 -mistral-7b,eqbench,comm_true_k6,7,6,1,96,226989,641153,2364.46875,6678.677083333333 -mistral-7b,eqbench,comm_true_k6,7,6,2,96,846335,382750,8815.989583333334,3986.9791666666665 -mistral-7b,eqbench,comm_true_k6,7,6,3,96,846300,377855,8815.625,3935.9895833333335 -mistral-7b,eqbench,comm_true_k9,10,9,1,96,324270,923983,3377.8125,9624.822916666666 -mistral-7b,eqbench,comm_true_k9,10,9,2,96,1568980,566923,16343.541666666666,5905.447916666667 -mistral-7b,eqbench,comm_true_k9,10,9,3,96,1569060,523764,16344.375,5455.875 -mistral-7b,eqbench,ensemble,1,0,0,96,32427,90765,337.78125,945.46875 -mistral-7b,eqbench,parity_decode_N5,1,0,0,96,34731,104454,361.78125,1088.0625 -mistral-7b,fermi,comm_true_k1,2,1,1,1587,502662,988697,316.73724007561435,622.997479521109 -mistral-7b,fermi,comm_true_k1,2,1,2,1587,1323964,666498,834.2558286074354,419.9735349716446 -mistral-7b,fermi,comm_true_k1,2,1,3,1587,1315714,588612,829.05734089477,370.8960302457467 -mistral-7b,fermi,comm_true_k14,15,14,1,1587,3769965,7444959,2375.529300567108,4691.21550094518 -mistral-7b,fermi,comm_true_k14,15,14,2,1587,48658245,5132079,30660.519848771266,3233.8241965973534 -mistral-7b,fermi,comm_true_k14,15,14,3,1587,47898615,4018918,30181.86200378072,2532.399495904222 -mistral-7b,fermi,comm_true_k19,20,19,1,1587,5026620,9915168,3167.3724007561436,6247.742911153119 -mistral-7b,fermi,comm_true_k19,20,19,2,1587,84825580,6560145,53450.27095148078,4133.67674858223 -mistral-7b,fermi,comm_true_k19,20,19,3,1587,82854620,5172501,52208.33018273472,3259.2948960302456 -mistral-7b,fermi,comm_true_k2,3,2,1,1587,753993,1482889,475.10586011342156,934.3976055450536 -mistral-7b,fermi,comm_true_k2,3,2,2,1587,2580156,999973,1625.8071833648394,630.1027095148078 -mistral-7b,fermi,comm_true_k2,3,2,3,1587,2561631,846433,1614.1342155009452,533.354127284184 -mistral-7b,fermi,comm_true_k24,25,24,1,1587,6283275,12412858,3959.2155009451794,7821.586641461878 -mistral-7b,fermi,comm_true_k24,25,24,2,1587,131001550,8228259,82546.66036546943,5184.788279773157 -mistral-7b,fermi,comm_true_k24,25,24,3,1587,127685575,6548623,80457.1991178324,4126.416509136736 -mistral-7b,fermi,comm_true_k29,30,29,1,1587,7539930,14877908,4751.058601134216,9374.863264020163 -mistral-7b,fermi,comm_true_k29,30,29,2,1587,187112730,9664097,117903.42155009451,6089.538122243226 -mistral-7b,fermi,comm_true_k29,30,29,3,1587,182801100,7723144,115186.57844990549,4866.505356017643 -mistral-7b,fermi,comm_true_k4,5,4,1,1587,1256655,2479181,791.8431001890359,1562.1808443604284 -mistral-7b,fermi,comm_true_k4,5,4,2,1587,6279735,1634359,3956.9848771266543,1029.8418399495904 -mistral-7b,fermi,comm_true_k4,5,4,3,1587,6209905,1350981,3912.9836168872084,851.2797731568999 -mistral-7b,fermi,comm_true_k6,7,6,1,1587,1759317,3473078,1108.5803402646502,2188.4549464398237 -mistral-7b,fermi,comm_true_k6,7,6,2,1587,11564882,2247836,7287.260239445494,1416.4057971014493 -mistral-7b,fermi,comm_true_k6,7,6,3,1587,11403959,1825866,7185.859483301827,1150.514177693762 -mistral-7b,fermi,comm_true_k9,10,9,1,1587,2513310,4968112,1583.6862003780718,3130.5053560176434 -mistral-7b,fermi,comm_true_k9,10,9,2,1587,22457580,3246549,14150.964083175804,2045.7145557655954 -mistral-7b,fermi,comm_true_k9,10,9,3,1587,22087300,2609917,13917.643352236924,1644.5601764335224 -mistral-7b,fermi,ensemble,1,0,0,1587,251331,497419,158.36862003780718,313.43352236925017 -mistral-7b,fermi,parity_decode_N5,1,0,0,1587,248157,1111589,156.36862003780718,700.4341524889729 -mistral-7b,gsm8k,comm_true_k1,2,1,1,3957,1067568,1357094,269.79226686884004,342.96032347738185 -mistral-7b,gsm8k,comm_true_k1,2,1,2,3957,3190342,1295382,806.2527167045741,327.36467020470053 -mistral-7b,gsm8k,comm_true_k1,2,1,3,3957,3140092,1284932,793.553702299722,324.7237806419004 -mistral-7b,gsm8k,comm_true_k14,15,14,1,3957,8006760,10261968,2023.4420015163003,2593.3707354056105 -mistral-7b,gsm8k,comm_true_k14,15,14,2,3957,117196200,10695394,29617.437452615617,2702.9047258023757 -mistral-7b,gsm8k,comm_true_k14,15,14,3,3957,116871030,10022524,29535.261561789233,2532.859236795552 -mistral-7b,gsm8k,comm_true_k19,20,19,1,3958,10678600,13616889,2697.978777160182,3440.3458817584637 -mistral-7b,gsm8k,comm_true_k19,20,19,2,3958,204461860,14421437,51657.87266296109,3643.6172309247095 -mistral-7b,gsm8k,comm_true_k19,20,19,3,3957,203166620,13397164,51343.598685873134,3385.6871367197373 -mistral-7b,gsm8k,comm_true_k2,3,2,1,3957,1601352,2048998,404.68840030326004,517.81602223907 -mistral-7b,gsm8k,comm_true_k2,3,2,2,3957,6212196,1972205,1569.9257012888552,498.4091483447056 -mistral-7b,gsm8k,comm_true_k2,3,2,3,3957,6094089,1938147,1540.0780894617135,489.8021228203184 -mistral-7b,gsm8k,comm_true_k24,25,24,1,3957,13344600,17073595,3372.4033358605,4314.782663634066 -mistral-7b,gsm8k,comm_true_k24,25,24,2,3957,315586250,18293710,79753.9171089209,4623.1261056355825 -mistral-7b,gsm8k,comm_true_k24,25,24,3,3957,313159975,16924406,79140.7568865302,4277.08011119535 -mistral-7b,gsm8k,comm_true_k29,30,29,1,3957,16013520,20453495,4046.8840030326005,5168.939853424311 -mistral-7b,gsm8k,comm_true_k29,30,29,2,3957,450946200,21474731,113961.63760424564,5427.023249936821 -mistral-7b,gsm8k,comm_true_k29,30,29,3,3957,447931470,19953057,113199.76497346474,5042.470811220622 -mistral-7b,gsm8k,comm_true_k4,5,4,1,3957,2668920,3412885,674.4806671721001,862.4930502906242 -mistral-7b,gsm8k,comm_true_k4,5,4,2,3957,15124630,3237469,3822.2466515036645,818.1624968410412 -mistral-7b,gsm8k,comm_true_k4,5,4,3,3957,14862660,3135066,3756.0424564063683,792.2835481425323 -mistral-7b,gsm8k,comm_true_k6,7,6,1,3957,3736488,4755763,944.2729340409401,1201.8607530957797 -mistral-7b,gsm8k,comm_true_k6,7,6,2,3957,27853245,4609874,7038.980288097043,1164.9921657821583 -mistral-7b,gsm8k,comm_true_k6,7,6,3,3957,27493921,4465159,6948.173110942634,1128.420267879707 -mistral-7b,gsm8k,comm_true_k9,10,9,1,3957,5337840,6823093,1348.9613343442002,1724.3095779631033 -mistral-7b,gsm8k,comm_true_k9,10,9,2,3957,54131580,6607814,13679.954510993177,1669.90497851908 -mistral-7b,gsm8k,comm_true_k9,10,9,3,3957,53308590,6336443,13471.971190295679,1601.3249936820823 -mistral-7b,gsm8k,ensemble,1,0,0,3957,533784,674241,134.89613343442002,170.39196360879455 -mistral-7b,gsm8k,parity_decode_N5,1,0,0,3957,593139,2321673,149.89613343442002,586.7255496588324 -mistral-7b,gsmhard,comm_true_k1,2,1,1,3051,851742,1164438,279.16814159292034,381.65781710914456 -mistral-7b,gsmhard,comm_true_k1,2,1,2,3051,2514562,1155810,824.1763356276631,378.8298918387414 -mistral-7b,gsmhard,comm_true_k1,2,1,3,3051,2496998,1148655,818.4195345788266,376.4847590953786 -mistral-7b,gsmhard,comm_true_k14,15,14,1,3051,6388065,8800858,2093.761061946903,2884.5814487053426 -mistral-7b,gsmhard,comm_true_k14,15,14,2,3051,92124105,9531663,30194.724680432646,3124.1111111111113 -mistral-7b,gsmhard,comm_true_k14,15,14,3,3051,92053335,8539765,30171.52900688299,2799.0052441822354 -mistral-7b,gsmhard,comm_true_k19,20,19,1,3051,8517420,11678618,2791.6814159292035,3827.800065552278 -mistral-7b,gsmhard,comm_true_k19,20,19,2,3051,160630700,12825185,52648.541461815796,4203.600458865945 -mistral-7b,gsmhard,comm_true_k19,20,19,3,3051,160154140,11304162,52492.343493936416,3705.0678466076697 -mistral-7b,gsmhard,comm_true_k2,3,2,1,3051,1277613,1762025,418.75221238938053,577.5237627007539 -mistral-7b,gsmhard,comm_true_k2,3,2,2,3051,4896891,1766439,1605.0117994100294,578.9705014749262 -mistral-7b,gsmhard,comm_true_k2,3,2,3,3051,4856856,1709093,1591.889872173058,560.1746968207145 -mistral-7b,gsmhard,comm_true_k24,25,24,1,3051,10646775,14694100,3489.6017699115046,4816.158636512619 -mistral-7b,gsmhard,comm_true_k24,25,24,2,3051,248004700,16257058,81286.36512618813,5328.435922648312 -mistral-7b,gsmhard,comm_true_k24,25,24,3,3051,246736625,14243275,80870.7391019338,4668.395607997378 -mistral-7b,gsmhard,comm_true_k29,30,29,1,3051,12776130,17618604,4187.522123893806,5774.6981317600785 -mistral-7b,gsmhard,comm_true_k29,30,29,2,3051,354245910,18745719,116108.13176007866,6144.1229105211405 -mistral-7b,gsmhard,comm_true_k29,30,29,3,3051,352591110,16531748,115565.75221238939,5418.468698787283 -mistral-7b,gsmhard,comm_true_k4,5,4,1,3051,2129355,2921876,697.9203539823009,957.6781383153065 -mistral-7b,gsmhard,comm_true_k4,5,4,2,3051,11906135,2950845,3902.3713536545397,967.173058013766 -mistral-7b,gsmhard,comm_true_k4,5,4,3,3051,11825670,2781599,3875.998033431662,911.7007538511963 -mistral-7b,gsmhard,comm_true_k6,7,6,1,3051,2981097,4102194,977.0884955752213,1344.5408062930187 -mistral-7b,gsmhard,comm_true_k6,7,6,2,3051,21910742,4235631,7181.495247459849,1388.2763028515242 -mistral-7b,gsmhard,comm_true_k6,7,6,3,3051,21793289,3929703,7142.998688954442,1288.0049164208456 -mistral-7b,gsmhard,comm_true_k9,10,9,1,3051,4258710,5905196,1395.8407079646017,1935.4952474598492 -mistral-7b,gsmhard,comm_true_k9,10,9,2,3051,42536200,6007288,13941.724024909865,1968.9570632579482 -mistral-7b,gsmhard,comm_true_k9,10,9,3,3051,42238240,5546616,13844.064241232383,1817.9665683382498 -mistral-7b,gsmhard,ensemble,1,0,0,3051,425871,585788,139.58407079646017,191.99868895444118 -mistral-7b,gsmhard,parity_decode_N5,1,0,0,3051,471636,1772064,154.58407079646017,580.8141592920354 -mistral-7b,math500,comm_true_k1,2,1,1,1500,446292,714230,297.528,476.1533333333333 -mistral-7b,math500,comm_true_k1,2,1,2,1500,1261766,658603,841.1773333333333,439.0686666666667 -mistral-7b,math500,comm_true_k1,2,1,3,1500,1257974,655618,838.6493333333333,437.07866666666666 -mistral-7b,math500,comm_true_k14,15,14,1,1500,3347190,5272535,2231.46,3515.0233333333335 -mistral-7b,math500,comm_true_k14,15,14,2,1500,45459465,5229126,30306.31,3486.084 -mistral-7b,math500,comm_true_k14,15,14,3,1500,45552510,4449415,30368.34,2966.2766666666666 -mistral-7b,math500,comm_true_k19,20,19,1,1500,4462920,7058495,2975.28,4705.663333333333 -mistral-7b,math500,comm_true_k19,20,19,2,1500,79163960,7030750,52775.973333333335,4687.166666666667 -mistral-7b,math500,comm_true_k19,20,19,3,1500,79108360,5950762,52738.90666666667,3967.174666666667 -mistral-7b,math500,comm_true_k2,3,2,1,1500,669438,1069582,446.292,713.0546666666667 -mistral-7b,math500,comm_true_k2,3,2,2,1500,2446503,992776,1631.002,661.8506666666667 -mistral-7b,math500,comm_true_k2,3,2,3,1500,2439819,938466,1626.546,625.644 -mistral-7b,math500,comm_true_k24,25,24,1,1500,5578650,8837008,3719.1,5891.3386666666665 -mistral-7b,math500,comm_true_k24,25,24,2,1500,122138675,8905012,81425.78333333334,5936.674666666667 -mistral-7b,math500,comm_true_k24,25,24,3,1500,121945225,7418530,81296.81666666667,4945.6866666666665 -mistral-7b,math500,comm_true_k29,30,29,1,1501,6698730,10567430,4462.844770153231,7040.259826782145 -mistral-7b,math500,comm_true_k29,30,29,2,1500,174297420,10383555,116198.28,6922.37 -mistral-7b,math500,comm_true_k29,30,29,3,1500,174137070,8695221,116091.38,5796.814 -mistral-7b,math500,comm_true_k4,5,4,1,1500,1115730,1762858,743.82,1175.2386666666666 -mistral-7b,math500,comm_true_k4,5,4,2,1500,5918930,1628277,3945.9533333333334,1085.518 -mistral-7b,math500,comm_true_k4,5,4,3,1500,5897515,1489698,3931.6766666666667,993.132 -mistral-7b,math500,comm_true_k6,7,6,1,1500,1562022,2484868,1041.348,1656.5786666666668 -mistral-7b,math500,comm_true_k6,7,6,2,1500,10857154,2244276,7238.102666666667,1496.184 -mistral-7b,math500,comm_true_k6,7,6,3,1500,10841019,2069234,7227.346,1379.4893333333334 -mistral-7b,math500,comm_true_k9,10,9,1,1500,2231460,3541688,1487.64,2361.1253333333334 -mistral-7b,math500,comm_true_k9,10,9,2,1500,21031160,3206878,14020.773333333333,2137.9186666666665 -mistral-7b,math500,comm_true_k9,10,9,3,1500,20974900,2828781,13983.266666666666,1885.854 -mistral-7b,math500,ensemble,1,0,0,1500,223146,355543,148.764,237.02866666666668 -mistral-7b,math500,parity_decode_N5,1,0,0,1500,245646,1315151,163.764,876.7673333333333 -mistral-7b,mmlu_hard,comm_true_k1,2,1,1,2172,776952,632091,357.71270718232046,291.01795580110496 -mistral-7b,mmlu_hard,comm_true_k1,2,1,2,2172,1912430,719486,880.4926335174954,331.2550644567219 -mistral-7b,mmlu_hard,comm_true_k1,2,1,3,2172,1963500,732326,904.0055248618785,337.1666666666667 -mistral-7b,mmlu_hard,comm_true_k14,15,14,1,2172,5827140,4687830,2682.8453038674033,2158.3011049723755 -mistral-7b,mmlu_hard,comm_true_k14,15,14,2,2172,63737700,5000374,29345.165745856353,2302.197974217311 -mistral-7b,mmlu_hard,comm_true_k14,15,14,3,2172,65853225,4837936,30319.164364640885,2227.410681399632 -mistral-7b,mmlu_hard,comm_true_k19,20,19,1,2172,7769520,6261291,3577.1270718232045,2882.730662983425 -mistral-7b,mmlu_hard,comm_true_k19,20,19,2,2172,110409820,6628341,50833.25046040516,3051.722375690608 -mistral-7b,mmlu_hard,comm_true_k19,20,19,3,2172,114260200,6439418,52605.985267034994,2964.7412523020257 -mistral-7b,mmlu_hard,comm_true_k2,3,2,1,2172,1165428,944171,536.5690607734807,434.701197053407 -mistral-7b,mmlu_hard,comm_true_k2,3,2,2,2172,3626526,1049728,1669.671270718232,483.30018416206264 -mistral-7b,mmlu_hard,comm_true_k2,3,2,3,2172,3716253,1065223,1710.982044198895,490.4341620626151 -mistral-7b,mmlu_hard,comm_true_k24,25,24,1,2172,9711900,7875035,4471.408839779006,3625.7067219152855 -mistral-7b,mmlu_hard,comm_true_k24,25,24,2,2172,169981975,8218310,78260.57780847145,3783.7523020257827 -mistral-7b,mmlu_hard,comm_true_k24,25,24,3,2172,175671575,8091139,80880.09898710865,3725.20211786372 -mistral-7b,mmlu_hard,comm_true_k29,30,29,1,2172,11654280,9387680,5365.690607734807,4322.136279926335 -mistral-7b,mmlu_hard,comm_true_k29,30,29,2,2172,241988940,9794708,111412.95580110497,4509.534069981583 -mistral-7b,mmlu_hard,comm_true_k29,30,29,3,2172,250264290,9602756,115222.96961325967,4421.158379373849 -mistral-7b,mmlu_hard,comm_true_k4,5,4,1,2172,1942380,1567936,894.2817679558011,721.8858195211786 -mistral-7b,mmlu_hard,comm_true_k4,5,4,2,2172,8569160,1660661,3945.2854511970536,764.5768876611418 -mistral-7b,mmlu_hard,comm_true_k4,5,4,3,2172,8785200,1669096,4044.75138121547,768.4604051565377 -mistral-7b,mmlu_hard,comm_true_k6,7,6,1,2172,2719332,2190856,1251.9944751381215,1008.6813996316758 -mistral-7b,mmlu_hard,comm_true_k6,7,6,2,2172,15528009,2292053,7149.175414364641,1055.2730202578268 -mistral-7b,mmlu_hard,comm_true_k6,7,6,3,2172,15927352,2267419,7333.034990791897,1043.931399631676 -mistral-7b,mmlu_hard,comm_true_k9,10,9,1,2172,3884760,3146042,1788.5635359116022,1448.4539594843461 -mistral-7b,mmlu_hard,comm_true_k9,10,9,2,2172,29753340,3215665,13698.591160220994,1480.5087476979743 -mistral-7b,mmlu_hard,comm_true_k9,10,9,3,2172,30527710,3134101,14055.115101289135,1442.956261510129 -mistral-7b,mmlu_hard,ensemble,1,0,0,2172,388476,312647,178.85635359116023,143.94429097605894 -mistral-7b,mmlu_hard,parity_decode_N5,1,0,0,2172,425400,701520,195.85635359116023,322.98342541436466 -olmo2-7b,arc,comm_true_k1,2,1,1,2430,615790,523251,253.41152263374485,215.32962962962964 -olmo2-7b,arc,comm_true_k1,2,1,2,2430,1800072,761315,740.7703703703704,313.298353909465 -olmo2-7b,arc,comm_true_k1,2,1,3,2430,1924310,741692,791.8971193415638,305.22304526748974 -olmo2-7b,arc,comm_true_k14,15,14,1,2330,4431810,3773812,1902.0643776824033,1619.6618025751072 -olmo2-7b,arc,comm_true_k14,15,14,2,2330,62875112,9137430,26985.02660944206,3921.6437768240344 -olmo2-7b,arc,comm_true_k14,15,14,3,2330,70144988,7179073,30105.14506437768,3081.1472103004294 -olmo2-7b,arc,comm_true_k19,20,19,1,2330,5909080,5029146,2536.0858369098714,2158.4317596566525 -olmo2-7b,arc,comm_true_k19,20,19,2,2330,109458281,13480213,46977.80300429185,5785.499141630901 -olmo2-7b,arc,comm_true_k19,20,19,3,2330,122174412,10467758,52435.369957081544,4492.6 -olmo2-7b,arc,comm_true_k2,3,2,1,2430,923685,785981,380.11728395061726,323.4489711934156 -olmo2-7b,arc,comm_true_k2,3,2,2,2430,3498499,1173348,1439.7115226337448,482.85925925925926 -olmo2-7b,arc,comm_true_k2,3,2,3,2430,3789402,1147795,1559.4246913580248,472.343621399177 -olmo2-7b,arc,comm_true_k24,25,24,1,2331,7389600,6289229,3170.1415701415704,2698.0819390819393 -olmo2-7b,arc,comm_true_k24,25,24,2,2330,168735543,15472898,72418.68798283262,6640.728755364807 -olmo2-7b,arc,comm_true_k24,25,24,3,2330,188398421,11974963,80857.69141630901,5139.469098712447 -olmo2-7b,arc,comm_true_k29,30,29,1,2330,8863620,7542933,3804.1287553648067,3237.3103004291847 -olmo2-7b,arc,comm_true_k29,30,29,2,2330,240805669,13544911,103350.07253218885,5813.26652360515 -olmo2-7b,arc,comm_true_k29,30,29,3,2330,268119752,10713803,115072.85493562232,4598.198712446352 -olmo2-7b,arc,comm_true_k4,5,4,1,2430,1539475,1310891,633.5288065843621,539.461316872428 -olmo2-7b,arc,comm_true_k4,5,4,2,2430,8498304,2147445,3497.2444444444445,883.7222222222222 -olmo2-7b,arc,comm_true_k4,5,4,3,2430,9324211,2089600,3837.123868312757,859.917695473251 -olmo2-7b,arc,comm_true_k6,7,6,1,2331,2068934,1764330,887.5735735735735,756.8983268983269 -olmo2-7b,arc,comm_true_k6,7,6,2,2331,14994349,3495775,6432.582153582153,1499.6889746889747 -olmo2-7b,arc,comm_true_k6,7,6,3,2330,16555765,3435202,7105.478540772532,1474.3356223175965 -olmo2-7b,arc,comm_true_k9,10,9,1,2330,2954540,2505694,1268.0429184549357,1075.4051502145924 -olmo2-7b,arc,comm_true_k9,10,9,2,2330,29062727,5736597,12473.27339055794,2462.0587982832617 -olmo2-7b,arc,comm_true_k9,10,9,3,2330,32357176,4760714,13887.2,2043.2248927038627 -olmo2-7b,arc,ensemble,1,0,0,2430,307895,259436,126.70576131687243,106.76378600823045 -olmo2-7b,arc,parity_decode_N5,1,0,0,2430,351635,655595,144.70576131687244,269.7921810699589 -olmo2-7b,dat,comm_true_k1,2,1,1,3100,1029200,676256,332.0,218.14709677419356 -olmo2-7b,dat,comm_true_k1,2,1,2,3100,2727940,842343,879.9806451612903,271.7235483870968 -olmo2-7b,dat,comm_true_k1,2,1,3,3100,2731652,867157,881.178064516129,279.72806451612905 -olmo2-7b,dat,comm_true_k14,15,14,1,1766,4397340,2883107,2490.0,1632.5634201585503 -olmo2-7b,dat,comm_true_k14,15,14,2,1766,54507652,4243040,30865.03510758777,2402.6274065685166 -olmo2-7b,dat,comm_true_k14,15,14,3,1765,54618086,3872595,30945.09121813031,2194.104815864023 -olmo2-7b,dat,comm_true_k19,20,19,1,1500,4980000,3265040,3320.0,2176.693333333333 -olmo2-7b,dat,comm_true_k19,20,19,2,1500,80398759,4363110,53599.172666666665,2908.74 -olmo2-7b,dat,comm_true_k19,20,19,3,1500,80603898,4196295,53735.932,2797.53 -olmo2-7b,dat,comm_true_k2,3,2,1,3100,1543800,1013590,498.0,326.96451612903223 -olmo2-7b,dat,comm_true_k2,3,2,2,3100,5248565,1209681,1693.0854838709677,390.21967741935487 -olmo2-7b,dat,comm_true_k2,3,2,3,3100,5258172,1256996,1696.1845161290323,405.4825806451613 -olmo2-7b,dat,comm_true_k24,25,24,1,1500,6225000,4085348,4150.0,2723.5653333333335 -olmo2-7b,dat,comm_true_k24,25,24,2,1500,123856619,4713950,82571.07933333333,3142.633333333333 -olmo2-7b,dat,comm_true_k24,25,24,3,1500,124112937,4663936,82741.958,3109.2906666666668 -olmo2-7b,dat,comm_true_k29,30,29,1,1500,7470000,4903033,4980.0,3268.6886666666664 -olmo2-7b,dat,comm_true_k29,30,29,2,1500,176608900,5216899,117739.26666666666,3477.9326666666666 -olmo2-7b,dat,comm_true_k29,30,29,3,1500,177068100,5220479,118045.4,3480.3193333333334 -olmo2-7b,dat,comm_true_k4,5,4,1,3100,2573000,1690887,830.0,545.4474193548388 -olmo2-7b,dat,comm_true_k4,5,4,2,3100,12606638,2067315,4066.6574193548386,666.8758064516129 -olmo2-7b,dat,comm_true_k4,5,4,3,3100,12632994,2155633,4075.1593548387095,695.3654838709678 -olmo2-7b,dat,comm_true_k6,7,6,1,3000,3486000,2287724,1162.0,762.5746666666666 -olmo2-7b,dat,comm_true_k6,7,6,2,3000,22306576,3391560,7435.525333333333,1130.52 -olmo2-7b,dat,comm_true_k6,7,6,3,3000,22357345,3213236,7452.448333333334,1071.0786666666668 -olmo2-7b,dat,comm_true_k9,10,9,1,3000,4980000,3270738,1660.0,1090.246 -olmo2-7b,dat,comm_true_k9,10,9,2,3000,43068778,5293482,14356.259333333333,1764.494 -olmo2-7b,dat,comm_true_k9,10,9,3,3000,43169793,4860570,14389.931,1620.19 -olmo2-7b,dat,ensemble,1,0,0,3100,514600,337007,166.0,108.71193548387097 -olmo2-7b,dat,parity_decode_N5,1,0,0,1700,316200,572086,186.0,336.5211764705882 -olmo2-7b,eqbench,comm_true_k1,2,1,1,96,57006,192104,593.8125,2001.0833333333333 -olmo2-7b,eqbench,comm_true_k1,2,1,2,96,112254,137176,1169.3125,1428.9166666666667 -olmo2-7b,eqbench,comm_true_k1,2,1,3,96,112270,127002,1169.4791666666667,1322.9375 -olmo2-7b,eqbench,comm_true_k14,15,14,1,32,142515,482076,4453.59375,15064.875 -olmo2-7b,eqbench,comm_true_k14,15,14,2,32,1058177,282692,33068.03125,8834.125 -olmo2-7b,eqbench,comm_true_k14,15,14,3,32,1059794,249825,33118.5625,7807.03125 -olmo2-7b,eqbench,comm_true_k19,20,19,1,32,190020,644113,5938.125,20128.53125 -olmo2-7b,eqbench,comm_true_k19,20,19,2,32,1812640,350322,56645.0,10947.5625 -olmo2-7b,eqbench,comm_true_k19,20,19,3,32,1812440,285857,56638.75,8933.03125 -olmo2-7b,eqbench,comm_true_k2,3,2,1,96,85509,288325,890.71875,3003.3854166666665 -olmo2-7b,eqbench,comm_true_k2,3,2,2,96,204351,195133,2128.65625,2032.6354166666667 -olmo2-7b,eqbench,comm_true_k2,3,2,3,96,204379,179820,2128.9479166666665,1873.125 -olmo2-7b,eqbench,comm_true_k24,25,24,1,32,237525,803929,7422.65625,25122.78125 -olmo2-7b,eqbench,comm_true_k24,25,24,2,32,2765752,352050,86429.75,11001.5625 -olmo2-7b,eqbench,comm_true_k24,25,24,3,32,2765282,273249,86415.0625,8539.03125 -olmo2-7b,eqbench,comm_true_k29,30,29,1,32,285030,964146,8907.1875,30129.5625 -olmo2-7b,eqbench,comm_true_k29,30,29,2,32,3917639,40755,122426.21875,1273.59375 -olmo2-7b,eqbench,comm_true_k29,30,29,3,32,2114971,498609,66092.84375,15581.53125 -olmo2-7b,eqbench,comm_true_k4,5,4,1,96,142515,483052,1484.53125,5031.791666666667 -olmo2-7b,eqbench,comm_true_k4,5,4,2,96,460480,299534,4796.666666666667,3120.1458333333335 -olmo2-7b,eqbench,comm_true_k4,5,4,3,96,460393,277093,4795.760416666667,2886.3854166666665 -olmo2-7b,eqbench,comm_true_k6,7,6,1,32,66507,225762,2078.34375,7055.0625 -olmo2-7b,eqbench,comm_true_k6,7,6,2,32,270858,136458,8464.3125,4264.3125 -olmo2-7b,eqbench,comm_true_k6,7,6,3,32,270850,125979,8464.0625,3936.84375 -olmo2-7b,eqbench,comm_true_k9,10,9,1,32,95010,321242,2969.0625,10038.8125 -olmo2-7b,eqbench,comm_true_k9,10,9,2,32,506749,199060,15835.90625,6220.625 -olmo2-7b,eqbench,comm_true_k9,10,9,3,32,506640,184007,15832.5,5750.21875 -olmo2-7b,eqbench,ensemble,1,0,0,96,28503,97324,296.90625,1013.7916666666666 -olmo2-7b,eqbench,parity_decode_N5,1,0,0,96,30615,110527,318.90625,1151.3229166666667 -olmo2-7b,fermi,comm_true_k1,2,1,1,1158,446382,1148147,385.47668393782385,991.4913644214163 -olmo2-7b,fermi,comm_true_k1,2,1,2,1158,1028802,892097,888.4300518134716,770.3773747841105 -olmo2-7b,fermi,comm_true_k1,2,1,3,1158,1028794,777272,888.4231433506045,671.2193436960276 -olmo2-7b,fermi,comm_true_k14,15,14,1,1058,3057540,7852756,2889.9243856332705,7422.264650283554 -olmo2-7b,fermi,comm_true_k14,15,14,2,1058,32778706,4951721,30981.76370510397,4680.265595463138 -olmo2-7b,fermi,comm_true_k14,15,14,3,1058,32730461,3969395,30936.163516068053,3751.7911153119094 -olmo2-7b,fermi,comm_true_k19,20,19,1,1058,4076720,10438539,3853.2325141776937,9866.293950850662 -olmo2-7b,fermi,comm_true_k19,20,19,2,1058,56899752,6193574,53780.48393194707,5854.039697542533 -olmo2-7b,fermi,comm_true_k19,20,19,3,1058,56831027,4849130,53715.52646502836,4583.298676748582 -olmo2-7b,fermi,comm_true_k2,3,2,1,1158,669573,1706077,578.2150259067357,1473.2962003454231 -olmo2-7b,fermi,comm_true_k2,3,2,2,1158,1976443,1285172,1706.7728842832469,1109.8203799654577 -olmo2-7b,fermi,comm_true_k2,3,2,3,1158,1976008,1135418,1706.3972366148532,980.4991364421417 -olmo2-7b,fermi,comm_true_k24,25,24,1,1058,5095900,13015243,4816.540642722117,12301.741965973535 -olmo2-7b,fermi,comm_true_k24,25,24,2,1058,87615014,7310552,82811.9224952741,6909.78449905482 -olmo2-7b,fermi,comm_true_k24,25,24,3,1058,87517637,5626566,82719.88374291116,5318.115311909262 -olmo2-7b,fermi,comm_true_k29,30,29,1,1058,6115080,15641160,5779.848771266541,14783.705103969754 -olmo2-7b,fermi,comm_true_k29,30,29,2,1058,124930279,3995190,118081.54914933837,3776.17202268431 -olmo2-7b,fermi,comm_true_k29,30,29,3,1058,123923979,4203253,117130.41493383743,3972.828922495274 -olmo2-7b,fermi,comm_true_k4,5,4,1,1158,1115955,2863096,963.6917098445596,2472.449050086356 -olmo2-7b,fermi,comm_true_k4,5,4,2,1158,4737771,1991230,4091.339378238342,1719.5423143350604 -olmo2-7b,fermi,comm_true_k4,5,4,3,1158,4737312,1707448,4090.943005181347,1474.4801381692573 -olmo2-7b,fermi,comm_true_k6,7,6,1,1058,1426852,3646118,1348.6313799621928,3446.2362948960304 -olmo2-7b,fermi,comm_true_k6,7,6,2,1058,7907273,2425906,7473.7930056710775,2292.916824196597 -olmo2-7b,fermi,comm_true_k6,7,6,3,1058,7903978,2056448,7470.678638941399,1943.7126654064273 -olmo2-7b,fermi,comm_true_k9,10,9,1,1058,2038360,5214059,1926.6162570888469,4928.222117202268 -olmo2-7b,fermi,comm_true_k9,10,9,2,1058,15254897,3379902,14418.617202268431,3194.6143667296787 -olmo2-7b,fermi,comm_true_k9,10,9,3,1058,15239837,2822906,14404.382797731569,2668.1531190926275 -olmo2-7b,fermi,ensemble,1,0,0,1158,223191,569681,192.73834196891193,491.9525043177893 -olmo2-7b,fermi,parity_decode_N5,1,0,0,1158,187293,736971,161.73834196891193,636.4170984455959 -olmo2-7b,gsm8k,comm_true_k1,2,1,1,2738,660834,857519,241.35646457268078,313.1917457998539 -olmo2-7b,gsm8k,comm_true_k1,2,1,2,2738,2010106,852134,734.1512052593133,311.2249817384953 -olmo2-7b,gsm8k,comm_true_k1,2,1,3,2738,2082170,802993,760.4711468224982,293.2772096420745 -olmo2-7b,gsm8k,comm_true_k14,15,14,1,2638,4776120,6185396,1810.5079605761941,2344.729340409401 -olmo2-7b,gsm8k,comm_true_k14,15,14,2,2638,72479437,6972812,27475.146702047005,2643.219105382866 -olmo2-7b,gsm8k,comm_true_k14,15,14,3,2638,77673814,6480922,29444.20545868082,2456.755875663381 -olmo2-7b,gsm8k,comm_true_k19,20,19,1,2638,6368160,8213110,2414.010614101592,3113.385140257771 -olmo2-7b,gsm8k,comm_true_k19,20,19,2,2638,126202078,9082904,47840.05989385898,3443.1023502653525 -olmo2-7b,gsm8k,comm_true_k19,20,19,3,2638,135245905,8348449,51268.34912812737,3164.688779378317 -olmo2-7b,gsm8k,comm_true_k2,3,2,1,2738,991251,1282154,362.0346968590212,468.2812271731191 -olmo2-7b,gsm8k,comm_true_k2,3,2,2,2738,3940264,1351334,1439.1029948867786,493.5478451424397 -olmo2-7b,gsm8k,comm_true_k2,3,2,3,2738,4127250,1270654,1507.3959094229365,464.0810810810811 -olmo2-7b,gsm8k,comm_true_k24,25,24,1,2639,7963400,10269559,3017.5824175824177,3891.458507010231 -olmo2-7b,gsm8k,comm_true_k24,25,24,2,2638,194968161,10603467,73907.56671721001,4019.5098559514786 -olmo2-7b,gsm8k,comm_true_k24,25,24,3,2638,208370595,10013760,78988.09514783927,3795.9666413949963 -olmo2-7b,gsm8k,comm_true_k29,30,29,1,2638,9552240,12357880,3621.0159211523883,4684.564063684609 -olmo2-7b,gsm8k,comm_true_k29,30,29,2,2638,278561331,10502704,105595.65238817286,3981.3131159969676 -olmo2-7b,gsm8k,comm_true_k29,30,29,3,2638,295522843,10230615,112025.33851402577,3878.1709628506446 -olmo2-7b,gsm8k,comm_true_k4,5,4,1,2738,1652085,2148186,603.391161431702,784.582176771366 -olmo2-7b,gsm8k,comm_true_k4,5,4,2,2738,9640281,2404067,3520.920745069394,878.0376186997809 -olmo2-7b,gsm8k,comm_true_k4,5,4,3,2738,10210331,2235522,3729.1201607012417,816.4799123447772 -olmo2-7b,gsm8k,comm_true_k6,7,6,1,2638,2228856,2878855,844.9037149355572,1091.3021228203183 -olmo2-7b,gsm8k,comm_true_k6,7,6,2,2638,17185340,3332836,6514.533737680061,1263.3949962092495 -olmo2-7b,gsm8k,comm_true_k6,7,6,3,2638,18276764,3048029,6928.265352539803,1155.4317664897649 -olmo2-7b,gsm8k,comm_true_k9,10,9,1,2638,3184080,4077567,1207.005307050796,1545.7039423805913 -olmo2-7b,gsm8k,comm_true_k9,10,9,2,2638,33435463,4781976,12674.550037907506,1812.7278241091735 -olmo2-7b,gsm8k,comm_true_k9,10,9,3,2638,35784777,4388825,13565.116376042457,1663.694086429113 -olmo2-7b,gsm8k,ensemble,1,0,0,2738,330417,423596,120.67823228634039,154.7100073046019 -olmo2-7b,gsm8k,parity_decode_N5,1,0,0,2738,371487,870688,135.6782322863404,318.0014609203798 -olmo2-7b,gsmhard,comm_true_k1,2,1,1,2134,523972,705989,245.53514526710404,330.8289597000937 -olmo2-7b,gsmhard,comm_true_k1,2,1,2,2134,1599018,772619,749.3055295220244,362.052014995314 -olmo2-7b,gsmhard,comm_true_k1,2,1,3,2134,1653662,722496,774.9119025304592,338.56419868791005 -olmo2-7b,gsmhard,comm_true_k14,15,14,1,2034,3743760,5123244,1840.589970501475,2518.802359882006 -olmo2-7b,gsmhard,comm_true_k14,15,14,2,2034,57259260,6201711,28151.061946902653,3049.022123893805 -olmo2-7b,gsmhard,comm_true_k14,15,14,3,2034,60795523,5559543,29889.63765978368,2733.3053097345132 -olmo2-7b,gsmhard,comm_true_k19,20,19,1,2034,4991680,6821288,2454.119960668633,3353.6322517207473 -olmo2-7b,gsmhard,comm_true_k19,20,19,2,2034,99900525,8290222,49115.30235988201,4075.8220255653882 -olmo2-7b,gsmhard,comm_true_k19,20,19,3,2034,105893115,7243901,52061.51179941003,3561.406588003933 -olmo2-7b,gsmhard,comm_true_k2,3,2,1,2135,786363,1082906,368.319906323185,507.215925058548 -olmo2-7b,gsmhard,comm_true_k2,3,2,2,2134,3142137,1202939,1472.4165885660732,563.7014995313964 -olmo2-7b,gsmhard,comm_true_k2,3,2,3,2134,3275901,1129000,1535.0988753514528,529.0534208059981 -olmo2-7b,gsmhard,comm_true_k24,25,24,1,2034,6239600,8518392,3067.6499508357915,4188.0 -olmo2-7b,gsmhard,comm_true_k24,25,24,2,2034,154259717,9325847,75840.56882989184,4584.978859390364 -olmo2-7b,gsmhard,comm_true_k24,25,24,3,2034,163317100,8359319,80293.55948869223,4109.7930186824 -olmo2-7b,gsmhard,comm_true_k29,30,29,1,2034,7487520,10222689,3681.17994100295,5025.90412979351 -olmo2-7b,gsmhard,comm_true_k29,30,29,2,2034,220487147,8694732,108400.76057030482,4274.69616519174 -olmo2-7b,gsmhard,comm_true_k29,30,29,3,2034,232042589,8248460,114081.90216322517,4055.290068829892 -olmo2-7b,gsmhard,comm_true_k4,5,4,1,2134,1309930,1790861,613.83786316776,839.2038425492034 -olmo2-7b,gsmhard,comm_true_k4,5,4,2,2134,7707980,2216612,3611.9868791002814,1038.7122774133084 -olmo2-7b,gsmhard,comm_true_k4,5,4,3,2134,8101145,2022070,3796.225398313027,947.5492033739456 -olmo2-7b,gsmhard,comm_true_k6,7,6,1,2034,1747088,2403859,858.9419862340217,1181.838249754179 -olmo2-7b,gsmhard,comm_true_k6,7,6,2,2034,13578130,3073015,6675.580137659784,1510.823500491642 -olmo2-7b,gsmhard,comm_true_k6,7,6,3,2034,14323854,2721465,7042.209439528024,1337.986725663717 -olmo2-7b,gsmhard,comm_true_k9,10,9,1,2034,2495840,3389003,1227.0599803343166,1666.176499508358 -olmo2-7b,gsmhard,comm_true_k9,10,9,2,2034,26439130,4408748,12998.588987217307,2167.5260570304818 -olmo2-7b,gsmhard,comm_true_k9,10,9,3,2034,28022538,3935873,13777.058997050148,1935.0408062930187 -olmo2-7b,gsmhard,ensemble,1,0,0,2134,261986,352422,122.76757263355202,165.14620431115276 -olmo2-7b,gsmhard,parity_decode_N5,1,0,0,2134,293996,777447,137.76757263355202,364.3144329896907 -olmo2-7b,math500,comm_true_k1,2,1,1,1100,294452,788072,267.68363636363637,716.4290909090909 -olmo2-7b,math500,comm_true_k1,2,1,2,1100,875868,829678,796.2436363636364,754.2527272727273 -olmo2-7b,math500,comm_true_k1,2,1,3,1100,884694,792633,804.2672727272727,720.5754545454546 -olmo2-7b,math500,comm_true_k14,15,14,1,1000,2006430,5304130,2006.43,5304.13 -olmo2-7b,math500,comm_true_k14,15,14,2,1000,29646796,5654461,29646.796,5654.461 -olmo2-7b,math500,comm_true_k14,15,14,3,1000,30235997,5241291,30235.997,5241.291 -olmo2-7b,math500,comm_true_k19,20,19,1,1000,2675240,7040951,2675.24,7040.951 -olmo2-7b,math500,comm_true_k19,20,19,2,1000,51610769,7149531,51610.769,7149.531 -olmo2-7b,math500,comm_true_k19,20,19,3,1000,52673681,6637842,52673.681,6637.842 -olmo2-7b,math500,comm_true_k2,3,2,1,1100,441678,1163038,401.5254545454545,1057.3072727272727 -olmo2-7b,math500,comm_true_k2,3,2,2,1100,1714311,1234454,1558.4645454545455,1122.230909090909 -olmo2-7b,math500,comm_true_k2,3,2,3,1100,1737761,1157206,1579.7827272727272,1052.0054545454545 -olmo2-7b,math500,comm_true_k24,25,24,1,1000,3344050,8788816,3344.05,8788.816 -olmo2-7b,math500,comm_true_k24,25,24,2,1000,79672281,7889850,79672.281,7889.85 -olmo2-7b,math500,comm_true_k24,25,24,3,1000,81113074,7413745,81113.074,7413.745 -olmo2-7b,math500,comm_true_k29,30,29,1,1000,4012860,10565179,4012.86,10565.179 -olmo2-7b,math500,comm_true_k29,30,29,2,1000,113794267,4766313,113794.267,4766.313 -olmo2-7b,math500,comm_true_k29,30,29,3,1000,111171412,5528120,111171.412,5528.12 -olmo2-7b,math500,comm_true_k4,5,4,1,1100,736130,1938060,669.209090909091,1761.8727272727272 -olmo2-7b,math500,comm_true_k4,5,4,2,1100,4186405,2043120,3805.822727272727,1857.3818181818183 -olmo2-7b,math500,comm_true_k4,5,4,3,1100,4258922,1860116,3871.7472727272725,1691.0145454545454 -olmo2-7b,math500,comm_true_k6,7,6,1,1000,936334,2479538,936.334,2479.538 -olmo2-7b,math500,comm_true_k6,7,6,2,1000,7030372,2648413,7030.372,2648.413 -olmo2-7b,math500,comm_true_k6,7,6,3,1000,7161530,2391562,7161.53,2391.562 -olmo2-7b,math500,comm_true_k9,10,9,1,1000,1337620,3527952,1337.62,3527.952 -olmo2-7b,math500,comm_true_k9,10,9,2,1000,13677877,3790044,13677.877,3790.044 -olmo2-7b,math500,comm_true_k9,10,9,3,1000,13955511,3497194,13955.511,3497.194 -olmo2-7b,math500,ensemble,1,0,0,1100,147226,382959,133.84181818181818,348.14454545454544 -olmo2-7b,math500,parity_decode_N5,1,0,0,1100,163726,898359,148.84181818181818,816.69 -olmo2-7b,mmlu_hard,comm_true_k1,2,1,1,1548,488830,593319,315.78165374677,383.281007751938 -olmo2-7b,mmlu_hard,comm_true_k1,2,1,2,1548,1310802,718041,846.7713178294574,463.8507751937984 -olmo2-7b,mmlu_hard,comm_true_k1,2,1,3,1548,1332530,684815,860.8074935400517,442.3869509043928 -olmo2-7b,mmlu_hard,comm_true_k14,15,14,1,1448,3425160,4081218,2365.441988950276,2818.520718232044 -olmo2-7b,mmlu_hard,comm_true_k14,15,14,2,1448,43253775,6106609,29871.391574585636,4217.271408839779 -olmo2-7b,mmlu_hard,comm_true_k14,15,14,3,1448,44526041,5152111,30750.028314917126,3558.0877071823206 -olmo2-7b,mmlu_hard,comm_true_k19,20,19,1,1448,4566880,5473931,3153.9226519337017,3780.33908839779 -olmo2-7b,mmlu_hard,comm_true_k19,20,19,2,1448,75136034,8801344,51889.526243093926,6078.276243093923 -olmo2-7b,mmlu_hard,comm_true_k19,20,19,3,1448,77359474,7114106,53425.05110497238,4913.056629834254 -olmo2-7b,mmlu_hard,comm_true_k2,3,2,1,1548,733245,879529,473.67248062015506,568.171188630491 -olmo2-7b,mmlu_hard,comm_true_k2,3,2,2,1548,2521807,1092568,1629.0742894056848,705.7932816537468 -olmo2-7b,mmlu_hard,comm_true_k2,3,2,3,1548,2577682,1029005,1665.169250645995,664.7319121447028 -olmo2-7b,mmlu_hard,comm_true_k24,25,24,1,1448,5708600,6860732,3942.403314917127,4738.074585635359 -olmo2-7b,mmlu_hard,comm_true_k24,25,24,2,1448,115759820,9772717,79944.6270718232,6749.1139502762435 -olmo2-7b,mmlu_hard,comm_true_k24,25,24,3,1448,119172950,8166802,82301.76104972376,5640.056629834254 -olmo2-7b,mmlu_hard,comm_true_k29,30,29,1,1448,6850320,8235897,4730.883977900552,5687.774171270718 -olmo2-7b,mmlu_hard,comm_true_k29,30,29,2,1448,165194814,6584160,114084.81629834254,4547.07182320442 -olmo2-7b,mmlu_hard,comm_true_k29,30,29,3,1448,165926242,6448782,114589.94613259668,4453.578729281768 -olmo2-7b,mmlu_hard,comm_true_k4,5,4,1,1548,1222075,1461122,789.454134366925,943.8772609819122 -olmo2-7b,mmlu_hard,comm_true_k4,5,4,2,1548,6079469,1914809,3927.3055555555557,1236.9567183462532 -olmo2-7b,mmlu_hard,comm_true_k4,5,4,3,1548,6227652,1809586,4023.031007751938,1168.9832041343668 -olmo2-7b,mmlu_hard,comm_true_k6,7,6,1,1448,1598408,1906991,1103.8729281767955,1316.9827348066299 -olmo2-7b,mmlu_hard,comm_true_k6,7,6,2,1448,10404526,2808334,7185.446132596685,1939.457182320442 -olmo2-7b,mmlu_hard,comm_true_k6,7,6,3,1448,10682767,2543228,7377.601519337017,1756.3729281767955 -olmo2-7b,mmlu_hard,comm_true_k9,10,9,1,1448,2283440,2726622,1576.9613259668508,1883.0262430939226 -olmo2-7b,mmlu_hard,comm_true_k9,10,9,2,1448,20098256,3987818,13880.011049723757,2754.017955801105 -olmo2-7b,mmlu_hard,comm_true_k9,10,9,3,1448,20674758,3497324,14278.147790055249,2415.279005524862 -olmo2-7b,mmlu_hard,ensemble,1,0,0,1548,244415,293489,157.890826873385,189.59237726098192 -olmo2-7b,mmlu_hard,parity_decode_N5,1,0,0,1548,272279,723754,175.890826873385,467.54134366925064 -phi4-14b,arc,comm_true_k1,2,1,1,3495,858402,368414,245.60858369098713,105.41173104434907 -phi4-14b,arc,comm_true_k1,2,1,2,3495,1956856,651942,559.901573676681,186.53562231759656 -phi4-14b,arc,comm_true_k1,2,1,3,3495,2498486,708534,714.8743919885551,202.72789699570816 -phi4-14b,arc,comm_true_k14,15,14,1,3495,6438015,2763378,1842.0643776824033,790.6660944206009 -phi4-14b,arc,comm_true_k14,15,14,2,3495,60051874,4869655,17182.224320457797,1393.3204577968527 -phi4-14b,arc,comm_true_k14,15,14,3,3495,89990043,5271213,25748.224034334766,1508.2154506437769 -phi4-14b,arc,comm_true_k19,20,19,1,3495,8584020,3685282,2456.0858369098714,1054.4440629470673 -phi4-14b,arc,comm_true_k19,20,19,2,3495,103313918,6567325,29560.49155937053,1879.0629470672388 -phi4-14b,arc,comm_true_k19,20,19,3,3495,157606932,7105983,45094.97339055794,2033.185407725322 -phi4-14b,arc,comm_true_k2,3,2,1,3495,1287603,553242,368.41287553648067,158.29527896995708 -phi4-14b,arc,comm_true_k2,3,2,2,3495,3635243,933249,1040.1267525035764,267.02403433476394 -phi4-14b,arc,comm_true_k2,3,2,3,3495,4735355,1002408,1354.8941344778254,286.81201716738195 -phi4-14b,arc,comm_true_k24,25,24,1,3496,10733375,4608460,3070.187356979405,1318.2093821510298 -phi4-14b,arc,comm_true_k24,25,24,2,3496,158302735,8308657,45281.102688787185,2376.6181350114416 -phi4-14b,arc,comm_true_k24,25,24,3,3495,245060108,8915638,70117.34134477825,2550.969384835479 -phi4-14b,arc,comm_true_k29,30,29,1,3495,12876030,5532680,3684.1287553648067,1583.027181688126 -phi4-14b,arc,comm_true_k29,30,29,2,3495,224874723,9948838,64341.837768240344,2846.5917024320456 -phi4-14b,arc,comm_true_k29,30,29,3,3495,349581155,10695741,100023.22031473534,3060.2978540772533 -phi4-14b,arc,comm_true_k4,5,4,1,3495,2146005,923812,614.0214592274679,264.3238912732475 -phi4-14b,arc,comm_true_k4,5,4,2,3495,8386527,1524508,2399.578540772532,436.19685264663804 -phi4-14b,arc,comm_true_k4,5,4,3,3495,11295330,1650817,3231.8540772532187,472.3367668097282 -phi4-14b,arc,comm_true_k6,7,6,1,3495,3004407,1287896,859.6300429184549,368.4967095851216 -phi4-14b,arc,comm_true_k6,7,6,2,3495,14983284,2165938,4287.062660944206,619.7247496423462 -phi4-14b,arc,comm_true_k6,7,6,3,3495,20882371,2340845,5974.927324749642,669.7696709585122 -phi4-14b,arc,comm_true_k9,10,9,1,3495,4292010,1842054,1228.0429184549357,527.0540772532189 -phi4-14b,arc,comm_true_k9,10,9,2,3495,28387502,3180133,8122.31816881259,909.9092989985694 -phi4-14b,arc,comm_true_k9,10,9,3,3495,41120213,3420292,11765.440057224607,978.6243204577969 -phi4-14b,arc,ensemble,1,0,0,3495,429201,184168,122.80429184549357,52.69470672389127 -phi4-14b,arc,parity_decode_N5,1,0,0,3495,492111,925835,140.80429184549357,264.9027181688126 -phi4-14b,dat,comm_true_k1,2,1,1,4500,1458000,841890,324.0,187.08666666666667 -phi4-14b,dat,comm_true_k1,2,1,2,4500,3878898,1180099,861.9773333333334,262.2442222222222 -phi4-14b,dat,comm_true_k1,2,1,3,4500,3928610,1184910,873.0244444444445,263.31333333333333 -phi4-14b,dat,comm_true_k14,15,14,1,4500,10935000,6302555,2430.0,1400.5677777777778 -phi4-14b,dat,comm_true_k14,15,14,2,4500,136040317,8946982,30231.181555555555,1988.2182222222223 -phi4-14b,dat,comm_true_k14,15,14,3,4500,138952162,9333744,30878.258222222223,2074.1653333333334 -phi4-14b,dat,comm_true_k19,20,19,1,4500,14580000,8401266,3240.0,1866.948 -phi4-14b,dat,comm_true_k19,20,19,2,4500,236232588,11667920,52496.130666666664,2592.871111111111 -phi4-14b,dat,comm_true_k19,20,19,3,4500,241447214,12268462,53654.93644444444,2726.324888888889 -phi4-14b,dat,comm_true_k2,3,2,1,4500,2187000,1261752,486.0,280.38933333333335 -phi4-14b,dat,comm_true_k2,3,2,2,4500,7463850,1792034,1658.6333333333334,398.22977777777777 -phi4-14b,dat,comm_true_k2,3,2,3,4500,7578022,1793206,1684.004888888889,398.4902222222222 -phi4-14b,dat,comm_true_k24,25,24,1,4500,18225000,10506855,4050.0,2334.8566666666666 -phi4-14b,dat,comm_true_k24,25,24,2,4500,363861812,14535681,80858.18044444444,3230.1513333333332 -phi4-14b,dat,comm_true_k24,25,24,3,4500,372000230,15344765,82666.71777777778,3409.947777777778 -phi4-14b,dat,comm_true_k29,30,29,1,4501,21874860,12595919,4860.0,2798.4712286158633 -phi4-14b,dat,comm_true_k29,30,29,2,4501,518858952,16834181,115276.3723616974,3740.0979782270606 -phi4-14b,dat,comm_true_k29,30,29,3,4500,530777221,17494625,117950.49355555556,3887.6944444444443 -phi4-14b,dat,comm_true_k4,5,4,1,4500,3645000,2102260,810.0,467.1688888888889 -phi4-14b,dat,comm_true_k4,5,4,2,4500,17929524,3057513,3984.3386666666665,679.4473333333333 -phi4-14b,dat,comm_true_k4,5,4,3,4500,18244602,3077113,4054.356,683.8028888888889 -phi4-14b,dat,comm_true_k6,7,6,1,4500,5103000,2938920,1134.0,653.0933333333334 -phi4-14b,dat,comm_true_k6,7,6,2,4500,32756694,4386722,7279.265333333334,974.8271111111111 -phi4-14b,dat,comm_true_k6,7,6,3,4500,33393679,4525362,7420.817555555555,1005.636 -phi4-14b,dat,comm_true_k9,10,9,1,4500,7290000,4201169,1620.0,933.593111111111 -phi4-14b,dat,comm_true_k9,10,9,2,4500,63253775,6168188,14056.394444444444,1370.7084444444445 -phi4-14b,dat,comm_true_k9,10,9,3,4500,64549351,6415918,14344.300222222222,1425.7595555555556 -phi4-14b,dat,ensemble,1,0,0,4500,729000,419459,162.0,93.21311111111112 -phi4-14b,dat,parity_decode_N5,1,0,0,4500,819000,1679998,182.0,373.3328888888889 -phi4-14b,eqbench,comm_true_k1,2,1,1,96,56238,188796,585.8125,1966.625 -phi4-14b,eqbench,comm_true_k1,2,1,2,96,111240,176240,1158.75,1835.8333333333333 -phi4-14b,eqbench,comm_true_k1,2,1,3,96,111474,175590,1161.1875,1829.0625 -phi4-14b,eqbench,comm_true_k14,15,14,1,96,421785,1414294,4393.59375,14732.229166666666 -phi4-14b,eqbench,comm_true_k14,15,14,2,96,3160036,1143371,32917.041666666664,11910.114583333334 -phi4-14b,eqbench,comm_true_k14,15,14,3,96,3173320,1152157,33055.416666666664,12001.635416666666 -phi4-14b,eqbench,comm_true_k19,20,19,1,96,562380,1883532,5858.125,19620.125 -phi4-14b,eqbench,comm_true_k19,20,19,2,96,5406262,1539146,56315.229166666664,16032.770833333334 -phi4-14b,eqbench,comm_true_k19,20,19,3,96,5430529,1541709,56568.010416666664,16059.46875 -phi4-14b,eqbench,comm_true_k2,3,2,1,96,84357,282967,878.71875,2947.5729166666665 -phi4-14b,eqbench,comm_true_k2,3,2,2,96,202536,259330,2109.75,2701.3541666666665 -phi4-14b,eqbench,comm_true_k2,3,2,3,96,203204,263032,2116.7083333333335,2739.9166666666665 -phi4-14b,eqbench,comm_true_k24,25,24,1,96,702975,2357355,7322.65625,24555.78125 -phi4-14b,eqbench,comm_true_k24,25,24,2,96,8256165,1919099,86001.71875,19990.614583333332 -phi4-14b,eqbench,comm_true_k24,25,24,3,96,8286205,1922710,86314.63541666667,20028.229166666668 -phi4-14b,eqbench,comm_true_k29,30,29,1,96,843570,2827201,8787.1875,29450.010416666668 -phi4-14b,eqbench,comm_true_k29,30,29,2,96,11697750,2295568,121851.5625,23912.166666666668 -phi4-14b,eqbench,comm_true_k29,30,29,3,96,11742132,2281219,122313.875,23762.697916666668 -phi4-14b,eqbench,comm_true_k4,5,4,1,96,140595,472387,1464.53125,4920.697916666667 -phi4-14b,eqbench,comm_true_k4,5,4,2,96,457235,415899,4762.864583333333,4332.28125 -phi4-14b,eqbench,comm_true_k4,5,4,3,96,458407,421137,4775.072916666667,4386.84375 -phi4-14b,eqbench,comm_true_k6,7,6,1,96,196833,658742,2050.34375,6861.895833333333 -phi4-14b,eqbench,comm_true_k6,7,6,2,96,806783,563235,8403.989583333334,5867.03125 -phi4-14b,eqbench,comm_true_k6,7,6,3,96,809669,572197,8434.052083333334,5960.385416666667 -phi4-14b,eqbench,comm_true_k9,10,9,1,96,281190,941071,2929.0625,9802.822916666666 -phi4-14b,eqbench,comm_true_k9,10,9,2,96,1511738,792083,15747.270833333334,8250.864583333334 -phi4-14b,eqbench,comm_true_k9,10,9,3,96,1516448,793016,15796.333333333334,8260.583333333334 -phi4-14b,eqbench,ensemble,1,0,0,96,28119,94702,292.90625,986.4791666666666 -phi4-14b,eqbench,parity_decode_N5,1,0,0,96,30231,128257,314.90625,1336.0104166666667 -phi4-14b,fermi,comm_true_k1,2,1,1,1587,429532,749548,270.6565847511027,472.3049779458097 -phi4-14b,fermi,comm_true_k1,2,1,2,1587,1228052,752677,773.8197857592943,474.2766225582861 -phi4-14b,fermi,comm_true_k1,2,1,3,1587,1239072,691626,780.7637051039698,435.8071833648393 -phi4-14b,fermi,comm_true_k14,15,14,1,1587,3221490,5646609,2029.9243856332703,3558.0396975425333 -phi4-14b,fermi,comm_true_k14,15,14,2,1587,46905031,5066305,29555.785129174543,3192.378701953371 -phi4-14b,fermi,comm_true_k14,15,14,3,1587,47732013,4425724,30076.88279773157,2788.735979836169 -phi4-14b,fermi,comm_true_k19,20,19,1,1587,4295320,7502904,2706.565847511027,4727.727788279773 -phi4-14b,fermi,comm_true_k19,20,19,2,1587,81861452,6489007,51582.51543793321,4088.851291745432 -phi4-14b,fermi,comm_true_k19,20,19,3,1587,83309068,5734818,52494.6868304978,3613.6219281663516 -phi4-14b,fermi,comm_true_k2,3,2,1,1587,644298,1123542,405.98487712665406,707.9659735349717 -phi4-14b,fermi,comm_true_k2,3,2,2,1587,2419208,1090372,1524.3906742281033,687.064902331443 -phi4-14b,fermi,comm_true_k2,3,2,3,1587,2451150,960886,1544.5179584120983,605.4732199117832 -phi4-14b,fermi,comm_true_k24,25,24,1,1588,5373525,9408810,3383.831863979849,5924.943324937028 -phi4-14b,fermi,comm_true_k24,25,24,2,1587,126475153,7848772,79694.48834278513,4945.666036546944 -phi4-14b,fermi,comm_true_k24,25,24,3,1587,128607100,6949129,81037.87019533712,4378.783238815375 -phi4-14b,fermi,comm_true_k29,30,29,1,1587,6442980,11253219,4059.8487712665406,7090.875236294896 -phi4-14b,fermi,comm_true_k29,30,29,2,1587,180785613,9039260,113916.58034026466,5695.816005040958 -phi4-14b,fermi,comm_true_k29,30,29,3,1587,183465598,8221142,115605.29174543163,5180.303717706364 -phi4-14b,fermi,comm_true_k4,5,4,1,1587,1073830,1887203,676.6414618777568,1189.1638311279144 -phi4-14b,fermi,comm_true_k4,5,4,2,1587,5967606,1850147,3760.306238185255,1165.8141146817895 -phi4-14b,fermi,comm_true_k4,5,4,3,1587,6058551,1604577,3817.61247637051,1011.0756143667297 -phi4-14b,fermi,comm_true_k6,7,6,1,1587,1503362,2632650,947.2980466288595,1658.8846880907372 -phi4-14b,fermi,comm_true_k6,7,6,2,1587,11065126,2562582,6972.354127284184,1614.733459357278 -phi4-14b,fermi,comm_true_k6,7,6,3,1587,11242992,2240314,7084.431001890359,1411.6660365469438 -phi4-14b,fermi,comm_true_k9,10,9,1,1587,2147660,3745835,1353.2829237555136,2360.3245116572148 -phi4-14b,fermi,comm_true_k9,10,9,2,1587,21595246,3530844,13607.590422180214,2224.8544423440453 -phi4-14b,fermi,comm_true_k9,10,9,3,1587,21981505,3082389,13850.979836168872,1942.274102079395 -phi4-14b,fermi,ensemble,1,0,0,1587,214766,376295,135.32829237555134,237.11090107120353 -phi4-14b,fermi,parity_decode_N5,1,0,0,1587,212121,895444,133.66162570888469,564.236925015753 -phi4-14b,gsm8k,comm_true_k1,2,1,1,3957,923568,749837,233.4010614101592,189.49633560778366 -phi4-14b,gsm8k,comm_true_k1,2,1,2,3957,2623790,893892,663.0755622946676,225.90144048521609 -phi4-14b,gsm8k,comm_true_k1,2,1,3,3957,2882564,900129,728.4720748041445,227.47763457164518 -phi4-14b,gsm8k,comm_true_k14,15,14,1,3957,6926760,5629003,1750.5079605761941,1422.54308819813 -phi4-14b,gsm8k,comm_true_k14,15,14,2,3957,94462721,7690577,23872.307556229465,1943.5372757139246 -phi4-14b,gsm8k,comm_true_k14,15,14,3,3957,111449186,7271944,28165.071013393987,1837.7417235279252 -phi4-14b,gsm8k,comm_true_k19,20,19,1,3957,9235680,7514433,2334.010614101592,1899.0227445034118 -phi4-14b,gsm8k,comm_true_k19,20,19,2,3957,164388520,10269947,41543.72504422542,2595.3871619914075 -phi4-14b,gsm8k,comm_true_k19,20,19,3,3957,194908018,9663079,49256.51200404347,2442.0214809198887 -phi4-14b,gsm8k,comm_true_k2,3,2,1,3957,1385352,1143862,350.1015921152388,289.07303512762195 -phi4-14b,gsm8k,comm_true_k2,3,2,2,3957,5088634,1342295,1285.982815264089,339.22036896638866 -phi4-14b,gsm8k,comm_true_k2,3,2,3,3957,5661736,1327485,1430.8152640889564,335.4776345716452 -phi4-14b,gsm8k,comm_true_k24,25,24,1,3957,11544600,9374765,2917.51326762699,2369.159716957291 -phi4-14b,gsm8k,comm_true_k24,25,24,2,3957,253394845,12712177,64037.110184483194,3212.5794794035887 -phi4-14b,gsm8k,comm_true_k24,25,24,3,3957,300922872,12179779,76048.23654283548,3078.0336113217086 -phi4-14b,gsm8k,comm_true_k29,30,29,1,3957,13853520,11325390,3501.0159211523883,2862.115238817286 -phi4-14b,gsm8k,comm_true_k29,30,29,2,3957,361933906,15088122,91466.74399797826,3813.0204700530703 -phi4-14b,gsm8k,comm_true_k29,30,29,3,3957,428912806,14615598,108393.4308819813,3693.6057619408643 -phi4-14b,gsm8k,comm_true_k4,5,4,1,3957,2308920,1891227,583.502653525398,477.94465504169824 -phi4-14b,gsm8k,comm_true_k4,5,4,2,3957,12326604,2339006,3115.138741470811,591.1058882992165 -phi4-14b,gsm8k,comm_true_k4,5,4,3,3957,14019744,2237738,3543.0235026535256,565.5137730603993 -phi4-14b,gsm8k,comm_true_k6,7,6,1,3957,3232488,2618824,816.9037149355572,661.8205711397524 -phi4-14b,gsm8k,comm_true_k6,7,6,2,3957,22622482,3370785,5717.079100328532,851.8536770280516 -phi4-14b,gsm8k,comm_true_k6,7,6,3,3957,26055651,3204309,6584.698256254738,809.7824109173616 -phi4-14b,gsm8k,comm_true_k9,10,9,1,3957,4617840,3767772,1167.005307050796,952.1789234268385 -phi4-14b,gsm8k,comm_true_k9,10,9,2,3957,43843850,4965530,11080.073287844327,1254.8723780641901 -phi4-14b,gsm8k,comm_true_k9,10,9,3,3957,51099726,4677379,12913.754359363154,1182.0518069244376 -phi4-14b,gsm8k,ensemble,1,0,0,3957,461784,378389,116.7005307050796,95.6252211271165 -phi4-14b,gsm8k,parity_decode_N5,1,0,0,3957,521139,1680757,131.70053070507961,424.7553702299722 -phi4-14b,gsmhard,comm_true_k1,2,1,1,3051,724344,692644,237.4119960668633,227.02196001311046 -phi4-14b,gsmhard,comm_true_k1,2,1,2,3051,2130388,872170,698.2589314978695,285.86365126188133 -phi4-14b,gsmhard,comm_true_k1,2,1,3,3051,2300588,883225,754.0439200262209,289.4870534251065 -phi4-14b,gsmhard,comm_true_k14,15,14,1,3051,5432580,5164825,1780.589970501475,1692.8302196001312 -phi4-14b,gsmhard,comm_true_k14,15,14,2,3051,78321876,7077989,25670.88692232055,2319.8915109800064 -phi4-14b,gsmhard,comm_true_k14,15,14,3,3051,89200523,6772230,29236.487381186496,2219.675516224189 -phi4-14b,gsmhard,comm_true_k19,20,19,1,3051,7243440,6912718,2374.119960668633,2265.7220583415274 -phi4-14b,gsmhard,comm_true_k19,20,19,2,3051,136432473,9350367,44717.29695181907,3064.6892822025566 -phi4-14b,gsmhard,comm_true_k19,20,19,3,3051,155991980,8981095,51128.148148148146,2943.656178302196 -phi4-14b,gsmhard,comm_true_k2,3,2,1,3051,1086516,1022881,356.117994100295,335.2608980662078 -phi4-14b,gsmhard,comm_true_k2,3,2,2,3051,4154373,1231058,1361.6430678466077,403.493280891511 -phi4-14b,gsmhard,comm_true_k2,3,2,3,3051,4537284,1209552,1487.1465093411996,396.44444444444446 -phi4-14b,gsmhard,comm_true_k24,25,24,1,3051,9054300,8557917,2967.6499508357915,2804.9547689282203 -phi4-14b,gsmhard,comm_true_k24,25,24,2,3051,210385104,11486715,68956.11406096362,3764.9016715830876 -phi4-14b,gsmhard,comm_true_k24,25,24,3,3051,240925517,11176526,78966.08226810882,3663.233693870862 -phi4-14b,gsmhard,comm_true_k29,30,29,1,3051,10865160,10314854,3561.17994100295,3380.8108816781382 -phi4-14b,gsmhard,comm_true_k29,30,29,2,3051,300326005,13570578,98435.26876433956,4447.911504424779 -phi4-14b,gsmhard,comm_true_k29,30,29,3,3051,343950661,13439915,112733.74664044575,4405.085217961324 -phi4-14b,gsmhard,comm_true_k4,5,4,1,3051,1810860,1716959,593.5299901671583,562.7528679121599 -phi4-14b,gsmhard,comm_true_k4,5,4,2,3051,10114459,2164161,3315.129137987545,709.3284169124877 -phi4-14b,gsmhard,comm_true_k4,5,4,3,3051,11249098,2100499,3687.0199934447724,688.4624713208784 -phi4-14b,gsmhard,comm_true_k6,7,6,1,3051,2535204,2415528,830.9419862340217,791.7168141592921 -phi4-14b,gsmhard,comm_true_k6,7,6,2,3051,18640141,3123443,6109.518518518518,1023.7440183546378 -phi4-14b,gsmhard,comm_true_k6,7,6,3,3051,20903428,3016581,6851.336610947231,988.7187807276302 -phi4-14b,gsmhard,comm_true_k9,10,9,1,3051,3621720,3451669,1187.0599803343166,1131.3238282530317 -phi4-14b,gsmhard,comm_true_k9,10,9,2,3051,36238629,4554172,11877.623402163224,1492.6817436905933 -phi4-14b,gsmhard,comm_true_k9,10,9,3,3051,40963777,4393911,13426.344477220584,1440.1543756145527 -phi4-14b,gsmhard,ensemble,1,0,0,3051,362172,340713,118.70599803343165,111.67256637168141 -phi4-14b,gsmhard,parity_decode_N5,1,0,0,3051,407937,1781186,133.70599803343165,583.8039986889545 -phi4-14b,math500,comm_true_k1,2,1,1,1500,389286,616989,259.524,411.326 -phi4-14b,math500,comm_true_k1,2,1,2,1500,1157826,929817,771.884,619.878 -phi4-14b,math500,comm_true_k1,2,1,3,1500,1188858,963249,792.572,642.166 -phi4-14b,math500,comm_true_k14,15,14,1,1500,2919645,4619058,1946.43,3079.372 -phi4-14b,math500,comm_true_k14,15,14,2,1500,43057296,5912456,28704.864,3941.637333333333 -phi4-14b,math500,comm_true_k14,15,14,3,1500,44867537,5491514,29911.691333333332,3661.0093333333334 -phi4-14b,math500,comm_true_k19,20,19,1,1500,3892860,6103067,2595.24,4068.711333333333 -phi4-14b,math500,comm_true_k19,20,19,2,1500,75017408,7540116,50011.60533333333,5026.744 -phi4-14b,math500,comm_true_k19,20,19,3,1500,78232322,6839579,52154.88133333333,4559.719333333333 -phi4-14b,math500,comm_true_k2,3,2,1,1500,583929,917693,389.286,611.7953333333334 -phi4-14b,math500,comm_true_k2,3,2,2,1500,2266987,1198788,1511.3246666666666,799.192 -phi4-14b,math500,comm_true_k2,3,2,3,1500,2333922,1194950,1555.948,796.6333333333333 -phi4-14b,math500,comm_true_k24,25,24,1,1500,4866075,7642974,3244.05,5095.316 -phi4-14b,math500,comm_true_k24,25,24,2,1500,115787860,8987709,77191.90666666666,5991.806 -phi4-14b,math500,comm_true_k24,25,24,3,1500,120744578,8147090,80496.38533333334,5431.393333333333 -phi4-14b,math500,comm_true_k29,30,29,1,1500,5839290,9202818,3892.86,6135.212 -phi4-14b,math500,comm_true_k29,30,29,2,1500,165386491,10511330,110257.66066666666,7007.553333333333 -phi4-14b,math500,comm_true_k29,30,29,3,1500,172482897,9615259,114988.598,6410.172666666666 -phi4-14b,math500,comm_true_k4,5,4,1,1500,973215,1532284,648.81,1021.5226666666666 -phi4-14b,math500,comm_true_k4,5,4,2,1500,5540419,1962679,3693.612666666667,1308.4526666666666 -phi4-14b,math500,comm_true_k4,5,4,3,1500,5729031,1878731,3819.354,1252.4873333333333 -phi4-14b,math500,comm_true_k6,7,6,1,1500,1362501,2162787,908.334,1441.858 -phi4-14b,math500,comm_true_k6,7,6,2,1500,10228517,2738483,6819.011333333333,1825.6553333333334 -phi4-14b,math500,comm_true_k6,7,6,3,1500,10595286,2533189,7063.524,1688.7926666666667 -phi4-14b,math500,comm_true_k9,10,9,1,1500,1946430,3079140,1297.62,2052.76 -phi4-14b,math500,comm_true_k9,10,9,2,1500,19869584,3823960,13246.389333333333,2549.306666666667 -phi4-14b,math500,comm_true_k9,10,9,3,1500,20680556,3500855,13787.037333333334,2333.903333333333 -phi4-14b,math500,ensemble,1,0,0,1500,194643,302014,129.762,201.34266666666667 -phi4-14b,math500,parity_decode_N5,1,0,0,1500,217143,1399125,144.762,932.75 -phi4-14b,mmlu_hard,comm_true_k1,2,1,1,2172,667656,410370,307.3922651933702,188.9364640883978 -phi4-14b,mmlu_hard,comm_true_k1,2,1,2,2172,1573724,644353,724.5506445672191,296.6634438305709 -phi4-14b,mmlu_hard,comm_true_k1,2,1,3,2172,1792140,683523,825.110497237569,314.6975138121547 -phi4-14b,mmlu_hard,comm_true_k14,15,14,1,2172,5007420,3015709,2305.441988950276,1388.4479742173112 -phi4-14b,mmlu_hard,comm_true_k14,15,14,2,2172,50974624,4489292,23468.979742173113,2066.8931860036832 -phi4-14b,mmlu_hard,comm_true_k14,15,14,3,2172,63171600,4555987,29084.53038674033,2097.5999079189687 -phi4-14b,mmlu_hard,comm_true_k19,20,19,1,2172,6676560,4010262,3073.9226519337017,1846.3453038674033 -phi4-14b,mmlu_hard,comm_true_k19,20,19,2,2172,87980984,5937524,40506.898710865564,2733.6666666666665 -phi4-14b,mmlu_hard,comm_true_k19,20,19,3,2172,109872197,5961403,50585.72605893186,2744.660681399632 -phi4-14b,mmlu_hard,comm_true_k2,3,2,1,2172,1001484,601944,461.08839779005524,277.1381215469613 -phi4-14b,mmlu_hard,comm_true_k2,3,2,2,2172,2962768,903192,1364.073664825046,415.83425414364643 -phi4-14b,mmlu_hard,comm_true_k2,3,2,3,2172,3430443,931681,1579.3936464088397,428.9507366482505 -phi4-14b,mmlu_hard,comm_true_k24,25,24,1,2172,8345700,5009773,3842.403314917127,2306.5253222836095 -phi4-14b,mmlu_hard,comm_true_k24,25,24,2,2172,135120887,7371855,62210.353130755066,3394.040055248619 -phi4-14b,mmlu_hard,comm_true_k24,25,24,3,2172,169451535,7384757,78016.36049723758,3399.9802025782687 -phi4-14b,mmlu_hard,comm_true_k29,30,29,1,2172,10014840,6020351,4610.883977900552,2771.800644567219 -phi4-14b,mmlu_hard,comm_true_k29,30,29,2,2172,192216632,8786568,88497.52854511971,4045.3812154696134 -phi4-14b,mmlu_hard,comm_true_k29,30,29,3,2172,241432914,8765477,111156.95856353591,4035.6708103130754 -phi4-14b,mmlu_hard,comm_true_k4,5,4,1,2172,1669140,1003004,768.4806629834254,461.78821362799266 -phi4-14b,mmlu_hard,comm_true_k4,5,4,2,2172,6943055,1456035,3196.61832412523,670.3660220994475 -phi4-14b,mmlu_hard,comm_true_k4,5,4,3,2172,8188196,1485876,3769.887661141805,684.1049723756906 -phi4-14b,mmlu_hard,comm_true_k6,7,6,1,2172,2336796,1406643,1075.8729281767955,647.6256906077348 -phi4-14b,mmlu_hard,comm_true_k6,7,6,2,2172,12538430,2078125,5772.757826887661,956.7794659300184 -phi4-14b,mmlu_hard,comm_true_k6,7,6,3,2172,15066504,2118670,6936.696132596685,975.4465930018416 -phi4-14b,mmlu_hard,comm_true_k9,10,9,1,2172,3338280,2002196,1536.9613259668508,921.8213627992634 -phi4-14b,mmlu_hard,comm_true_k9,10,9,2,2172,23918876,2957136,11012.373848987108,1361.4806629834254 -phi4-14b,mmlu_hard,comm_true_k9,10,9,3,2172,29233991,2978368,13459.480202578268,1371.255985267035 -phi4-14b,mmlu_hard,ensemble,1,0,0,2172,333828,202058,153.6961325966851,93.02854511970534 -phi4-14b,mmlu_hard,parity_decode_N5,1,0,0,2172,372924,922528,171.6961325966851,424.7366482504604 -qwen2.5-3b,arc,comm_true_k1,2,1,1,1165,343028,168007,294.444635193133,144.212017167382 -qwen2.5-3b,arc,comm_true_k1,2,1,2,1165,798780,239914,685.6480686695279,205.93476394849785 -qwen2.5-3b,arc,comm_true_k1,2,1,3,1165,910184,262957,781.2738197424893,225.71416309012875 -qwen2.5-3b,arc,comm_true_k14,15,14,1,1165,2572710,1256675,2208.3347639484978,1078.6909871244636 -qwen2.5-3b,arc,comm_true_k14,15,14,2,1165,25494418,1882050,21883.62060085837,1615.4935622317596 -qwen2.5-3b,arc,comm_true_k14,15,14,3,1165,32460222,2137188,27862.851502145924,1834.4961373390558 -qwen2.5-3b,arc,comm_true_k19,20,19,1,1165,3430280,1675412,2944.4463519313304,1438.1218884120171 -qwen2.5-3b,arc,comm_true_k19,20,19,2,1165,44070587,2472328,37828.83004291845,2122.169957081545 -qwen2.5-3b,arc,comm_true_k19,20,19,3,1165,56156824,2774924,48203.28240343348,2381.9090128755365 -qwen2.5-3b,arc,comm_true_k2,3,2,1,1165,514542,251049,441.66695278969956,215.49270386266093 -qwen2.5-3b,arc,comm_true_k2,3,2,2,1165,1493168,384912,1281.6892703862661,330.39656652360514 -qwen2.5-3b,arc,comm_true_k2,3,2,3,1165,1791706,424220,1537.9450643776825,364.137339055794 -qwen2.5-3b,arc,comm_true_k24,25,24,1,1165,4287850,2092703,3680.557939914163,1796.3115879828326 -qwen2.5-3b,arc,comm_true_k24,25,24,2,1165,67644721,3113893,58064.138197424894,2672.869527896996 -qwen2.5-3b,arc,comm_true_k24,25,24,3,1165,86359001,3520308,74127.89785407725,3021.7236051502146 -qwen2.5-3b,arc,comm_true_k29,30,29,1,1165,5145420,2510292,4416.6695278969955,2154.7570815450645 -qwen2.5-3b,arc,comm_true_k29,30,29,2,1165,96255355,3728346,82622.62231759657,3200.2969957081546 -qwen2.5-3b,arc,comm_true_k29,30,29,3,1165,122357911,4233935,105028.24978540772,3634.2789699570817 -qwen2.5-3b,arc,comm_true_k4,5,4,1,1165,857570,418935,736.1115879828326,359.6008583690987 -qwen2.5-3b,arc,comm_true_k4,5,4,2,1165,3483359,656665,2990.0077253218883,563.6609442060086 -qwen2.5-3b,arc,comm_true_k4,5,4,3,1165,4342949,716383,3727.85321888412,614.9210300429185 -qwen2.5-3b,arc,comm_true_k6,7,6,1,1165,1200598,585135,1030.5562231759657,502.2618025751073 -qwen2.5-3b,arc,comm_true_k6,7,6,2,1165,6271217,910102,5383.018884120172,781.2034334763948 -qwen2.5-3b,arc,comm_true_k6,7,6,3,1165,7898849,1006234,6780.127896995708,863.7201716738198 -qwen2.5-3b,arc,comm_true_k9,10,9,1,1165,1715140,835983,1472.2231759656652,717.581974248927 -qwen2.5-3b,arc,comm_true_k9,10,9,2,1165,11936098,1259191,10245.577682403433,1080.850643776824 -qwen2.5-3b,arc,comm_true_k9,10,9,3,1165,15059645,1413767,12926.7339055794,1213.533905579399 -qwen2.5-3b,arc,ensemble,1,0,0,1165,171514,83552,147.2223175965665,71.71845493562232 -qwen2.5-3b,arc,parity_decode_N5,1,0,0,1165,192484,333517,165.2223175965665,286.280686695279 -qwen2.5-3b,dat,comm_true_k1,2,1,1,1500,564000,410766,376.0,273.844 -qwen2.5-3b,dat,comm_true_k1,2,1,2,1500,1382494,431215,921.6626666666667,287.4766666666667 -qwen2.5-3b,dat,comm_true_k1,2,1,3,1500,1385750,434774,923.8333333333334,289.84933333333333 -qwen2.5-3b,dat,comm_true_k14,15,14,1,1040,2932800,2141702,2820.0,2059.3288461538464 -qwen2.5-3b,dat,comm_true_k14,15,14,2,1039,32374480,2345436,31159.26852743022,2257.3974975938404 -qwen2.5-3b,dat,comm_true_k14,15,14,3,1039,32537494,2410884,31316.16361886429,2320.388835418672 -qwen2.5-3b,dat,comm_true_k2,3,2,1,1500,846000,617136,564.0,411.424 -qwen2.5-3b,dat,comm_true_k2,3,2,2,1500,2631639,652207,1754.426,434.80466666666666 -qwen2.5-3b,dat,comm_true_k2,3,2,3,1500,2639982,647397,1759.988,431.598 -qwen2.5-3b,dat,comm_true_k4,5,4,1,1500,1410000,1026599,940.0,684.3993333333333 -qwen2.5-3b,dat,comm_true_k4,5,4,2,1500,6241650,1118732,4161.1,745.8213333333333 -qwen2.5-3b,dat,comm_true_k4,5,4,3,1500,6269852,1113923,4179.901333333333,742.6153333333333 -qwen2.5-3b,dat,comm_true_k6,7,6,1,1500,1974000,1445984,1316.0,963.9893333333333 -qwen2.5-3b,dat,comm_true_k6,7,6,2,1500,11346124,1574078,7564.082666666666,1049.3853333333334 -qwen2.5-3b,dat,comm_true_k6,7,6,3,1500,11395340,1561082,7596.893333333333,1040.7213333333334 -qwen2.5-3b,dat,comm_true_k9,10,9,1,1500,2820000,2056045,1880.0,1370.6966666666667 -qwen2.5-3b,dat,comm_true_k9,10,9,2,1500,21783755,2266939,14522.503333333334,1511.2926666666667 -qwen2.5-3b,dat,comm_true_k9,10,9,3,1500,21889041,2288708,14592.694,1525.8053333333332 -qwen2.5-3b,dat,ensemble,1,0,0,1500,282000,206121,188.0,137.414 -qwen2.5-3b,fermi,comm_true_k1,2,1,1,529,226300,554434,427.7882797731569,1048.0793950850662 -qwen2.5-3b,fermi,comm_true_k1,2,1,2,529,491360,389969,928.8468809073725,737.1814744801512 -qwen2.5-3b,fermi,comm_true_k1,2,1,3,529,490166,283564,926.5897920604915,536.0378071833649 -qwen2.5-3b,fermi,comm_true_k14,15,14,1,529,1697250,4154124,3208.412098298677,7852.786389413988 -qwen2.5-3b,fermi,comm_true_k14,15,14,2,529,16593937,1761522,31368.500945179585,3329.9092627599243 -qwen2.5-3b,fermi,comm_true_k14,15,14,3,529,16182630,1176468,30590.982986767485,2223.947069943289 -qwen2.5-3b,fermi,comm_true_k19,20,19,1,529,2263000,5553287,4277.882797731569,10497.706994328923 -qwen2.5-3b,fermi,comm_true_k19,20,19,2,529,28777423,2310275,54399.66540642722,4367.2495274102075 -qwen2.5-3b,fermi,comm_true_k19,20,19,3,529,28178839,1417413,53268.126654064275,2679.41965973535 -qwen2.5-3b,fermi,comm_true_k2,3,2,1,529,339450,838715,641.6824196597354,1585.4725897920605 -qwen2.5-3b,fermi,comm_true_k2,3,2,2,529,935105,510195,1767.6843100189035,964.4517958412098 -qwen2.5-3b,fermi,comm_true_k2,3,2,3,529,932245,353937,1762.2778827977315,669.0680529300568 -qwen2.5-3b,fermi,comm_true_k24,25,24,1,529,2828750,6918025,5347.353497164461,13077.551984877127 -qwen2.5-3b,fermi,comm_true_k24,25,24,2,529,44286965,2903681,83718.27032136107,5489.0 -qwen2.5-3b,fermi,comm_true_k24,25,24,3,529,42976580,1813387,81241.17202268432,3427.952741020794 -qwen2.5-3b,fermi,comm_true_k29,30,29,1,529,3394500,8311616,6416.824196597354,15711.939508506617 -qwen2.5-3b,fermi,comm_true_k29,30,29,2,529,63122068,2725285,119323.37996219282,5151.767485822306 -qwen2.5-3b,fermi,comm_true_k29,30,29,3,529,59307086,1951923,112111.69376181475,3689.835538752363 -qwen2.5-3b,fermi,comm_true_k4,5,4,1,529,565750,1386060,1069.4706994328922,2620.1512287334594 -qwen2.5-3b,fermi,comm_true_k4,5,4,2,529,2218319,727571,4193.419659735349,1375.3705103969755 -qwen2.5-3b,fermi,comm_true_k4,5,4,3,529,2205349,480378,4168.9017013232515,908.0869565217391 -qwen2.5-3b,fermi,comm_true_k6,7,6,1,529,792050,1941668,1497.2589792060492,3670.4499054820417 -qwen2.5-3b,fermi,comm_true_k6,7,6,2,529,4029569,1005107,7617.33270321361,1900.0132325141776 -qwen2.5-3b,fermi,comm_true_k6,7,6,3,529,3992042,627623,7546.393194706994,1186.4328922495274 -qwen2.5-3b,fermi,comm_true_k9,10,9,1,529,1131500,2779054,2138.9413988657843,5253.410207939509 -qwen2.5-3b,fermi,comm_true_k9,10,9,2,529,7736377,1319322,14624.531190926276,2493.992438563327 -qwen2.5-3b,fermi,comm_true_k9,10,9,3,529,7659066,820961,14478.385633270322,1551.9111531190927 -qwen2.5-3b,fermi,ensemble,1,0,0,529,113150,279401,213.89413988657844,528.1682419659735 -qwen2.5-3b,fermi,parity_decode_N5,1,0,0,529,96222,330491,181.89413988657844,624.7466918714556 -qwen2.5-3b,gsm8k,comm_true_k1,2,1,1,1319,377316,401359,286.0621683093253,304.2903714935557 -qwen2.5-3b,gsm8k,comm_true_k1,2,1,2,1319,1056750,455188,801.1751326762699,345.1008339651251 -qwen2.5-3b,gsm8k,comm_true_k1,2,1,3,1319,1078764,439852,817.8650492797574,333.47384382107657 -qwen2.5-3b,gsm8k,comm_true_k14,15,14,1,1319,2829870,3040140,2145.4662623199392,2304.882486732373 -qwen2.5-3b,gsm8k,comm_true_k14,15,14,2,1319,38457412,3457122,29156.49128127369,2621.0174374526155 -qwen2.5-3b,gsm8k,comm_true_k14,15,14,3,1319,39827800,3393557,30195.451099317666,2572.825625473844 -qwen2.5-3b,gsm8k,comm_true_k19,20,19,1,1319,3773160,4047559,2860.6216830932526,3068.6573161485976 -qwen2.5-3b,gsm8k,comm_true_k19,20,19,2,1319,67032056,4535964,50820.36087945413,3438.9416224412435 -qwen2.5-3b,gsm8k,comm_true_k19,20,19,3,1319,69401484,4444782,52616.74298711145,3369.811978771797 -qwen2.5-3b,gsm8k,comm_true_k2,3,2,1,1319,565974,608118,429.0932524639879,461.04473085670963 -qwen2.5-3b,gsm8k,comm_true_k2,3,2,2,1319,2054339,699092,1557.497346474602,530.0166793025019 -qwen2.5-3b,gsm8k,comm_true_k2,3,2,3,1319,2110327,684604,1599.9446550416983,519.03260045489 -qwen2.5-3b,gsm8k,comm_true_k24,25,24,1,1319,4716450,5064020,3575.7771038665655,3839.2873388931007 -qwen2.5-3b,gsm8k,comm_true_k24,25,24,2,1319,103462581,5634449,78440.16755117514,4271.758150113723 -qwen2.5-3b,gsm8k,comm_true_k24,25,24,3,1319,107000320,5507039,81122.30477634571,4175.162244124336 -qwen2.5-3b,gsm8k,comm_true_k29,30,29,1,1319,5659740,6078749,4290.9325246398785,4608.604245640637 -qwen2.5-3b,gsm8k,comm_true_k29,30,29,2,1319,147779232,6622279,112038.84154662624,5020.681576952236 -qwen2.5-3b,gsm8k,comm_true_k29,30,29,3,1319,152711090,6443551,115777.93025018954,4885.178923426838 -qwen2.5-3b,gsm8k,comm_true_k4,5,4,1,1319,943290,1015397,715.1554207733132,769.8233510235027 -qwen2.5-3b,gsm8k,comm_true_k4,5,4,2,1319,4981335,1167447,3776.5996967399547,885.1000758150113 -qwen2.5-3b,gsm8k,comm_true_k4,5,4,3,1319,5136624,1142601,3894.3320697498107,866.2630780894617 -qwen2.5-3b,gsm8k,comm_true_k6,7,6,1,1319,1320606,1426509,1001.2175890826384,1081.5079605761941 -qwen2.5-3b,gsm8k,comm_true_k6,7,6,2,1319,9168359,1648434,6950.992418498863,1249.7604245640637 -qwen2.5-3b,gsm8k,comm_true_k6,7,6,3,1319,9468926,1614851,7178.867323730098,1224.2994692949203 -qwen2.5-3b,gsm8k,comm_true_k9,10,9,1,1319,1886580,2027330,1430.3108415466263,1537.0204700530705 -qwen2.5-3b,gsm8k,comm_true_k9,10,9,2,1319,17788737,2323357,13486.532979529948,1761.453373768006 -qwen2.5-3b,gsm8k,comm_true_k9,10,9,3,1319,18403221,2272928,13952.404094010613,1723.2206216830932 -qwen2.5-3b,gsm8k,ensemble,1,0,0,1319,188658,203228,143.03108415466264,154.0773313115997 -qwen2.5-3b,gsm8k,parity_decode_N5,1,0,0,1319,208443,619661,158.03108415466264,469.79605761940866 -qwen2.5-3b,gsmhard,comm_true_k1,2,1,1,1017,300242,390747,295.22320550639137,384.2153392330383 -qwen2.5-3b,gsmhard,comm_true_k1,2,1,2,1017,839958,435526,825.9174041297935,428.2458210422812 -qwen2.5-3b,gsmhard,comm_true_k1,2,1,3,1017,846982,433429,832.8239921337266,426.18387413962637 -qwen2.5-3b,gsmhard,comm_true_k14,15,14,1,1017,2251815,2919381,2214.174041297935,2870.581120943953 -qwen2.5-3b,gsmhard,comm_true_k14,15,14,2,1017,30594028,3071280,30082.623402163226,3019.9410029498526 -qwen2.5-3b,gsmhard,comm_true_k14,15,14,3,1017,31034381,3027052,30515.615535889872,2976.4523107177974 -qwen2.5-3b,gsmhard,comm_true_k19,20,19,1,1017,3002420,3906384,2952.2320550639133,3841.085545722714 -qwen2.5-3b,gsmhard,comm_true_k19,20,19,2,1017,53355931,4080320,52464.04228121927,4012.1140609636186 -qwen2.5-3b,gsmhard,comm_true_k19,20,19,3,1017,54072246,3996483,53168.383480825956,3929.678466076696 -qwen2.5-3b,gsmhard,comm_true_k2,3,2,1,1017,450363,590191,442.834808259587,580.3254670599804 -qwen2.5-3b,gsmhard,comm_true_k2,3,2,2,1017,1632384,661044,1605.0973451327434,649.9941002949853 -qwen2.5-3b,gsmhard,comm_true_k2,3,2,3,1017,1651111,637975,1623.511307767945,627.3107177974434 -qwen2.5-3b,gsmhard,comm_true_k24,25,24,1,1017,3753025,4901459,3690.290068829892,4819.527040314651 -qwen2.5-3b,gsmhard,comm_true_k24,25,24,2,1017,82323200,5055700,80947.09931170108,4971.189773844641 -qwen2.5-3b,gsmhard,comm_true_k24,25,24,3,1017,83402981,5001506,82008.8308751229,4917.901671583088 -qwen2.5-3b,gsmhard,comm_true_k29,30,29,1,1017,4503630,5837546,4428.34808259587,5739.96656833825 -qwen2.5-3b,gsmhard,comm_true_k29,30,29,2,1017,117554773,5942360,115589.74729596854,5843.028515240904 -qwen2.5-3b,gsmhard,comm_true_k29,30,29,3,1017,118986671,5687052,116997.7099311701,5591.988200589971 -qwen2.5-3b,gsmhard,comm_true_k4,5,4,1,1017,750605,980966,738.0580137659783,964.5683382497542 -qwen2.5-3b,gsmhard,comm_true_k4,5,4,2,1017,3963672,1099854,3897.4159292035397,1081.4690265486727 -qwen2.5-3b,gsmhard,comm_true_k4,5,4,3,1017,4012891,1078822,3945.812192723697,1060.7885939036382 -qwen2.5-3b,gsmhard,comm_true_k6,7,6,1,1017,1050847,1363963,1033.2812192723698,1341.1632251720748 -qwen2.5-3b,gsmhard,comm_true_k6,7,6,2,1017,7289359,1502907,7167.511307767945,1477.7846607669617 -qwen2.5-3b,gsmhard,comm_true_k6,7,6,3,1017,7384065,1476443,7260.634218289086,1451.7630285152409 -qwen2.5-3b,gsmhard,comm_true_k9,10,9,1,1017,1501210,1947825,1476.1160275319567,1915.2654867256638 -qwen2.5-3b,gsmhard,comm_true_k9,10,9,2,1017,14141764,2098215,13905.372664700099,2063.141592920354 -qwen2.5-3b,gsmhard,comm_true_k9,10,9,3,1017,14337393,2065148,14097.731563421828,2030.6273352999017 -qwen2.5-3b,gsmhard,ensemble,1,0,0,1017,150121,192982,147.61160275319568,189.75614552605703 -qwen2.5-3b,gsmhard,parity_decode_N5,1,0,0,1017,165376,581951,162.61160275319568,572.2232055063913 -qwen2.5-3b,math500,comm_true_k1,2,1,1,500,156098,362146,312.196,724.292 -qwen2.5-3b,math500,comm_true_k1,2,1,2,500,422352,345515,844.704,691.03 -qwen2.5-3b,math500,comm_true_k1,2,1,3,500,424110,329166,848.22,658.332 -qwen2.5-3b,math500,comm_true_k14,15,14,1,500,1170735,2683231,2341.47,5366.462 -qwen2.5-3b,math500,comm_true_k14,15,14,2,500,15153972,2331192,30307.944,4662.384 -qwen2.5-3b,math500,comm_true_k14,15,14,3,500,15283688,2106008,30567.376,4212.016 -qwen2.5-3b,math500,comm_true_k19,20,19,1,500,1560980,3601706,3121.96,7203.412 -qwen2.5-3b,math500,comm_true_k19,20,19,2,500,26399643,3109850,52799.286,6219.7 -qwen2.5-3b,math500,comm_true_k19,20,19,3,500,26585229,2857966,53170.458,5715.932 -qwen2.5-3b,math500,comm_true_k2,3,2,1,500,234147,541304,468.294,1082.608 -qwen2.5-3b,math500,comm_true_k2,3,2,2,500,817125,513749,1634.25,1027.498 -qwen2.5-3b,math500,comm_true_k2,3,2,3,500,822807,488335,1645.614,976.67 -qwen2.5-3b,math500,comm_true_k24,25,24,1,500,1951225,4462771,3902.45,8925.542 -qwen2.5-3b,math500,comm_true_k24,25,24,2,500,40720013,3763427,81440.026,7526.854 -qwen2.5-3b,math500,comm_true_k24,25,24,3,500,41008684,3297476,82017.368,6594.952 -qwen2.5-3b,math500,comm_true_k29,30,29,1,500,2341470,5393846,4682.94,10787.692 -qwen2.5-3b,math500,comm_true_k29,30,29,2,500,58149944,4341016,116299.888,8682.032 -qwen2.5-3b,math500,comm_true_k29,30,29,3,500,58477505,3744501,116955.01,7489.002 -qwen2.5-3b,math500,comm_true_k4,5,4,1,500,390245,894021,780.49,1788.042 -qwen2.5-3b,math500,comm_true_k4,5,4,2,500,1975654,851173,3951.308,1702.346 -qwen2.5-3b,math500,comm_true_k4,5,4,3,500,1989593,810773,3979.186,1621.546 -qwen2.5-3b,math500,comm_true_k6,7,6,1,500,546343,1260233,1092.686,2520.466 -qwen2.5-3b,math500,comm_true_k6,7,6,2,500,3625780,1179024,7251.56,2358.048 -qwen2.5-3b,math500,comm_true_k6,7,6,3,500,3652695,1111127,7305.39,2222.254 -qwen2.5-3b,math500,comm_true_k9,10,9,1,500,780490,1785284,1560.98,3570.568 -qwen2.5-3b,math500,comm_true_k9,10,9,2,500,7017673,1585446,14035.346,3170.892 -qwen2.5-3b,math500,comm_true_k9,10,9,3,500,7073950,1451764,14147.9,2903.528 -qwen2.5-3b,math500,ensemble,1,0,0,500,78049,177745,156.098,355.49 -qwen2.5-3b,math500,parity_decode_N5,1,0,0,500,85549,410829,171.098,821.658 -qwen2.5-3b,mmlu_hard,comm_true_k1,2,1,1,724,262748,182109,362.91160220994476,251.53176795580112 -qwen2.5-3b,mmlu_hard,comm_true_k1,2,1,2,724,602840,241762,832.6519337016574,333.92541436464086 -qwen2.5-3b,mmlu_hard,comm_true_k1,2,1,3,724,637802,251683,880.9419889502763,347.628453038674 -qwen2.5-3b,mmlu_hard,comm_true_k14,15,14,1,724,1970610,1359746,2721.8370165745855,1878.1022099447514 -qwen2.5-3b,mmlu_hard,comm_true_k14,15,14,2,724,19516447,1708752,26956.418508287294,2360.1546961325967 -qwen2.5-3b,mmlu_hard,comm_true_k14,15,14,3,724,21706829,1773783,29981.808011049725,2449.9765193370167 -qwen2.5-3b,mmlu_hard,comm_true_k19,20,19,1,724,2627480,1818965,3629.1160220994475,2512.3825966850827 -qwen2.5-3b,mmlu_hard,comm_true_k19,20,19,2,724,33683929,2252331,46524.7638121547,3110.954419889503 -qwen2.5-3b,mmlu_hard,comm_true_k19,20,19,3,724,37564326,2328942,51884.42817679558,3216.770718232044 -qwen2.5-3b,mmlu_hard,comm_true_k2,3,2,1,724,394122,274429,544.3674033149172,379.0455801104972 -qwen2.5-3b,mmlu_hard,comm_true_k2,3,2,2,724,1135139,368663,1567.871546961326,509.20303867403317 -qwen2.5-3b,mmlu_hard,comm_true_k2,3,2,3,724,1226463,381500,1694.0096685082874,526.9337016574585 -qwen2.5-3b,mmlu_hard,comm_true_k24,25,24,1,724,3284350,2273231,4536.3950276243095,3139.82182320442 -qwen2.5-3b,mmlu_hard,comm_true_k24,25,24,2,724,51775130,2805373,71512.61049723758,3874.824585635359 -qwen2.5-3b,mmlu_hard,comm_true_k24,25,24,3,724,57633891,2903420,79604.82182320442,4010.24861878453 -qwen2.5-3b,mmlu_hard,comm_true_k29,30,29,1,724,3941220,2706579,5443.674033149171,3738.3687845303866 -qwen2.5-3b,mmlu_hard,comm_true_k29,30,29,2,724,73768112,3339882,101889.65745856354,4613.096685082873 -qwen2.5-3b,mmlu_hard,comm_true_k29,30,29,3,724,82150445,3453003,113467.46546961326,4769.341160220994 -qwen2.5-3b,mmlu_hard,comm_true_k4,5,4,1,724,656870,451276,907.2790055248619,623.3093922651934 -qwen2.5-3b,mmlu_hard,comm_true_k4,5,4,2,724,2653799,603894,3665.468232044199,834.1077348066299 -qwen2.5-3b,mmlu_hard,comm_true_k4,5,4,3,724,2908691,616374,4017.529005524862,851.3453038674033 -qwen2.5-3b,mmlu_hard,comm_true_k6,7,6,1,724,919618,633614,1270.1906077348067,875.1574585635359 -qwen2.5-3b,mmlu_hard,comm_true_k6,7,6,2,724,4795316,832756,6623.364640883978,1150.2154696132598 -qwen2.5-3b,mmlu_hard,comm_true_k6,7,6,3,724,5285633,868280,7300.598066298343,1199.2817679558011 -qwen2.5-3b,mmlu_hard,comm_true_k9,10,9,1,724,1313740,905181,1814.5580110497237,1250.25 -qwen2.5-3b,mmlu_hard,comm_true_k9,10,9,2,724,9155583,1140590,12645.832872928177,1575.4005524861877 -qwen2.5-3b,mmlu_hard,comm_true_k9,10,9,3,724,10134862,1180266,13998.42817679558,1630.2016574585634 -qwen2.5-3b,mmlu_hard,ensemble,1,0,0,724,131374,91861,181.45580110497238,126.87983425414365 -qwen2.5-3b,mmlu_hard,parity_decode_N5,1,0,0,724,144406,321860,199.45580110497238,444.55801104972375 -qwen2.5-72b-awq,arc,comm_true_k1,2,1,1,300,88696,27385,295.6533333333333,91.28333333333333 -qwen2.5-72b-awq,arc,comm_true_k1,2,1,2,300,176356,50353,587.8533333333334,167.84333333333333 -qwen2.5-72b-awq,arc,comm_true_k1,2,1,3,300,220242,55809,734.14,186.03 -qwen2.5-72b-awq,arc,comm_true_k14,15,14,1,300,665220,206370,2217.4,687.9 -qwen2.5-72b-awq,arc,comm_true_k14,15,14,2,300,4923990,344398,16413.3,1147.9933333333333 -qwen2.5-72b-awq,arc,comm_true_k14,15,14,3,300,6975884,386618,23252.946666666667,1288.7266666666667 -qwen2.5-72b-awq,arc,comm_true_k19,20,19,1,300,886960,274319,2956.5333333333333,914.3966666666666 -qwen2.5-72b-awq,arc,comm_true_k19,20,19,2,300,8409417,453518,28031.39,1511.7266666666667 -qwen2.5-72b-awq,arc,comm_true_k19,20,19,3,300,11969335,509368,39897.78333333333,1697.8933333333334 -qwen2.5-72b-awq,arc,comm_true_k2,3,2,1,300,133044,40976,443.48,136.58666666666667 -qwen2.5-72b-awq,arc,comm_true_k2,3,2,2,300,318990,66173,1063.3,220.57666666666665 -qwen2.5-72b-awq,arc,comm_true_k2,3,2,3,300,393792,73093,1312.64,243.64333333333335 -qwen2.5-72b-awq,arc,comm_true_k24,25,24,1,300,1108700,343124,3695.6666666666665,1143.7466666666667 -qwen2.5-72b-awq,arc,comm_true_k24,25,24,2,300,12842525,569776,42808.416666666664,1899.2533333333333 -qwen2.5-72b-awq,arc,comm_true_k24,25,24,3,300,18463595,644127,61545.316666666666,2147.09 -qwen2.5-72b-awq,arc,comm_true_k29,30,29,1,300,1330440,412443,4434.8,1374.81 -qwen2.5-72b-awq,arc,comm_true_k29,30,29,2,300,18225958,683383,60753.193333333336,2277.943333333333 -qwen2.5-72b-awq,arc,comm_true_k29,30,29,3,300,26292353,767528,87641.17666666667,2558.4266666666667 -qwen2.5-72b-awq,arc,comm_true_k4,5,4,1,300,221740,68939,739.1333333333333,229.79666666666665 -qwen2.5-72b-awq,arc,comm_true_k4,5,4,2,300,716580,112677,2388.6,375.59 -qwen2.5-72b-awq,arc,comm_true_k4,5,4,3,300,932780,125472,3109.266666666667,418.24 -qwen2.5-72b-awq,arc,comm_true_k6,7,6,1,300,310436,95772,1034.7866666666666,319.24 -qwen2.5-72b-awq,arc,comm_true_k6,7,6,2,300,1256626,158762,4188.753333333333,529.2066666666667 -qwen2.5-72b-awq,arc,comm_true_k6,7,6,3,300,1691788,179383,5639.293333333333,597.9433333333334 -qwen2.5-72b-awq,arc,comm_true_k9,10,9,1,300,443480,136936,1478.2666666666667,456.4533333333333 -qwen2.5-72b-awq,arc,comm_true_k9,10,9,2,300,2344730,226025,7815.766666666666,753.4166666666666 -qwen2.5-72b-awq,arc,comm_true_k9,10,9,3,300,3225730,253709,10752.433333333332,845.6966666666667 -qwen2.5-72b-awq,arc,ensemble,1,0,0,300,44348,13726,147.82666666666665,45.75333333333333 -qwen2.5-72b-awq,arc,parity_decode_N5,1,0,0,300,49748,90145,165.82666666666665,300.48333333333335 -qwen2.5-72b-awq,gsm8k,comm_true_k1,2,1,1,1319,377316,237335,286.0621683093253,179.9355572403336 -qwen2.5-72b-awq,gsm8k,comm_true_k1,2,1,2,1319,968036,323385,733.9166034874905,245.17437452615619 -qwen2.5-72b-awq,gsm8k,comm_true_k1,2,1,3,1319,1059184,326790,803.0204700530705,247.75587566338135 -qwen2.5-72b-awq,gsm8k,comm_true_k14,15,14,1,1319,2829870,1776836,2145.4662623199392,1347.1084154662624 -qwen2.5-72b-awq,gsm8k,comm_true_k14,15,14,2,1319,33463705,2424424,25370.51175132676,1838.0773313115997 -qwen2.5-72b-awq,gsm8k,comm_true_k14,15,14,3,1319,38114319,2381882,28896.375284306294,1805.8241091736163 -qwen2.5-72b-awq,gsm8k,comm_true_k19,20,19,1,1319,3773160,2368559,2860.6216830932526,1795.7232752084913 -qwen2.5-72b-awq,gsm8k,comm_true_k19,20,19,2,1319,58141507,3237260,44079.99014404852,2454.3290371493554 -qwen2.5-72b-awq,gsm8k,comm_true_k19,20,19,3,1319,66444747,3180811,50375.092494313874,2411.532221379833 -qwen2.5-72b-awq,gsm8k,comm_true_k2,3,2,1,1319,565974,352840,429.0932524639879,267.50568612585295 -qwen2.5-72b-awq,gsm8k,comm_true_k2,3,2,2,1319,1851706,486189,1403.871114480667,368.60424564063686 -qwen2.5-72b-awq,gsm8k,comm_true_k2,3,2,3,1319,2046583,483426,1551.6171341925701,366.50947687642156 -qwen2.5-72b-awq,gsm8k,comm_true_k24,25,24,1,1319,4716450,2961806,3575.7771038665655,2245.4935557240333 -qwen2.5-72b-awq,gsm8k,comm_true_k24,25,24,2,1319,89520411,4041129,67869.90978013647,3063.7824109173616 -qwen2.5-72b-awq,gsm8k,comm_true_k24,25,24,3,1319,102563930,3981780,77758.85519332827,3018.786959818044 -qwen2.5-72b-awq,gsm8k,comm_true_k29,30,29,1,434,1860930,1153797,4287.857142857143,2658.5184331797236 -qwen2.5-72b-awq,gsm8k,comm_true_k29,30,29,2,434,41743009,1577166,96182.04838709677,3634.0230414746543 -qwen2.5-72b-awq,gsm8k,comm_true_k29,30,29,3,433,47868238,1542881,110550.20323325635,3563.235565819861 -qwen2.5-72b-awq,gsm8k,comm_true_k4,5,4,1,1319,943290,587868,715.1554207733132,445.69219105382865 -qwen2.5-72b-awq,gsm8k,comm_true_k4,5,4,2,1319,4424177,815695,3354.1902956785443,618.4192570128886 -qwen2.5-72b-awq,gsm8k,comm_true_k4,5,4,3,1319,4953517,799881,3755.5094768764216,606.4298711144806 -qwen2.5-72b-awq,gsm8k,comm_true_k6,7,6,1,1319,1320606,829091,1001.2175890826384,628.5754359363154 -qwen2.5-72b-awq,gsm8k,comm_true_k6,7,6,2,1319,8072699,1154613,6120.31766489765,875.3699772554966 -qwen2.5-72b-awq,gsm8k,comm_true_k6,7,6,3,1319,9109158,1126715,6906.109173616376,854.2191053828658 -qwen2.5-72b-awq,gsm8k,comm_true_k9,10,9,1,1319,1886580,1180735,1430.3108415466263,895.1743745261562 -qwen2.5-72b-awq,gsm8k,comm_true_k9,10,9,2,1319,15532178,1640106,11775.722517058377,1243.4465504169825 -qwen2.5-72b-awq,gsm8k,comm_true_k9,10,9,3,1319,17648587,1602304,13380.278241091737,1214.786959818044 -qwen2.5-72b-awq,gsm8k,ensemble,1,0,0,1319,188658,117739,143.03108415466264,89.26383623957544 -qwen2.5-72b-awq,gsm8k,parity_decode_N5,1,0,0,300,47140,139601,157.13333333333333,465.33666666666664 -qwen2.5-72b-awq,gsmhard,comm_true_k1,2,1,1,300,88236,64730,294.12,215.76666666666668 -qwen2.5-72b-awq,gsmhard,comm_true_k1,2,1,2,300,236824,89628,789.4133333333333,298.76 -qwen2.5-72b-awq,gsmhard,comm_true_k1,2,1,3,300,248014,89562,826.7133333333334,298.54 -qwen2.5-72b-awq,gsmhard,comm_true_k14,15,14,1,300,661770,495846,2205.9,1652.82 -qwen2.5-72b-awq,gsmhard,comm_true_k14,15,14,2,300,8442752,670062,28142.506666666668,2233.54 -qwen2.5-72b-awq,gsmhard,comm_true_k14,15,14,3,300,9006469,644113,30021.56333333333,2147.0433333333335 -qwen2.5-72b-awq,gsmhard,comm_true_k19,20,19,1,300,882360,659683,2941.2,2198.943333333333 -qwen2.5-72b-awq,gsmhard,comm_true_k19,20,19,2,300,14698027,889680,48993.42333333333,2965.6 -qwen2.5-72b-awq,gsmhard,comm_true_k19,20,19,3,300,15688793,859399,52295.97666666667,2864.6633333333334 -qwen2.5-72b-awq,gsmhard,comm_true_k2,3,2,1,300,132354,100409,441.18,334.69666666666666 -qwen2.5-72b-awq,gsmhard,comm_true_k2,3,2,2,300,457828,136513,1526.0933333333332,455.04333333333335 -qwen2.5-72b-awq,gsmhard,comm_true_k2,3,2,3,300,481081,134842,1603.6033333333332,449.47333333333336 -qwen2.5-72b-awq,gsmhard,comm_true_k24,25,24,1,300,1102950,821601,3676.5,2738.67 -qwen2.5-72b-awq,gsmhard,comm_true_k24,25,24,2,300,22655765,1107321,75519.21666666666,3691.07 -qwen2.5-72b-awq,gsmhard,comm_true_k24,25,24,3,300,24212261,1072319,80707.53666666667,3574.3966666666665 -qwen2.5-72b-awq,gsmhard,comm_true_k29,30,29,1,300,1323540,989406,4411.8,3298.02 -qwen2.5-72b-awq,gsmhard,comm_true_k29,30,29,2,300,32347116,1322920,107823.72,4409.733333333334 -qwen2.5-72b-awq,gsmhard,comm_true_k29,30,29,3,300,34567730,1287115,115225.76666666666,4290.383333333333 -qwen2.5-72b-awq,gsmhard,comm_true_k4,5,4,1,300,220590,164566,735.3,548.5533333333333 -qwen2.5-72b-awq,gsmhard,comm_true_k4,5,4,2,300,1105361,231225,3684.536666666667,770.75 -qwen2.5-72b-awq,gsmhard,comm_true_k4,5,4,3,300,1166920,222151,3889.733333333333,740.5033333333333 -qwen2.5-72b-awq,gsmhard,comm_true_k6,7,6,1,300,308826,232451,1029.42,774.8366666666667 -qwen2.5-72b-awq,gsmhard,comm_true_k6,7,6,2,300,2019976,326570,6733.253333333333,1088.5666666666666 -qwen2.5-72b-awq,gsmhard,comm_true_k6,7,6,3,300,2146148,314613,7153.826666666667,1048.71 -qwen2.5-72b-awq,gsmhard,comm_true_k9,10,9,1,300,441180,329805,1470.6,1099.35 -qwen2.5-72b-awq,gsmhard,comm_true_k9,10,9,2,300,3911494,457105,13038.313333333334,1523.6833333333334 -qwen2.5-72b-awq,gsmhard,comm_true_k9,10,9,3,300,4163201,448721,13877.336666666666,1495.7366666666667 -qwen2.5-72b-awq,gsmhard,ensemble,1,0,0,300,44118,32517,147.06,108.39 -qwen2.5-72b-awq,gsmhard,parity_decode_N5,1,0,0,300,48618,189926,162.06,633.0866666666667 -qwen2.5-72b-awq,math500,comm_true_k1,2,1,1,300,92490,72414,308.3,241.38 -qwen2.5-72b-awq,math500,comm_true_k1,2,1,2,300,240620,135564,802.0666666666667,451.88 -qwen2.5-72b-awq,math500,comm_true_k1,2,1,3,300,252116,141964,840.3866666666667,473.2133333333333 -qwen2.5-72b-awq,math500,comm_true_k14,15,14,1,300,693675,545097,2312.25,1816.99 -qwen2.5-72b-awq,math500,comm_true_k14,15,14,2,300,8430955,830943,28103.183333333334,2769.81 -qwen2.5-72b-awq,math500,comm_true_k14,15,14,3,300,9017032,788173,30056.773333333334,2627.2433333333333 -qwen2.5-72b-awq,math500,comm_true_k19,20,19,1,124,389680,295247,3142.5806451612902,2381.0241935483873 -qwen2.5-72b-awq,math500,comm_true_k19,20,19,2,124,6085059,449748,49073.0564516129,3627.0 -qwen2.5-72b-awq,math500,comm_true_k19,20,19,3,123,6455885,438499,52486.86991869919,3565.032520325203 -qwen2.5-72b-awq,math500,comm_true_k2,3,2,1,300,138735,108989,462.45,363.2966666666667 -qwen2.5-72b-awq,math500,comm_true_k2,3,2,2,300,462721,202021,1542.4033333333334,673.4033333333333 -qwen2.5-72b-awq,math500,comm_true_k2,3,2,3,300,487320,208399,1624.4,694.6633333333333 -qwen2.5-72b-awq,math500,comm_true_k4,5,4,1,300,231225,181948,770.75,606.4933333333333 -qwen2.5-72b-awq,math500,comm_true_k4,5,4,2,300,1111460,319710,3704.866666666667,1065.7 -qwen2.5-72b-awq,math500,comm_true_k4,5,4,3,300,1175508,309782,3918.36,1032.6066666666666 -qwen2.5-72b-awq,math500,comm_true_k6,7,6,1,300,323715,250926,1079.05,836.42 -qwen2.5-72b-awq,math500,comm_true_k6,7,6,2,300,2025850,432073,6752.833333333333,1440.2433333333333 -qwen2.5-72b-awq,math500,comm_true_k6,7,6,3,300,2153944,409658,7179.8133333333335,1365.5266666666666 -qwen2.5-72b-awq,math500,comm_true_k9,10,9,1,300,462450,357173,1541.5,1190.5766666666666 -qwen2.5-72b-awq,math500,comm_true_k9,10,9,2,300,3910220,592286,13034.066666666668,1974.2866666666666 -qwen2.5-72b-awq,math500,comm_true_k9,10,9,3,300,4175367,562903,13917.89,1876.3433333333332 -qwen2.5-72b-awq,math500,ensemble,1,0,0,300,46245,35230,154.15,117.43333333333334 -qwen2.5-72b-awq,mmlu_hard,comm_true_k1,2,1,1,300,108724,41577,362.41333333333336,138.59 -qwen2.5-72b-awq,mmlu_hard,comm_true_k1,2,1,2,300,221396,75702,737.9866666666667,252.34 -qwen2.5-72b-awq,mmlu_hard,comm_true_k1,2,1,3,300,258906,83766,863.02,279.22 -qwen2.5-72b-awq,mmlu_hard,comm_true_k14,15,14,1,300,815430,313784,2718.1,1045.9466666666667 -qwen2.5-72b-awq,mmlu_hard,comm_true_k14,15,14,2,300,6486879,499685,21622.93,1665.6166666666666 -qwen2.5-72b-awq,mmlu_hard,comm_true_k14,15,14,3,300,8293243,531377,27644.143333333333,1771.2566666666667 -qwen2.5-72b-awq,mmlu_hard,comm_true_k19,20,19,1,300,1087240,420290,3624.133333333333,1400.9666666666667 -qwen2.5-72b-awq,mmlu_hard,comm_true_k19,20,19,2,300,11162992,655452,37209.973333333335,2184.84 -qwen2.5-72b-awq,mmlu_hard,comm_true_k19,20,19,3,300,14331316,704424,47771.05333333334,2348.08 -qwen2.5-72b-awq,mmlu_hard,comm_true_k2,3,2,1,300,163086,62616,543.62,208.72 -qwen2.5-72b-awq,mmlu_hard,comm_true_k2,3,2,2,300,406307,106544,1354.3566666666666,355.14666666666665 -qwen2.5-72b-awq,mmlu_hard,comm_true_k2,3,2,3,300,479134,115735,1597.1133333333332,385.78333333333336 -qwen2.5-72b-awq,mmlu_hard,comm_true_k24,25,24,1,300,1359050,523119,4530.166666666667,1743.73 -qwen2.5-72b-awq,mmlu_hard,comm_true_k24,25,24,2,300,17042537,826041,56808.456666666665,2753.47 -qwen2.5-72b-awq,mmlu_hard,comm_true_k24,25,24,3,300,22070631,876174,73568.77,2920.58 -qwen2.5-72b-awq,mmlu_hard,comm_true_k29,30,29,1,300,1630860,628338,5436.2,2094.46 -qwen2.5-72b-awq,mmlu_hard,comm_true_k29,30,29,2,300,24196675,977526,80655.58333333333,3258.42 -qwen2.5-72b-awq,mmlu_hard,comm_true_k29,30,29,3,300,31365064,1046578,104550.21333333333,3488.5933333333332 -qwen2.5-72b-awq,mmlu_hard,comm_true_k4,5,4,1,300,271810,104467,906.0333333333333,348.22333333333336 -qwen2.5-72b-awq,mmlu_hard,comm_true_k4,5,4,2,300,923645,172161,3078.8166666666666,573.87 -qwen2.5-72b-awq,mmlu_hard,comm_true_k4,5,4,3,300,1123817,184085,3746.056666666667,613.6166666666667 -qwen2.5-72b-awq,mmlu_hard,comm_true_k6,7,6,1,300,380534,148102,1268.4466666666667,493.67333333333335 -qwen2.5-72b-awq,mmlu_hard,comm_true_k6,7,6,2,300,1641670,236804,5472.233333333334,789.3466666666667 -qwen2.5-72b-awq,mmlu_hard,comm_true_k6,7,6,3,300,2021340,251937,6737.8,839.79 -qwen2.5-72b-awq,mmlu_hard,comm_true_k9,10,9,1,300,543620,209339,1812.0666666666666,697.7966666666666 -qwen2.5-72b-awq,mmlu_hard,comm_true_k9,10,9,2,300,3079574,337709,10265.246666666666,1125.6966666666667 -qwen2.5-72b-awq,mmlu_hard,comm_true_k9,10,9,3,300,3867036,361782,12890.12,1205.94 -qwen2.5-72b-awq,mmlu_hard,ensemble,1,0,0,300,54362,21027,181.20666666666668,70.09 -qwen2.5-72b-awq,mmlu_hard,parity_decode_N5,1,0,0,300,59762,135670,199.20666666666668,452.23333333333335 -qwen2.5-7b,arc,comm_true_k1,2,1,1,2330,686056,179478,294.444635193133,77.02918454935622 -qwen2.5-7b,arc,comm_true_k1,2,1,2,2330,1297008,283671,556.6557939914163,121.74721030042919 -qwen2.5-7b,arc,comm_true_k1,2,1,3,2330,1501080,324280,644.2403433476395,139.17596566523605 -qwen2.5-7b,arc,comm_true_k14,15,14,1,2330,5145420,1345886,2208.3347639484978,577.6334763948498 -qwen2.5-7b,arc,comm_true_k14,15,14,2,2330,34229497,2332283,14690.771244635192,1000.9798283261803 -qwen2.5-7b,arc,comm_true_k14,15,14,3,2330,48539704,2733305,20832.490987124464,1173.0922746781116 -qwen2.5-7b,arc,comm_true_k19,20,19,1,2330,6860560,1797294,2944.4463519313304,771.3708154506438 -qwen2.5-7b,arc,comm_true_k19,20,19,2,2330,58400287,3257756,25064.500858369098,1398.1785407725322 -qwen2.5-7b,arc,comm_true_k19,20,19,3,2330,86537751,3788301,37140.665665236054,1625.8802575107295 -qwen2.5-7b,arc,comm_true_k2,3,2,1,2330,1029084,269421,441.66695278969956,115.631330472103 -qwen2.5-7b,arc,comm_true_k2,3,2,2,2330,2320350,400542,995.8583690987125,171.90643776824035 -qwen2.5-7b,arc,comm_true_k2,3,2,3,2330,2707982,436172,1162.224034334764,187.19828326180257 -qwen2.5-7b,arc,comm_true_k24,25,24,1,2330,8575700,2244995,3680.557939914163,963.5171673819742 -qwen2.5-7b,arc,comm_true_k24,25,24,2,2330,88864532,4219371,38139.28412017167,1810.8888412017168 -qwen2.5-7b,arc,comm_true_k24,25,24,3,2330,136257256,4850622,58479.50901287554,2081.812017167382 -qwen2.5-7b,arc,comm_true_k29,30,29,1,2331,10295730,2693332,4416.872586872587,1155.4405834405834 -qwen2.5-7b,arc,comm_true_k29,30,29,2,2331,125709096,5102039,53929.25611325612,2188.77691977692 -qwen2.5-7b,arc,comm_true_k29,30,29,3,2330,195192432,5834598,83773.57596566524,2504.119313304721 -qwen2.5-7b,arc,comm_true_k4,5,4,1,2330,1715140,449013,736.1115879828326,192.70944206008585 -qwen2.5-7b,arc,comm_true_k4,5,4,2,2330,5114720,683675,2195.1587982832616,293.4227467811159 -qwen2.5-7b,arc,comm_true_k4,5,4,3,2330,6268773,775532,2690.4605150214593,332.84635193133045 -qwen2.5-7b,arc,comm_true_k6,7,6,1,2330,2401196,627572,1030.5562231759657,269.3442060085837 -qwen2.5-7b,arc,comm_true_k6,7,6,2,2330,8901418,995786,3820.3510729613736,427.37596566523604 -qwen2.5-7b,arc,comm_true_k6,7,6,3,2330,11402448,1140551,4893.754506437768,489.5068669527897 -qwen2.5-7b,arc,comm_true_k9,10,9,1,2330,3430280,898798,1472.2231759656652,385.7502145922747 -qwen2.5-7b,arc,comm_true_k9,10,9,2,2330,16479339,1505772,7072.677682403433,646.2540772532188 -qwen2.5-7b,arc,comm_true_k9,10,9,3,2330,22344401,1731338,9589.871673819742,743.0635193133047 -qwen2.5-7b,arc,ensemble,1,0,0,2330,343028,89564,147.2223175965665,38.439484978540776 -qwen2.5-7b,arc,parity_decode_N5,1,0,0,2330,384968,379077,165.2223175965665,162.69399141630902 -qwen2.5-7b,dat,comm_true_k1,2,1,1,3000,1128000,376002,376.0,125.334 -qwen2.5-7b,dat,comm_true_k1,2,1,2,3000,2426042,567611,808.6806666666666,189.20366666666666 -qwen2.5-7b,dat,comm_true_k1,2,1,3,3000,2729184,637378,909.728,212.45933333333332 -qwen2.5-7b,dat,comm_true_k14,15,14,1,3000,8460000,2821233,2820.0,940.411 -qwen2.5-7b,dat,comm_true_k14,15,14,2,3000,74406840,4026761,24802.28,1342.2536666666667 -qwen2.5-7b,dat,comm_true_k14,15,14,3,3000,89473103,4905110,29824.367666666665,1635.0366666666666 -qwen2.5-7b,dat,comm_true_k19,20,19,1,3000,11280000,3759943,3760.0,1253.3143333333333 -qwen2.5-7b,dat,comm_true_k19,20,19,2,3000,128184820,5591310,42728.27333333333,1863.77 -qwen2.5-7b,dat,comm_true_k19,20,19,3,3000,156467359,6465835,52155.78633333333,2155.278333333333 -qwen2.5-7b,dat,comm_true_k2,3,2,1,3000,1692000,564979,564.0,188.32633333333334 -qwen2.5-7b,dat,comm_true_k2,3,2,2,3000,4503087,840881,1501.029,280.29366666666664 -qwen2.5-7b,dat,comm_true_k2,3,2,3,3000,5155701,975359,1718.567,325.11966666666666 -qwen2.5-7b,dat,comm_true_k24,25,24,1,3000,14100000,4704703,4700.0,1568.2343333333333 -qwen2.5-7b,dat,comm_true_k24,25,24,2,3000,196601675,6836394,65533.89166666667,2278.798 -qwen2.5-7b,dat,comm_true_k24,25,24,3,3000,239479247,7908826,79826.41566666667,2636.2753333333335 -qwen2.5-7b,dat,comm_true_k29,30,29,1,3000,16920000,5647344,5640.0,1882.448 -qwen2.5-7b,dat,comm_true_k29,30,29,2,3000,279504420,8018863,93168.14,2672.954333333333 -qwen2.5-7b,dat,comm_true_k29,30,29,3,3000,338825954,9132905,112941.98466666667,3044.3016666666667 -qwen2.5-7b,dat,comm_true_k4,5,4,1,3000,2820000,940868,940.0,313.62266666666665 -qwen2.5-7b,dat,comm_true_k4,5,4,2,3000,10374788,1336307,3458.2626666666665,445.4356666666667 -qwen2.5-7b,dat,comm_true_k4,5,4,3,3000,12044451,1604217,4014.817,534.739 -qwen2.5-7b,dat,comm_true_k6,7,6,1,3000,3948000,1317923,1316.0,439.30766666666665 -qwen2.5-7b,dat,comm_true_k6,7,6,2,3000,18550185,1846426,6183.395,615.4753333333333 -qwen2.5-7b,dat,comm_true_k6,7,6,3,3000,21730113,2225742,7243.371,741.914 -qwen2.5-7b,dat,comm_true_k9,10,9,1,3000,5640000,1879896,1880.0,626.632 -qwen2.5-7b,dat,comm_true_k9,10,9,2,3000,35090770,2665597,11696.923333333334,888.5323333333333 -qwen2.5-7b,dat,comm_true_k9,10,9,3,3000,41817172,3284126,13939.057333333334,1094.7086666666667 -qwen2.5-7b,dat,ensemble,1,0,0,3000,564000,187927,188.0,62.64233333333333 -qwen2.5-7b,dat,parity_decode_N5,1,0,0,3000,624000,829760,208.0,276.58666666666664 -qwen2.5-7b,eqbench,comm_true_k1,2,1,1,64,40636,127279,634.9375,1988.734375 -qwen2.5-7b,eqbench,comm_true_k1,2,1,2,64,77434,102842,1209.90625,1606.90625 -qwen2.5-7b,eqbench,comm_true_k1,2,1,3,64,77440,103312,1210.0,1614.25 -qwen2.5-7b,eqbench,comm_true_k14,15,14,1,64,304770,954808,4762.03125,14918.875 -qwen2.5-7b,eqbench,comm_true_k14,15,14,2,64,2144023,678306,33500.359375,10598.53125 -qwen2.5-7b,eqbench,comm_true_k14,15,14,3,64,2143996,653806,33499.9375,10215.71875 -qwen2.5-7b,eqbench,comm_true_k19,20,19,1,64,406360,1274669,6349.375,19916.703125 -qwen2.5-7b,eqbench,comm_true_k19,20,19,2,64,3664369,891803,57255.765625,13934.421875 -qwen2.5-7b,eqbench,comm_true_k19,20,19,3,64,3664423,878758,57256.609375,13730.59375 -qwen2.5-7b,eqbench,comm_true_k2,3,2,1,64,60954,189617,952.40625,2962.765625 -qwen2.5-7b,eqbench,comm_true_k2,3,2,2,64,140160,150220,2190.0,2347.1875 -qwen2.5-7b,eqbench,comm_true_k2,3,2,3,64,140166,148458,2190.09375,2319.65625 -qwen2.5-7b,eqbench,comm_true_k24,25,24,1,64,507950,1591271,7936.71875,24863.609375 -qwen2.5-7b,eqbench,comm_true_k24,25,24,2,64,5587189,1104030,87299.828125,17250.46875 -qwen2.5-7b,eqbench,comm_true_k24,25,24,3,64,5586879,1072464,87294.984375,16757.25 -qwen2.5-7b,eqbench,comm_true_k29,30,29,1,64,609540,1906127,9524.0625,29783.234375 -qwen2.5-7b,eqbench,comm_true_k29,30,29,2,64,7912109,1283564,123626.703125,20055.6875 -qwen2.5-7b,eqbench,comm_true_k29,30,29,3,64,7912446,1242747,123631.96875,19417.921875 -qwen2.5-7b,eqbench,comm_true_k4,5,4,1,64,101590,317322,1587.34375,4958.15625 -qwen2.5-7b,eqbench,comm_true_k4,5,4,2,64,313500,247140,4898.4375,3861.5625 -qwen2.5-7b,eqbench,comm_true_k4,5,4,3,64,313472,238842,4898.0,3731.90625 -qwen2.5-7b,eqbench,comm_true_k6,7,6,1,64,142226,445680,2222.28125,6963.75 -qwen2.5-7b,eqbench,comm_true_k6,7,6,2,64,550843,335024,8606.921875,5234.75 -qwen2.5-7b,eqbench,comm_true_k6,7,6,3,64,550851,336039,8607.046875,5250.609375 -qwen2.5-7b,eqbench,comm_true_k9,10,9,1,64,203180,636004,3174.6875,9937.5625 -qwen2.5-7b,eqbench,comm_true_k9,10,9,2,64,1026588,465230,16040.4375,7269.21875 -qwen2.5-7b,eqbench,comm_true_k9,10,9,3,64,1026868,458485,16044.8125,7163.828125 -qwen2.5-7b,eqbench,ensemble,1,0,0,64,20318,63389,317.46875,990.453125 -qwen2.5-7b,eqbench,parity_decode_N5,1,0,0,64,21726,84714,339.46875,1323.65625 -qwen2.5-7b,fermi,comm_true_k1,2,1,1,1058,452600,976039,427.7882797731569,922.5321361058601 -qwen2.5-7b,fermi,comm_true_k1,2,1,2,1058,982878,439797,928.9962192816636,415.687145557656 -qwen2.5-7b,fermi,comm_true_k1,2,1,3,1058,974562,326578,921.1361058601134,308.67485822306236 -qwen2.5-7b,fermi,comm_true_k14,15,14,1,1058,3394500,7330465,3208.412098298677,6928.605860113422 -qwen2.5-7b,fermi,comm_true_k14,15,14,2,1058,33194340,3263900,31374.612476370512,3084.971644612476 -qwen2.5-7b,fermi,comm_true_k14,15,14,3,1058,32807318,2543223,31008.80718336484,2403.802457466919 -qwen2.5-7b,fermi,comm_true_k19,20,19,1,1058,4526000,9763742,4277.882797731569,9228.489603024575 -qwen2.5-7b,fermi,comm_true_k19,20,19,2,1058,57564517,4449715,54408.80623818526,4205.7797731569 -qwen2.5-7b,fermi,comm_true_k19,20,19,3,1058,56724071,3512241,53614.43383742911,3319.6984877126656 -qwen2.5-7b,fermi,comm_true_k2,3,2,1,1058,678900,1463385,641.6824196597354,1383.1616257088847 -qwen2.5-7b,fermi,comm_true_k2,3,2,2,1058,1870436,748195,1767.897920604915,707.1786389413988 -qwen2.5-7b,fermi,comm_true_k2,3,2,3,1058,1856521,504691,1754.7457466918715,477.023629489603 -qwen2.5-7b,fermi,comm_true_k24,25,24,1,1059,5663000,12215189,5347.497639282342,11534.644948064211 -qwen2.5-7b,fermi,comm_true_k24,25,24,2,1058,88589471,4868923,83732.95935727788,4602.006616257089 -qwen2.5-7b,fermi,comm_true_k24,25,24,3,1058,87335627,3843578,82547.8516068053,3632.8714555765596 -qwen2.5-7b,fermi,comm_true_k29,30,29,1,1058,6789000,14669476,6416.824196597354,13865.289224952741 -qwen2.5-7b,fermi,comm_true_k29,30,29,2,1058,126271083,5361828,119348.84971644613,5067.890359168242 -qwen2.5-7b,fermi,comm_true_k29,30,29,3,1058,123433452,4485606,116666.77882797731,4239.703213610586 -qwen2.5-7b,fermi,comm_true_k4,5,4,1,1058,1131500,2443149,1069.4706994328922,2309.2145557655954 -qwen2.5-7b,fermi,comm_true_k4,5,4,2,1058,4437151,1236754,4193.904536862004,1168.9546313799622 -qwen2.5-7b,fermi,comm_true_k4,5,4,3,1058,4411580,863377,4169.735349716446,816.0463137996219 -qwen2.5-7b,fermi,comm_true_k6,7,6,1,1058,1584100,3401121,1497.2589792060492,3214.670132325142 -qwen2.5-7b,fermi,comm_true_k6,7,6,2,1058,8060406,1703898,7618.531190926276,1610.4896030245748 -qwen2.5-7b,fermi,comm_true_k6,7,6,3,1058,8007713,1184971,7568.726843100189,1120.0103969754252 -qwen2.5-7b,fermi,comm_true_k9,10,9,1,1058,2263000,4872174,2138.9413988657843,4605.079395085067 -qwen2.5-7b,fermi,comm_true_k9,10,9,2,1058,15475393,2149693,14627.025519848772,2031.8459357277882 -qwen2.5-7b,fermi,comm_true_k9,10,9,3,1058,15293288,1657232,14454.90359168242,1566.3818525519848 -qwen2.5-7b,fermi,ensemble,1,0,0,1058,226300,490858,213.89413988657844,463.94896030245746 -qwen2.5-7b,fermi,parity_decode_N5,1,0,0,1058,192444,601167,181.89413988657844,568.210775047259 -qwen2.5-7b,gsm8k,comm_true_k1,2,1,1,3957,1131948,731759,286.0621683093253,184.9277230224918 -qwen2.5-7b,gsm8k,comm_true_k1,2,1,2,3957,2904140,874937,733.9246904220369,221.11119535001262 -qwen2.5-7b,gsm8k,comm_true_k1,2,1,3,3957,3090138,856277,780.9294920394238,216.3955016426586 -qwen2.5-7b,gsm8k,comm_true_k14,15,14,1,3957,8489610,5478948,2145.4662623199392,1384.6216830932524 -qwen2.5-7b,gsm8k,comm_true_k14,15,14,2,3957,100404242,7144805,25373.829163507708,1805.6115744250694 -qwen2.5-7b,gsm8k,comm_true_k14,15,14,3,3957,113556963,7165390,28697.74147081122,1810.8137477887287 -qwen2.5-7b,gsm8k,comm_true_k19,20,19,1,3957,11319480,7297050,2860.6216830932526,1844.0864291129644 -qwen2.5-7b,gsm8k,comm_true_k19,20,19,2,3957,174516833,9618116,44103.318928481174,2430.6585797321204 -qwen2.5-7b,gsm8k,comm_true_k19,20,19,3,3957,197966702,9702080,50029.49254485722,2451.877685114986 -qwen2.5-7b,gsm8k,comm_true_k2,3,2,1,3957,1697922,1093500,429.0932524639879,276.34571645185747 -qwen2.5-7b,gsm8k,comm_true_k2,3,2,2,3957,5558577,1361067,1404.7452615617892,343.96436694465507 -qwen2.5-7b,gsm8k,comm_true_k2,3,2,3,3957,6002532,1339312,1516.940106141016,338.4665150366439 -qwen2.5-7b,gsm8k,comm_true_k24,25,24,1,3957,14149350,9106189,3575.7771038665655,2301.286075309578 -qwen2.5-7b,gsm8k,comm_true_k24,25,24,2,3957,268828493,12095411,67937.45084660096,3056.7124083901945 -qwen2.5-7b,gsm8k,comm_true_k24,25,24,3,3957,305755828,12202690,77269.60525650745,3083.823603740207 -qwen2.5-7b,gsm8k,comm_true_k29,30,29,1,3957,16979220,10930871,4290.9325246398785,2762.413697245388 -qwen2.5-7b,gsm8k,comm_true_k29,30,29,2,3957,383353142,14489643,96879.74273439475,3661.7748294162243 -qwen2.5-7b,gsm8k,comm_true_k29,30,29,3,3957,437317249,14302137,110517.37402072277,3614.3889310083396 -qwen2.5-7b,gsm8k,comm_true_k4,5,4,1,3957,2829870,1827862,715.1554207733132,461.9312610563558 -qwen2.5-7b,gsm8k,comm_true_k4,5,4,2,3957,13282935,2321187,3356.8195602729343,586.6027293404094 -qwen2.5-7b,gsm8k,comm_true_k4,5,4,3,3957,14572329,2282010,3682.6709628506446,576.702047005307 -qwen2.5-7b,gsm8k,comm_true_k6,7,6,1,3957,3961818,2566326,1001.2175890826384,648.5534495830175 -qwen2.5-7b,gsm8k,comm_true_k6,7,6,2,3957,24256424,3256029,6130.003538033864,822.8529188779378 -qwen2.5-7b,gsm8k,comm_true_k6,7,6,3,3957,26861641,3215830,6788.3853929744755,812.6939600707607 -qwen2.5-7b,gsm8k,comm_true_k9,10,9,1,3957,5659740,3637908,1430.3108415466263,919.3601213040182 -qwen2.5-7b,gsm8k,comm_true_k9,10,9,2,3957,46645138,4700851,11788.0055597675,1187.9835734142027 -qwen2.5-7b,gsm8k,comm_true_k9,10,9,3,3957,52122781,4679845,13172.297447561285,1182.6750063179177 -qwen2.5-7b,gsm8k,ensemble,1,0,0,3957,565974,365758,143.03108415466264,92.43315643164013 -qwen2.5-7b,gsm8k,parity_decode_N5,1,0,0,3957,625329,1797136,158.03108415466264,454.16628759160983 -qwen2.5-7b,gsmhard,comm_true_k1,2,1,1,3051,900726,664455,295.22320550639137,217.7826941986234 -qwen2.5-7b,gsmhard,comm_true_k1,2,1,2,3051,2373948,861471,778.0884955752213,282.35693215339234 -qwen2.5-7b,gsmhard,comm_true_k1,2,1,3,3051,2485956,854717,814.8003933136677,280.1432317273025 -qwen2.5-7b,gsmhard,comm_true_k14,15,14,1,3051,6755445,4975348,2214.174041297935,1630.726974762373 -qwen2.5-7b,gsmhard,comm_true_k14,15,14,2,3051,83673137,6960573,27424.823664372336,2281.407079646018 -qwen2.5-7b,gsmhard,comm_true_k14,15,14,3,3051,91070181,6978760,29849.289085545723,2287.3680760406423 -qwen2.5-7b,gsmhard,comm_true_k19,20,19,1,3051,9007260,6636547,2952.2320550639133,2175.2038675843987 -qwen2.5-7b,gsmhard,comm_true_k19,20,19,2,3051,145451245,9369045,47673.302196001314,3070.811209439528 -qwen2.5-7b,gsmhard,comm_true_k19,20,19,3,3051,158773580,9436145,52039.84922976074,3092.8039986889544 -qwen2.5-7b,gsmhard,comm_true_k2,3,2,1,3051,1351089,990193,442.834808259587,324.54703375942313 -qwen2.5-7b,gsmhard,comm_true_k2,3,2,2,3051,4567307,1332143,1496.9868895444117,436.6250409701737 -qwen2.5-7b,gsmhard,comm_true_k2,3,2,3,3051,4832127,1345530,1583.7846607669617,441.0127826941986 -qwen2.5-7b,gsmhard,comm_true_k24,25,24,1,2134,7878275,5803060,3691.787722586692,2719.3345829428304 -qwen2.5-7b,gsmhard,comm_true_k24,25,24,2,2133,156853416,8263228,73536.5288326301,3873.993436474449 -qwen2.5-7b,gsmhard,comm_true_k24,25,24,3,2133,171364112,8275056,80339.48054383497,3879.5386779184246 -qwen2.5-7b,gsmhard,comm_true_k29,30,29,1,2034,9007260,6629113,4428.34808259587,3259.1509341199608 -qwen2.5-7b,gsmhard,comm_true_k29,30,29,2,2034,213380091,9316379,104906.63274336283,4580.323992133726 -qwen2.5-7b,gsmhard,comm_true_k29,30,29,3,2034,233558740,9068988,114827.3058013766,4458.69616519174 -qwen2.5-7b,gsmhard,comm_true_k4,5,4,1,3051,2251815,1662224,738.0580137659783,544.8128482464766 -qwen2.5-7b,gsmhard,comm_true_k4,5,4,2,3051,10994524,2274127,3603.5804654211734,745.3710258931497 -qwen2.5-7b,gsmhard,comm_true_k4,5,4,3,3051,11743014,2282714,3848.906588003933,748.1855129465749 -qwen2.5-7b,gsmhard,comm_true_k6,7,6,1,3051,3152541,2316840,1033.2812192723698,759.37069813176 -qwen2.5-7b,gsmhard,comm_true_k6,7,6,2,3051,20082313,3170134,6582.206817436906,1039.0475254015078 -qwen2.5-7b,gsmhard,comm_true_k6,7,6,3,3051,21614722,3198406,7084.471320878401,1048.3139954113406 -qwen2.5-7b,gsmhard,comm_true_k9,10,9,1,3051,4503630,3312180,1476.1160275319567,1085.6047197640119 -qwen2.5-7b,gsmhard,comm_true_k9,10,9,2,3051,38790278,4552459,12713.955424451,1492.120288430023 -qwen2.5-7b,gsmhard,comm_true_k9,10,9,3,3051,41932997,4575221,13744.017371353655,1499.5807931825632 -qwen2.5-7b,gsmhard,ensemble,1,0,0,3051,450363,329268,147.61160275319568,107.92133726647 -qwen2.5-7b,gsmhard,parity_decode_N5,1,0,0,2034,330752,1129227,162.61160275319568,555.1755162241888 -qwen2.5-7b,math500,comm_true_k1,2,1,1,1000,312196,189634,312.196,189.634 -qwen2.5-7b,math500,comm_true_k1,2,1,2,1000,753882,343377,753.882,343.377 -qwen2.5-7b,math500,comm_true_k1,2,1,3,1000,820688,361517,820.688,361.517 -qwen2.5-7b,math500,comm_true_k14,15,14,1,1000,2341470,1422929,2341.47,1422.929 -qwen2.5-7b,math500,comm_true_k14,15,14,2,1000,25240094,2732537,25240.094,2732.537 -qwen2.5-7b,math500,comm_true_k14,15,14,3,1000,29606434,2739275,29606.434,2739.275 -qwen2.5-7b,math500,comm_true_k19,20,19,1,1000,3121960,1896143,3121.96,1896.143 -qwen2.5-7b,math500,comm_true_k19,20,19,2,1000,43691887,3750881,43691.887,3750.881 -qwen2.5-7b,math500,comm_true_k19,20,19,3,1000,51594022,3717094,51594.022,3717.094 -qwen2.5-7b,math500,comm_true_k2,3,2,1,1000,468294,287335,468.294,287.335 -qwen2.5-7b,math500,comm_true_k2,3,2,2,1000,1430361,537585,1430.361,537.585 -qwen2.5-7b,math500,comm_true_k2,3,2,3,1000,1585782,563677,1585.782,563.677 -qwen2.5-7b,math500,comm_true_k24,25,24,1,1000,3902450,2358456,3902.45,2358.456 -qwen2.5-7b,math500,comm_true_k24,25,24,2,1000,67237360,4582991,67237.36,4582.991 -qwen2.5-7b,math500,comm_true_k24,25,24,3,1000,79789439,4522633,79789.439,4522.633 -qwen2.5-7b,math500,comm_true_k29,30,29,1,1000,4682940,2850782,4682.94,2850.782 -qwen2.5-7b,math500,comm_true_k29,30,29,2,1000,95995724,5353779,95995.724,5353.779 -qwen2.5-7b,math500,comm_true_k29,30,29,3,1000,113750423,5119618,113750.423,5119.618 -qwen2.5-7b,math500,comm_true_k4,5,4,1,1000,780490,470554,780.49,470.554 -qwen2.5-7b,math500,comm_true_k4,5,4,2,1000,3382034,898193,3382.034,898.193 -qwen2.5-7b,math500,comm_true_k4,5,4,3,1000,3826415,923933,3826.415,923.933 -qwen2.5-7b,math500,comm_true_k6,7,6,1,1000,1092686,657170,1092.686,657.17 -qwen2.5-7b,math500,comm_true_k6,7,6,2,1000,6133058,1220240,6133.058,1220.24 -qwen2.5-7b,math500,comm_true_k6,7,6,3,1000,7038016,1214169,7038.016,1214.169 -qwen2.5-7b,math500,comm_true_k9,10,9,1,1000,1560980,951753,1560.98,951.753 -qwen2.5-7b,math500,comm_true_k9,10,9,2,1000,11769648,1762426,11769.648,1762.426 -qwen2.5-7b,math500,comm_true_k9,10,9,3,1000,13657001,1767261,13657.001,1767.261 -qwen2.5-7b,math500,ensemble,1,0,0,1000,156098,94201,156.098,94.201 -qwen2.5-7b,math500,parity_decode_N5,1,0,0,1000,171098,744799,171.098,744.799 -qwen2.5-7b,mmlu_hard,comm_true_k1,2,1,1,1448,525496,179742,362.91160220994476,124.13121546961327 -qwen2.5-7b,mmlu_hard,comm_true_k1,2,1,2,1448,1029132,260806,710.7265193370166,180.1146408839779 -qwen2.5-7b,mmlu_hard,comm_true_k1,2,1,3,1448,1149176,279977,793.6298342541437,193.3542817679558 -qwen2.5-7b,mmlu_hard,comm_true_k14,15,14,1,1448,3941220,1345787,2721.8370165745855,929.4109116022099 -qwen2.5-7b,mmlu_hard,comm_true_k14,15,14,2,1448,28949711,2242225,19992.894337016576,1548.4979281767955 -qwen2.5-7b,mmlu_hard,comm_true_k14,15,14,3,1448,38146758,2473766,26344.446132596684,1708.4019337016575 -qwen2.5-7b,mmlu_hard,comm_true_k19,20,19,1,1448,5254960,1797203,3629.1160220994475,1241.1622928176796 -qwen2.5-7b,mmlu_hard,comm_true_k19,20,19,2,1448,49645015,3081421,34285.23135359116,2128.05317679558 -qwen2.5-7b,mmlu_hard,comm_true_k19,20,19,3,1448,66982688,3376963,46258.76243093923,2332.156767955801 -qwen2.5-7b,mmlu_hard,comm_true_k2,3,2,1,1448,788244,269282,544.3674033149172,185.96823204419888 -qwen2.5-7b,mmlu_hard,comm_true_k2,3,2,2,1448,1867589,400539,1289.771408839779,276.6153314917127 -qwen2.5-7b,mmlu_hard,comm_true_k2,3,2,3,1448,2142855,428924,1479.8722375690609,296.2182320441989 -qwen2.5-7b,mmlu_hard,comm_true_k24,25,24,1,1448,6568700,2238513,4536.3950276243095,1545.9343922651933 -qwen2.5-7b,mmlu_hard,comm_true_k24,25,24,2,1448,75612683,3894843,52218.70372928177,2689.8087016574586 -qwen2.5-7b,mmlu_hard,comm_true_k24,25,24,3,1448,103825776,4242314,71702.88397790055,2929.774861878453 -qwen2.5-7b,mmlu_hard,comm_true_k29,30,29,1,1448,7882440,2684623,5443.674033149171,1854.021408839779 -qwen2.5-7b,mmlu_hard,comm_true_k29,30,29,2,1448,107187209,4670242,74024.31560773481,3225.3052486187844 -qwen2.5-7b,mmlu_hard,comm_true_k29,30,29,3,1448,148496270,4995758,102552.6726519337,3450.1091160220994 -qwen2.5-7b,mmlu_hard,comm_true_k4,5,4,1,1448,1313740,446994,907.2790055248619,308.6975138121547 -qwen2.5-7b,mmlu_hard,comm_true_k4,5,4,2,1448,4191872,690445,2894.939226519337,476.82665745856355 -qwen2.5-7b,mmlu_hard,comm_true_k4,5,4,3,1448,5026071,750315,3471.0435082872928,518.1733425414365 -qwen2.5-7b,mmlu_hard,comm_true_k6,7,6,1,1448,1839236,625753,1270.1906077348067,432.14986187845307 -qwen2.5-7b,mmlu_hard,comm_true_k6,7,6,2,1448,7384222,993588,5099.600828729282,686.1795580110497 -qwen2.5-7b,mmlu_hard,comm_true_k6,7,6,3,1448,9134790,1089860,6308.556629834254,752.6657458563536 -qwen2.5-7b,mmlu_hard,comm_true_k9,10,9,1,1448,2627480,897427,1814.5580110497237,619.7700276243094 -qwen2.5-7b,mmlu_hard,comm_true_k9,10,9,2,1448,13803523,1443458,9532.819751381216,996.8632596685082 -qwen2.5-7b,mmlu_hard,comm_true_k9,10,9,3,1448,17631689,1584637,12176.580801104972,1094.3625690607735 -qwen2.5-7b,mmlu_hard,ensemble,1,0,0,1448,262748,90100,181.45580110497238,62.22375690607735 -qwen2.5-7b,mmlu_hard,parity_decode_N5,1,0,0,1448,288812,480458,199.45580110497238,331.80801104972375 -qwen3-14b,arc,comm_true_k1,2,1,1,3495,910254,416139,260.444635193133,119.06695278969957 -qwen3-14b,arc,comm_true_k1,2,1,2,3495,2117532,557022,605.874678111588,159.37682403433476 -qwen3-14b,arc,comm_true_k1,2,1,3,3495,2395920,558856,685.5278969957081,159.90157367668098 -qwen3-14b,arc,comm_true_k14,15,14,1,3495,6826905,3111426,1953.3347639484978,890.2506437768241 -qwen3-14b,arc,comm_true_k14,15,14,2,3495,66779997,4257101,19107.295278969956,1218.0546494992848 -qwen3-14b,arc,comm_true_k14,15,14,3,3495,83643745,4509851,23932.40200286123,1290.3722460658082 -qwen3-14b,arc,comm_true_k19,20,19,1,3495,9102540,4142584,2604.4463519313304,1185.2886981402003 -qwen3-14b,arc,comm_true_k19,20,19,2,3495,115232742,5795836,32970.741630901284,1658.3221745350502 -qwen3-14b,arc,comm_true_k19,20,19,3,3495,147732613,6092872,42269.70329041488,1743.3110157367669 -qwen3-14b,arc,comm_true_k2,3,2,1,3495,1365381,622260,390.66695278969956,178.04291845493563 -qwen3-14b,arc,comm_true_k2,3,2,2,3495,3954515,803479,1131.47782546495,229.89384835479257 -qwen3-14b,arc,comm_true_k2,3,2,3,3495,4489187,828469,1284.459799713877,237.04406294706723 -qwen3-14b,arc,comm_true_k24,25,24,1,3495,11378175,5183699,3255.557939914163,1483.1756795422032 -qwen3-14b,arc,comm_true_k24,25,24,2,3495,177031015,7303872,50652.65092989986,2089.806008583691 -qwen3-14b,arc,comm_true_k24,25,24,3,3495,229080815,7670194,65545.29756795421,2194.6191702432047 -qwen3-14b,arc,comm_true_k29,30,29,1,3495,13653810,6220989,3906.6695278969955,1779.9682403433476 -qwen3-14b,arc,comm_true_k29,30,29,2,3495,251921378,8788295,72080.50872675251,2514.533619456366 -qwen3-14b,arc,comm_true_k29,30,29,3,3495,327430493,9217900,93685.40572246065,2637.453505007153 -qwen3-14b,arc,comm_true_k4,5,4,1,3495,2275635,1037495,651.1115879828326,296.85121602288984 -qwen3-14b,arc,comm_true_k4,5,4,2,3495,9183093,1348058,2627.4944206008586,385.710443490701 -qwen3-14b,arc,comm_true_k4,5,4,3,3495,10725465,1419841,3068.802575107296,406.2492131616595 -qwen3-14b,arc,comm_true_k6,7,6,1,3495,3185889,1452289,911.5562231759657,415.53333333333336 -qwen3-14b,arc,comm_true_k6,7,6,2,3495,16501608,1902376,4721.490128755365,544.3135908440629 -qwen3-14b,arc,comm_true_k6,7,6,3,3495,19620681,2007275,5613.928755364806,574.3276108726752 -qwen3-14b,arc,comm_true_k9,10,9,1,3495,4551270,2073457,1302.2231759656652,593.2638054363376 -qwen3-14b,arc,comm_true_k9,10,9,2,3495,31341149,2750382,8967.42460658083,786.9476394849786 -qwen3-14b,arc,comm_true_k9,10,9,3,3495,38042275,2895823,10884.771101573677,828.5616595135908 -qwen3-14b,arc,ensemble,1,0,0,3495,455127,206997,130.2223175965665,59.22660944206009 -qwen3-14b,arc,parity_decode_N5,1,0,0,3495,518037,862638,148.2223175965665,246.8206008583691 -qwen3-14b,dat,comm_true_k1,2,1,1,4500,1539000,718800,342.0,159.73333333333332 -qwen3-14b,dat,comm_true_k1,2,1,2,4500,3795258,1056235,843.3906666666667,234.7188888888889 -qwen3-14b,dat,comm_true_k1,2,1,3,4500,4003316,1021516,889.6257777777778,227.00355555555555 -qwen3-14b,dat,comm_true_k14,15,14,1,4500,11542500,5386643,2565.0,1197.0317777777777 -qwen3-14b,dat,comm_true_k14,15,14,2,4500,127755975,8004666,28390.216666666667,1778.8146666666667 -qwen3-14b,dat,comm_true_k14,15,14,3,4500,139933084,8128662,31096.24088888889,1806.3693333333333 -qwen3-14b,dat,comm_true_k19,20,19,1,4500,15390000,7179055,3420.0,1595.3455555555556 -qwen3-14b,dat,comm_true_k19,20,19,2,4500,221484400,10706559,49218.75555555556,2379.2353333333335 -qwen3-14b,dat,comm_true_k19,20,19,3,4500,243193771,10691361,54043.06022222222,2375.858 -qwen3-14b,dat,comm_true_k2,3,2,1,4500,2308500,1077183,513.0,239.374 -qwen3-14b,dat,comm_true_k2,3,2,2,4500,7213308,1658115,1602.9573333333333,368.47 -qwen3-14b,dat,comm_true_k2,3,2,3,4500,7688849,1641872,1708.633111111111,364.86044444444445 -qwen3-14b,dat,comm_true_k24,25,24,1,4501,19241775,8980836,4275.0,1995.2979337924905 -qwen3-14b,dat,comm_true_k24,25,24,2,4500,340941650,13259303,75764.8111111111,2946.5117777777778 -qwen3-14b,dat,comm_true_k24,25,24,3,4500,374758158,13412770,83279.59066666667,2980.6155555555556 -qwen3-14b,dat,comm_true_k29,30,29,1,4500,23085000,10772637,5130.0,2393.9193333333333 -qwen3-14b,dat,comm_true_k29,30,29,2,4500,485908350,15934126,107979.63333333333,3540.916888888889 -qwen3-14b,dat,comm_true_k29,30,29,3,4500,534720699,15936147,118826.822,3541.366 -qwen3-14b,dat,comm_true_k4,5,4,1,4500,3847500,1796350,855.0,399.18888888888887 -qwen3-14b,dat,comm_true_k4,5,4,2,4500,17101620,2794642,3800.36,621.0315555555555 -qwen3-14b,dat,comm_true_k4,5,4,3,4500,18436774,2757415,4097.060888888889,612.7588888888889 -qwen3-14b,dat,comm_true_k6,7,6,1,4500,5386500,2513604,1197.0,558.5786666666667 -qwen3-14b,dat,comm_true_k6,7,6,2,4500,31041542,3941057,6898.120444444445,875.7904444444445 -qwen3-14b,dat,comm_true_k6,7,6,3,4500,33684250,3807571,7485.388888888889,846.1268888888889 -qwen3-14b,dat,comm_true_k9,10,9,1,4500,7695000,3590519,1710.0,797.8931111111111 -qwen3-14b,dat,comm_true_k9,10,9,2,4500,59564000,5588030,13236.444444444445,1241.7844444444445 -qwen3-14b,dat,comm_true_k9,10,9,3,4500,64978447,5411136,14439.654888888888,1202.4746666666667 -qwen3-14b,dat,ensemble,1,0,0,4500,769500,358862,171.0,79.74711111111111 -qwen3-14b,dat,parity_decode_N5,1,0,0,4500,859500,1613841,191.0,358.6313333333333 -qwen3-14b,eqbench,comm_true_k1,2,1,1,96,57690,195567,600.9375,2037.15625 -qwen3-14b,eqbench,comm_true_k1,2,1,2,96,112944,194795,1176.5,2029.1145833333333 -qwen3-14b,eqbench,comm_true_k1,2,1,3,96,112942,195081,1176.4791666666667,2032.09375 -qwen3-14b,eqbench,comm_true_k14,15,14,1,96,432675,1469235,4507.03125,15304.53125 -qwen3-14b,eqbench,comm_true_k14,15,14,2,96,3191693,1420677,33246.802083333336,14798.71875 -qwen3-14b,eqbench,comm_true_k14,15,14,3,96,3191017,1422766,33239.760416666664,14820.479166666666 -qwen3-14b,eqbench,comm_true_k19,20,19,1,96,576900,1958679,6009.375,20402.90625 -qwen3-14b,eqbench,comm_true_k19,20,19,2,96,5464101,1863332,56917.71875,19409.708333333332 -qwen3-14b,eqbench,comm_true_k19,20,19,3,96,5463662,1880999,56913.145833333336,19593.739583333332 -qwen3-14b,eqbench,comm_true_k2,3,2,1,96,86535,293297,901.40625,3055.1770833333335 -qwen3-14b,eqbench,comm_true_k2,3,2,2,96,205374,292321,2139.3125,3045.0104166666665 -qwen3-14b,eqbench,comm_true_k2,3,2,3,96,205371,291974,2139.28125,3041.3958333333335 -qwen3-14b,eqbench,comm_true_k24,25,24,1,96,721125,2446834,7511.71875,25487.854166666668 -qwen3-14b,eqbench,comm_true_k24,25,24,2,96,8340553,2312982,86880.76041666667,24093.5625 -qwen3-14b,eqbench,comm_true_k24,25,24,3,96,8340660,2325552,86881.875,24224.5 -qwen3-14b,eqbench,comm_true_k29,30,29,1,96,865350,2935860,9014.0625,30581.875 -qwen3-14b,eqbench,comm_true_k29,30,29,2,96,11819843,2770940,123123.36458333333,28863.958333333332 -qwen3-14b,eqbench,comm_true_k29,30,29,3,96,11820167,2778880,123126.73958333333,28946.666666666668 -qwen3-14b,eqbench,comm_true_k4,5,4,1,96,144225,489380,1502.34375,5097.708333333333 -qwen3-14b,eqbench,comm_true_k4,5,4,2,96,462177,486164,4814.34375,5064.208333333333 -qwen3-14b,eqbench,comm_true_k4,5,4,3,96,462130,486115,4813.854166666667,5063.697916666667 -qwen3-14b,eqbench,comm_true_k6,7,6,1,96,201915,686128,2103.28125,7147.166666666667 -qwen3-14b,eqbench,comm_true_k6,7,6,2,96,814832,676271,8487.833333333334,7044.489583333333 -qwen3-14b,eqbench,comm_true_k6,7,6,3,96,814792,676284,8487.416666666666,7044.625 -qwen3-14b,eqbench,comm_true_k9,10,9,1,96,288450,978499,3004.6875,10192.697916666666 -qwen3-14b,eqbench,comm_true_k9,10,9,2,96,1523472,959260,15869.5,9992.291666666666 -qwen3-14b,eqbench,comm_true_k9,10,9,3,96,1523473,959061,15869.510416666666,9990.21875 -qwen3-14b,eqbench,ensemble,1,0,0,96,28845,97759,300.46875,1018.3229166666666 -qwen3-14b,eqbench,parity_decode_N5,1,0,0,96,30957,135217,322.46875,1408.5104166666667 -qwen3-14b,fermi,comm_true_k1,2,1,1,1587,453546,602971,285.7882797731569,379.94391934467546 -qwen3-14b,fermi,comm_true_k1,2,1,2,1587,1196726,398429,754.0806553245117,251.05797101449275 -qwen3-14b,fermi,comm_true_k1,2,1,3,1587,1236584,371317,779.1959672337745,233.97416509136735 -qwen3-14b,fermi,comm_true_k14,15,14,1,1587,3401595,4580996,2143.412098298677,2886.575929426591 -qwen3-14b,fermi,comm_true_k14,15,14,2,1587,44129418,3491568,27806.81663516068,2200.1058601134214 -qwen3-14b,fermi,comm_true_k14,15,14,3,1587,47763667,3308078,30096.82860743541,2084.4851921865156 -qwen3-14b,fermi,comm_true_k19,20,19,1,1587,4535460,6099840,2857.882797731569,3843.6294896030245 -qwen3-14b,fermi,comm_true_k19,20,19,2,1587,76984523,4726607,48509.466288594835,2978.3282923755514 -qwen3-14b,fermi,comm_true_k19,20,19,3,1587,83562408,4477625,52654.3213610586,2821.4398235664776 -qwen3-14b,fermi,comm_true_k2,3,2,1,1587,680319,925021,428.68241965973533,582.8739760554505 -qwen3-14b,fermi,comm_true_k2,3,2,2,1587,2331504,630326,1469.1266540642723,397.18084436042847 -qwen3-14b,fermi,comm_true_k2,3,2,3,1587,2444389,583285,1540.2577189666038,367.5393824826717 -qwen3-14b,fermi,comm_true_k24,25,24,1,1587,5669325,7650364,3572.3534971644613,4820.645242596093 -qwen3-14b,fermi,comm_true_k24,25,24,2,1587,118851272,5849531,74890.53056080655,3685.904851921865 -qwen3-14b,fermi,comm_true_k24,25,24,3,1587,129126439,5588009,81365.11594202899,3521.11468178954 -qwen3-14b,fermi,comm_true_k29,30,29,1,1587,6803190,9151017,4286.824196597354,5766.23629489603 -qwen3-14b,fermi,comm_true_k29,30,29,2,1587,169776297,6999813,106979.393194707,4410.7202268431 -qwen3-14b,fermi,comm_true_k29,30,29,3,1587,184697507,6720725,116381.54190296156,4234.861373660996 -qwen3-14b,fermi,comm_true_k4,5,4,1,1587,1133865,1517034,714.4706994328923,955.9130434782609 -qwen3-14b,fermi,comm_true_k4,5,4,2,1587,5687853,1066334,3584.028355387524,671.9180844360428 -qwen3-14b,fermi,comm_true_k4,5,4,3,1587,6021279,985148,3794.1266540642723,620.7611846250787 -qwen3-14b,fermi,comm_true_k6,7,6,1,1587,1587411,2139873,1000.2589792060492,1348.37618147448 -qwen3-14b,fermi,comm_true_k6,7,6,2,1587,10485279,1521943,6606.981096408317,959.0063011972275 -qwen3-14b,fermi,comm_true_k6,7,6,3,1587,11182536,1433993,7046.336483931947,903.5872715816005 -qwen3-14b,fermi,comm_true_k9,10,9,1,1587,2267730,3048770,1428.9413988657845,1921.090107120353 -qwen3-14b,fermi,comm_true_k9,10,9,2,1587,20364607,2271785,12832.140516698173,1431.496534341525 -qwen3-14b,fermi,comm_true_k9,10,9,3,1587,21889885,2130960,13793.248267170762,1342.7599243856332 -qwen3-14b,fermi,ensemble,1,0,0,1587,226773,307394,142.89413988657844,193.6950220541903 -qwen3-14b,fermi,parity_decode_N5,1,0,0,1587,223599,1604116,140.89413988657844,1010.7851291745432 -qwen3-14b,gsm8k,comm_true_k1,2,1,1,3957,997410,987298,252.06216830932524,249.5066969926712 -qwen3-14b,gsm8k,comm_true_k1,2,1,2,3957,2774940,822904,701.2736921910538,207.96158706090472 -qwen3-14b,gsm8k,comm_true_k1,2,1,3,3957,2893360,793132,731.2004043467273,200.43770533232248 -qwen3-14b,gsm8k,comm_true_k14,15,14,1,3957,7480575,7402512,1890.4662623199395,1870.7384382107657 -qwen3-14b,gsm8k,comm_true_k14,15,14,2,3957,99586020,6727113,25167.050796057618,1700.0538286580743 -qwen3-14b,gsm8k,comm_true_k14,15,14,3,3957,111202576,6565753,28102.74854687895,1659.275461207986 -qwen3-14b,gsm8k,comm_true_k19,20,19,1,3957,9974100,9850895,2520.6216830932526,2489.4857215061916 -qwen3-14b,gsm8k,comm_true_k19,20,19,2,3957,173390767,9057158,43818.74323982815,2288.895122567602 -qwen3-14b,gsm8k,comm_true_k19,20,19,3,3957,194564418,8814429,49169.67854435178,2227.5534495830175 -qwen3-14b,gsm8k,comm_true_k2,3,2,1,3957,1496115,1456899,378.0932524639879,368.18271417740715 -qwen3-14b,gsm8k,comm_true_k2,3,2,2,3957,5366139,1300636,1356.1129643669447,328.69244377053326 -qwen3-14b,gsm8k,comm_true_k2,3,2,3,3957,5734416,1231695,1449.1827141774072,311.2699014404852 -qwen3-14b,gsm8k,comm_true_k24,25,24,1,3957,12467625,12344466,3150.7771038665655,3119.652767247915 -qwen3-14b,gsm8k,comm_true_k24,25,24,2,3957,267601346,11268556,67627.33030073288,2847.752337629517 -qwen3-14b,gsm8k,comm_true_k24,25,24,3,3957,300992812,11044882,76065.9115491534,2791.2261814505937 -qwen3-14b,gsm8k,comm_true_k29,30,29,1,3957,14961150,14836477,3780.932524639879,3749.425574930503 -qwen3-14b,gsm8k,comm_true_k29,30,29,2,3957,382065459,13469598,96554.32373009856,3403.9924184988627 -qwen3-14b,gsm8k,comm_true_k29,30,29,3,3957,429777308,13215796,108611.90497851909,3339.852413444529 -qwen3-14b,gsm8k,comm_true_k4,5,4,1,3957,2493525,2460747,630.1554207733132,621.8718726307809 -qwen3-14b,gsm8k,comm_true_k4,5,4,2,3957,12964156,2189581,3276.258781905484,553.3436947182208 -qwen3-14b,gsm8k,comm_true_k4,5,4,3,3957,14097964,2068037,3562.791003285317,522.6274955774577 -qwen3-14b,gsm8k,comm_true_k6,7,6,1,3957,3490935,3448652,882.2175890826384,871.5319686631286 -qwen3-14b,gsm8k,comm_true_k6,7,6,2,3957,23788746,3061025,6011.813495072024,773.5721506191559 -qwen3-14b,gsm8k,comm_true_k6,7,6,3,3957,26061043,2945709,6586.060904725802,744.4298711144806 -qwen3-14b,gsm8k,comm_true_k9,10,9,1,3957,4987050,4853227,1260.3108415466263,1226.4915339903969 -qwen3-14b,gsm8k,comm_true_k9,10,9,2,3957,46061105,4417974,11640.410664644933,1116.4958301743745 -qwen3-14b,gsm8k,comm_true_k9,10,9,3,3957,50646784,4282667,12799.28834975992,1082.301491028557 -qwen3-14b,gsm8k,ensemble,1,0,0,3957,498705,485728,126.03108415466262,122.75157947940359 -qwen3-14b,gsm8k,parity_decode_N5,1,0,0,3957,558060,2080000,141.03108415466264,525.6507455142785 -qwen3-14b,gsmhard,comm_true_k1,2,1,1,3051,796992,787472,261.22320550639137,258.1029170763684 -qwen3-14b,gsmhard,comm_true_k1,2,1,2,3051,2296796,825904,752.8010488364471,270.6994428056375 -qwen3-14b,gsmhard,comm_true_k1,2,1,3,3051,2366538,786187,775.6597836774828,257.68174369059324 -qwen3-14b,gsmhard,comm_true_k14,15,14,1,3052,5979120,6016350,1959.0825688073394,1971.281127129751 -qwen3-14b,gsmhard,comm_true_k14,15,14,2,3051,84345888,6538963,27645.32546705998,2143.2196001311045 -qwen3-14b,gsmhard,comm_true_k14,15,14,3,3051,89776293,6254259,29425.20255653884,2049.904621435595 -qwen3-14b,gsmhard,comm_true_k19,20,19,1,3052,7973040,8022288,2612.3984272608127,2628.534731323722 -qwen3-14b,gsmhard,comm_true_k19,20,19,2,3051,147038611,8726699,48193.57948213701,2860.2749918059653 -qwen3-14b,gsmhard,comm_true_k19,20,19,3,3051,156997382,8339299,51457.68010488364,2733.3002294329726 -qwen3-14b,gsmhard,comm_true_k2,3,2,1,3051,1195488,1196538,391.834808259587,392.17895771878074 -qwen3-14b,gsmhard,comm_true_k2,3,2,2,3051,4471971,1278936,1465.7394296951818,419.1858407079646 -qwen3-14b,gsmhard,comm_true_k2,3,2,3,3051,4652705,1196746,1524.9770567027203,392.24713208784004 -qwen3-14b,gsmhard,comm_true_k24,25,24,1,3051,9962400,9967082,3265.290068829892,3266.824647656506 -qwen3-14b,gsmhard,comm_true_k24,25,24,2,3051,227033462,10860253,74412.80301540479,3559.5716158636515 -qwen3-14b,gsmhard,comm_true_k24,25,24,3,3051,242674533,10337301,79539.34218289086,3388.16814159292 -qwen3-14b,gsmhard,comm_true_k29,30,29,1,3051,11954880,12058203,3918.34808259587,3952.2133726647003 -qwen3-14b,gsmhard,comm_true_k29,30,29,2,3051,324413225,12972687,106330.12946574893,4251.945919370698 -qwen3-14b,gsmhard,comm_true_k29,30,29,3,3051,346500144,12437400,113569.36873156342,4076.4995083579156 -qwen3-14b,gsmhard,comm_true_k4,5,4,1,3051,1992480,2001125,653.0580137659783,655.8915109800065 -qwen3-14b,gsmhard,comm_true_k4,5,4,2,3051,10889918,2128005,3569.2946574893476,697.4778761061947 -qwen3-14b,gsmhard,comm_true_k4,5,4,3,3051,11406148,1974004,3738.4949196984594,647.002294329728 -qwen3-14b,gsmhard,comm_true_k6,7,6,1,3051,2789472,2787477,914.2812192723698,913.6273352999017 -qwen3-14b,gsmhard,comm_true_k6,7,6,2,3051,20025580,2968909,6563.611930514586,973.0937397574565 -qwen3-14b,gsmhard,comm_true_k6,7,6,3,3051,21088564,2779673,6912.017043592265,911.0694854146182 -qwen3-14b,gsmhard,comm_true_k9,10,9,1,3051,3984960,3999688,1306.1160275319567,1310.9432972795805 -qwen3-14b,gsmhard,comm_true_k9,10,9,2,3051,38919257,4299879,12756.229760734186,1409.3343166175025 -qwen3-14b,gsmhard,comm_true_k9,10,9,3,3051,41118346,4056499,13477.006227466405,1329.5637495902984 -qwen3-14b,gsmhard,ensemble,1,0,0,3051,398496,406111,130.61160275319568,133.10750573582433 -qwen3-14b,gsmhard,parity_decode_N5,1,0,0,3051,444261,2279702,145.61160275319568,747.1982956407735 -qwen3-14b,math500,comm_true_k1,2,1,1,1500,417294,467322,278.196,311.548 -qwen3-14b,math500,comm_true_k1,2,1,2,1500,1138802,442592,759.2013333333333,295.0613333333333 -qwen3-14b,math500,comm_true_k1,2,1,3,1500,1187660,420412,791.7733333333333,280.2746666666667 -qwen3-14b,math500,comm_true_k14,15,14,1,1500,3129705,3404234,2086.47,2269.4893333333334 -qwen3-14b,math500,comm_true_k14,15,14,2,1500,40703052,3340701,27135.368,2227.134 -qwen3-14b,math500,comm_true_k14,15,14,3,1500,43859021,3146995,29239.347333333335,2097.9966666666664 -qwen3-14b,math500,comm_true_k19,20,19,1,1500,4172940,4557132,2781.96,3038.088 -qwen3-14b,math500,comm_true_k19,20,19,2,1500,70870337,4611323,47246.89133333333,3074.2153333333335 -qwen3-14b,math500,comm_true_k19,20,19,3,1500,76785677,4370472,51190.45133333333,2913.648 -qwen3-14b,math500,comm_true_k2,3,2,1,1500,625941,674570,417.294,449.7133333333333 -qwen3-14b,math500,comm_true_k2,3,2,2,1500,2201906,664054,1467.9373333333333,442.7026666666667 -qwen3-14b,math500,comm_true_k2,3,2,3,1500,2301257,629680,1534.1713333333332,419.7866666666667 -qwen3-14b,math500,comm_true_k24,25,24,1,1500,5216175,5709221,3477.45,3806.1473333333333 -qwen3-14b,math500,comm_true_k24,25,24,2,1500,109385089,5660526,72923.39266666667,3773.684 -qwen3-14b,math500,comm_true_k24,25,24,3,1500,118649340,5278812,79099.56,3519.208 -qwen3-14b,math500,comm_true_k29,30,29,1,1500,6259410,6777133,4172.94,4518.0886666666665 -qwen3-14b,math500,comm_true_k29,30,29,2,1500,156071281,6676111,104047.52066666666,4450.740666666667 -qwen3-14b,math500,comm_true_k29,30,29,3,1500,169295748,6284896,112863.832,4189.930666666667 -qwen3-14b,math500,comm_true_k4,5,4,1,1500,1043235,1140950,695.49,760.6333333333333 -qwen3-14b,math500,comm_true_k4,5,4,2,1500,5314533,1063277,3543.022,708.8513333333333 -qwen3-14b,math500,comm_true_k4,5,4,3,1500,5596276,968672,3730.8506666666667,645.7813333333334 -qwen3-14b,math500,comm_true_k6,7,6,1,1500,1460529,1604338,973.686,1069.5586666666666 -qwen3-14b,math500,comm_true_k6,7,6,2,1500,9752743,1490034,6501.828666666666,993.356 -qwen3-14b,math500,comm_true_k6,7,6,3,1500,10320007,1389512,6880.004666666667,926.3413333333333 -qwen3-14b,math500,comm_true_k9,10,9,1,1500,2086470,2275189,1390.98,1516.7926666666667 -qwen3-14b,math500,comm_true_k9,10,9,2,1500,18859786,2168123,12573.190666666667,1445.4153333333334 -qwen3-14b,math500,comm_true_k9,10,9,3,1500,20125938,2066087,13417.292,1377.3913333333333 -qwen3-14b,math500,ensemble,1,0,0,1500,208647,219051,139.098,146.034 -qwen3-14b,math500,parity_decode_N5,1,0,0,1500,231147,2064914,154.098,1376.6093333333333 -qwen3-14b,mmlu_hard,comm_true_k1,2,1,1,2172,714396,364379,328.91160220994476,167.76197053406997 -qwen3-14b,mmlu_hard,comm_true_k1,2,1,2,2172,1577208,471224,726.1546961325967,216.95395948434623 -qwen3-14b,mmlu_hard,comm_true_k1,2,1,3,2172,1769412,471506,814.646408839779,217.08379373848987 -qwen3-14b,mmlu_hard,comm_true_k14,15,14,1,2172,5357970,2743419,2466.8370165745855,1263.0842541436464 -qwen3-14b,mmlu_hard,comm_true_k14,15,14,2,2172,49041336,3448563,22578.883977900554,1587.736187845304 -qwen3-14b,mmlu_hard,comm_true_k14,15,14,3,2172,59413581,3523663,27354.31906077348,1622.312615101289 -qwen3-14b,mmlu_hard,comm_true_k19,20,19,1,2172,7143960,3657748,3289.1160220994475,1684.0460405156539 -qwen3-14b,mmlu_hard,comm_true_k19,20,19,2,2172,84559129,4683082,38931.45902394107,2156.1151012891346 -qwen3-14b,mmlu_hard,comm_true_k19,20,19,3,2172,103852299,4741222,47814.13397790055,2182.8830570902396 -qwen3-14b,mmlu_hard,comm_true_k2,3,2,1,2172,1071594,546200,493.3674033149171,251.4732965009208 -qwen3-14b,mmlu_hard,comm_true_k2,3,2,2,2172,2939580,671175,1353.3977900552486,309.01243093922653 -qwen3-14b,mmlu_hard,comm_true_k2,3,2,3,2172,3297064,678571,1517.9852670349908,312.41758747697975 -qwen3-14b,mmlu_hard,comm_true_k24,25,24,1,2172,8929950,4556575,4111.3950276243095,2097.870626151013 -qwen3-14b,mmlu_hard,comm_true_k24,25,24,2,2172,129757613,5850217,59741.0741252302,2693.470073664825 -qwen3-14b,mmlu_hard,comm_true_k24,25,24,3,2172,160467795,5921686,73880.2002762431,2726.374769797422 -qwen3-14b,mmlu_hard,comm_true_k29,30,29,1,2172,10715940,5507672,4933.674033149171,2535.7605893186005 -qwen3-14b,mmlu_hard,comm_true_k29,30,29,2,2172,184631226,7039451,85005.16850828729,3240.9995395948436 -qwen3-14b,mmlu_hard,comm_true_k29,30,29,3,2172,229291240,7145946,105566.86924493554,3290.0303867403313 -qwen3-14b,mmlu_hard,comm_true_k4,5,4,1,2172,1785990,913575,822.2790055248619,420.6146408839779 -qwen3-14b,mmlu_hard,comm_true_k4,5,4,2,2172,6790403,1078738,3126.336556169429,496.65653775322284 -qwen3-14b,mmlu_hard,comm_true_k4,5,4,3,2172,7742354,1101121,3564.6197053407,506.96178637200734 -qwen3-14b,mmlu_hard,comm_true_k6,7,6,1,2172,2500386,1273283,1151.1906077348067,586.22605893186 -qwen3-14b,mmlu_hard,comm_true_k6,7,6,2,2172,12171957,1520778,5604.031767955801,700.1740331491712 -qwen3-14b,mmlu_hard,comm_true_k6,7,6,3,2172,14059138,1548800,6472.899631675875,713.0755064456722 -qwen3-14b,mmlu_hard,comm_true_k9,10,9,1,2172,3571980,1842290,1644.5580110497237,848.1998158379374 -qwen3-14b,mmlu_hard,comm_true_k9,10,9,2,2172,23084050,2214915,10628.015653775323,1019.7582872928177 -qwen3-14b,mmlu_hard,comm_true_k9,10,9,3,2172,27183450,2260154,12515.400552486188,1040.586556169429 -qwen3-14b,mmlu_hard,ensemble,1,0,0,2172,357198,186118,164.45580110497238,85.68968692449356 -qwen3-14b,mmlu_hard,parity_decode_N5,1,0,0,2172,396294,1358583,182.45580110497238,625.4986187845304 -qwen3-4b,arc,comm_true_k1,2,1,1,3495,910254,379115,260.444635193133,108.47353361945636 -qwen3-4b,arc,comm_true_k1,2,1,2,3495,2031842,459752,581.3567954220315,131.5456366237482 -qwen3-4b,arc,comm_true_k1,2,1,3,3495,2202642,479150,630.2266094420601,137.09585121602288 -qwen3-4b,arc,comm_true_k14,15,14,1,3495,6826905,2824597,1953.3347639484978,808.18226037196 -qwen3-4b,arc,comm_true_k14,15,14,2,3495,61906919,3749600,17712.995422031472,1072.8469241773962 -qwen3-4b,arc,comm_true_k14,15,14,3,3495,75965768,4018940,21735.555937052934,1149.9113018597998 -qwen3-4b,arc,comm_true_k19,20,19,1,3495,9102540,3773182,2604.4463519313304,1079.5942775393419 -qwen3-4b,arc,comm_true_k19,20,19,2,3495,106794612,4991523,30556.398283261802,1428.1896995708155 -qwen3-4b,arc,comm_true_k19,20,19,3,3495,131476407,5392662,37618.428326180256,1542.9648068669528 -qwen3-4b,arc,comm_true_k2,3,2,1,3495,1365381,566245,390.66695278969956,162.01573676680974 -qwen3-4b,arc,comm_true_k2,3,2,2,3495,3760261,700331,1075.8972818311875,200.38082975679544 -qwen3-4b,arc,comm_true_k2,3,2,3,3495,4177110,717171,1195.1673819742489,205.1991416309013 -qwen3-4b,arc,comm_true_k24,25,24,1,3495,11378175,4711096,3255.557939914163,1347.9530758226038 -qwen3-4b,arc,comm_true_k24,25,24,2,3495,163648569,6448337,46823.62489270386,1845.01773962804 -qwen3-4b,arc,comm_true_k24,25,24,3,3495,207122294,7014189,59262.458941344776,2006.9210300429183 -qwen3-4b,arc,comm_true_k29,30,29,1,3495,13653810,5657937,3906.6695278969955,1618.8660944206008 -qwen3-4b,arc,comm_true_k29,30,29,2,3495,232606287,7765275,66554.01630901288,2221.824034334764 -qwen3-4b,arc,comm_true_k29,30,29,3,3495,295934518,8408836,84673.68183118741,2405.961659513591 -qwen3-4b,arc,comm_true_k4,5,4,1,3495,2275635,942841,651.1115879828326,269.76852646638054 -qwen3-4b,arc,comm_true_k4,5,4,2,3495,8650903,1228500,2475.2226037195996,351.50214592274676 -qwen3-4b,arc,comm_true_k4,5,4,3,3495,10110589,1278463,2892.872389127325,365.79771101573675 -qwen3-4b,arc,comm_true_k6,7,6,1,3495,3185889,1325493,911.5562231759657,379.2540772532189 -qwen3-4b,arc,comm_true_k6,7,6,2,3495,15441942,1713958,4418.2952789699575,490.40286123032905 -qwen3-4b,arc,comm_true_k6,7,6,3,3495,18264242,1796346,5225.820314735336,513.975965665236 -qwen3-4b,arc,comm_true_k9,10,9,1,3495,4551270,1880235,1302.2231759656652,537.9785407725321 -qwen3-4b,arc,comm_true_k9,10,9,2,3495,29210270,2430098,8357.73104434907,695.3070100143061 -qwen3-4b,arc,comm_true_k9,10,9,3,3495,34784900,2565719,9952.761087267525,734.1113018597997 -qwen3-4b,arc,ensemble,1,0,0,3495,455127,187110,130.2223175965665,53.53648068669528 -qwen3-4b,arc,parity_decode_N5,1,0,0,3495,518037,852276,148.2223175965665,243.8557939914163 -qwen3-4b,dat,comm_true_k1,2,1,1,4500,1539000,669674,342.0,148.81644444444444 -qwen3-4b,dat,comm_true_k1,2,1,2,4500,3693614,892384,820.8031111111111,198.30755555555555 -qwen3-4b,dat,comm_true_k1,2,1,3,4500,3984634,924631,885.4742222222222,205.47355555555555 -qwen3-4b,dat,comm_true_k14,15,14,1,4500,11542500,5025591,2565.0,1116.798 -qwen3-4b,dat,comm_true_k14,15,14,2,4500,122145015,7423713,27143.336666666666,1649.714 -qwen3-4b,dat,comm_true_k14,15,14,3,4500,139526462,8242774,31005.880444444443,1831.7275555555555 -qwen3-4b,dat,comm_true_k19,20,19,1,4500,15390000,6707106,3420.0,1490.468 -qwen3-4b,dat,comm_true_k19,20,19,2,4500,211693620,10186608,47043.026666666665,2263.690666666667 -qwen3-4b,dat,comm_true_k19,20,19,3,4500,242784753,11123615,53952.16733333333,2471.9144444444446 -qwen3-4b,dat,comm_true_k2,3,2,1,4500,2308500,1006432,513.0,223.65155555555555 -qwen3-4b,dat,comm_true_k2,3,2,2,4500,6993906,1424869,1554.2013333333334,316.63755555555554 -qwen3-4b,dat,comm_true_k2,3,2,3,4500,7683023,1476493,1707.3384444444443,328.1095555555556 -qwen3-4b,dat,comm_true_k24,25,24,1,4502,19246050,8396178,4275.0,1864.9884495779654 -qwen3-4b,dat,comm_true_k24,25,24,2,4502,325851725,12747286,72379.32585517548,2831.4717903154155 -qwen3-4b,dat,comm_true_k24,25,24,3,4500,373820075,13932628,83071.12777777777,3096.1395555555555 -qwen3-4b,dat,comm_true_k29,30,29,1,4500,23085000,10078881,5130.0,2239.751333333333 -qwen3-4b,dat,comm_true_k29,30,29,2,4500,464260410,15212891,103168.98,3380.6424444444447 -qwen3-4b,dat,comm_true_k29,30,29,3,4500,533158419,16973776,118479.64866666666,3771.9502222222222 -qwen3-4b,dat,comm_true_k4,5,4,1,4500,3847500,1677450,855.0,372.76666666666665 -qwen3-4b,dat,comm_true_k4,5,4,2,4500,16486990,2428869,3663.7755555555555,539.7486666666666 -qwen3-4b,dat,comm_true_k4,5,4,3,4500,18426010,2511496,4094.668888888889,558.1102222222222 -qwen3-4b,dat,comm_true_k6,7,6,1,4500,5386500,2349452,1197.0,522.1004444444444 -qwen3-4b,dat,comm_true_k6,7,6,2,4500,29847496,3404880,6632.776888888889,756.64 -qwen3-4b,dat,comm_true_k6,7,6,3,4500,33641296,3631197,7475.843555555555,806.9326666666667 -qwen3-4b,dat,comm_true_k9,10,9,1,4500,7695000,3354937,1710.0,745.5415555555555 -qwen3-4b,dat,comm_true_k9,10,9,2,4500,57115950,4824990,12692.433333333332,1072.22 -qwen3-4b,dat,comm_true_k9,10,9,3,4500,64805995,5338653,14401.332222222221,1186.3673333333334 -qwen3-4b,dat,ensemble,1,0,0,4500,769500,334940,171.0,74.43111111111111 -qwen3-4b,dat,parity_decode_N5,1,0,0,4500,859500,1725513,191.0,383.44733333333335 -qwen3-4b,eqbench,comm_true_k1,2,1,1,96,57690,193385,600.9375,2014.4270833333333 -qwen3-4b,eqbench,comm_true_k1,2,1,2,96,112924,185850,1176.2916666666667,1935.9375 -qwen3-4b,eqbench,comm_true_k1,2,1,3,96,112926,185395,1176.3125,1931.1979166666667 -qwen3-4b,eqbench,comm_true_k14,15,14,1,96,432675,1444609,4507.03125,15048.010416666666 -qwen3-4b,eqbench,comm_true_k14,15,14,2,96,3191899,1293313,33248.947916666664,13472.010416666666 -qwen3-4b,eqbench,comm_true_k14,15,14,3,96,3191965,1318617,33249.635416666664,13735.59375 -qwen3-4b,eqbench,comm_true_k19,20,19,1,96,576900,1926920,6009.375,20072.083333333332 -qwen3-4b,eqbench,comm_true_k19,20,19,2,96,5464223,1715933,56918.989583333336,17874.302083333332 -qwen3-4b,eqbench,comm_true_k19,20,19,3,96,5463666,1724139,56913.1875,17959.78125 -qwen3-4b,eqbench,comm_true_k2,3,2,1,96,86535,289478,901.40625,3015.3958333333335 -qwen3-4b,eqbench,comm_true_k2,3,2,2,96,205341,276686,2138.96875,2882.1458333333335 -qwen3-4b,eqbench,comm_true_k2,3,2,3,96,205341,280289,2138.96875,2919.6770833333335 -qwen3-4b,eqbench,comm_true_k24,25,24,1,96,721125,2410061,7511.71875,25104.802083333332 -qwen3-4b,eqbench,comm_true_k24,25,24,2,96,8339861,2082632,86873.55208333333,21694.083333333332 -qwen3-4b,eqbench,comm_true_k24,25,24,3,96,8339444,2098022,86869.20833333333,21854.395833333332 -qwen3-4b,eqbench,comm_true_k29,30,29,1,96,865350,2889662,9014.0625,30100.645833333332 -qwen3-4b,eqbench,comm_true_k29,30,29,2,96,11821781,2488151,123143.55208333333,25918.239583333332 -qwen3-4b,eqbench,comm_true_k29,30,29,3,96,11822211,2490051,123148.03125,25938.03125 -qwen3-4b,eqbench,comm_true_k4,5,4,1,96,144225,481321,1502.34375,5013.760416666667 -qwen3-4b,eqbench,comm_true_k4,5,4,2,96,462173,458241,4814.302083333333,4773.34375 -qwen3-4b,eqbench,comm_true_k4,5,4,3,96,462199,457711,4814.572916666667,4767.822916666667 -qwen3-4b,eqbench,comm_true_k6,7,6,1,96,201915,675741,2103.28125,7038.96875 -qwen3-4b,eqbench,comm_true_k6,7,6,2,96,814822,636138,8487.729166666666,6626.4375 -qwen3-4b,eqbench,comm_true_k6,7,6,3,96,814887,634852,8488.40625,6613.041666666667 -qwen3-4b,eqbench,comm_true_k9,10,9,1,96,288450,963309,3004.6875,10034.46875 -qwen3-4b,eqbench,comm_true_k9,10,9,2,96,1523747,894427,15872.364583333334,9316.947916666666 -qwen3-4b,eqbench,comm_true_k9,10,9,3,96,1523580,905114,15870.625,9428.270833333334 -qwen3-4b,eqbench,ensemble,1,0,0,96,28845,96699,300.46875,1007.28125 -qwen3-4b,eqbench,parity_decode_N5,1,0,0,96,30957,112823,322.46875,1175.2395833333333 -qwen3-4b,fermi,comm_true_k1,2,1,1,1587,367848,303162,231.7882797731569,191.02835538752362 -qwen3-4b,fermi,comm_true_k1,2,1,2,1587,1086682,377956,684.7397605545053,238.15752993068682 -qwen3-4b,fermi,comm_true_k1,2,1,3,1587,1153682,350322,726.9577819785759,220.74480151228732 -qwen3-4b,fermi,comm_true_k14,15,14,1,1587,2758860,2287906,1738.4120982986767,1441.6546943919345 -qwen3-4b,fermi,comm_true_k14,15,14,2,1587,41656630,3384533,26248.664146187777,2132.6609955891618 -qwen3-4b,fermi,comm_true_k14,15,14,3,1587,47215702,3363267,29751.545053560178,2119.2608695652175 -qwen3-4b,fermi,comm_true_k19,20,19,1,1587,3678480,3043981,2317.882797731569,1918.072463768116 -qwen3-4b,fermi,comm_true_k19,20,19,2,1587,72747792,4558362,45839.81852551985,2872.313799621928 -qwen3-4b,fermi,comm_true_k19,20,19,3,1587,82855563,4519466,52208.92438563327,2847.8046628859483 -qwen3-4b,fermi,comm_true_k2,3,2,1,1587,551772,456165,347.68241965973533,287.4385633270321 -qwen3-4b,fermi,comm_true_k2,3,2,2,1587,2142578,601549,1350.0806553245116,379.0478890989288 -qwen3-4b,fermi,comm_true_k2,3,2,3,1587,2323141,561447,1463.8569628229363,353.7788279773157 -qwen3-4b,fermi,comm_true_k24,25,24,1,1587,4598100,3800944,2897.3534971644613,2395.049779458097 -qwen3-4b,fermi,comm_true_k24,25,24,2,1587,112585560,5772318,70942.38185255199,3637.251417769376 -qwen3-4b,fermi,comm_true_k24,25,24,3,1587,128381096,5718571,80895.46061751732,3603.384373030876 -qwen3-4b,fermi,comm_true_k29,30,29,1,1587,5517720,4581520,3476.8241965973534,2886.906112161311 -qwen3-4b,fermi,comm_true_k29,30,29,2,1587,161021277,6969120,101462.68241965973,4391.3799621928165 -qwen3-4b,fermi,comm_true_k29,30,29,3,1587,183938859,6850034,115903.50283553875,4316.341524889729 -qwen3-4b,fermi,comm_true_k4,5,4,1,1587,919620,754162,579.4706994328923,475.21235034656587 -qwen3-4b,fermi,comm_true_k4,5,4,2,1587,5279532,1028127,3326.7372400756144,647.8431001890359 -qwen3-4b,fermi,comm_true_k4,5,4,3,1587,5839724,993464,3679.725267800882,626.0012602394455 -qwen3-4b,fermi,comm_true_k6,7,6,1,1587,1287468,1068133,811.2589792060492,673.0516698172653 -qwen3-4b,fermi,comm_true_k6,7,6,2,1587,9797434,1470085,6173.556395715186,926.3295526149968 -qwen3-4b,fermi,comm_true_k6,7,6,3,1587,10921105,1435470,6881.6036546943915,904.5179584120983 -qwen3-4b,fermi,comm_true_k9,10,9,1,1587,1839240,1516847,1158.9413988657845,955.7952110901072 -qwen3-4b,fermi,comm_true_k9,10,9,2,1587,19114706,2155916,12044.553245116573,1358.4851921865154 -qwen3-4b,fermi,comm_true_k9,10,9,3,1587,21514833,2128191,13556.920604914934,1341.015122873346 -qwen3-4b,fermi,ensemble,1,0,0,1587,183924,154391,115.89413988657844,97.2848141146818 -qwen3-4b,fermi,parity_decode_N5,1,0,0,1587,204555,1827155,128.89413988657844,1151.326402016383 -qwen3-4b,gsm8k,comm_true_k1,2,1,1,3957,997410,1501337,252.06216830932524,379.4129390952742 -qwen3-4b,gsm8k,comm_true_k1,2,1,2,3957,2791714,825152,705.512762193581,208.52969421278746 -qwen3-4b,gsm8k,comm_true_k1,2,1,3,3957,2880368,793047,727.9171089208996,200.41622441243365 -qwen3-4b,gsm8k,comm_true_k14,15,14,1,3957,7480575,11243615,1890.4662623199395,2841.449330300733 -qwen3-4b,gsm8k,comm_true_k14,15,14,2,3957,100609471,6778703,25425.69396007076,1713.091483447056 -qwen3-4b,gsm8k,comm_true_k14,15,14,3,3957,109892879,6617070,27771.76623704827,1672.2441243366186 -qwen3-4b,gsm8k,comm_true_k19,20,19,1,3957,9974100,15015057,2520.6216830932526,3794.5557240333587 -qwen3-4b,gsm8k,comm_true_k19,20,19,2,3957,175285549,9016014,44297.586302754615,2278.497346474602 -qwen3-4b,gsm8k,comm_true_k19,20,19,3,3957,192218883,8766708,48576.9226686884,2215.4935557240333 -qwen3-4b,gsm8k,comm_true_k2,3,2,1,3957,1496115,2263755,378.0932524639879,572.0887035633056 -qwen3-4b,gsm8k,comm_true_k2,3,2,2,3957,5407587,1309696,1366.587566338135,330.98205711397526 -qwen3-4b,gsm8k,comm_true_k2,3,2,3,3957,5706576,1261914,1442.1470811220622,318.9067475360121 -qwen3-4b,gsm8k,comm_true_k24,25,24,1,3957,12467625,18790504,3150.7771038665655,4748.674248167804 -qwen3-4b,gsm8k,comm_true_k24,25,24,2,3957,270573154,11321794,68378.35582512004,2861.206469547637 -qwen3-4b,gsm8k,comm_true_k24,25,24,3,3957,296708659,11010614,74983.23452110184,2782.566085418246 -qwen3-4b,gsm8k,comm_true_k29,30,29,1,3957,14961150,22489806,3780.932524639879,5683.549658832449 -qwen3-4b,gsm8k,comm_true_k29,30,29,2,3957,386421347,13575461,97655.12939095274,3430.745766995198 -qwen3-4b,gsm8k,comm_true_k29,30,29,3,3957,424058905,13163917,107166.76901693201,3326.7417235279254 -qwen3-4b,gsm8k,comm_true_k4,5,4,1,3957,2493525,3735399,630.1554207733132,943.9977255496589 -qwen3-4b,gsm8k,comm_true_k4,5,4,2,3957,13085023,2183384,3306.8038918372504,551.7776092999748 -qwen3-4b,gsm8k,comm_true_k4,5,4,3,3957,13995520,2100373,3536.901693201921,530.7993429365681 -qwen3-4b,gsm8k,comm_true_k6,7,6,1,3957,3490935,5287157,882.2175890826384,1336.1528936062673 -qwen3-4b,gsm8k,comm_true_k6,7,6,2,3957,24030057,3095283,6072.796815769522,782.229719484458 -qwen3-4b,gsm8k,comm_true_k6,7,6,3,3957,25842247,3002244,6530.767500631791,758.7172100075815 -qwen3-4b,gsm8k,comm_true_k9,10,9,1,3957,4987050,7499331,1260.3108415466263,1895.2062168309326 -qwen3-4b,gsm8k,comm_true_k9,10,9,2,3957,46536785,4465152,11760.622946676775,1128.4184988627749 -qwen3-4b,gsm8k,comm_true_k9,10,9,3,3957,50539795,4344918,12772.250442254233,1098.0333586050037 -qwen3-4b,gsm8k,ensemble,1,0,0,3957,498705,750727,126.03108415466262,189.72125347485468 -qwen3-4b,gsm8k,parity_decode_N5,1,0,0,3957,558060,3929940,141.03108415466264,993.1614859742228 -qwen3-4b,gsmhard,comm_true_k1,2,1,1,3051,796992,1132929,261.22320550639137,371.33038348082596 -qwen3-4b,gsmhard,comm_true_k1,2,1,2,3051,2290830,766345,750.8456243854474,251.1783021960013 -qwen3-4b,gsmhard,comm_true_k1,2,1,3,3051,2343834,732369,768.2182890855457,240.04228121927238 -qwen3-4b,gsmhard,comm_true_k14,15,14,1,3051,5977440,8602859,1959.174041297935,2819.6850213044904 -qwen3-4b,gsmhard,comm_true_k14,15,14,2,3051,84078789,6363261,27557.780727630285,2085.631268436578 -qwen3-4b,gsmhard,comm_true_k14,15,14,3,3051,88735900,6063941,29084.201901016062,1987.5257292690922 -qwen3-4b,gsmhard,comm_true_k19,20,19,1,3051,7969920,11478757,2612.2320550639133,3762.293346443789 -qwen3-4b,gsmhard,comm_true_k19,20,19,2,3051,146626309,8414979,48058.442805637496,2758.1052114060963 -qwen3-4b,gsmhard,comm_true_k19,20,19,3,3051,154768192,8035203,50727.03769255982,2633.62930186824 -qwen3-4b,gsmhard,comm_true_k2,3,2,1,3051,1195488,1741950,391.834808259587,570.9439528023598 -qwen3-4b,gsmhard,comm_true_k2,3,2,2,3051,4463104,1229052,1462.8331694526385,402.83579154375616 -qwen3-4b,gsmhard,comm_true_k2,3,2,3,3051,4623426,1173857,1515.3805309734514,384.74500163880697 -qwen3-4b,gsmhard,comm_true_k24,25,24,1,3051,9962400,14253843,3265.290068829892,4671.859390363815 -qwen3-4b,gsmhard,comm_true_k24,25,24,2,3051,226296936,10559341,74171.3982300885,3460.9442805637495 -qwen3-4b,gsmhard,comm_true_k24,25,24,3,3051,239214788,10084143,78405.37135365454,3305.1927236971483 -qwen3-4b,gsmhard,comm_true_k29,30,29,1,3051,11954880,17178812,3918.34808259587,5630.551294657489 -qwen3-4b,gsmhard,comm_true_k29,30,29,2,3051,323183773,12647777,105927.1625696493,4145.452966240577 -qwen3-4b,gsmhard,comm_true_k29,30,29,3,3051,341897825,11947458,112060.90626024254,3915.9154375614553 -qwen3-4b,gsmhard,comm_true_k4,5,4,1,3051,1992480,2878525,653.0580137659783,943.4693543100623 -qwen3-4b,gsmhard,comm_true_k4,5,4,2,3051,10859412,2053434,3559.2959685349065,673.0363815142576 -qwen3-4b,gsmhard,comm_true_k4,5,4,3,3051,11335831,1953933,3715.4477220583417,640.4237954768928 -qwen3-4b,gsmhard,comm_true_k6,7,6,1,3051,2789472,4020834,914.2812192723698,1317.874139626352 -qwen3-4b,gsmhard,comm_true_k6,7,6,2,3051,19993475,2921061,6553.089151098001,957.4110127826942 -qwen3-4b,gsmhard,comm_true_k6,7,6,3,3051,20914677,2769462,6855.023598820059,907.7227138643068 -qwen3-4b,gsmhard,comm_true_k9,10,9,1,3051,3984960,5751056,1306.1160275319567,1884.974106850213 -qwen3-4b,gsmhard,comm_true_k9,10,9,2,3051,38834945,4165178,12728.5955424451,1365.1845296624058 -qwen3-4b,gsmhard,comm_true_k9,10,9,3,3051,40735348,3981947,13351.474270730909,1305.1284824647657 -qwen3-4b,gsmhard,ensemble,1,0,0,3051,398496,567812,130.61160275319568,186.1068502130449 -qwen3-4b,gsmhard,parity_decode_N5,1,0,0,3051,444261,3531687,145.61160275319568,1157.5506391347099 -qwen3-4b,math500,comm_true_k1,2,1,1,1500,417294,607372,278.196,404.9146666666667 -qwen3-4b,math500,comm_true_k1,2,1,2,1500,1137906,454216,758.604,302.8106666666667 -qwen3-4b,math500,comm_true_k1,2,1,3,1500,1178676,432359,785.784,288.2393333333333 -qwen3-4b,math500,comm_true_k14,15,14,1,1500,3129705,4390803,2086.47,2927.202 -qwen3-4b,math500,comm_true_k14,15,14,2,1500,40690013,3411540,27126.675333333333,2274.36 -qwen3-4b,math500,comm_true_k14,15,14,3,1500,43550104,3242113,29033.402666666665,2161.4086666666667 -qwen3-4b,math500,comm_true_k19,20,19,1,1500,4172940,5920171,2781.96,3946.7806666666665 -qwen3-4b,math500,comm_true_k19,20,19,2,1500,70893645,4534222,47262.43,3022.8146666666667 -qwen3-4b,math500,comm_true_k19,20,19,3,1500,75976613,4256465,50651.075333333334,2837.6433333333334 -qwen3-4b,math500,comm_true_k2,3,2,1,1500,625941,879103,417.294,586.0686666666667 -qwen3-4b,math500,comm_true_k2,3,2,2,1500,2201274,694195,1467.516,462.7966666666667 -qwen3-4b,math500,comm_true_k2,3,2,3,1500,2304883,666802,1536.5886666666668,444.5346666666667 -qwen3-4b,math500,comm_true_k24,25,24,1,1500,5216175,7353031,3477.45,4902.020666666666 -qwen3-4b,math500,comm_true_k24,25,24,2,1500,109435453,5694422,72956.96866666667,3796.2813333333334 -qwen3-4b,math500,comm_true_k24,25,24,3,1500,117445031,5348185,78296.68733333334,3565.4566666666665 -qwen3-4b,math500,comm_true_k29,30,29,1,1500,6259410,8879850,4172.94,5919.9 -qwen3-4b,math500,comm_true_k29,30,29,2,1500,156178815,6821991,104119.21,4547.994 -qwen3-4b,math500,comm_true_k29,30,29,3,1500,167992053,6361408,111994.702,4240.938666666667 -qwen3-4b,math500,comm_true_k4,5,4,1,1500,1043235,1464003,695.49,976.002 -qwen3-4b,math500,comm_true_k4,5,4,2,1500,5310940,1118396,3540.6266666666666,745.5973333333334 -qwen3-4b,math500,comm_true_k4,5,4,3,1500,5597794,1087801,3731.862666666667,725.2006666666666 -qwen3-4b,math500,comm_true_k6,7,6,1,1500,1460529,2043232,973.686,1362.1546666666666 -qwen3-4b,math500,comm_true_k6,7,6,2,1500,9732833,1590010,6488.555333333334,1060.0066666666667 -qwen3-4b,math500,comm_true_k6,7,6,3,1500,10313186,1511466,6875.457333333334,1007.644 -qwen3-4b,math500,comm_true_k9,10,9,1,1500,2086470,2927619,1390.98,1951.746 -qwen3-4b,math500,comm_true_k9,10,9,2,1500,18843923,2246123,12562.615333333333,1497.4153333333334 -qwen3-4b,math500,comm_true_k9,10,9,3,1500,20043310,2128459,13362.206666666667,1418.9726666666666 -qwen3-4b,math500,ensemble,1,0,0,1500,208647,287719,139.098,191.81266666666667 -qwen3-4b,math500,parity_decode_N5,1,0,0,1500,231147,2287900,154.098,1525.2666666666667 -qwen3-4b,mmlu_hard,comm_true_k1,2,1,1,2172,714396,359495,328.91160220994476,165.5133517495396 -qwen3-4b,mmlu_hard,comm_true_k1,2,1,2,2172,1562426,408064,719.3489871086556,187.87476979742172 -qwen3-4b,mmlu_hard,comm_true_k1,2,1,3,2172,1679926,420680,773.4465930018416,193.68324125230203 -qwen3-4b,mmlu_hard,comm_true_k14,15,14,1,2172,5357970,2714474,2466.8370165745855,1249.7578268876612 -qwen3-4b,mmlu_hard,comm_true_k14,15,14,2,2172,48106994,3130085,22148.708103130753,1441.1072744014732 -qwen3-4b,mmlu_hard,comm_true_k14,15,14,3,2172,55822785,3227278,25701.098066298342,1485.8554327808472 -qwen3-4b,mmlu_hard,comm_true_k19,20,19,1,2172,7143960,3623913,3289.1160220994475,1668.4682320441989 -qwen3-4b,mmlu_hard,comm_true_k19,20,19,2,2172,82847499,4198784,38143.41574585636,1933.1418047882137 -qwen3-4b,mmlu_hard,comm_true_k19,20,19,3,2172,97005989,4343600,44662.05755064457,1999.815837937385 -qwen3-4b,mmlu_hard,comm_true_k2,3,2,1,2172,1071594,556389,493.3674033149171,256.164364640884 -qwen3-4b,mmlu_hard,comm_true_k2,3,2,2,2172,2903097,610746,1336.6008287292818,281.19060773480663 -qwen3-4b,mmlu_hard,comm_true_k2,3,2,3,2172,3161624,625715,1455.6279926335176,288.08241252302025 -qwen3-4b,mmlu_hard,comm_true_k24,25,24,1,2172,8929950,4510315,4111.3950276243095,2076.5722836095765 -qwen3-4b,mmlu_hard,comm_true_k24,25,24,2,2172,127101991,5294422,58518.4120626151,2437.5791896869246 -qwen3-4b,mmlu_hard,comm_true_k24,25,24,3,2172,150368301,5470475,69230.341160221,2518.6348987108654 -qwen3-4b,mmlu_hard,comm_true_k29,30,29,1,2172,10715940,5441477,4933.674033149171,2505.284069981584 -qwen3-4b,mmlu_hard,comm_true_k29,30,29,2,2172,180798998,6332063,83240.79097605894,2915.314456721915 -qwen3-4b,mmlu_hard,comm_true_k29,30,29,3,2172,213331207,6486293,98218.78775322283,2986.322744014733 -qwen3-4b,mmlu_hard,comm_true_k4,5,4,1,2172,1785990,906315,822.2790055248619,417.2720994475138 -qwen3-4b,mmlu_hard,comm_true_k4,5,4,2,2172,6694211,1033024,3082.0492633517497,475.609576427256 -qwen3-4b,mmlu_hard,comm_true_k4,5,4,3,2172,7511103,1044635,3458.1505524861877,480.95534069981585 -qwen3-4b,mmlu_hard,comm_true_k6,7,6,1,2172,2500386,1260544,1151.1906077348067,580.3609576427256 -qwen3-4b,mmlu_hard,comm_true_k6,7,6,2,2172,11978662,1439224,5515.037753222836,662.6261510128913 -qwen3-4b,mmlu_hard,comm_true_k6,7,6,3,2172,13552413,1467249,6239.600828729282,675.5290055248619 -qwen3-4b,mmlu_hard,comm_true_k9,10,9,1,2172,3571980,1801089,1644.5580110497237,829.2306629834254 -qwen3-4b,mmlu_hard,comm_true_k9,10,9,2,2172,22654866,2043100,10430.417127071823,940.6537753222836 -qwen3-4b,mmlu_hard,comm_true_k9,10,9,3,2172,25811599,2097500,11883.793278084715,965.6998158379374 -qwen3-4b,mmlu_hard,ensemble,1,0,0,2172,357198,180594,164.45580110497238,83.146408839779 -qwen3-4b,mmlu_hard,parity_decode_N5,1,0,0,2172,396294,1459805,182.45580110497238,672.1017495395948 -qwen3-8b,arc,comm_true_k1,2,1,1,3495,910254,344614,260.444635193133,98.60200286123033 -qwen3-8b,arc,comm_true_k1,2,1,2,3495,1969216,469217,563.4380543633763,134.253791130186 -qwen3-8b,arc,comm_true_k1,2,1,3,3495,2226168,485584,636.957939914163,138.93676680972817 -qwen3-8b,arc,comm_true_k14,15,14,1,3495,6826905,2582472,1953.3347639484978,738.9047210300429 -qwen3-8b,arc,comm_true_k14,15,14,2,3495,58583790,3694005,16762.171673819743,1056.9399141630902 -qwen3-8b,arc,comm_true_k14,15,14,3,3495,75421160,3954714,21579.73104434907,1131.5347639484978 -qwen3-8b,arc,comm_true_k19,20,19,1,3495,9102540,3441796,2604.4463519313304,984.7771101573677 -qwen3-8b,arc,comm_true_k19,20,19,2,3495,100810880,4901650,28844.314735336193,1402.474964234621 -qwen3-8b,arc,comm_true_k19,20,19,3,3495,130265382,5305553,37271.92618025751,1518.0409155937052 -qwen3-8b,arc,comm_true_k2,3,2,1,3495,1365381,514829,390.66695278969956,147.30443490701 -qwen3-8b,arc,comm_true_k2,3,2,2,3495,3628275,724343,1038.1330472103004,207.25121602288985 -qwen3-8b,arc,comm_true_k2,3,2,3,3495,4257446,725213,1218.1533619456366,207.50014306151644 -qwen3-8b,arc,comm_true_k24,25,24,1,3495,11378175,4307035,3255.557939914163,1232.3419170243205 -qwen3-8b,arc,comm_true_k24,25,24,2,3495,154357275,6228117,44165.17167381974,1782.0077253218883 -qwen3-8b,arc,comm_true_k24,25,24,3,3495,202831727,6707809,58034.82889842632,1919.2586552217454 -qwen3-8b,arc,comm_true_k29,30,29,1,3495,13653810,5153998,3906.6695278969955,1474.677539341917 -qwen3-8b,arc,comm_true_k29,30,29,2,3495,219193680,7528469,62716.36051502146,2154.068383404864 -qwen3-8b,arc,comm_true_k29,30,29,3,3495,290743210,8079149,83188.32904148784,2311.630615164521 -qwen3-8b,arc,comm_true_k4,5,4,1,3495,2275635,862803,651.1115879828326,246.86781115879828 -qwen3-8b,arc,comm_true_k4,5,4,2,3495,8279965,1205510,2369.0886981402004,344.9241773962804 -qwen3-8b,arc,comm_true_k4,5,4,3,3495,10022756,1274240,2867.7413447782546,364.58941344778253 -qwen3-8b,arc,comm_true_k6,7,6,1,3495,3185889,1206887,911.5562231759657,345.3181688125894 -qwen3-8b,arc,comm_true_k6,7,6,2,3495,14711151,1714513,4209.199141630901,490.5616595135908 -qwen3-8b,arc,comm_true_k6,7,6,3,3495,18323772,1833792,5242.85321888412,524.6901287553648 -qwen3-8b,arc,comm_true_k9,10,9,1,3495,4551270,1716045,1302.2231759656652,491.0 -qwen3-8b,arc,comm_true_k9,10,9,2,3495,27711700,2464691,7928.9556509299,705.2048640915593 -qwen3-8b,arc,comm_true_k9,10,9,3,3495,35231440,2640308,10080.526466380543,755.4529327610873 -qwen3-8b,arc,ensemble,1,0,0,3495,455127,173810,130.2223175965665,49.7310443490701 -qwen3-8b,arc,parity_decode_N5,1,0,0,3495,518037,916834,148.2223175965665,262.32732474964234 -qwen3-8b,dat,comm_true_k1,2,1,1,4500,1539000,711917,342.0,158.2037777777778 -qwen3-8b,dat,comm_true_k1,2,1,2,4500,3796640,1041502,843.6977777777778,231.4448888888889 -qwen3-8b,dat,comm_true_k1,2,1,3,4500,4006558,1033857,890.3462222222222,229.746 -qwen3-8b,dat,comm_true_k14,15,14,1,4500,11542500,5339396,2565.0,1186.5324444444445 -qwen3-8b,dat,comm_true_k14,15,14,2,4500,127820399,7494808,28404.53311111111,1665.512888888889 -qwen3-8b,dat,comm_true_k14,15,14,3,4500,139775808,7980742,31061.290666666668,1773.4982222222222 -qwen3-8b,dat,comm_true_k19,20,19,1,4501,15393420,7111035,3420.0,1579.8789157964898 -qwen3-8b,dat,comm_true_k19,20,19,2,4501,221702950,9645616,49256.37636080871,2142.994001333037 -qwen3-8b,dat,comm_true_k19,20,19,3,4500,242824765,10567645,53961.05888888889,2348.3655555555556 -qwen3-8b,dat,comm_true_k2,3,2,1,4500,2308500,1062667,513.0,236.14822222222222 -qwen3-8b,dat,comm_true_k2,3,2,2,4500,7206530,1543057,1601.451111111111,342.90155555555555 -qwen3-8b,dat,comm_true_k2,3,2,3,4500,7692003,1540244,1709.334,342.27644444444445 -qwen3-8b,dat,comm_true_k24,25,24,1,4500,19237500,8936598,4275.0,1985.9106666666667 -qwen3-8b,dat,comm_true_k24,25,24,2,4500,342035007,12226051,76007.77933333334,2716.900222222222 -qwen3-8b,dat,comm_true_k24,25,24,3,4500,374249910,13064751,83166.64666666667,2903.278 -qwen3-8b,dat,comm_true_k29,30,29,1,4500,23085000,10714541,5130.0,2381.009111111111 -qwen3-8b,dat,comm_true_k29,30,29,2,4500,487297796,14547391,108288.39911111111,3232.7535555555555 -qwen3-8b,dat,comm_true_k29,30,29,3,4500,533759537,15552150,118613.23044444445,3456.0333333333333 -qwen3-8b,dat,comm_true_k4,5,4,1,4500,3847500,1767444,855.0,392.76533333333333 -qwen3-8b,dat,comm_true_k4,5,4,2,4500,17055418,2496665,3790.092888888889,554.8144444444445 -qwen3-8b,dat,comm_true_k4,5,4,3,4500,18417838,2564914,4092.852888888889,569.9808888888889 -qwen3-8b,dat,comm_true_k6,7,6,1,4500,5386500,2473254,1197.0,549.612 -qwen3-8b,dat,comm_true_k6,7,6,2,4500,30971192,3417291,6882.4871111111115,759.398 -qwen3-8b,dat,comm_true_k6,7,6,3,4500,33623847,3593853,7471.966,798.634 -qwen3-8b,dat,comm_true_k9,10,9,1,4500,7695000,3558574,1710.0,790.7942222222222 -qwen3-8b,dat,comm_true_k9,10,9,2,4500,59582157,4882751,13240.479333333333,1085.0557777777778 -qwen3-8b,dat,comm_true_k9,10,9,3,4500,64868272,5235376,14415.171555555555,1163.416888888889 -qwen3-8b,dat,ensemble,1,0,0,4500,769500,355619,171.0,79.02644444444445 -qwen3-8b,dat,parity_decode_N5,1,0,0,4500,859500,1453331,191.0,322.96244444444443 -qwen3-8b,eqbench,comm_true_k1,2,1,1,96,57690,195309,600.9375,2034.46875 -qwen3-8b,eqbench,comm_true_k1,2,1,2,96,112934,190399,1176.3958333333333,1983.3229166666667 -qwen3-8b,eqbench,comm_true_k1,2,1,3,96,112936,190399,1176.4166666666667,1983.3229166666667 -qwen3-8b,eqbench,comm_true_k14,15,14,1,96,432675,1465182,4507.03125,15262.3125 -qwen3-8b,eqbench,comm_true_k14,15,14,2,96,3192053,1284206,33250.552083333336,13377.145833333334 -qwen3-8b,eqbench,comm_true_k14,15,14,3,96,3191871,1303444,33248.65625,13577.541666666666 -qwen3-8b,eqbench,comm_true_k19,20,19,1,96,576900,1952794,6009.375,20341.604166666668 -qwen3-8b,eqbench,comm_true_k19,20,19,2,96,5464524,1695727,56922.125,17663.822916666668 -qwen3-8b,eqbench,comm_true_k19,20,19,3,96,5463930,1712592,56915.9375,17839.5 -qwen3-8b,eqbench,comm_true_k2,3,2,1,96,86535,292474,901.40625,3046.6041666666665 -qwen3-8b,eqbench,comm_true_k2,3,2,2,96,205359,282609,2139.15625,2943.84375 -qwen3-8b,eqbench,comm_true_k2,3,2,3,96,205342,286132,2138.9791666666665,2980.5416666666665 -qwen3-8b,eqbench,comm_true_k24,25,24,1,96,721125,2440440,7511.71875,25421.25 -qwen3-8b,eqbench,comm_true_k24,25,24,2,96,8341034,2070790,86885.77083333333,21570.729166666668 -qwen3-8b,eqbench,comm_true_k24,25,24,3,96,8340986,2109879,86885.27083333333,21977.90625 -qwen3-8b,eqbench,comm_true_k29,30,29,1,96,865350,2928566,9014.0625,30505.895833333332 -qwen3-8b,eqbench,comm_true_k29,30,29,2,96,11822016,2441687,123146.0,25434.239583333332 -qwen3-8b,eqbench,comm_true_k29,30,29,3,96,11822076,2466486,123146.625,25692.5625 -qwen3-8b,eqbench,comm_true_k4,5,4,1,96,144225,488063,1502.34375,5083.989583333333 -qwen3-8b,eqbench,comm_true_k4,5,4,2,96,462118,463552,4813.729166666667,4828.666666666667 -qwen3-8b,eqbench,comm_true_k4,5,4,3,96,462074,465302,4813.270833333333,4846.895833333333 -qwen3-8b,eqbench,comm_true_k6,7,6,1,96,201915,682731,2103.28125,7111.78125 -qwen3-8b,eqbench,comm_true_k6,7,6,2,96,814854,633316,8488.0625,6597.041666666667 -qwen3-8b,eqbench,comm_true_k6,7,6,3,96,814870,639353,8488.229166666666,6659.927083333333 -qwen3-8b,eqbench,comm_true_k9,10,9,1,96,288450,976966,3004.6875,10176.729166666666 -qwen3-8b,eqbench,comm_true_k9,10,9,2,96,1523818,884797,15873.104166666666,9216.635416666666 -qwen3-8b,eqbench,comm_true_k9,10,9,3,96,1523700,887580,15871.875,9245.625 -qwen3-8b,eqbench,ensemble,1,0,0,96,28845,97687,300.46875,1017.5729166666666 -qwen3-8b,eqbench,parity_decode_N5,1,0,0,96,30957,130642,322.46875,1360.8541666666667 -qwen3-8b,fermi,comm_true_k1,2,1,1,1587,453546,625743,285.7882797731569,394.2930056710775 -qwen3-8b,fermi,comm_true_k1,2,1,2,1587,1218622,449749,767.877756773787,283.3957151858853 -qwen3-8b,fermi,comm_true_k1,2,1,3,1587,1248224,418670,786.5305608065532,263.8122243226213 -qwen3-8b,fermi,comm_true_k14,15,14,1,1587,3401595,4665902,2143.412098298677,2940.0768746061754 -qwen3-8b,fermi,comm_true_k14,15,14,2,1587,45240684,3759692,28507.047258979204,2369.0560806553244 -qwen3-8b,fermi,comm_true_k14,15,14,3,1587,47814755,3633524,30129.020163831126,2289.5551354757404 -qwen3-8b,fermi,comm_true_k19,20,19,1,1587,4535460,6238235,2857.882797731569,3930.8349086326402 -qwen3-8b,fermi,comm_true_k19,20,19,2,1587,78995367,4888388,49776.53875236295,3080.2696912413357 -qwen3-8b,fermi,comm_true_k19,20,19,3,1587,83422070,4760741,52565.89161940769,2999.8367989918083 -qwen3-8b,fermi,comm_true_k2,3,2,1,1587,680319,928569,428.68241965973533,585.109640831758 -qwen3-8b,fermi,comm_true_k2,3,2,2,1587,2377534,716874,1498.1310649023314,451.7164461247637 -qwen3-8b,fermi,comm_true_k2,3,2,3,1587,2470369,659063,1556.628229363579,415.2885948330183 -qwen3-8b,fermi,comm_true_k24,25,24,1,1587,5669325,7763673,3572.3534971644613,4892.04347826087 -qwen3-8b,fermi,comm_true_k24,25,24,2,1587,121997209,6106088,76872.8475110271,3847.56647763075 -qwen3-8b,fermi,comm_true_k24,25,24,3,1587,128839556,5963539,81184.34530560806,3757.743541272842 -qwen3-8b,fermi,comm_true_k29,30,29,1,1588,6806310,9337666,4286.089420654912,5880.142317380352 -qwen3-8b,fermi,comm_true_k29,30,29,2,1587,174300115,7158277,109829.94013862633,4510.571518588532 -qwen3-8b,fermi,comm_true_k29,30,29,3,1587,183973798,7063390,115925.51858853182,4450.781348456207 -qwen3-8b,fermi,comm_true_k4,5,4,1,1587,1133865,1565117,714.4706994328923,986.2110901071203 -qwen3-8b,fermi,comm_true_k4,5,4,2,1587,5811869,1209705,3662.1732829237553,762.2589792060492 -qwen3-8b,fermi,comm_true_k4,5,4,3,1587,6071027,1153002,3825.473850031506,726.5293005671077 -qwen3-8b,fermi,comm_true_k6,7,6,1,1587,1587411,2171578,1000.2589792060492,1368.354127284184 -qwen3-8b,fermi,comm_true_k6,7,6,2,1587,10722284,1733209,6756.322621298046,1092.1291745431631 -qwen3-8b,fermi,comm_true_k6,7,6,3,1587,11255269,1673634,7092.166981726528,1054.5897920604914 -qwen3-8b,fermi,comm_true_k9,10,9,1,1587,2267730,3128317,1428.9413988657845,1971.214240705734 -qwen3-8b,fermi,comm_true_k9,10,9,2,1587,20850099,2419773,13138.058601134215,1524.7466918714556 -qwen3-8b,fermi,comm_true_k9,10,9,3,1587,21936005,2332844,13822.30938878387,1469.9710144927535 -qwen3-8b,fermi,ensemble,1,0,0,1587,226773,313514,142.89413988657844,197.5513547574039 -qwen3-8b,fermi,parity_decode_N5,1,0,0,1587,223599,1635267,140.89413988657844,1030.413988657845 -qwen3-8b,gsm8k,comm_true_k1,2,1,1,3957,997410,1639683,252.06216830932524,414.37528430629266 -qwen3-8b,gsm8k,comm_true_k1,2,1,2,3957,2895280,1082268,731.6856204195097,273.50720242608037 -qwen3-8b,gsm8k,comm_true_k1,2,1,3,3957,2977854,984822,752.5534495830175,248.88097043214557 -qwen3-8b,gsm8k,comm_true_k14,15,14,1,3957,7480575,12397434,1890.4662623199395,3133.0386656558 -qwen3-8b,gsm8k,comm_true_k14,15,14,2,3957,106435557,8159518,26898.04321455648,2062.0464998736416 -qwen3-8b,gsm8k,comm_true_k14,15,14,3,3957,113621379,7657446,28714.02047005307,1935.1645185746777 -qwen3-8b,gsm8k,comm_true_k19,20,19,1,3957,9974100,16346134,2520.6216830932526,4130.941117007834 -qwen3-8b,gsm8k,comm_true_k19,20,19,2,3957,185532358,10810880,46887.12610563558,2732.0899671468283 -qwen3-8b,gsm8k,comm_true_k19,20,19,3,3957,198782438,10143684,50235.64265857973,2563.478392721759 -qwen3-8b,gsm8k,comm_true_k2,3,2,1,3957,1496115,2465548,378.0932524639879,623.0851655294415 -qwen3-8b,gsm8k,comm_true_k2,3,2,2,3957,5641398,1658888,1425.6755117513267,419.22870861763965 -qwen3-8b,gsm8k,comm_true_k2,3,2,3,3957,5899835,1523877,1490.9868587313622,385.10917361637604 -qwen3-8b,gsm8k,comm_true_k24,25,24,1,3957,12467625,20524932,3150.7771038665655,5186.993176648976 -qwen3-8b,gsm8k,comm_true_k24,25,24,2,3957,286492761,13271871,72401.50644427596,3354.0235026535256 -qwen3-8b,gsm8k,comm_true_k24,25,24,3,3957,307031944,12527210,77592.10108668183,3165.8352287086177 -qwen3-8b,gsm8k,comm_true_k29,30,29,1,3957,14961150,24595002,3780.932524639879,6215.567854435179 -qwen3-8b,gsm8k,comm_true_k29,30,29,2,3957,409443155,15711894,103473.12484205206,3970.658074298711 -qwen3-8b,gsm8k,comm_true_k29,30,29,3,3957,438336106,14771060,110774.8562041951,3732.8936062673743 -qwen3-8b,gsm8k,comm_true_k4,5,4,1,3957,2493525,4132328,630.1554207733132,1044.3083143795805 -qwen3-8b,gsm8k,comm_true_k4,5,4,2,3957,13731374,2748190,3470.147586555471,694.5135203436947 -qwen3-8b,gsm8k,comm_true_k4,5,4,3,3957,14458266,2570269,3653.8453373768007,649.5499115491534 -qwen3-8b,gsm8k,comm_true_k6,7,6,1,3957,3490935,5700702,882.2175890826384,1440.6626231993935 -qwen3-8b,gsm8k,comm_true_k6,7,6,2,3957,25261057,3855024,6383.891079100328,974.2289613343441 -qwen3-8b,gsm8k,comm_true_k6,7,6,3,3957,26796620,3614526,6771.953500126358,913.4510993176649 -qwen3-8b,gsm8k,comm_true_k9,10,9,1,3957,4987050,8151596,1260.3108415466263,2060.044478140005 -qwen3-8b,gsm8k,comm_true_k9,10,9,2,3957,49093488,5428611,12406.744503411675,1371.9006823351024 -qwen3-8b,gsm8k,comm_true_k9,10,9,3,3957,52247964,5114179,13203.933282789993,1292.4384634824362 -qwen3-8b,gsm8k,ensemble,1,0,0,3957,498705,819982,126.03108415466262,207.22314885013898 -qwen3-8b,gsm8k,parity_decode_N5,1,0,0,3957,558060,2072619,141.03108415466264,523.7854435178165 -qwen3-8b,gsmhard,comm_true_k1,2,1,1,3051,796992,1169349,261.22320550639137,383.26745329400194 -qwen3-8b,gsmhard,comm_true_k1,2,1,2,3051,2350612,1054896,770.4398557849885,345.7541789577188 -qwen3-8b,gsmhard,comm_true_k1,2,1,3,3051,2397456,970080,785.7935103244838,317.95476892822023 -qwen3-8b,gsmhard,comm_true_k14,15,14,1,3051,5977440,8835589,1959.174041297935,2895.964929531301 -qwen3-8b,gsmhard,comm_true_k14,15,14,2,3051,87411198,7847383,28650.015732546704,2572.0691576532286 -qwen3-8b,gsmhard,comm_true_k14,15,14,3,3051,90839046,7451031,29773.531956735496,2442.1602753195675 -qwen3-8b,gsmhard,comm_true_k19,20,19,1,3051,7969920,11835438,2612.2320550639133,3879.1996066863326 -qwen3-8b,gsmhard,comm_true_k19,20,19,2,3051,152511760,10336251,49987.466404457555,3387.8239921337267 -qwen3-8b,gsmhard,comm_true_k19,20,19,3,3051,158716696,9838962,52021.20485086857,3224.83185840708 -qwen3-8b,gsmhard,comm_true_k2,3,2,1,3051,1195488,1795500,391.834808259587,588.4955752212389 -qwen3-8b,gsmhard,comm_true_k2,3,2,2,3051,4598132,1597586,1507.0901343821697,523.627007538512 -qwen3-8b,gsmhard,comm_true_k2,3,2,3,3051,4725875,1509445,1548.959357587676,494.73779088823335 -qwen3-8b,gsmhard,comm_true_k24,25,24,1,3051,9962400,14747836,3265.290068829892,4833.771222549984 -qwen3-8b,gsmhard,comm_true_k24,25,24,2,3051,235547247,12689453,77203.2930186824,4159.112749918059 -qwen3-8b,gsmhard,comm_true_k24,25,24,3,3051,245114379,11963373,80339.02949852508,3921.131760078663 -qwen3-8b,gsmhard,comm_true_k29,30,29,1,3051,11954880,17762896,3918.34808259587,5821.991478203868 -qwen3-8b,gsmhard,comm_true_k29,30,29,2,3051,336658680,14962957,110343.7168141593,4904.2795804654215 -qwen3-8b,gsmhard,comm_true_k29,30,29,3,3051,350017435,14052188,114722.2009177319,4605.76466732219 -qwen3-8b,gsmhard,comm_true_k4,5,4,1,3051,1992480,2949438,653.0580137659783,966.7118977384464 -qwen3-8b,gsmhard,comm_true_k4,5,4,2,3051,11226992,2614062,3679.7745001638805,856.7885939036381 -qwen3-8b,gsmhard,comm_true_k4,5,4,3,3051,11579916,2459464,3795.44936086529,806.1173385775156 -qwen3-8b,gsmhard,comm_true_k6,7,6,1,3051,2789472,4113791,914.2812192723698,1348.3418551294658 -qwen3-8b,gsmhard,comm_true_k6,7,6,2,3051,20702101,3680735,6785.349393641429,1206.4028187479514 -qwen3-8b,gsmhard,comm_true_k6,7,6,3,3051,21442021,3503386,7027.866601114389,1148.2746640445755 -qwen3-8b,gsmhard,comm_true_k9,10,9,1,3051,3984960,5906893,1306.1160275319567,1936.0514585381843 -qwen3-8b,gsmhard,comm_true_k9,10,9,2,3051,40291844,5215086,13206.11078334972,1709.3038348082596 -qwen3-8b,gsmhard,comm_true_k9,10,9,3,3051,41780870,4919217,13694.156014421502,1612.329400196657 -qwen3-8b,gsmhard,ensemble,1,0,0,3051,398496,602427,130.61160275319568,197.45231071779745 -qwen3-8b,gsmhard,parity_decode_N5,1,0,0,3051,444261,1847139,145.61160275319568,605.4208456243855 -qwen3-8b,math500,comm_true_k1,2,1,1,1500,417294,1272540,278.196,848.36 -qwen3-8b,math500,comm_true_k1,2,1,2,1500,1194712,714815,796.4746666666666,476.54333333333335 -qwen3-8b,math500,comm_true_k1,2,1,3,1500,1201260,580896,800.84,387.264 -qwen3-8b,math500,comm_true_k14,15,14,1,1500,3129705,9599071,2086.47,6399.380666666667 -qwen3-8b,math500,comm_true_k14,15,14,2,1500,43915727,4992843,29277.151333333335,3328.562 -qwen3-8b,math500,comm_true_k14,15,14,3,1500,44798405,4204345,29865.603333333333,2802.8966666666665 -qwen3-8b,math500,comm_true_k19,20,19,1,1500,4172940,12809775,2781.96,8539.85 -qwen3-8b,math500,comm_true_k19,20,19,2,1500,76574140,6532200,51049.426666666666,4354.8 -qwen3-8b,math500,comm_true_k19,20,19,3,1500,78289675,5520096,52193.11666666667,3680.064 -qwen3-8b,math500,comm_true_k2,3,2,1,1500,625941,1918817,417.294,1279.2113333333334 -qwen3-8b,math500,comm_true_k2,3,2,2,1500,2327840,1061270,1551.8933333333334,707.5133333333333 -qwen3-8b,math500,comm_true_k2,3,2,3,1500,2347294,858673,1564.8626666666667,572.4486666666667 -qwen3-8b,math500,comm_true_k24,25,24,1,1500,5216175,15989645,3477.45,10659.763333333334 -qwen3-8b,math500,comm_true_k24,25,24,2,1500,118219360,8063794,78812.90666666666,5375.862666666667 -qwen3-8b,math500,comm_true_k24,25,24,3,1500,120915149,6954091,80610.09933333333,4636.060666666666 -qwen3-8b,math500,comm_true_k29,30,29,1,1500,6259410,19206569,4172.94,12804.379333333334 -qwen3-8b,math500,comm_true_k29,30,29,2,1500,168944611,9390614,112629.74066666666,6260.409333333333 -qwen3-8b,math500,comm_true_k29,30,29,3,1500,172743383,8063492,115162.25533333333,5375.6613333333335 -qwen3-8b,math500,comm_true_k4,5,4,1,1500,1043235,3193453,695.49,2128.9686666666666 -qwen3-8b,math500,comm_true_k4,5,4,2,1500,5666862,1656404,3777.908,1104.2693333333334 -qwen3-8b,math500,comm_true_k4,5,4,3,1500,5731324,1380048,3820.882666666667,920.032 -qwen3-8b,math500,comm_true_k6,7,6,1,1500,1460529,4481709,973.686,2987.806 -qwen3-8b,math500,comm_true_k6,7,6,2,1500,10432227,2355027,6954.818,1570.018 -qwen3-8b,math500,comm_true_k6,7,6,3,1500,10579018,1951870,7052.678666666667,1301.2466666666667 -qwen3-8b,math500,comm_true_k9,10,9,1,1500,2086470,6392948,1390.98,4261.965333333334 -qwen3-8b,math500,comm_true_k9,10,9,2,1500,20272584,3310547,13515.056,2207.0313333333334 -qwen3-8b,math500,comm_true_k9,10,9,3,1500,20605324,2763123,13736.882666666666,1842.082 -qwen3-8b,math500,ensemble,1,0,0,1500,208647,639203,139.098,426.13533333333334 -qwen3-8b,math500,parity_decode_N5,1,0,0,1500,231147,1783622,154.098,1189.0813333333333 -qwen3-8b,mmlu_hard,comm_true_k1,2,1,1,2172,714396,385077,328.91160220994476,177.2914364640884 -qwen3-8b,mmlu_hard,comm_true_k1,2,1,2,2172,1580064,456429,727.4696132596686,210.14226519337018 -qwen3-8b,mmlu_hard,comm_true_k1,2,1,3,2172,1724262,459565,793.8591160220994,211.58609576427256 -qwen3-8b,mmlu_hard,comm_true_k14,15,14,1,2172,5357970,2867749,2466.8370165745855,1320.3264272559852 -qwen3-8b,mmlu_hard,comm_true_k14,15,14,2,2172,49078059,3457905,22595.79143646409,1592.0372928176796 -qwen3-8b,mmlu_hard,comm_true_k14,15,14,3,2172,57953099,3508012,26681.90561694291,1615.1068139963168 -qwen3-8b,mmlu_hard,comm_true_k19,20,19,1,2172,7143960,3826354,3289.1160220994475,1761.6731123388581 -qwen3-8b,mmlu_hard,comm_true_k19,20,19,2,2172,84734263,4540223,39012.09162062615,2090.3420810313078 -qwen3-8b,mmlu_hard,comm_true_k19,20,19,3,2172,100360114,4651780,46206.313996316756,2141.70349907919 -qwen3-8b,mmlu_hard,comm_true_k2,3,2,1,2172,1071594,575270,493.3674033149171,264.8572744014733 -qwen3-8b,mmlu_hard,comm_true_k2,3,2,2,2172,2939806,700505,1353.5018416206262,322.51611418047884 -qwen3-8b,mmlu_hard,comm_true_k2,3,2,3,2172,3280714,690911,1510.4576427255986,318.0989871086556 -qwen3-8b,mmlu_hard,comm_true_k24,25,24,1,2172,8929950,4766214,4111.3950276243095,2194.3895027624308 -qwen3-8b,mmlu_hard,comm_true_k24,25,24,2,2172,129927999,5696631,59819.520718232045,2622.758287292818 -qwen3-8b,mmlu_hard,comm_true_k24,25,24,3,2172,155277715,5799206,71490.66068139963,2669.9843462246777 -qwen3-8b,mmlu_hard,comm_true_k29,30,29,1,2172,10715940,5703603,4933.674033149171,2625.968232044199 -qwen3-8b,mmlu_hard,comm_true_k29,30,29,2,2172,184814130,6809010,85089.37845303868,3134.903314917127 -qwen3-8b,mmlu_hard,comm_true_k29,30,29,3,2172,221316534,6907783,101895.27348066299,3180.3789134438307 -qwen3-8b,mmlu_hard,comm_true_k4,5,4,1,2172,1785990,954300,822.2790055248619,439.3646408839779 -qwen3-8b,mmlu_hard,comm_true_k4,5,4,2,2172,6799109,1137147,3130.344843462247,523.5483425414365 -qwen3-8b,mmlu_hard,comm_true_k4,5,4,3,2172,7753956,1145103,3569.961325966851,527.2113259668508 -qwen3-8b,mmlu_hard,comm_true_k6,7,6,1,2172,2500386,1351700,1151.1906077348067,622.329650092081 -qwen3-8b,mmlu_hard,comm_true_k6,7,6,2,2172,12188980,1601417,5611.869244935543,737.3006445672191 -qwen3-8b,mmlu_hard,comm_true_k6,7,6,3,2172,14074773,1627083,6480.098066298343,749.1174033149172 -qwen3-8b,mmlu_hard,comm_true_k9,10,9,1,2172,3571980,1924716,1644.5580110497237,886.1491712707183 -qwen3-8b,mmlu_hard,comm_true_k9,10,9,2,2172,23112867,2277600,10641.283149171271,1048.6187845303868 -qwen3-8b,mmlu_hard,comm_true_k9,10,9,3,2172,27006183,2320793,12433.78591160221,1068.5050644567218 -qwen3-8b,mmlu_hard,ensemble,1,0,0,2172,357198,188699,164.45580110497238,86.8779926335175 -qwen3-8b,mmlu_hard,parity_decode_N5,1,0,0,2172,396294,1254916,182.45580110497238,577.7697974217311 -qwen3.6-27b,arc,comm_true_k1,2,1,1,300,80730,39630,269.1,132.1 -qwen3.6-27b,arc,comm_true_k1,2,1,2,300,196032,64585,653.44,215.28333333333333 -qwen3.6-27b,arc,comm_true_k1,2,1,3,300,232184,69839,773.9466666666667,232.79666666666665 -qwen3.6-27b,arc,comm_true_k2,3,2,1,300,121095,60035,403.65,200.11666666666667 -qwen3.6-27b,arc,comm_true_k2,3,2,2,300,368529,95278,1228.43,317.5933333333333 -qwen3.6-27b,arc,comm_true_k2,3,2,3,300,447378,101599,1491.26,338.66333333333336 -qwen3.6-27b,arc,comm_true_k4,5,4,1,300,201825,99398,672.75,331.32666666666665 -qwen3.6-27b,arc,comm_true_k4,5,4,2,300,862795,156164,2875.983333333333,520.5466666666666 -qwen3.6-27b,arc,comm_true_k4,5,4,3,300,1083065,166383,3610.2166666666667,554.61 -qwen3.6-27b,arc,ensemble,1,0,0,300,40365,19998,134.55,66.66 -qwen3.6-27b,arc,parity_decode_N5,1,0,0,300,45765,131178,152.55,437.26 -qwen3.6-27b,dat,comm_true_k1,2,1,1,300,109200,47777,364.0,159.25666666666666 -qwen3.6-27b,dat,comm_true_k1,2,1,2,300,261552,96359,871.84,321.19666666666666 -qwen3.6-27b,dat,comm_true_k1,2,1,3,300,275982,90009,919.94,300.03 -qwen3.6-27b,dat,comm_true_k2,3,2,1,300,163800,71689,546.0,238.96333333333334 -qwen3.6-27b,dat,comm_true_k2,3,2,2,300,493581,156545,1645.27,521.8166666666667 -qwen3.6-27b,dat,comm_true_k2,3,2,3,300,526485,152691,1754.95,508.97 -qwen3.6-27b,dat,comm_true_k4,5,4,1,300,273000,119953,910.0,399.8433333333333 -qwen3.6-27b,dat,comm_true_k4,5,4,2,300,1163510,279837,3878.366666666667,932.79 -qwen3.6-27b,dat,comm_true_k4,5,4,3,300,1252375,276462,4174.583333333333,921.54 -qwen3.6-27b,dat,ensemble,1,0,0,300,54600,23723,182.0,79.07666666666667 -qwen3.6-27b,dat,parity_decode_N5,1,0,0,300,60600,135056,202.0,450.18666666666667 -qwen3.6-27b,eqbench,comm_true_k1,2,1,1,96,58434,196608,608.6875,2048.0 -qwen3.6-27b,eqbench,comm_true_k1,2,1,2,96,113910,196605,1186.5625,2047.96875 -qwen3.6-27b,eqbench,comm_true_k1,2,1,3,96,113916,196608,1186.625,2048.0 -qwen3.6-27b,eqbench,comm_true_k2,3,2,1,96,87651,294912,913.03125,3072.0 -qwen3.6-27b,eqbench,comm_true_k2,3,2,2,96,206874,294797,2154.9375,3070.8020833333335 -qwen3.6-27b,eqbench,comm_true_k2,3,2,3,96,206880,294912,2155.0,3072.0 -qwen3.6-27b,eqbench,comm_true_k4,5,4,1,96,146085,491520,1521.71875,5120.0 -qwen3.6-27b,eqbench,comm_true_k4,5,4,2,96,464750,491146,4841.145833333333,5116.104166666667 -qwen3.6-27b,eqbench,comm_true_k4,5,4,3,96,464760,491470,4841.25,5119.479166666667 -qwen3.6-27b,eqbench,ensemble,1,0,0,96,29217,98304,304.34375,1024.0 -qwen3.6-27b,eqbench,parity_decode_N5,1,0,0,96,31329,139248,326.34375,1450.5 -qwen3.6-27b,fermi,comm_true_k1,2,1,1,300,124914,529473,416.38,1764.91 -qwen3.6-27b,fermi,comm_true_k1,2,1,2,300,274824,323036,916.08,1076.7866666666666 -qwen3.6-27b,fermi,comm_true_k1,2,1,3,300,274704,259273,915.68,864.2433333333333 -qwen3.6-27b,fermi,comm_true_k2,3,2,1,300,187371,787196,624.57,2623.9866666666667 -qwen3.6-27b,fermi,comm_true_k2,3,2,2,300,524658,519205,1748.86,1730.6833333333334 -qwen3.6-27b,fermi,comm_true_k2,3,2,3,300,524475,408626,1748.25,1362.0866666666666 -qwen3.6-27b,fermi,comm_true_k4,5,4,1,300,312285,1320336,1040.95,4401.12 -qwen3.6-27b,fermi,comm_true_k4,5,4,2,300,1249095,911769,4163.65,3039.23 -qwen3.6-27b,fermi,comm_true_k4,5,4,3,300,1248895,698511,4162.983333333334,2328.37 -qwen3.6-27b,fermi,ensemble,1,0,0,300,62457,265859,208.19,886.1966666666667 -qwen3.6-27b,fermi,parity_decode_N5,1,0,0,300,51657,601737,172.19,2005.79 -qwen3.6-27b,gsm8k,comm_true_k1,2,1,1,1519,397496,862429,261.68268597761687,567.7610269914418 -qwen3.6-27b,gsm8k,comm_true_k1,2,1,2,1519,1160204,528482,763.7946017116524,347.9144173798552 -qwen3.6-27b,gsm8k,comm_true_k1,2,1,3,1519,1186758,490074,781.2758393680052,322.62936142198816 -qwen3.6-27b,gsm8k,comm_true_k14,15,14,1,1319,2590860,5680881,1964.2608036391205,4306.960576194087 -qwen3.6-27b,gsm8k,comm_true_k14,15,14,2,1319,36949665,3735442,28013.392721758908,2832.0257771038664 -qwen3.6-27b,gsm8k,comm_true_k14,15,14,3,1319,38943060,3293035,29524.685367702805,2496.614859742229 -qwen3.6-27b,gsm8k,comm_true_k19,20,19,1,1319,3454480,7585656,2619.0144048521606,5751.065959059893 -qwen3.6-27b,gsm8k,comm_true_k19,20,19,2,1319,64408220,5044655,48831.0993176649,3824.6057619408643 -qwen3.6-27b,gsm8k,comm_true_k19,20,19,3,1319,68089240,4588546,51621.86504927976,3478.806671721001 -qwen3.6-27b,gsm8k,comm_true_k2,3,2,1,1519,596244,1301025,392.5240289664253,856.500987491771 -qwen3.6-27b,gsm8k,comm_true_k2,3,2,2,1519,2255445,816835,1484.8222514812376,537.7452271231073 -qwen3.6-27b,gsm8k,comm_true_k2,3,2,3,1519,2337024,731662,1538.5279789335088,481.6734693877551 -qwen3.6-27b,gsm8k,comm_true_k24,25,24,1,1319,4318100,9481362,3273.768006065201,7188.295678544352 -qwen3.6-27b,gsm8k,comm_true_k24,25,24,2,1319,99440925,6324566,75391.14859742229,4794.970432145565 -qwen3.6-27b,gsm8k,comm_true_k24,25,24,3,1319,105077725,5652594,79664.68915845337,4285.514783927218 -qwen3.6-27b,gsm8k,comm_true_k29,30,29,1,1319,5181720,11328439,3928.521607278241,8588.657316148598 -qwen3.6-27b,gsm8k,comm_true_k29,30,29,2,1319,142060860,7624444,107703.45716451858,5780.473085670963 -qwen3.6-27b,gsm8k,comm_true_k29,30,29,3,1319,150304380,6937201,113953.28278999242,5259.439727065959 -qwen3.6-27b,gsm8k,comm_true_k4,5,4,1,1519,993740,2147789,654.2067149440421,1413.9493087557603 -qwen3.6-27b,gsm8k,comm_true_k4,5,4,2,1519,5481880,1349425,3608.8742593811717,888.3640552995391 -qwen3.6-27b,gsm8k,comm_true_k4,5,4,3,1519,5668180,1211212,3731.5207373271887,797.3745885450954 -qwen3.6-27b,gsm8k,comm_true_k6,7,6,1,1319,1209068,2653340,916.6550416982562,2011.6300227445033 -qwen3.6-27b,gsm8k,comm_true_k6,7,6,2,1319,8787226,1671924,6662.036391205459,1267.5693707354055 -qwen3.6-27b,gsm8k,comm_true_k6,7,6,3,1319,9118620,1476667,6913.282789992419,1119.5352539802882 -qwen3.6-27b,gsm8k,comm_true_k9,10,9,1,1319,1727240,3760588,1309.5072024260803,2851.090219863533 -qwen3.6-27b,gsm8k,comm_true_k9,10,9,2,1319,17053050,2427381,12928.77179681577,1840.3191811978772 -qwen3.6-27b,gsm8k,comm_true_k9,10,9,3,1319,17831170,2152858,13518.703563305535,1632.1895375284307 -qwen3.6-27b,gsm8k,ensemble,1,0,0,1519,198748,427829,130.84134298880844,281.65174456879527 -qwen3.6-27b,gsm8k,parity_decode_N5,1,0,0,1519,223052,1581039,146.84134298880844,1040.8420013166558 -qwen3.6-27b,gsmhard,comm_true_k1,2,1,1,1217,330406,1126422,271.4921939194741,925.5727198027937 -qwen3.6-27b,gsmhard,comm_true_k1,2,1,2,1217,975386,838842,801.4675431388661,689.2703368940016 -qwen3.6-27b,gsmhard,comm_true_k1,2,1,3,1217,985062,783578,809.41824157765,643.8603122432211 -qwen3.6-27b,gsmhard,comm_true_k2,3,2,1,620,253032,848945,408.1161290322581,1369.266129032258 -qwen3.6-27b,gsmhard,comm_true_k2,3,2,2,620,972849,666014,1569.1112903225805,1074.216129032258 -qwen3.6-27b,gsmhard,comm_true_k2,3,2,3,620,985647,586337,1589.7532258064516,945.7048387096775 -qwen3.6-27b,gsmhard,comm_true_k4,5,4,1,300,205260,662475,684.2,2208.25 -qwen3.6-27b,gsmhard,comm_true_k4,5,4,2,300,1146325,561181,3821.0833333333335,1870.6033333333332 -qwen3.6-27b,gsmhard,comm_true_k4,5,4,3,300,1161865,522261,3872.883333333333,1740.87 -qwen3.6-27b,gsmhard,ensemble,1,0,0,1217,165203,548290,135.74609695973706,450.52588331963847 -qwen3.6-27b,gsmhard,parity_decode_N5,1,0,0,300,45852,539656,152.84,1798.8533333333332 -qwen3.6-27b,math500,comm_true_k1,2,1,1,300,87786,286096,292.62,953.6533333333333 -qwen3.6-27b,math500,comm_true_k1,2,1,2,300,247760,313377,825.8666666666667,1044.59 -qwen3.6-27b,math500,comm_true_k1,2,1,3,300,251116,326938,837.0533333333333,1089.7933333333333 -qwen3.6-27b,math500,comm_true_k2,3,2,1,300,131679,425650,438.93,1418.8333333333333 -qwen3.6-27b,math500,comm_true_k2,3,2,2,300,481284,471905,1604.28,1573.0166666666667 -qwen3.6-27b,math500,comm_true_k2,3,2,3,300,488754,478693,1629.18,1595.6433333333334 -qwen3.6-27b,math500,comm_true_k4,5,4,1,300,219465,710721,731.55,2369.07 -qwen3.6-27b,math500,comm_true_k4,5,4,2,300,1166165,784763,3887.2166666666667,2615.8766666666666 -qwen3.6-27b,math500,comm_true_k4,5,4,3,300,1184815,762916,3949.383333333333,2543.0533333333333 -qwen3.6-27b,math500,ensemble,1,0,0,300,43893,142456,146.31,474.85333333333335 -qwen3.6-27b,math500,parity_decode_N5,1,0,0,300,48693,626155,162.31,2087.1833333333334 -qwen3.6-27b,mmlu_hard,comm_true_k1,2,1,1,300,104280,151837,347.6,506.12333333333333 -qwen3.6-27b,mmlu_hard,comm_true_k1,2,1,2,300,249798,231494,832.66,771.6466666666666 -qwen3.6-27b,mmlu_hard,comm_true_k1,2,1,3,300,269098,246074,896.9933333333333,820.2466666666667 -qwen3.6-27b,mmlu_hard,comm_true_k2,3,2,1,300,156420,219307,521.4,731.0233333333333 -qwen3.6-27b,mmlu_hard,comm_true_k2,3,2,2,300,470109,344899,1567.03,1149.6633333333334 -qwen3.6-27b,mmlu_hard,comm_true_k2,3,2,3,300,515289,365173,1717.63,1217.2433333333333 -qwen3.6-27b,mmlu_hard,comm_true_k4,5,4,1,300,260700,368328,869.0,1227.76 -qwen3.6-27b,mmlu_hard,comm_true_k4,5,4,2,300,1106970,554118,3689.9,1847.06 -qwen3.6-27b,mmlu_hard,comm_true_k4,5,4,3,300,1229625,588142,4098.75,1960.4733333333334 -qwen3.6-27b,mmlu_hard,ensemble,1,0,0,300,52140,70828,173.8,236.09333333333333 -qwen3.6-27b,mmlu_hard,parity_decode_N5,1,0,0,300,57540,364717,191.8,1215.7233333333334 -r1-distill-qwen-14b,arc,comm_true_k1,2,1,1,1165,289438,1024195,248.44463519313305,879.137339055794 -r1-distill-qwen-14b,arc,comm_true_k1,2,1,2,1165,927218,1004034,795.8952789699571,861.8317596566524 -r1-distill-qwen-14b,arc,comm_true_k1,2,1,3,1165,927242,961952,795.9158798283262,825.7098712446352 -r1-distill-qwen-14b,arc,comm_true_k14,15,14,1,1165,2170785,7659265,1863.3347639484978,6574.476394849786 -r1-distill-qwen-14b,arc,comm_true_k14,15,14,2,1165,35405428,7354358,30390.92532188841,6312.7536480686695 -r1-distill-qwen-14b,arc,comm_true_k14,15,14,3,1165,35405948,7111749,30391.371673819744,6104.505579399141 -r1-distill-qwen-14b,arc,comm_true_k19,20,19,1,1165,2894380,10171961,2484.4463519313304,8731.296995708155 -r1-distill-qwen-14b,arc,comm_true_k19,20,19,2,1165,61870600,9761858,53107.811158798286,8379.277253218885 -r1-distill-qwen-14b,arc,comm_true_k19,20,19,3,1165,61870669,9494720,53107.87038626609,8149.974248927038 -r1-distill-qwen-14b,arc,comm_true_k2,3,2,1,1165,434157,1525933,372.66695278969956,1309.8137339055795 -r1-distill-qwen-14b,arc,comm_true_k2,3,2,2,1165,1827177,1509342,1568.3922746781116,1295.5725321888413 -r1-distill-qwen-14b,arc,comm_true_k2,3,2,3,1165,1827287,1451974,1568.48669527897,1246.3296137339055 -r1-distill-qwen-14b,arc,comm_true_k24,25,24,1,1166,3620800,12704310,3105.3173241852487,10895.634648370498 -r1-distill-qwen-14b,arc,comm_true_k24,25,24,2,1165,95664218,12149817,82115.20858369098,10429.027467811158 -r1-distill-qwen-14b,arc,comm_true_k24,25,24,3,1165,95668689,11776462,82119.04635193133,10108.551072961372 -r1-distill-qwen-14b,arc,comm_true_k29,30,29,1,1165,4341570,15265169,3726.6695278969955,13103.149356223175 -r1-distill-qwen-14b,arc,comm_true_k29,30,29,2,1165,136789969,14768945,117416.28240343348,12677.206008583691 -r1-distill-qwen-14b,arc,comm_true_k29,30,29,3,1165,136794764,14332822,117420.3982832618,12302.851502145922 -r1-distill-qwen-14b,arc,comm_true_k4,5,4,1,1165,723595,2546374,621.1115879828326,2185.728755364807 -r1-distill-qwen-14b,arc,comm_true_k4,5,4,2,1165,4499795,2495151,3862.4849785407723,2141.760515021459 -r1-distill-qwen-14b,arc,comm_true_k4,5,4,3,1165,4499866,2415465,3862.545922746781,2073.3605150214594 -r1-distill-qwen-14b,arc,comm_true_k6,7,6,1,1165,1013033,3560078,869.5562231759657,3055.8609442060088 -r1-distill-qwen-14b,arc,comm_true_k6,7,6,2,1165,8336498,3499200,7155.792274678111,3003.6051502145924 -r1-distill-qwen-14b,arc,comm_true_k6,7,6,3,1165,8336529,3369466,7155.818884120172,2892.245493562232 -r1-distill-qwen-14b,arc,comm_true_k9,10,9,1,1165,1447190,5086637,1242.2231759656652,4366.212017167382 -r1-distill-qwen-14b,arc,comm_true_k9,10,9,2,1165,16272371,4985238,13967.70042918455,4279.174248927038 -r1-distill-qwen-14b,arc,comm_true_k9,10,9,3,1165,16273030,4823369,13968.2660944206,4140.230901287554 -r1-distill-qwen-14b,arc,ensemble,1,0,0,1165,144719,512451,124.22231759656653,439.87210300429183 -r1-distill-qwen-14b,arc,parity_decode_N5,1,0,0,1165,165689,755999,142.2223175965665,648.9261802575107 -r1-distill-qwen-14b,dat,comm_true_k1,2,1,1,1500,495000,1172404,330.0,781.6026666666667 -r1-distill-qwen-14b,dat,comm_true_k1,2,1,2,1500,1318314,1646967,878.876,1097.978 -r1-distill-qwen-14b,dat,comm_true_k1,2,1,3,1500,1318366,1624320,878.9106666666667,1082.88 -r1-distill-qwen-14b,dat,comm_true_k14,15,14,1,1500,3712500,8750120,2475.0,5833.413333333333 -r1-distill-qwen-14b,dat,comm_true_k14,15,14,2,1500,46479347,10540315,30986.231333333333,7026.876666666667 -r1-distill-qwen-14b,dat,comm_true_k14,15,14,3,1500,46480800,11130833,30987.2,7420.555333333334 -r1-distill-qwen-14b,dat,comm_true_k19,20,19,1,1500,4950000,11672659,3300.0,7781.772666666667 -r1-distill-qwen-14b,dat,comm_true_k19,20,19,2,1500,80828262,14393785,53885.508,9595.856666666667 -r1-distill-qwen-14b,dat,comm_true_k19,20,19,3,1500,80831152,15585689,53887.43466666667,10390.459333333334 -r1-distill-qwen-14b,dat,comm_true_k2,3,2,1,1500,742500,1744321,495.0,1162.8806666666667 -r1-distill-qwen-14b,dat,comm_true_k2,3,2,2,1500,2538594,2424678,1692.396,1616.452 -r1-distill-qwen-14b,dat,comm_true_k2,3,2,3,1500,2538660,2489823,1692.44,1659.882 -r1-distill-qwen-14b,dat,comm_true_k24,25,24,1,1500,6187500,14625265,4125.0,9750.176666666666 -r1-distill-qwen-14b,dat,comm_true_k24,25,24,2,1500,124602749,16985515,83068.49933333334,11323.676666666666 -r1-distill-qwen-14b,dat,comm_true_k24,25,24,3,1500,124604785,17368970,83069.85666666667,11579.313333333334 -r1-distill-qwen-14b,dat,comm_true_k29,30,29,1,1500,7425000,17616813,4950.0,11744.542 -r1-distill-qwen-14b,dat,comm_true_k29,30,29,2,1500,177820639,20225946,118547.09266666666,13483.964 -r1-distill-qwen-14b,dat,comm_true_k29,30,29,3,1500,177813429,20828835,118542.286,13885.89 -r1-distill-qwen-14b,dat,comm_true_k4,5,4,1,1501,1238325,2932247,825.0,1953.528980679547 -r1-distill-qwen-14b,dat,comm_true_k4,5,4,2,1501,6106100,3874944,4068.021319120586,2581.5749500333113 -r1-distill-qwen-14b,dat,comm_true_k4,5,4,3,1500,6102324,3991127,4068.216,2660.751333333333 -r1-distill-qwen-14b,dat,comm_true_k6,7,6,1,1500,1732500,4093586,1155.0,2729.057333333333 -r1-distill-qwen-14b,dat,comm_true_k6,7,6,2,1500,11161554,5199155,7441.036,3466.1033333333335 -r1-distill-qwen-14b,dat,comm_true_k6,7,6,3,1500,11161232,5341133,7440.821333333333,3560.7553333333335 -r1-distill-qwen-14b,dat,comm_true_k9,10,9,1,1500,2475000,5848640,1650.0,3899.0933333333332 -r1-distill-qwen-14b,dat,comm_true_k9,10,9,2,1500,21557987,7189885,14371.991333333333,4793.256666666667 -r1-distill-qwen-14b,dat,comm_true_k9,10,9,3,1500,21557977,7504106,14371.984666666667,5002.7373333333335 -r1-distill-qwen-14b,dat,ensemble,1,0,0,1500,247500,586431,165.0,390.954 -r1-distill-qwen-14b,dat,parity_decode_N5,1,0,0,1500,277500,862224,185.0,574.816 -r1-distill-qwen-14b,eqbench,comm_true_k1,2,1,1,32,18846,65535,588.9375,2047.96875 -r1-distill-qwen-14b,eqbench,comm_true_k1,2,1,2,32,37264,65536,1164.5,2048.0 -r1-distill-qwen-14b,eqbench,comm_true_k1,2,1,3,32,37260,65486,1164.375,2046.4375 -r1-distill-qwen-14b,eqbench,comm_true_k14,15,14,1,32,141345,491520,4417.03125,15360.0 -r1-distill-qwen-14b,eqbench,comm_true_k14,15,14,2,32,1060965,491313,33155.15625,15353.53125 -r1-distill-qwen-14b,eqbench,comm_true_k14,15,14,3,32,1061070,490513,33158.4375,15328.53125 -r1-distill-qwen-14b,eqbench,comm_true_k19,20,19,1,32,188460,655360,5889.375,20480.0 -r1-distill-qwen-14b,eqbench,comm_true_k19,20,19,2,32,1817580,654289,56799.375,20446.53125 -r1-distill-qwen-14b,eqbench,comm_true_k19,20,19,3,32,1817538,654567,56798.0625,20455.21875 -r1-distill-qwen-14b,eqbench,comm_true_k2,3,2,1,32,28269,98304,883.40625,3072.0 -r1-distill-qwen-14b,eqbench,comm_true_k2,3,2,2,32,67878,97908,2121.1875,3059.625 -r1-distill-qwen-14b,eqbench,comm_true_k2,3,2,3,32,67863,98012,2120.71875,3062.875 -r1-distill-qwen-14b,eqbench,comm_true_k24,25,24,1,32,235575,819103,7361.71875,25596.96875 -r1-distill-qwen-14b,eqbench,comm_true_k24,25,24,2,32,2775575,817551,86736.71875,25548.46875 -r1-distill-qwen-14b,eqbench,comm_true_k24,25,24,3,32,2775450,816465,86732.8125,25514.53125 -r1-distill-qwen-14b,eqbench,comm_true_k29,30,29,1,32,282690,982978,8834.0625,30718.0625 -r1-distill-qwen-14b,eqbench,comm_true_k29,30,29,2,32,3934708,982627,122959.625,30707.09375 -r1-distill-qwen-14b,eqbench,comm_true_k29,30,29,3,32,3934379,980464,122949.34375,30639.5 -r1-distill-qwen-14b,eqbench,comm_true_k4,5,4,1,32,47115,163840,1472.34375,5120.0 -r1-distill-qwen-14b,eqbench,comm_true_k4,5,4,2,32,153098,163521,4784.3125,5110.03125 -r1-distill-qwen-14b,eqbench,comm_true_k4,5,4,3,32,153056,163621,4783.0,5113.15625 -r1-distill-qwen-14b,eqbench,comm_true_k6,7,6,1,32,65961,229376,2061.28125,7168.0 -r1-distill-qwen-14b,eqbench,comm_true_k6,7,6,2,32,270277,229355,8446.15625,7167.34375 -r1-distill-qwen-14b,eqbench,comm_true_k6,7,6,3,32,270102,229024,8440.6875,7157.0 -r1-distill-qwen-14b,eqbench,comm_true_k9,10,9,1,32,94230,327680,2944.6875,10240.0 -r1-distill-qwen-14b,eqbench,comm_true_k9,10,9,2,32,506010,326540,15812.8125,10204.375 -r1-distill-qwen-14b,eqbench,comm_true_k9,10,9,3,32,506030,326919,15813.4375,10216.21875 -r1-distill-qwen-14b,eqbench,ensemble,1,0,0,32,9423,32768,294.46875,1024.0 -r1-distill-qwen-14b,eqbench,parity_decode_N5,1,0,0,32,10127,61706,316.46875,1928.3125 -r1-distill-qwen-14b,fermi,comm_true_k1,2,1,1,529,116268,918310,219.7882797731569,1735.9357277882798 -r1-distill-qwen-14b,fermi,comm_true_k1,2,1,2,529,392076,897481,741.1644612476371,1696.5614366729678 -r1-distill-qwen-14b,fermi,comm_true_k1,2,1,3,529,392080,873699,741.17202268431,1651.6049149338373 -r1-distill-qwen-14b,fermi,comm_true_k14,15,14,1,529,1514745,7856845,2863.412098298677,14852.25897920605 -r1-distill-qwen-14b,fermi,comm_true_k14,15,14,2,529,16421387,7005441,31042.319470699433,13242.799621928167 -r1-distill-qwen-14b,fermi,comm_true_k14,15,14,3,529,16419559,6690104,31038.863894139886,12646.69943289225 -r1-distill-qwen-14b,fermi,comm_true_k19,20,19,1,529,2019660,10472847,3817.882797731569,19797.44234404537 -r1-distill-qwen-14b,fermi,comm_true_k19,20,19,2,529,28553401,9332194,53976.18336483932,17641.198487712667 -r1-distill-qwen-14b,fermi,comm_true_k19,20,19,3,529,28549297,8892163,53968.425330812854,16809.381852551986 -r1-distill-qwen-14b,fermi,comm_true_k2,3,2,1,529,174402,1358052,329.68241965973533,2567.2060491493385 -r1-distill-qwen-14b,fermi,comm_true_k2,3,2,2,529,786159,1327648,1486.1228733459357,2509.7315689981097 -r1-distill-qwen-14b,fermi,comm_true_k2,3,2,3,529,786105,1297611,1486.0207939508507,2452.9508506616257 -r1-distill-qwen-14b,fermi,comm_true_k24,25,24,1,529,2524575,13107329,4772.353497164461,24777.559546313798 -r1-distill-qwen-14b,fermi,comm_true_k24,25,24,2,529,44011658,11722753,83197.84120982987,22160.21361058601 -r1-distill-qwen-14b,fermi,comm_true_k24,25,24,3,529,44008893,11101170,83192.61436672969,20985.198487712667 -r1-distill-qwen-14b,fermi,comm_true_k29,30,29,1,529,3029490,15695609,5726.824196597354,29670.338374291114 -r1-distill-qwen-14b,fermi,comm_true_k29,30,29,2,529,62801454,13869188,118717.30434782608,26217.746691871456 -r1-distill-qwen-14b,fermi,comm_true_k29,30,29,3,529,62797616,13141232,118710.04914933837,24841.648393194708 -r1-distill-qwen-14b,fermi,comm_true_k4,5,4,1,529,290670,2269006,549.4706994328923,4289.23629489603 -r1-distill-qwen-14b,fermi,comm_true_k4,5,4,2,529,1970665,2199185,3725.2646502835537,4157.2495274102075 -r1-distill-qwen-14b,fermi,comm_true_k4,5,4,3,529,1970760,2105360,3725.444234404537,3979.8865784499053 -r1-distill-qwen-14b,fermi,comm_true_k6,7,6,1,529,406938,3171093,769.2589792060492,5994.50472589792 -r1-distill-qwen-14b,fermi,comm_true_k6,7,6,2,529,3683229,3110451,6962.625708884688,5879.869565217391 -r1-distill-qwen-14b,fermi,comm_true_k6,7,6,3,529,3683297,2989885,6962.754253308129,5651.95652173913 -r1-distill-qwen-14b,fermi,comm_true_k9,10,9,1,529,743340,4827421,1405.179584120983,9125.5595463138 -r1-distill-qwen-14b,fermi,comm_true_k9,10,9,2,529,7385801,4548018,13961.81663516068,8597.38752362949 -r1-distill-qwen-14b,fermi,comm_true_k9,10,9,3,529,7384792,4378601,13959.909262759924,8277.12854442344 -r1-distill-qwen-14b,fermi,ensemble,1,0,0,529,58134,456384,109.89413988657844,862.7296786389414 -r1-distill-qwen-14b,fermi,parity_decode_N5,1,0,0,529,84055,1614294,158.89413988657844,3051.5954631379964 -r1-distill-qwen-14b,gsm8k,comm_true_k1,2,1,1,2638,633284,2485265,240.06216830932524,942.1019711902957 -r1-distill-qwen-14b,gsm8k,comm_true_k1,2,1,2,2638,2060416,2443611,781.0523123578469,926.3119787717968 -r1-distill-qwen-14b,gsm8k,comm_true_k1,2,1,3,2638,2060202,2357557,780.9711902956785,893.691053828658 -r1-distill-qwen-14b,gsm8k,comm_true_k14,15,14,1,1319,2374815,9346810,1800.4662623199395,7086.28506444276 -r1-distill-qwen-14b,gsm8k,comm_true_k14,15,14,2,1319,39912884,9059434,30259.95754359363,6868.410917361638 -r1-distill-qwen-14b,gsm8k,comm_true_k14,15,14,3,1319,39910970,8657863,30258.506444275965,6563.9598180439725 -r1-distill-qwen-14b,gsm8k,comm_true_k19,20,19,1,1319,3166420,12442905,2400.6216830932526,9433.589840788476 -r1-distill-qwen-14b,gsm8k,comm_true_k19,20,19,2,1319,69804188,12010559,52922.05307050796,9105.806671721 -r1-distill-qwen-14b,gsm8k,comm_true_k19,20,19,3,1319,69799433,11463758,52918.44806671721,8691.249431387414 -r1-distill-qwen-14b,gsm8k,comm_true_k2,3,2,1,2638,949926,3744039,360.0932524639879,1419.2717968157694 -r1-distill-qwen-14b,gsm8k,comm_true_k2,3,2,2,2638,4078233,3659188,1545.9564063684609,1387.106899166035 -r1-distill-qwen-14b,gsm8k,comm_true_k2,3,2,3,2638,4077753,3513571,1545.7744503411675,1331.907126611069 -r1-distill-qwen-14b,gsm8k,comm_true_k24,25,24,1,1319,3958025,15569136,3000.7771038665655,11803.742228961335 -r1-distill-qwen-14b,gsm8k,comm_true_k24,25,24,2,1319,107988406,14756120,81871.42228961334,11187.354056103108 -r1-distill-qwen-14b,gsm8k,comm_true_k24,25,24,3,1319,107977427,13962741,81863.09855951478,10585.853677028052 -r1-distill-qwen-14b,gsm8k,comm_true_k29,30,29,1,1319,4749630,18681735,3600.932524639879,14163.559514783927 -r1-distill-qwen-14b,gsm8k,comm_true_k29,30,29,2,1319,154463939,17764129,117106.85291887794,13467.876421531464 -r1-distill-qwen-14b,gsm8k,comm_true_k29,30,29,3,1319,154452464,16843613,117098.15314632298,12769.987111448067 -r1-distill-qwen-14b,gsm8k,comm_true_k4,5,4,1,2380,1429735,5641558,600.7289915966387,2370.4025210084033 -r1-distill-qwen-14b,gsm8k,comm_true_k4,5,4,2,2379,9098600,5491205,3824.548129466162,2308.1988230348884 -r1-distill-qwen-14b,gsm8k,comm_true_k4,5,4,3,2379,9097773,5215871,3824.200504413619,2192.4636401849516 -r1-distill-qwen-14b,gsm8k,comm_true_k6,7,6,1,1319,1108247,4344627,840.2175890826384,3293.879454131918 -r1-distill-qwen-14b,gsm8k,comm_true_k6,7,6,2,1319,9365412,4244128,7100.3881728582255,3217.686125852919 -r1-distill-qwen-14b,gsm8k,comm_true_k6,7,6,3,1319,9364609,4016498,7099.779378316906,3045.108415466262 -r1-distill-qwen-14b,gsm8k,comm_true_k9,10,9,1,1319,1583210,6233861,1200.3108415466263,4726.202426080364 -r1-distill-qwen-14b,gsm8k,comm_true_k9,10,9,2,1319,18314452,6117410,13885.10386656558,4637.915087187263 -r1-distill-qwen-14b,gsm8k,comm_true_k9,10,9,3,1319,18313964,5851125,13884.733889310084,4436.0310841546625 -r1-distill-qwen-14b,gsm8k,ensemble,1,0,0,2638,316642,1246465,120.03108415466262,472.5037907505686 -r1-distill-qwen-14b,gsm8k,parity_decode_N5,1,0,0,1319,178106,1161256,135.03108415466264,880.4063684609553 -r1-distill-qwen-14b,gsmhard,comm_true_k1,2,1,1,1018,253664,1439087,249.1787819253438,1413.6414538310412 -r1-distill-qwen-14b,gsmhard,comm_true_k1,2,1,2,1018,804430,1334393,790.2062868369352,1310.7986247544204 -r1-distill-qwen-14b,gsmhard,comm_true_k1,2,1,3,1017,803668,1276345,790.2340216322517,1255.0098328416911 -r1-distill-qwen-14b,gsmhard,comm_true_k14,15,14,1,1017,1900950,10808605,1869.174041297935,10627.930186823993 -r1-distill-qwen-14b,gsmhard,comm_true_k14,15,14,2,1017,30849852,9761677,30334.171091445427,9598.502458210423 -r1-distill-qwen-14b,gsmhard,comm_true_k14,15,14,3,1017,30844931,9387569,30329.332350049164,9230.647984267453 -r1-distill-qwen-14b,gsmhard,comm_true_k19,20,19,1,1017,2534600,14397960,2492.2320550639133,14157.286135693215 -r1-distill-qwen-14b,gsmhard,comm_true_k19,20,19,2,1017,53922131,12996623,53020.77777777778,12779.373647984268 -r1-distill-qwen-14b,gsmhard,comm_true_k19,20,19,3,1017,53917658,12449891,53016.379547689285,12241.780727630285 -r1-distill-qwen-14b,gsmhard,comm_true_k2,3,2,1,1017,380190,2168696,373.834808259587,2132.4444444444443 -r1-distill-qwen-14b,gsmhard,comm_true_k2,3,2,2,1017,1586275,1981041,1559.7590953785643,1947.9262536873157 -r1-distill-qwen-14b,gsmhard,comm_true_k2,3,2,3,1017,1586143,1904146,1559.62930186824,1872.3166175024583 -r1-distill-qwen-14b,gsmhard,comm_true_k24,25,24,1,1017,3168250,18015312,3115.290068829892,17714.171091445427 -r1-distill-qwen-14b,gsmhard,comm_true_k24,25,24,2,1017,83391794,15886758,81997.8308751229,15621.197640117995 -r1-distill-qwen-14b,gsmhard,comm_true_k24,25,24,3,1017,83385672,15192071,81991.81120943953,14938.12291052114 -r1-distill-qwen-14b,gsmhard,comm_true_k29,30,29,1,1017,3801900,21616744,3738.34808259587,21255.40216322517 -r1-distill-qwen-14b,gsmhard,comm_true_k29,30,29,2,1017,119257148,19138564,117263.6656833825,18818.647000983285 -r1-distill-qwen-14b,gsmhard,comm_true_k29,30,29,3,1017,119245370,18254945,117252.08456243854,17949.79842674533 -r1-distill-qwen-14b,gsmhard,comm_true_k4,5,4,1,1017,633650,3606035,623.0580137659783,3545.7571288102263 -r1-distill-qwen-14b,gsmhard,comm_true_k4,5,4,2,1017,3912555,3276264,3847.1533923303837,3221.4985250737464 -r1-distill-qwen-14b,gsmhard,comm_true_k4,5,4,3,1017,3912338,3134261,3846.9400196656834,3081.8692232055064 -r1-distill-qwen-14b,gsmhard,comm_true_k6,7,6,1,1017,887110,5030556,872.2812192723698,4946.466076696165 -r1-distill-qwen-14b,gsmhard,comm_true_k6,7,6,2,1017,7253985,4578529,7132.728613569321,4501.995083579154 -r1-distill-qwen-14b,gsmhard,comm_true_k6,7,6,3,1017,7253586,4377304,7132.33628318584,4304.133726647001 -r1-distill-qwen-14b,gsmhard,comm_true_k9,10,9,1,1017,1267300,7205972,1246.1160275319567,7085.518190757129 -r1-distill-qwen-14b,gsmhard,comm_true_k9,10,9,2,1017,14170174,6590451,13933.307767944936,6480.286135693215 -r1-distill-qwen-14b,gsmhard,comm_true_k9,10,9,3,1017,14168563,6341614,13931.723697148476,6235.608652900688 -r1-distill-qwen-14b,gsmhard,ensemble,1,0,0,1017,126730,720824,124.61160275319567,708.7748279252704 -r1-distill-qwen-14b,gsmhard,parity_decode_N5,1,0,0,1017,141985,1986576,139.61160275319568,1953.3687315634218 -r1-distill-qwen-14b,math500,comm_true_k1,2,1,1,500,133098,814633,266.196,1629.266 -r1-distill-qwen-14b,math500,comm_true_k1,2,1,2,500,403632,795204,807.264,1590.408 -r1-distill-qwen-14b,math500,comm_true_k1,2,1,3,500,403558,786742,807.116,1573.484 -r1-distill-qwen-14b,math500,comm_true_k14,15,14,1,500,998235,6106098,1996.47,12212.196 -r1-distill-qwen-14b,math500,comm_true_k14,15,14,2,500,15229602,5902132,30459.204,11804.264 -r1-distill-qwen-14b,math500,comm_true_k14,15,14,3,500,15225770,5753578,30451.54,11507.156 -r1-distill-qwen-14b,math500,comm_true_k19,20,19,1,500,1330980,8150607,2661.96,16301.214 -r1-distill-qwen-14b,math500,comm_true_k19,20,19,2,500,26594007,7883285,53188.014,15766.57 -r1-distill-qwen-14b,math500,comm_true_k19,20,19,3,500,26590564,7700964,53181.128,15401.928 -r1-distill-qwen-14b,math500,comm_true_k2,3,2,1,500,199647,1219999,399.294,2439.998 -r1-distill-qwen-14b,math500,comm_true_k2,3,2,2,500,792649,1184698,1585.298,2369.396 -r1-distill-qwen-14b,math500,comm_true_k2,3,2,3,500,792544,1163813,1585.088,2327.626 -r1-distill-qwen-14b,math500,comm_true_k24,25,24,1,500,1663725,10172839,3327.45,20345.678 -r1-distill-qwen-14b,math500,comm_true_k24,25,24,2,500,41100394,9759797,82200.788,19519.594 -r1-distill-qwen-14b,math500,comm_true_k24,25,24,3,500,41096682,9514344,82193.364,19028.688 -r1-distill-qwen-14b,math500,comm_true_k29,30,29,1,500,1996470,12205496,3992.94,24410.992 -r1-distill-qwen-14b,math500,comm_true_k29,30,29,2,500,58754941,11781585,117509.882,23563.17 -r1-distill-qwen-14b,math500,comm_true_k29,30,29,3,500,58745881,11541544,117491.762,23083.088 -r1-distill-qwen-14b,math500,comm_true_k4,5,4,1,500,332745,2033164,665.49,4066.328 -r1-distill-qwen-14b,math500,comm_true_k4,5,4,2,500,1944871,1969674,3889.742,3939.348 -r1-distill-qwen-14b,math500,comm_true_k4,5,4,3,500,1944586,1922507,3889.172,3845.014 -r1-distill-qwen-14b,math500,comm_true_k6,7,6,1,500,465843,2854907,931.686,5709.814 -r1-distill-qwen-14b,math500,comm_true_k6,7,6,2,500,3596159,2749937,7192.318,5499.874 -r1-distill-qwen-14b,math500,comm_true_k6,7,6,3,500,3595787,2691819,7191.574,5383.638 -r1-distill-qwen-14b,math500,comm_true_k9,10,9,1,500,665490,4071327,1330.98,8142.654 -r1-distill-qwen-14b,math500,comm_true_k9,10,9,2,500,7008890,3962811,14017.78,7925.622 -r1-distill-qwen-14b,math500,comm_true_k9,10,9,3,500,7007989,3845166,14015.978,7690.332 -r1-distill-qwen-14b,math500,ensemble,1,0,0,500,66549,408133,133.098,816.266 -r1-distill-qwen-14b,math500,parity_decode_N5,1,0,0,500,74049,1179397,148.098,2358.794 -r1-distill-qwen-14b,mmlu_hard,comm_true_k1,2,1,1,724,229444,1025353,316.91160220994476,1416.2334254143645 -r1-distill-qwen-14b,mmlu_hard,comm_true_k1,2,1,2,724,625706,961168,864.2348066298342,1327.5801104972375 -r1-distill-qwen-14b,mmlu_hard,comm_true_k1,2,1,3,724,625690,924681,864.2127071823204,1277.1837016574586 -r1-distill-qwen-14b,mmlu_hard,comm_true_k14,15,14,1,724,1720830,7675789,2376.8370165745855,10601.918508287294 -r1-distill-qwen-14b,mmlu_hard,comm_true_k14,15,14,2,724,22372570,6948066,30901.339779005524,9596.776243093922 -r1-distill-qwen-14b,mmlu_hard,comm_true_k14,15,14,3,724,22370270,6751886,30898.162983425413,9325.809392265193 -r1-distill-qwen-14b,mmlu_hard,comm_true_k19,20,19,1,724,2294440,10233867,3169.1160220994475,14135.175414364641 -r1-distill-qwen-14b,mmlu_hard,comm_true_k19,20,19,2,724,38938052,9280887,53781.839779005524,12818.904696132597 -r1-distill-qwen-14b,mmlu_hard,comm_true_k19,20,19,3,724,38936984,9011942,53780.36464088398,12447.43370165746 -r1-distill-qwen-14b,mmlu_hard,comm_true_k2,3,2,1,724,344166,1536016,475.3674033149171,2121.5690607734805 -r1-distill-qwen-14b,mmlu_hard,comm_true_k2,3,2,2,724,1209766,1432750,1670.9475138121547,1978.9364640883978 -r1-distill-qwen-14b,mmlu_hard,comm_true_k2,3,2,3,724,1209679,1397305,1670.8273480662983,1929.9792817679559 -r1-distill-qwen-14b,mmlu_hard,comm_true_k24,25,24,1,724,2868050,12789244,3961.3950276243095,17664.701657458565 -r1-distill-qwen-14b,mmlu_hard,comm_true_k24,25,24,2,724,60063817,11523212,82961.0732044199,15916.038674033149 -r1-distill-qwen-14b,mmlu_hard,comm_true_k24,25,24,3,724,60061235,11119585,82957.50690607735,15358.542817679558 -r1-distill-qwen-14b,mmlu_hard,comm_true_k29,30,29,1,725,3447600,15376855,4755.310344827586,21209.455172413793 -r1-distill-qwen-14b,mmlu_hard,comm_true_k29,30,29,2,724,85741683,13912765,118427.73895027624,19216.526243093922 -r1-distill-qwen-14b,mmlu_hard,comm_true_k29,30,29,3,724,85734073,13500555,118417.22790055249,18647.17541436464 -r1-distill-qwen-14b,mmlu_hard,comm_true_k4,5,4,1,724,573610,2565000,792.2790055248619,3542.817679558011 -r1-distill-qwen-14b,mmlu_hard,comm_true_k4,5,4,2,724,2920219,2361728,4033.4516574585637,3262.0552486187844 -r1-distill-qwen-14b,mmlu_hard,comm_true_k4,5,4,3,724,2919812,2288918,4032.8895027624308,3161.488950276243 -r1-distill-qwen-14b,mmlu_hard,comm_true_k6,7,6,1,724,803054,3592350,1109.1906077348067,4961.809392265193 -r1-distill-qwen-14b,mmlu_hard,comm_true_k6,7,6,2,724,5353707,3301079,7394.6229281767955,4559.501381215469 -r1-distill-qwen-14b,mmlu_hard,comm_true_k6,7,6,3,724,5353148,3197859,7393.850828729282,4416.932320441989 -r1-distill-qwen-14b,mmlu_hard,comm_true_k9,10,9,1,724,1147220,5133749,1584.5580110497237,7090.813535911602 -r1-distill-qwen-14b,mmlu_hard,comm_true_k9,10,9,2,724,10358384,4694453,14307.160220994476,6484.0511049723755 -r1-distill-qwen-14b,mmlu_hard,comm_true_k9,10,9,3,724,10358870,4551903,14307.831491712706,6287.158839779006 -r1-distill-qwen-14b,mmlu_hard,ensemble,1,0,0,724,114722,513385,158.45580110497238,709.0953038674033 -r1-distill-qwen-14b,mmlu_hard,parity_decode_N5,1,0,0,724,127754,1020784,176.45580110497238,1409.9226519337017 -smollm3-3b,arc,comm_true_k1,2,1,1,3495,1284738,460663,367.5931330472103,131.8062947067239 -smollm3-3b,arc,comm_true_k1,2,1,2,3495,2406266,650695,688.4881258941344,186.1788268955651 -smollm3-3b,arc,comm_true_k1,2,1,3,3495,2833196,691061,810.6426323319027,197.72846924177395 -smollm3-3b,arc,comm_true_k14,15,14,1,3495,9635535,3409506,2756.9484978540772,975.5381974248927 -smollm3-3b,arc,comm_true_k14,15,14,2,3495,64356979,5015819,18414.01402002861,1435.1413447782547 -smollm3-3b,arc,comm_true_k14,15,14,3,3495,89052451,5783762,25479.957367668096,1654.8675250357653 -smollm3-3b,arc,comm_true_k19,20,19,1,3495,12847380,4605631,3675.931330472103,1317.7771101573676 -smollm3-3b,arc,comm_true_k19,20,19,2,3495,109632802,6686893,31368.469814020027,1913.2741058655222 -smollm3-3b,arc,comm_true_k19,20,19,3,3495,153531923,7759997,43929.01945636624,2220.313876967096 -smollm3-3b,arc,comm_true_k2,3,2,1,3495,1927107,703470,551.3896995708154,201.27896995708156 -smollm3-3b,arc,comm_true_k2,3,2,2,3495,4314451,977072,1234.4638054363377,279.56280400572246 -smollm3-3b,arc,comm_true_k2,3,2,3,3495,5284963,1053484,1512.1496423462088,301.4260371959943 -smollm3-3b,arc,comm_true_k24,25,24,1,3495,16059225,5746873,4594.914163090129,1644.3127324749642 -smollm3-3b,arc,comm_true_k24,25,24,2,3495,166742692,8337539,47708.92474964235,2385.561945636624 -smollm3-3b,arc,comm_true_k24,25,24,3,3495,234487220,9808990,67092.19456366237,2806.5779685264665 -smollm3-3b,arc,comm_true_k29,30,29,1,3495,19271070,6917464,5513.8969957081545,1979.2457796852646 -smollm3-3b,arc,comm_true_k29,30,29,2,3495,235893992,9837702,67494.704434907,2814.79313304721 -smollm3-3b,arc,comm_true_k29,30,29,3,3495,328487104,11817765,93987.72646638054,3381.3347639484978 -smollm3-3b,arc,comm_true_k4,5,4,1,3495,3211845,1152407,918.9828326180258,329.7301859799714 -smollm3-3b,arc,comm_true_k4,5,4,2,3495,9576185,1614898,2739.967095851216,462.05951359084406 -smollm3-3b,arc,comm_true_k4,5,4,3,3495,12242521,1755418,3502.8672389127323,502.265522174535 -smollm3-3b,arc,comm_true_k6,7,6,1,3495,4496583,1612479,1286.575965665236,461.36738197424893 -smollm3-3b,arc,comm_true_k6,7,6,2,3495,16711083,2262021,4781.425751072961,647.2163090128755 -smollm3-3b,arc,comm_true_k6,7,6,3,3495,21944646,2482946,6278.868669527897,710.4280400572246 -smollm3-3b,arc,comm_true_k9,10,9,1,3495,6423690,2327444,1837.9656652360516,665.9353361945637 -smollm3-3b,arc,comm_true_k9,10,9,2,3495,31015995,3315138,8874.390557939914,948.537339055794 -smollm3-3b,arc,comm_true_k9,10,9,3,3495,41787974,3669693,11956.501859799713,1049.9836909871244 -smollm3-3b,arc,ensemble,1,0,0,3495,642369,226928,183.79656652360515,64.92932761087268 -smollm3-3b,arc,parity_decode_N5,1,0,0,3495,705279,991239,201.79656652360515,283.61630901287555 -smollm3-3b,dat,comm_true_k1,2,1,1,4500,2007000,843670,446.0,187.48222222222222 -smollm3-3b,dat,comm_true_k1,2,1,2,4500,4348674,1164702,966.372,258.8226666666667 -smollm3-3b,dat,comm_true_k1,2,1,3,4500,4447528,1167110,988.3395555555555,259.35777777777776 -smollm3-3b,dat,comm_true_k14,15,14,1,4500,15052500,6279387,3345.0,1395.4193333333333 -smollm3-3b,dat,comm_true_k14,15,14,2,4500,135450440,7575692,30100.097777777777,1683.487111111111 -smollm3-3b,dat,comm_true_k14,15,14,3,4500,141821415,7816588,31515.87,1737.0195555555556 -smollm3-3b,dat,comm_true_k19,20,19,1,4500,20070000,8365278,4460.0,1858.9506666666666 -smollm3-3b,dat,comm_true_k19,20,19,2,4500,233255403,10151473,51834.534,2255.882888888889 -smollm3-3b,dat,comm_true_k19,20,19,3,4500,245140501,10439918,54475.66688888889,2319.9817777777776 -smollm3-3b,dat,comm_true_k2,3,2,1,4500,3010500,1263451,669.0,280.7668888888889 -smollm3-3b,dat,comm_true_k2,3,2,2,4500,8106351,1561029,1801.4113333333332,346.8953333333333 -smollm3-3b,dat,comm_true_k2,3,2,3,4500,8326003,1628094,1850.222888888889,361.7986666666667 -smollm3-3b,dat,comm_true_k24,25,24,1,4500,25087500,10471029,5575.0,2326.8953333333334 -smollm3-3b,dat,comm_true_k24,25,24,2,4500,357694402,12758321,79487.64488888888,2835.1824444444446 -smollm3-3b,dat,comm_true_k24,25,24,3,4500,376776939,13107569,83728.20866666667,2912.793111111111 -smollm3-3b,dat,comm_true_k29,30,29,1,4500,30105000,12574429,6690.0,2794.3175555555554 -smollm3-3b,dat,comm_true_k29,30,29,2,4500,508515596,15338716,113003.46577777778,3408.6035555555554 -smollm3-3b,dat,comm_true_k29,30,29,3,4500,535820893,15768108,119071.30955555555,3504.024 -smollm3-3b,dat,comm_true_k4,5,4,1,4500,5017500,2108460,1115.0,468.5466666666667 -smollm3-3b,dat,comm_true_k4,5,4,2,4500,18785664,2506559,4174.592,557.0131111111111 -smollm3-3b,dat,comm_true_k4,5,4,3,4500,19401345,2609380,4311.41,579.8622222222223 -smollm3-3b,dat,comm_true_k6,7,6,1,4500,7024500,2938533,1561.0,653.0073333333333 -smollm3-3b,dat,comm_true_k6,7,6,2,4500,33691222,3466347,7486.938222222222,770.2993333333334 -smollm3-3b,dat,comm_true_k6,7,6,3,4500,34918327,3624996,7759.628222222223,805.5546666666667 -smollm3-3b,dat,comm_true_k9,10,9,1,4500,10035000,4203704,2230.0,934.1564444444444 -smollm3-3b,dat,comm_true_k9,10,9,2,4500,63964464,4968935,14214.325333333334,1104.2077777777777 -smollm3-3b,dat,comm_true_k9,10,9,3,4500,66537083,5151880,14786.018444444444,1144.8622222222223 -smollm3-3b,dat,ensemble,1,0,0,4500,1003500,422115,223.0,93.80333333333333 -smollm3-3b,dat,parity_decode_N5,1,0,0,4500,1093500,1250688,243.0,277.93066666666664 -smollm3-3b,eqbench,comm_true_k1,2,1,1,96,67938,190725,707.6875,1986.71875 -smollm3-3b,eqbench,comm_true_k1,2,1,2,96,123180,177228,1283.125,1846.125 -smollm3-3b,eqbench,comm_true_k1,2,1,3,96,123178,170717,1283.1041666666667,1778.3020833333333 -smollm3-3b,eqbench,comm_true_k14,15,14,1,96,509535,1414585,5307.65625,14735.260416666666 -smollm3-3b,eqbench,comm_true_k14,15,14,2,96,3261277,1280677,33971.635416666664,13340.385416666666 -smollm3-3b,eqbench,comm_true_k14,15,14,3,96,3261307,1279275,33971.947916666664,13325.78125 -smollm3-3b,eqbench,comm_true_k19,20,19,1,96,679380,1891619,7076.875,19704.364583333332 -smollm3-3b,eqbench,comm_true_k19,20,19,2,96,5547342,1636047,57784.8125,17042.15625 -smollm3-3b,eqbench,comm_true_k19,20,19,3,96,5546666,1650698,57777.770833333336,17194.770833333332 -smollm3-3b,eqbench,comm_true_k2,3,2,1,96,101907,283017,1061.53125,2948.09375 -smollm3-3b,eqbench,comm_true_k2,3,2,2,96,220767,260503,2299.65625,2713.5729166666665 -smollm3-3b,eqbench,comm_true_k2,3,2,3,96,220732,254175,2299.2916666666665,2647.65625 -smollm3-3b,eqbench,comm_true_k24,25,24,1,96,849225,2369285,8846.09375,24680.052083333332 -smollm3-3b,eqbench,comm_true_k24,25,24,2,96,8432295,2068193,87836.40625,21543.677083333332 -smollm3-3b,eqbench,comm_true_k24,25,24,3,96,8431973,2064835,87833.05208333333,21508.697916666668 -smollm3-3b,eqbench,comm_true_k29,30,29,1,96,1019070,2839814,10615.3125,29581.395833333332 -smollm3-3b,eqbench,comm_true_k29,30,29,2,96,11915531,2470197,124120.11458333333,25731.21875 -smollm3-3b,eqbench,comm_true_k29,30,29,3,96,11915500,2516549,124119.79166666667,26214.052083333332 -smollm3-3b,eqbench,comm_true_k4,5,4,1,96,169845,474340,1769.21875,4941.041666666667 -smollm3-3b,eqbench,comm_true_k4,5,4,2,96,487715,429508,5080.364583333333,4474.041666666667 -smollm3-3b,eqbench,comm_true_k4,5,4,3,96,487745,427322,5080.677083333333,4451.270833333333 -smollm3-3b,eqbench,comm_true_k6,7,6,1,96,237783,661094,2476.90625,6886.395833333333 -smollm3-3b,eqbench,comm_true_k6,7,6,2,96,850836,590910,8862.875,6155.3125 -smollm3-3b,eqbench,comm_true_k6,7,6,3,96,850749,596008,8861.96875,6208.416666666667 -smollm3-3b,eqbench,comm_true_k9,10,9,1,96,339690,943104,3538.4375,9824.0 -smollm3-3b,eqbench,comm_true_k9,10,9,2,96,1574758,857952,16403.729166666668,8937.0 -smollm3-3b,eqbench,comm_true_k9,10,9,3,96,1574882,849535,16405.020833333332,8849.322916666666 -smollm3-3b,eqbench,ensemble,1,0,0,96,33969,95735,353.84375,997.2395833333334 -smollm3-3b,eqbench,parity_decode_N5,1,0,0,96,35985,318749,374.84375,3320.3020833333335 -smollm3-3b,fermi,comm_true_k1,2,1,1,1587,707786,1437489,445.98991808443606,905.7901701323251 -smollm3-3b,fermi,comm_true_k1,2,1,2,1587,1504634,763948,948.099558916194,481.37870195337115 -smollm3-3b,fermi,comm_true_k1,2,1,3,1587,1476276,580709,930.2306238185255,365.9161940768746 -smollm3-3b,fermi,comm_true_k14,15,14,1,1587,5308395,10688244,3344.9243856332705,6734.873345935728 -smollm3-3b,fermi,comm_true_k14,15,14,2,1587,49330283,8559661,31083.98424700693,5393.611216131065 -smollm3-3b,fermi,comm_true_k14,15,14,3,1587,48938068,5288036,30836.84183994959,3332.0957781978577 -smollm3-3b,fermi,comm_true_k19,20,19,1,1587,7077860,14285590,4459.899180844361,9001.632010081916 -smollm3-3b,fermi,comm_true_k19,20,19,2,1587,85274378,11231736,53733.06742281033,7077.3383742911155 -smollm3-3b,fermi,comm_true_k19,20,19,3,1587,84608316,6918526,53313.36862003781,4359.499684940139 -smollm3-3b,fermi,comm_true_k2,3,2,1,1587,1061679,2138261,668.984877126654,1347.3604284814114 -smollm3-3b,fermi,comm_true_k2,3,2,2,1587,2839589,1259061,1789.2810333963453,793.359168241966 -smollm3-3b,fermi,comm_true_k2,3,2,3,1587,2789921,894277,1757.9842470069314,563.5015752993069 -smollm3-3b,fermi,comm_true_k24,25,24,1,1587,8847325,17889547,5574.873976055451,11272.556395715186 -smollm3-3b,fermi,comm_true_k24,25,24,2,1587,130996058,14188553,82543.19974795211,8940.487082545684 -smollm3-3b,fermi,comm_true_k24,25,24,3,1587,130195808,8623444,82038.94643982357,5433.8021424070575 -smollm3-3b,fermi,comm_true_k29,30,29,1,1587,10616790,21570344,6689.848771266541,13591.89918084436 -smollm3-3b,fermi,comm_true_k29,30,29,2,1587,186456685,17030220,117490.03465658476,10731.077504725898 -smollm3-3b,fermi,comm_true_k29,30,29,3,1587,185262329,10328863,116737.44738500315,6508.420289855072 -smollm3-3b,fermi,comm_true_k4,5,4,1,1587,1769465,3532410,1114.9747952110902,2225.8412098298677 -smollm3-3b,fermi,comm_true_k4,5,4,2,1587,6681359,2429972,4210.056080655325,1531.1732829237556 -smollm3-3b,fermi,comm_true_k4,5,4,3,1587,6591291,1645141,4153.302457466919,1036.6357908002522 -smollm3-3b,fermi,comm_true_k6,7,6,1,1587,2477251,4966707,1560.964713295526,3129.6200378071835 -smollm3-3b,fermi,comm_true_k6,7,6,2,1587,12089277,3275844,7617.691871455577,2064.1739130434785 -smollm3-3b,fermi,comm_true_k6,7,6,3,1587,11875959,2059523,7483.275992438564,1297.746061751733 -smollm3-3b,fermi,comm_true_k9,10,9,1,1587,3538930,7162572,2229.9495904221803,4513.277882797732 -smollm3-3b,fermi,comm_true_k9,10,9,2,1587,23123283,5305274,14570.436672967864,3342.957781978576 -smollm3-3b,fermi,comm_true_k9,10,9,3,1587,22849215,3405694,14397.74102079395,2145.994959042218 -smollm3-3b,fermi,ensemble,1,0,0,1587,353893,703767,222.99495904221803,443.45746691871454 -smollm3-3b,fermi,parity_decode_N5,1,0,0,1587,327972,2741091,206.66162570888469,1727.2155009451797 -smollm3-3b,gsm8k,comm_true_k1,2,1,1,3957,1406286,4960104,355.3919636087945,1253.5011372251706 -smollm3-3b,gsm8k,comm_true_k1,2,1,2,3957,3368906,2374923,851.3788223401567,600.1827141774071 -smollm3-3b,gsm8k,comm_true_k1,2,1,3,3957,3395196,1821693,858.0227445034117,460.37225170583775 -smollm3-3b,gsm8k,comm_true_k14,15,14,1,3957,10547145,37027242,2665.4397270659592,9357.402577710387 -smollm3-3b,gsm8k,comm_true_k14,15,14,2,3957,112767003,23669537,28498.105382865808,5981.687389436442 -smollm3-3b,gsm8k,comm_true_k14,15,14,3,3957,115686481,16570617,29235.906242102603,4187.671721000758 -smollm3-3b,gsm8k,comm_true_k19,20,19,1,3957,14062860,49543235,3553.9196360879455,12520.403083143796 -smollm3-3b,gsm8k,comm_true_k19,20,19,2,3957,195216447,31702764,49334.45716451857,8011.818043972707 -smollm3-3b,gsm8k,comm_true_k19,20,19,3,3957,200807900,21980682,50747.51074045995,5554.885519332828 -smollm3-3b,gsm8k,comm_true_k2,3,2,1,3957,2109429,7432838,533.0879454131918,1878.402324993682 -smollm3-3b,gsm8k,comm_true_k2,3,2,2,3957,6398725,4043643,1617.064695476371,1021.89613343442 -smollm3-3b,gsm8k,comm_true_k2,3,2,3,3957,6483097,3094655,1638.3869092747032,782.0710133939854 -smollm3-3b,gsm8k,comm_true_k24,25,24,1,3958,17582250,61837134,4442.205659423951,15623.32844871147 -smollm3-3b,gsm8k,comm_true_k24,25,24,2,3958,300222286,40286347,75852.01768569984,10178.460586154624 -smollm3-3b,gsm8k,comm_true_k24,25,24,3,3957,309270555,27194513,78157.83548142533,6872.5077078594895 -smollm3-3b,gsm8k,comm_true_k29,30,29,1,3958,21099270,74196072,5330.790803436079,18745.849418898433 -smollm3-3b,gsm8k,comm_true_k29,30,29,2,3957,427354297,49093466,107999.56962345212,12406.738943644175 -smollm3-3b,gsm8k,comm_true_k29,30,29,3,3957,441049045,33084372,111460.46120798585,8360.973464746019 -smollm3-3b,gsm8k,comm_true_k4,5,4,1,3957,3515715,12390218,888.4799090219864,3131.2150619155927 -smollm3-3b,gsm8k,comm_true_k4,5,4,2,3957,15150270,7171753,3828.7263078089463,1812.4217841799343 -smollm3-3b,gsm8k,comm_true_k4,5,4,3,3957,15374423,5201489,3885.3735152893605,1314.5031589588073 -smollm3-3b,gsm8k,comm_true_k6,7,6,1,3957,4922001,17289517,1243.871872630781,4369.3497599191305 -smollm3-3b,gsm8k,comm_true_k6,7,6,2,3957,27477502,10227391,6944.02375537023,2584.632549911549 -smollm3-3b,gsm8k,comm_true_k6,7,6,3,3957,27925940,7301534,7057.351528936063,1845.219610816275 -smollm3-3b,gsm8k,comm_true_k9,10,9,1,3957,7031430,24681443,1776.9598180439727,6237.412939095274 -smollm3-3b,gsm8k,comm_true_k9,10,9,2,3957,52717300,15046891,13322.542330048016,3802.6007076067726 -smollm3-3b,gsm8k,comm_true_k9,10,9,3,3957,53788442,10662864,13593.237806419003,2694.6838514025776 -smollm3-3b,gsm8k,ensemble,1,0,0,3957,703143,2467890,177.69598180439726,623.6770280515542 -smollm3-3b,gsm8k,parity_decode_N5,1,0,0,3957,762498,14373120,192.69598180439726,3632.327520849128 -smollm3-3b,gsmhard,comm_true_k1,2,1,1,3051,1096554,3632473,359.4080629301868,1190.58439855785 -smollm3-3b,gsmhard,comm_true_k1,2,1,2,3051,2652516,1885576,869.3923303834808,618.0190101606031 -smollm3-3b,gsmhard,comm_true_k1,2,1,3,3051,2672128,1605323,875.8203867584399,526.162897410685 -smollm3-3b,gsmhard,comm_true_k14,15,14,1,3051,8224155,27026936,2695.560471976401,8858.386102917077 -smollm3-3b,gsmhard,comm_true_k14,15,14,2,3051,89397692,18195557,29301.11176663389,5963.801048836447 -smollm3-3b,gsmhard,comm_true_k14,15,14,3,3051,91158720,13036634,29878.308751229106,4272.905276958374 -smollm3-3b,gsmhard,comm_true_k19,20,19,1,3051,10965540,36272637,3594.080629301868,11888.769911504425 -smollm3-3b,gsmhard,comm_true_k19,20,19,2,3051,155048967,24581017,50819.06489675516,8056.70829236316 -smollm3-3b,gsmhard,comm_true_k19,20,19,3,3051,158149890,17315963,51835.42772861357,5675.504097017371 -smollm3-3b,gsmhard,comm_true_k2,3,2,1,3051,1644831,5443763,539.1120943952802,1784.2553261225828 -smollm3-3b,gsmhard,comm_true_k2,3,2,2,3051,5049709,3165928,1655.0996394624713,1037.6689609963946 -smollm3-3b,gsmhard,comm_true_k2,3,2,3,3051,5105373,2511546,1673.3441494591937,823.1878072763028 -smollm3-3b,gsmhard,comm_true_k24,25,24,1,3051,13706925,45194134,4492.600786627335,14812.892166502786 -smollm3-3b,gsmhard,comm_true_k24,25,24,2,3051,238382621,31403238,78132.61914126515,10292.768928220255 -smollm3-3b,gsmhard,comm_true_k24,25,24,3,3051,243388632,22221115,79773.39626352016,7283.223533267781 -smollm3-3b,gsmhard,comm_true_k29,30,29,1,3051,16448310,54185887,5391.120943952802,17760.041625696493 -smollm3-3b,gsmhard,comm_true_k29,30,29,2,3051,339550559,38277558,111291.56309406752,12545.905604719765 -smollm3-3b,gsmhard,comm_true_k29,30,29,3,3051,347963183,27245936,114048.89642740085,8930.165847263192 -smollm3-3b,gsmhard,comm_true_k4,5,4,1,3051,2741385,8986127,898.520157325467,2945.305473615208 -smollm3-3b,gsmhard,comm_true_k4,5,4,2,3051,11972370,5528244,3924.080629301868,1811.944936086529 -smollm3-3b,gsmhard,comm_true_k4,5,4,3,3051,12092814,4298607,3963.5575221238937,1408.9174041297936 -smollm3-3b,gsmhard,comm_true_k6,7,6,1,3051,3837939,12640123,1257.928220255654,4142.94428056375 -smollm3-3b,gsmhard,comm_true_k6,7,6,2,3051,21757501,7810602,7131.268764339561,2560.0137659783677 -smollm3-3b,gsmhard,comm_true_k6,7,6,3,3051,22020357,5914699,7217.4228121927235,1938.609963946247 -smollm3-3b,gsmhard,comm_true_k9,10,9,1,3051,5482770,18139398,1797.040314650934,5945.394296951819 -smollm3-3b,gsmhard,comm_true_k9,10,9,2,3051,41823212,11700183,13708.03408718453,3834.868239921337 -smollm3-3b,gsmhard,comm_true_k9,10,9,3,3051,42448064,8708174,13912.836447066535,2854.203212061619 -smollm3-3b,gsmhard,ensemble,1,0,0,3051,548277,1775178,179.7040314650934,581.834808259587 -smollm3-3b,gsmhard,parity_decode_N5,1,0,0,3051,594042,10556807,194.7040314650934,3460.1137332022286 -smollm3-3b,math500,comm_true_k1,2,1,1,1500,572274,1587829,381.516,1058.5526666666667 -smollm3-3b,math500,comm_true_k1,2,1,2,1500,1351224,1188018,900.816,792.012 -smollm3-3b,math500,comm_true_k1,2,1,3,1500,1351654,1046086,901.1026666666667,697.3906666666667 -smollm3-3b,math500,comm_true_k14,15,14,1,1500,4292055,11917698,2861.37,7945.132 -smollm3-3b,math500,comm_true_k14,15,14,2,1500,44965725,10112652,29977.15,6741.768 -smollm3-3b,math500,comm_true_k14,15,14,3,1500,45302352,8241773,30201.568,5494.515333333334 -smollm3-3b,math500,comm_true_k19,20,19,1,1500,5722740,15748881,3815.16,10499.254 -smollm3-3b,math500,comm_true_k19,20,19,2,1500,77783644,13447736,51855.76266666667,8965.157333333333 -smollm3-3b,math500,comm_true_k19,20,19,3,1500,78392807,10893339,52261.871333333336,7262.226 -smollm3-3b,math500,comm_true_k2,3,2,1,1500,858411,2391420,572.274,1594.28 -smollm3-3b,math500,comm_true_k2,3,2,2,1500,2563690,1862195,1709.1266666666668,1241.4633333333334 -smollm3-3b,math500,comm_true_k2,3,2,3,1500,2569627,1623792,1713.0846666666666,1082.528 -smollm3-3b,math500,comm_true_k24,25,24,1,1500,7153425,19859180,4768.95,13239.453333333333 -smollm3-3b,math500,comm_true_k24,25,24,2,1500,119619394,16566929,79746.26266666666,11044.619333333334 -smollm3-3b,math500,comm_true_k24,25,24,3,1500,120467265,13180616,80311.51,8787.077333333333 -smollm3-3b,math500,comm_true_k29,30,29,1,1500,8584110,23762346,5722.74,15841.564 -smollm3-3b,math500,comm_true_k29,30,29,2,1500,170324347,20525748,113549.56466666667,13683.832 -smollm3-3b,math500,comm_true_k29,30,29,3,1500,171989109,16300782,114659.406,10867.188 -smollm3-3b,math500,comm_true_k4,5,4,1,1500,1430685,3970965,953.79,2647.31 -smollm3-3b,math500,comm_true_k4,5,4,2,1500,6053013,3205452,4035.342,2136.968 -smollm3-3b,math500,comm_true_k4,5,4,3,1500,6055425,2775575,4036.95,1850.3833333333334 -smollm3-3b,math500,comm_true_k6,7,6,1,1500,2002959,5516736,1335.306,3677.824 -smollm3-3b,math500,comm_true_k6,7,6,2,1500,10990065,4525719,7326.71,3017.146 -smollm3-3b,math500,comm_true_k6,7,6,3,1500,10996403,3810385,7330.935333333333,2540.2566666666667 -smollm3-3b,math500,comm_true_k9,10,9,1,1501,2864340,7912258,1908.2878081279148,5271.324450366423 -smollm3-3b,math500,comm_true_k9,10,9,2,1500,21041844,6600230,14027.896,4400.153333333334 -smollm3-3b,math500,comm_true_k9,10,9,3,1500,21115095,5488265,14076.73,3658.8433333333332 -smollm3-3b,math500,ensemble,1,0,0,1500,286137,783860,190.758,522.5733333333334 -smollm3-3b,math500,parity_decode_N5,1,0,0,1500,308637,3479907,205.758,2319.938 -smollm3-3b,mmlu_hard,comm_true_k1,2,1,1,2172,931848,840519,429.02762430939225,386.9792817679558 -smollm3-3b,mmlu_hard,comm_true_k1,2,1,2,2172,1878616,791263,864.9244935543278,364.30156537753226 -smollm3-3b,mmlu_hard,comm_true_k1,2,1,3,2172,2018354,763984,929.2605893186004,351.74217311233883 -smollm3-3b,mmlu_hard,comm_true_k14,15,14,1,2172,6988860,6479024,3217.707182320442,2982.97605893186 -smollm3-3b,mmlu_hard,comm_true_k14,15,14,2,2172,55346446,6253492,25481.789134438306,2879.1399631675877 -smollm3-3b,mmlu_hard,comm_true_k14,15,14,3,2172,63318016,5846674,29151.941068139964,2691.8388581952117 -smollm3-3b,mmlu_hard,comm_true_k19,20,19,1,2172,9318480,8625907,4290.276243093923,3971.412062615101 -smollm3-3b,mmlu_hard,comm_true_k19,20,19,2,2172,94861884,8224364,43674.900552486186,3786.539594843462 -smollm3-3b,mmlu_hard,comm_true_k19,20,19,3,2172,108899755,7764782,50138.008747697975,3574.9456721915285 -smollm3-3b,mmlu_hard,comm_true_k2,3,2,1,2172,1397772,1283944,643.5414364640884,591.1344383057091 -smollm3-3b,mmlu_hard,comm_true_k2,3,2,2,2172,3450698,1202699,1588.719152854512,553.7288213627993 -smollm3-3b,mmlu_hard,comm_true_k2,3,2,3,2172,3784109,1198016,1742.223296500921,551.572744014733 -smollm3-3b,mmlu_hard,comm_true_k24,25,24,1,2172,11648100,10797343,5362.845303867403,4971.152394106814 -smollm3-3b,mmlu_hard,comm_true_k24,25,24,2,2172,144963519,10480889,66741.95165745856,4825.455340699816 -smollm3-3b,mmlu_hard,comm_true_k24,25,24,3,2172,166231741,9824014,76533.95073664826,4523.026703499079 -smollm3-3b,mmlu_hard,comm_true_k29,30,29,1,2172,13977720,12992651,6435.414364640884,5981.883517495396 -smollm3-3b,mmlu_hard,comm_true_k29,30,29,2,2172,205662872,12520884,94688.2467771639,5764.67955801105 -smollm3-3b,mmlu_hard,comm_true_k29,30,29,3,2172,235226906,12124057,108299.68047882136,5581.9783609576425 -smollm3-3b,mmlu_hard,comm_true_k4,5,4,1,2172,2329620,2146652,1072.5690607734807,988.329650092081 -smollm3-3b,mmlu_hard,comm_true_k4,5,4,2,2172,7873165,2018821,3624.84576427256,929.4755985267035 -smollm3-3b,mmlu_hard,comm_true_k4,5,4,3,2172,8763927,1878546,4034.957182320442,864.8922651933701 -smollm3-3b,mmlu_hard,comm_true_k6,7,6,1,2172,3261468,2970620,1501.596685082873,1367.6887661141805 -smollm3-3b,mmlu_hard,comm_true_k6,7,6,2,2172,13959433,2872471,6426.994935543278,1322.5004604051564 -smollm3-3b,mmlu_hard,comm_true_k6,7,6,3,2172,15716525,2617780,7235.969152854512,1205.2394106813997 -smollm3-3b,mmlu_hard,comm_true_k9,10,9,1,2172,4659240,4315676,2145.1381215469614,1986.9594843462246 -smollm3-3b,mmlu_hard,comm_true_k9,10,9,2,2172,26323290,4118218,12119.378453038675,1896.048802946593 -smollm3-3b,mmlu_hard,comm_true_k9,10,9,3,2172,29849491,3852138,13742.859576427256,1773.5441988950276 -smollm3-3b,mmlu_hard,ensemble,1,0,0,2172,465924,431490,214.51381215469613,198.66022099447514 -smollm3-3b,mmlu_hard,parity_decode_N5,1,0,0,2172,505020,2694437,232.51381215469613,1240.5326887661142 diff --git a/tests/_dual.py b/tests/_dual.py new file mode 100644 index 0000000..0a8a2d8 --- /dev/null +++ b/tests/_dual.py @@ -0,0 +1,56 @@ +"""Load the original module and the new package side by side for golden-master comparison. + +The original single-file model is frozen alongside the tests as +``tests/agentic_ecal_reference.py`` -- the golden master this suite proves the package against. +It and the package (``agentic_ecal_pkg``) are loaded together; to avoid the name clash they carry +different names: the package keeps its own name, and the reference is loaded by explicit file path +under the alias ``agentic_ecal_orig``. +""" + +from __future__ import annotations + +import importlib.util +import math +import struct +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent # .../a-eCAL/tests +ROOT = HERE.parent # .../a-eCAL (repo root) + +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +# NEW package (modular refactor). +import agentic_ecal_pkg as NEW # noqa: E402 + +# ORIGINAL model (frozen golden-master reference), loaded by path under a unique alias. +ORIG_PATH = HERE / "agentic_ecal_reference.py" +_spec = importlib.util.spec_from_file_location("agentic_ecal_orig", ORIG_PATH) +ORIG = importlib.util.module_from_spec(_spec) +sys.modules["agentic_ecal_orig"] = ORIG +_spec.loader.exec_module(ORIG) + + +def same(a, b) -> bool: + """True iff ``a`` and ``b`` are bit-identical, recursing into dict/list/tuple. + + Floats are compared by their IEEE-754 bytes, so ``-0.0`` differs from ``0.0`` and ``inf`` is + handled; two NaNs count as equal (both implementations should produce NaN in the same places). + """ + if isinstance(a, float) or isinstance(b, float): + fa, fb = float(a), float(b) + if math.isnan(fa) and math.isnan(fb): + return True + return struct.pack(" 0 diff --git a/tests/test_components.py b/tests/test_components.py new file mode 100644 index 0000000..8ab3de6 --- /dev/null +++ b/tests/test_components.py @@ -0,0 +1,51 @@ +"""Parity of Tool / Retrieval / TwoRateCalib behaviour.""" + +from __future__ import annotations + +import itertools + +import pytest + +from _dual import HW_KEYS, NEW, ORIG, same + +# Retrieval parameter grid, incl. the log2(max(n,2)) edge and both search modes. +_RET_GRID = list(itertools.product( + [True, False], # ann + [0, 5], # k_chunks + [0, 256], # chunk_tokens + [1, 2, 1e6], # n_vectors (1 exercises the log2 clamp) + [1.1e8, 7.615e9], # embed_params + [0, 64], # query_tokens +)) + + +@pytest.mark.parametrize("hk", HW_KEYS) +@pytest.mark.parametrize("ann,k_chunks,chunk_tokens,n_vectors,embed_params,query_tokens", _RET_GRID) +def test_retrieval_energy_and_context(hk, ann, k_chunks, chunk_tokens, n_vectors, + embed_params, query_tokens): + kw = dict(ann=ann, k_chunks=k_chunks, chunk_tokens=chunk_tokens, n_vectors=n_vectors, + embed_params=embed_params, query_tokens=query_tokens) + o = ORIG.Retrieval(**kw) + n = NEW.Retrieval(**kw) + assert same(o.energy(ORIG.HW[hk]), n.energy(NEW.HW[hk])) + assert same(o.added_context, n.added_context) + + +def test_tool_defaults_and_values(): + for kw in (dict(name="code"), dict(name="api", energy=2.0, obs_tokens=150)): + o, n = ORIG.Tool(**kw), NEW.Tool(**kw) + assert same(o.energy, n.energy) + assert same(o.obs_tokens, n.obs_tokens) + + +_CALIB = [("Q", 0.023, 3.2, 0.02), ("L", 0.028, 3.3, 0.06), ("z", 0.0, 0.0, 0.0)] + + +@pytest.mark.parametrize("name,c_pre,a_dec,kv_floor", _CALIB) +@pytest.mark.parametrize("batch", [0, 1, 32, 256]) +@pytest.mark.parametrize("p_in,p_out", [(0, 0), (64, 512), (300, 200)]) +def test_two_rate_calib(name, c_pre, a_dec, kv_floor, batch, p_in, p_out): + o = ORIG.TwoRateCalib(name, c_pre, a_dec, kv_floor) + n = NEW.TwoRateCalib(name, c_pre, a_dec, kv_floor) + assert same(o.c_dec(batch), n.c_dec(batch)) + assert same(o.energy(p_in, p_out, batch), n.energy(p_in, p_out, batch)) diff --git a/tests/test_paper_numbers.py b/tests/test_paper_numbers.py new file mode 100644 index 0000000..a2ec302 --- /dev/null +++ b/tests/test_paper_numbers.py @@ -0,0 +1,65 @@ +"""Pin the specific anchors and case-study figures the manuscript reports. + +Unlike the parity tests (new == original), these pin literal expected values, so a change that +drifts *both* implementations away from the numbers in the paper is still caught. All values are +computed from the NEW package; each is also asserted equal to the ORIGINAL as a cross-check. +""" + +from __future__ import annotations + +import pytest + +from _dual import NEW, ORIG, same + +# Two-rate check (paper Table II): derived c_pre/a_dec vs the measured A100-sweep targets. +EXPECTED_TWO_RATE = { + "Qwen2.5-7B": {"c_pre": (0.023, 0.023), "a_dec": (4.27, 3.2)}, + "Llama-3 8B": {"c_pre": (0.0242, 0.028), "a_dec": (4.5, 3.3)}, +} +EXPECTED_SAMSI = { + "J/token at b=1": 36.8, + "implied batch for 3 J/token": 13.3, + "implied batch for 4 J/token": 9.8, +} +EXPECTED_CURRENT_GEN = { + "llama3_70b implied batch at 0.70 J/token (Fig. 3b)": 30.6, + "llama3_8b implied batch at 0.084 J/token (released runs, active GPU)": 30.0, +} + + +def test_two_rate_anchors(): + got = NEW.validate_two_rate() + assert got == EXPECTED_TWO_RATE + assert same(got, ORIG.validate_two_rate()) + + +def test_samsi_anchor(): + got = NEW.validate_against_samsi() + assert got == EXPECTED_SAMSI + assert same(got, ORIG.validate_against_samsi()) + + +def test_current_gen_anchor(): + got = NEW.validate_current_gen() + assert got == EXPECTED_CURRENT_GEN + assert same(got, ORIG.validate_current_gen()) + + +def test_case_study_breakdown(): + """The single-pass four-agent RAG case study (Llama-3 8B on A100).""" + r = NEW.four_agent_rag_workflow().run() + # Token budget and topology are fixed, so these are exact. + assert r["prefill_tokens"] == 4990 + assert r["decode_tokens"] == 1020 + assert r["useful_output_tokens"] == 1020 + # Energies at the fixed operating point. + assert r["e_total"] == pytest.approx(5240.261244, rel=1e-9) + assert r["e_llm"] == pytest.approx(4761.512832, rel=1e-9) + # And identical to the golden master. + assert same(r, ORIG.four_agent_rag_workflow().run()) + + +def test_case_study_agentic_ecal(): + got = NEW.four_agent_rag_workflow().agentic_ecal() + assert got == pytest.approx(0.3022065308190462, rel=1e-12) + assert same(got, ORIG.four_agent_rag_workflow().agentic_ecal()) diff --git a/tests/test_rates.py b/tests/test_rates.py new file mode 100644 index 0000000..22ea845 --- /dev/null +++ b/tests/test_rates.py @@ -0,0 +1,74 @@ +"""Parity of the two-rate call model: rates, energy, latency, embedding FLOPs.""" + +from __future__ import annotations + +import pytest + +from _dual import HW_KEYS, MODEL_KEYS, NEW, ORIG, same + +P_IN = [0, 1, 50, 64, 300, 512, 2048] +P_OUT = [1, 120, 256, 382, 512] +BATCH = [0, 1, 32, 64, 256] +CONTEXT = [0, 64, 512, 4096] + + +@pytest.mark.parametrize("mk", MODEL_KEYS) +@pytest.mark.parametrize("hk", HW_KEYS) +def test_prefill_rate(mk, hk): + assert same(ORIG.prefill_rate(ORIG.LLMS[mk], ORIG.HW[hk]), + NEW.prefill_rate(NEW.LLMS[mk], NEW.HW[hk])) + + +@pytest.mark.parametrize("mk", MODEL_KEYS) +@pytest.mark.parametrize("hk", HW_KEYS) +@pytest.mark.parametrize("batch", BATCH) +@pytest.mark.parametrize("ctx", CONTEXT) +def test_decode_rate(mk, hk, batch, ctx): + assert same(ORIG.decode_rate(ORIG.LLMS[mk], ORIG.HW[hk], batch, ctx), + NEW.decode_rate(NEW.LLMS[mk], NEW.HW[hk], batch, ctx)) + + +def test_decode_rate_no_bandwidth_raises_same_message(): + oh = ORIG.Hardware("no-bw", flops_peak=1e12, power=100.0) # bandwidth defaults to 0 + nh = NEW.Hardware("no-bw", flops_peak=1e12, power=100.0) + with pytest.raises(ValueError) as oe: + ORIG.decode_rate(ORIG.LLMS["llama3_8b"], oh, 1, 0) + with pytest.raises(ValueError) as ne: + NEW.decode_rate(NEW.LLMS["llama3_8b"], nh, 1, 0) + assert str(oe.value) == str(ne.value) + + +@pytest.mark.parametrize("mk", MODEL_KEYS) +@pytest.mark.parametrize("hk", HW_KEYS) +@pytest.mark.parametrize("p_in", P_IN) +@pytest.mark.parametrize("p_out", P_OUT) +@pytest.mark.parametrize("batch", [0, 1, 32, 256]) +def test_call_latency_and_energy(mk, hk, p_in, p_out, batch): + om, oh = ORIG.LLMS[mk], ORIG.HW[hk] + nm, nh = NEW.LLMS[mk], NEW.HW[hk] + assert same(ORIG.call_latency(om, oh, p_in, p_out, batch), + NEW.call_latency(nm, nh, p_in, p_out, batch)) + assert same(ORIG.llm_call_energy(om, oh, p_in, p_out, batch), + NEW.llm_call_energy(nm, nh, p_in, p_out, batch)) + + +@pytest.mark.parametrize("mk", MODEL_KEYS) +@pytest.mark.parametrize("hk", HW_KEYS) +def test_joules_per_token_defaults(mk, hk): + assert same(ORIG.joules_per_token(ORIG.LLMS[mk], ORIG.HW[hk]), + NEW.joules_per_token(NEW.LLMS[mk], NEW.HW[hk])) + + +@pytest.mark.parametrize("p_out", [1, 256, 512]) +def test_joules_per_token_grid(p_out): + om, oh = ORIG.LLMS["qwen2_5_7b"], ORIG.HW["a100"] + nm, nh = NEW.LLMS["qwen2_5_7b"], NEW.HW["a100"] + assert same(ORIG.joules_per_token(om, oh, 64, p_out, 32), + NEW.joules_per_token(nm, nh, 64, p_out, 32)) + + +@pytest.mark.parametrize("n_params", [0.0, 1.1e8, 7.615e9]) +@pytest.mark.parametrize("n_tokens", [0, 64, 256, 512]) +def test_embedding_flops(n_params, n_tokens): + assert same(ORIG.embedding_flops(n_params, n_tokens), + NEW.embedding_flops(n_params, n_tokens)) diff --git a/tests/test_registries.py b/tests/test_registries.py new file mode 100644 index 0000000..38aa42a --- /dev/null +++ b/tests/test_registries.py @@ -0,0 +1,47 @@ +"""Parity of the HW / LLMS / TWO_RATE preset registries, field by field.""" + +from __future__ import annotations + +import dataclasses + +import pytest + +from _dual import NEW, ORIG, same + +BATCH = [0, 1, 32, 64, 128, 256] + + +def _fields_equal(o, n): + of = {f.name: getattr(o, f.name) for f in dataclasses.fields(o)} + nf = {f.name: getattr(n, f.name) for f in dataclasses.fields(n)} + assert of.keys() == nf.keys() + for k in of: + assert same(of[k], nf[k]), k + + +def test_registry_keys_match(): + assert ORIG.HW.keys() == NEW.HW.keys() + assert ORIG.LLMS.keys() == NEW.LLMS.keys() + assert ORIG.TWO_RATE.keys() == NEW.TWO_RATE.keys() + + +@pytest.mark.parametrize("k", list(ORIG.HW.keys())) +def test_hw_entry(k): + o, n = ORIG.HW[k], NEW.HW[k] + _fields_equal(o, n) + assert same(o.flops_eff, n.flops_eff) + + +@pytest.mark.parametrize("k", list(ORIG.LLMS.keys())) +def test_llm_entry(k): + o, n = ORIG.LLMS[k], NEW.LLMS[k] + _fields_equal(o, n) + assert same(o.kv_bytes_per_token, n.kv_bytes_per_token) + + +@pytest.mark.parametrize("k", list(ORIG.TWO_RATE.keys())) +def test_two_rate_entry(k): + o, n = ORIG.TWO_RATE[k], NEW.TWO_RATE[k] + _fields_equal(o, n) + for b in BATCH: + assert same(o.c_dec(b), n.c_dec(b)), b diff --git a/tests/test_validation.py b/tests/test_validation.py new file mode 100644 index 0000000..474113b --- /dev/null +++ b/tests/test_validation.py @@ -0,0 +1,32 @@ +"""Parity of the literature-anchor validators and the internal implied_batch helper.""" + +from __future__ import annotations + +import pytest + +from _dual import HW_KEYS, MODEL_KEYS, NEW, ORIG, same + +# implied_batch is package-internal (not exported); reach it on each module. +_orig_implied = ORIG.implied_batch +_new_implied = NEW.validation.implied_batch + + +def test_validate_two_rate(): + assert same(ORIG.validate_two_rate(), NEW.validate_two_rate()) + + +def test_validate_against_samsi(): + assert same(ORIG.validate_against_samsi(), NEW.validate_against_samsi()) + + +def test_validate_current_gen(): + assert same(ORIG.validate_current_gen(), NEW.validate_current_gen()) + + +@pytest.mark.parametrize("mk", MODEL_KEYS) +@pytest.mark.parametrize("hk", HW_KEYS) +@pytest.mark.parametrize("j", [0.084, 0.70, 3.0, 4.0, 25.6, 30.0]) +@pytest.mark.parametrize("p_in,p_out", [(64, 512), (50, 382)]) +def test_implied_batch(mk, hk, j, p_in, p_out): + assert same(_orig_implied(ORIG.LLMS[mk], ORIG.HW[hk], j, p_in, p_out), + _new_implied(NEW.LLMS[mk], NEW.HW[hk], j, p_in, p_out)) diff --git a/tests/test_workflow.py b/tests/test_workflow.py new file mode 100644 index 0000000..2b55494 --- /dev/null +++ b/tests/test_workflow.py @@ -0,0 +1,90 @@ +"""Parity of Workflow.run() (all 13 keys) and agentic_ecal() across the parameter factorial.""" + +from __future__ import annotations + +import itertools + +import pytest + +from _dual import NEW, ORIG, same + + +def _build_steps(mod, spec): + steps = [] + for s in spec: + tool = mod.Tool(**s["tool"]) if s.get("tool") else None + ret = mod.Retrieval(**s["retrieval"]) if s.get("retrieval") else None + steps.append(mod.Step(agent=s.get("agent", "main"), + p_in_local=s.get("p_in_local", 0), + p_out=s.get("p_out", 256), + tool=tool, retrieval=ret)) + return steps + + +EMPTY = [] +SINGLE = [{"p_in_local": 600, "p_out": 300}] +MULTI = [ + {"agent": "a", "p_in_local": 300, "p_out": 200}, + {"agent": "b", "p_in_local": 120, "p_out": 120, + "retrieval": {"k_chunks": 5, "chunk_tokens": 256}}, + {"agent": "a", "p_in_local": 200, "p_out": 300, + "tool": {"name": "code", "energy": 2.0, "obs_tokens": 150}}, + {"agent": "c", "p_in_local": 200, "p_out": 400}, + {"agent": "c", "p_in_local": 50, "p_out": 128}, +] +STEP_SPECS = {"empty": EMPTY, "single": SINGLE, "multi": MULTI} + +# A curated factorial over every run() axis (kept representative, not exhaustive). +_AXES = list(itertools.product( + [True, False], # carry_history + [None, "llama3_8b"], # calib key (None -> derived path) + [0, 1, 2, 4], # round_size + [None, 0, 1, 2], # history_rounds + [True, False], # exclude_self + [1, 32, 256], # serving_batch + [0, 400], # sys_tokens + [0.0, 0.10], # gamma_v + [0.0, 0.5], # tx_energy_per_message +)) +# Sample deterministically to keep the run fast while covering every axis value. +_COMBOS = _AXES[::37] + + +def _make(mod, spec_name, model_key, hw_key, axis): + ch, calib_key, rs, hr, ex, sb, sys_t, gv, tx = axis + calib = mod.TWO_RATE[calib_key] if calib_key else None + return mod.Workflow( + model=mod.LLMS[model_key], hw=mod.HW[hw_key], + steps=_build_steps(mod, STEP_SPECS[spec_name]), + sys_tokens=sys_t, carry_history=ch, history_rounds=hr, round_size=rs, + exclude_self=ex, serving_batch=sb, calib=calib, gamma_v=gv, + tx_energy_per_message=tx) + + +@pytest.mark.parametrize("spec_name", list(STEP_SPECS)) +@pytest.mark.parametrize("axis", _COMBOS) +def test_run_and_agentic_ecal(spec_name, axis): + ow = _make(ORIG, spec_name, "llama3_8b", "a100", axis) + nw = _make(NEW, spec_name, "llama3_8b", "a100", axis) + assert same(ow.run(), nw.run()) + for bpt in (17.0, 8.0): + for emb in (0.0, 1e12): + assert same(ow.agentic_ecal(bpt, emb), nw.agentic_ecal(bpt, emb)) + + +@pytest.mark.parametrize("model_key", ["llama3_8b", "llama3_70b", "qwen2_5_7b"]) +@pytest.mark.parametrize("hw_key", ["h100", "h100_nvl", "a100"]) +def test_run_across_models_and_hw(model_key, hw_key): + axis = _COMBOS[len(_COMBOS) // 2] + ow = _make(ORIG, "multi", model_key, hw_key, axis) + nw = _make(NEW, "multi", model_key, hw_key, axis) + assert same(ow.run(), nw.run()) + + +def test_reduce_case_one_shot_equals_direct_call(): + """The demo's self-consistency check, verified in the new package.""" + m, hw = NEW.LLMS["llama3_8b"], NEW.HW["h100"] + wf = NEW.Workflow(model=m, hw=hw, steps=[NEW.Step(p_in_local=600, p_out=300)], + sys_tokens=0, carry_history=True, gamma_v=0.0, serving_batch=64) + direct = NEW.llm_call_energy(m, hw, 600, 300, 64) + assert abs(wf.run()["e_total"] - direct) < 1e-6