From e0c6e86b4262f640cc9963dd02d9de97bda24a1e Mon Sep 17 00:00:00 2001
From: Rohan Dsouza <rohan@hotdata.dev>
Date: Mon, 26 Oct 2026 14:22:55 +0530
Subject: [PATCH 1/8] Phase 12A task 1: load the BEAM splits and cut the
 windows of capture

bench/beam.py pins the 100K and 1M files of Mohammadta/BEAM by revision
and SHA-256, parses each row into dated batches and probes, strips the
index markers from user turns, maps the gold field of each ability to
one field, and cuts each batch into windows of whole turn pairs of at
most 10,000 characters. A longer pair is cut at blank lines.

On the real files it counts 1,606 windows at 100K and 21,003 at 1M.
---
 bench/beam.py            | 385 +++++++++++++++++++++++++++++++++++++++
 bench/tests/test_beam.py | 177 ++++++++++++++++++
 2 files changed, 562 insertions(+)
 create mode 100644 bench/beam.py
 create mode 100644 bench/tests/test_beam.py

diff --git a/bench/beam.py b/bench/beam.py
new file mode 100644
index 0000000..93eb607
--- /dev/null
+++ b/bench/beam.py
@@ -0,0 +1,385 @@
+"""The BEAM data, splits `100K` and `1M`: the download, its checks, and the windows of capture.
+
+One row of a split is one conversation: dated batches of user and assistant turns, and 20
+probing questions, 2 for each of the 10 abilities. Print the counts of a split with:
+
+    uv run --group bench python -m bench.beam --split 1M
+"""
+
+from __future__ import annotations
+
+import argparse
+import ast
+import re
+import statistics
+import urllib.request
+from collections.abc import Iterator, Mapping, Sequence
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from bench.data import DATA_DIR, sha256
+
+DATA_REVISION = "3205395e897e7318c7b094ef4e6047b9b82dbb03"
+DATA_URL = f"https://huggingface.co/datasets/Mohammadta/BEAM/resolve/{DATA_REVISION}/data/"
+SPLITS = {
+    "100K": (
+        "100K-00000-of-00001.parquet",
+        "c0519be25907005ba873c927c50877471d550873039d96c041554d0075a78ace",
+    ),
+    "1M": (
+        "1M-00000-of-00001.parquet",
+        "41b5acbbb55a586b1305514ef9d9fb03365d9b3331b598a1c2dd7603d93ef533",
+    ),
+}
+ABILITIES = (
+    "abstention",
+    "contradiction_resolution",
+    "event_ordering",
+    "information_extraction",
+    "instruction_following",
+    "knowledge_update",
+    "multi_session_reasoning",
+    "preference_following",
+    "summarization",
+    "temporal_reasoning",
+)
+"""The 10 abilities, in the column order of the official report."""
+GOLD_FIELDS = {
+    "abstention": "ideal_response",
+    "contradiction_resolution": "ideal_answer",
+    "event_ordering": "answer",
+    "information_extraction": "answer",
+    "instruction_following": "expected_compliance",
+    "knowledge_update": "answer",
+    "multi_session_reasoning": "answer",
+    "preference_following": "expected_compliance",
+    "summarization": "ideal_summary",
+    "temporal_reasoning": "answer",
+}
+"""The field of each ability that holds the gold answer."""
+WINDOW_CHARS = 10_000
+DATE_FORMAT = "%B-%d-%Y"
+_MARKER = re.compile(r"\s*->->\s*\d+\s*,\s*\d+\s*$")
+_PARAGRAPH = "\n\n"
+
+
+@dataclass(frozen=True)
+class Message:
+    """One turn of a batch: its id in the conversation, its role, and its text."""
+
+    id: int
+    role: str
+    content: str
+
+
+@dataclass(frozen=True)
+class Batch:
+    """One batch of a conversation: its number from 1, its date in UTC, and its turns."""
+
+    number: int
+    date: datetime
+    messages: tuple[Message, ...]
+
+
+@dataclass(frozen=True)
+class Probe:
+    """One probing question, with its gold answer, its rubric, and its source turn ids."""
+
+    id: str
+    conversation: str
+    ability: str
+    question: str
+    gold: str
+    rubric: tuple[str, ...]
+    difficulty: str
+    sources: tuple[int, ...]
+
+
+@dataclass(frozen=True)
+class Window:
+    """One `capture` call: whole turn pairs of one batch, as `role: content` paragraphs.
+
+    `messages` holds the ids of the turns in the window. A turn pair longer than the
+    limit is cut into pieces, so its ids can be in more than one window.
+    """
+
+    batch: int
+    number: int
+    date: datetime
+    text: str
+    messages: tuple[int, ...]
+
+    @property
+    def key(self) -> str:
+        """The episode key: `window-<batch in 2 digits>-<number in 4 digits>`."""
+        return f"window-{self.batch:02d}-{self.number:04d}"
+
+
+@dataclass(frozen=True)
+class Conversation:
+    """One row of a split: the domain, the batches in order, and the probes."""
+
+    id: str
+    domain: str
+    batches: tuple[Batch, ...]
+    probes: tuple[Probe, ...]
+
+    @property
+    def chars(self) -> int:
+        """The characters of every turn."""
+        return sum(len(m.content) for batch in self.batches for m in batch.messages)
+
+    def windows(self, limit: int = WINDOW_CHARS) -> list[Window]:
+        """Return the windows of every batch, in order, numbered from 1 in each batch.
+
+        A window holds whole turn pairs, a user turn and the turns after it, of at most
+        `limit` characters in all. A pair longer than `limit` is cut at blank lines into
+        pieces of at most `limit` characters, and each piece is packed as a pair.
+        """
+        found = []
+        for batch in self.batches:
+            parts = [
+                (piece, ids)
+                for text, ids in _pairs(batch.messages)
+                for piece in (cut(text, limit) if len(text) > limit else [text])
+            ]
+            for number, (text, ids) in enumerate(_pack(parts, limit), start=1):
+                found.append(Window(batch.number, number, batch.date, text, ids))
+        return found
+
+
+def strip_marker(text: str) -> str:
+    """Return `text` without the index marker at its end, such as `->-> 1,1`."""
+    return _MARKER.sub("", text)
+
+
+def cut(text: str, limit: int) -> list[str]:
+    """Cut `text` at blank lines into pieces of at most `limit` characters.
+
+    A paragraph longer than `limit` is cut every `limit` characters.
+    """
+    pieces: list[str] = []
+    current = ""
+    for paragraph in text.split(_PARAGRAPH):
+        while len(paragraph) > limit:
+            if current:
+                pieces.append(current)
+                current = ""
+            pieces.append(paragraph[:limit])
+            paragraph = paragraph[limit:]
+        joined = current + _PARAGRAPH + paragraph if current else paragraph
+        if len(joined) > limit:
+            pieces.append(current)
+            current = paragraph
+        else:
+            current = joined
+    if current:
+        pieces.append(current)
+    return pieces
+
+
+def _pairs(messages: Sequence[Message]) -> Iterator[tuple[str, tuple[int, ...]]]:
+    """Yield the text and the turn ids of each turn pair, a user turn and the turns after it."""
+    pair: list[Message] = []
+    for message in messages:
+        if message.role == "user" and pair:
+            yield _paragraphs(pair), tuple(m.id for m in pair)
+            pair = []
+        pair.append(message)
+    if pair:
+        yield _paragraphs(pair), tuple(m.id for m in pair)
+
+
+def _paragraphs(messages: Sequence[Message]) -> str:
+    """Return `messages` as `role: content` paragraphs, as `bench.prompts.session_text` does."""
+    return _PARAGRAPH.join(f"{m.role}: {m.content.strip()}" for m in messages)
+
+
+def _pack(
+    parts: Sequence[tuple[str, tuple[int, ...]]], limit: int
+) -> Iterator[tuple[str, tuple[int, ...]]]:
+    """Join `parts` in order into windows of at most `limit` characters, one blank line apart."""
+    text = ""
+    ids: tuple[int, ...] = ()
+    for part, part_ids in parts:
+        if text and len(text) + len(_PARAGRAPH) + len(part) > limit:
+            yield text, ids
+            text, ids = "", ()
+        text = text + _PARAGRAPH + part if text else part
+        ids = ids + tuple(i for i in part_ids if i not in ids)
+    if text:
+        yield text, ids
+
+
+def parse_date(value: str) -> datetime:
+    """Return a date such as `March-15-2024` as midnight UTC."""
+    return datetime.strptime(value, DATE_FORMAT).replace(tzinfo=UTC)
+
+
+def parse_conversation(row: Mapping[str, Any]) -> Conversation:
+    """Return the conversation of one row, with the index markers stripped from user turns.
+
+    The date of a batch is the `time_anchor` of its first turn that has one. Raises
+    ValueError if a batch has no date or two dates, or if a probe has no gold field.
+    """
+    conversation = str(row["conversation_id"])
+    batches = []
+    for number, turns in enumerate(row["chat"], start=1):
+        anchors = {t["time_anchor"] for t in turns if t["time_anchor"]}
+        if len(anchors) != 1:
+            raise ValueError(f"conversation {conversation} batch {number}: dates {anchors}")
+        messages = tuple(
+            Message(
+                int(t["id"]),
+                t["role"],
+                strip_marker(t["content"]) if t["role"] == "user" else t["content"],
+            )
+            for t in turns
+        )
+        batches.append(Batch(number, parse_date(anchors.pop()), messages))
+    probes = _probes(conversation, row["probing_questions"])
+    return Conversation(conversation, row["conversation_seed"]["category"], tuple(batches), probes)
+
+
+def _probes(conversation: str, value: str | Mapping[str, Any]) -> tuple[Probe, ...]:
+    """Return the probes of `value`, a dict or its Python literal, in the order of ABILITIES."""
+    by_ability = ast.literal_eval(value) if isinstance(value, str) else value
+    if set(by_ability) != set(ABILITIES):
+        raise ValueError(f"conversation {conversation}: abilities {sorted(by_ability)}")
+    probes = []
+    for ability in ABILITIES:
+        for number, entry in enumerate(by_ability[ability], start=1):
+            field = GOLD_FIELDS[ability]
+            if field not in entry:
+                raise ValueError(f"conversation {conversation} {ability} {number}: no {field}")
+            probes.append(
+                Probe(
+                    id=f"{conversation}-{ability}-{number}",
+                    conversation=conversation,
+                    ability=ability,
+                    question=entry["question"],
+                    gold=str(entry[field]),
+                    rubric=tuple(str(item) for item in entry["rubric"]),
+                    difficulty=str(entry.get("difficulty", "")),
+                    sources=_ids(entry.get("source_chat_ids")),
+                )
+            )
+    return tuple(probes)
+
+
+def _ids(value: Any) -> tuple[int, ...]:
+    """Return the turn ids in `value`, at any depth of lists and dicts, sorted, without repeats.
+
+    An item that is not a whole number, such as a note, is left out.
+    """
+    found: set[int] = set()
+
+    def walk(item: Any) -> None:
+        if isinstance(item, Mapping):
+            for inner in item.values():
+                walk(inner)
+        elif isinstance(item, list | tuple):
+            for inner in item:
+                walk(inner)
+        elif isinstance(item, int) or (isinstance(item, str) and item.strip().isdigit()):
+            found.add(int(item))
+
+    walk(value)
+    return tuple(sorted(found))
+
+
+def download(split: str, directory: Path = DATA_DIR) -> Path:
+    """Download the file of `split` into `directory` unless it is there, and return its path.
+
+    Raises ValueError, and keeps no file, if the download has another SHA-256.
+    """
+    name, expected = SPLITS[split]
+    path = directory / f"beam-{name}"
+    if path.exists():
+        return path
+    directory.mkdir(parents=True, exist_ok=True)
+    partial = path.with_suffix(".part")
+    with urllib.request.urlopen(DATA_URL + name) as response, partial.open("wb") as file:
+        while block := response.read(1 << 20):
+            file.write(block)
+    found = sha256(partial)
+    if found != expected:
+        partial.unlink()
+        raise ValueError(f"{DATA_URL + name} has SHA-256 {found}, expected {expected}")
+    partial.rename(path)
+    return path
+
+
+def load(path: Path, expected_sha256: str | None) -> list[Conversation]:
+    """Return the conversations of the parquet file at `path`, in file order.
+
+    Raises ValueError if `expected_sha256` is not None and the file has another SHA-256,
+    or if two conversations share an id.
+    """
+    import pyarrow.parquet
+
+    if expected_sha256 is not None:
+        found = sha256(path)
+        if found != expected_sha256:
+            raise ValueError(f"{path} has SHA-256 {found}, expected {expected_sha256}")
+    rows = pyarrow.parquet.read_table(path).to_pylist()
+    conversations = [parse_conversation(row) for row in rows]
+    ids = [conversation.id for conversation in conversations]
+    if len(set(ids)) != len(ids):
+        raise ValueError(f"{path} repeats a conversation id")
+    return conversations
+
+
+def load_split(split: str, directory: Path = DATA_DIR) -> list[Conversation]:
+    """Download the file of `split` if it is not there, and return its conversations."""
+    return load(download(split, directory), SPLITS[split][1])
+
+
+def counts(conversations: Sequence[Conversation], limit: int = WINDOW_CHARS) -> dict[str, Any]:
+    """Return the counts of `conversations`: questions, nuggets, characters, and windows."""
+    windows = [conversation.windows(limit) for conversation in conversations]
+    per_conversation = [len(found) for found in windows]
+    sizes = [len(window.text) for found in windows for window in found]
+    probes = [probe for conversation in conversations for probe in conversation.probes]
+    long_pairs = sum(
+        1
+        for conversation in conversations
+        for batch in conversation.batches
+        for text, _ in _pairs(batch.messages)
+        if len(text) > limit
+    )
+    return {
+        "conversations": len(conversations),
+        "questions": len(probes),
+        "nuggets": sum(len(probe.rubric) for probe in probes),
+        "characters": sum(conversation.chars for conversation in conversations),
+        "windows": sum(per_conversation),
+        "windows_per_conversation": {
+            "min": min(per_conversation),
+            "median": statistics.median(per_conversation),
+            "mean": round(statistics.mean(per_conversation), 1),
+            "max": max(per_conversation),
+        },
+        "window_chars": {
+            "median": statistics.median(sizes),
+            "mean": round(statistics.mean(sizes)),
+            "max": max(sizes),
+        },
+        "long_pairs": long_pairs,
+        "domains": sorted({conversation.domain for conversation in conversations}),
+    }
+
+
+def main(argv: Sequence[str] | None = None) -> None:
+    """Print the counts of one split."""
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0] if __doc__ else "")
+    parser.add_argument("--split", choices=sorted(SPLITS), required=True)
+    args = parser.parse_args(argv)
+    for name, value in counts(load_split(args.split)).items():
+        print(f"{name}: {value}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/bench/tests/test_beam.py b/bench/tests/test_beam.py
new file mode 100644
index 0000000..961d953
--- /dev/null
+++ b/bench/tests/test_beam.py
@@ -0,0 +1,177 @@
+import hashlib
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from bench.beam import (
+    ABILITIES,
+    GOLD_FIELDS,
+    Conversation,
+    counts,
+    cut,
+    load,
+    parse_conversation,
+    strip_marker,
+)
+
+
+def turn(id: int, role: str, content: str, anchor: str | None = None) -> dict[str, Any]:
+    return {
+        "id": id,
+        "role": role,
+        "content": content,
+        "time_anchor": anchor,
+        "index": None,
+        "question_type": None,
+    }
+
+
+def probes() -> dict[str, list[dict[str, Any]]]:
+    found = {}
+    for ability in ABILITIES:
+        found[ability] = [
+            {
+                "question": f"{ability} question {n}?",
+                GOLD_FIELDS[ability]: f"{ability} gold {n}",
+                "rubric": [f"{ability} nugget {n}a", f"{ability} nugget {n}b"],
+                "difficulty": "easy",
+                "source_chat_ids": {"first": [3, 1], "second": [[1, 5]]} if n == 1 else [2],
+            }
+            for n in (1, 2)
+        ]
+    return found
+
+
+def row(conversation: str = "7", chat: list[list[dict[str, Any]]] | None = None) -> dict[str, Any]:
+    if chat is None:
+        chat = [
+            [
+                turn(0, "user", "First question? ->-> 1,1", "March-15-2024"),
+                turn(1, "assistant", "First answer ->-> 1,1"),
+                turn(2, "user", "Second question ->->2,3"),
+                turn(3, "assistant", "Second answer"),
+            ],
+            [
+                turn(4, "user", "Later question", "April-02-2024"),
+                turn(5, "assistant", "Later answer"),
+            ],
+        ]
+    return {
+        "conversation_id": conversation,
+        "conversation_seed": {"category": "Coding", "id": 1, "theme": "t", "title": "t"},
+        "chat": chat,
+        "probing_questions": repr(probes()),
+    }
+
+
+@pytest.fixture
+def conversation() -> Conversation:
+    return parse_conversation(row())
+
+
+def test_strip_marker_removes_only_the_end_marker() -> None:
+    assert strip_marker("Hello ->-> 1,1") == "Hello"
+    assert strip_marker("Hello ->->12, 3 ") == "Hello"
+    assert strip_marker("a ->-> b is kept") == "a ->-> b is kept"
+
+
+def test_batches_have_their_dates_and_stripped_user_turns(conversation: Conversation) -> None:
+    assert conversation.id == "7"
+    assert conversation.domain == "Coding"
+    first, later = conversation.batches
+    assert (first.number, first.date) == (1, datetime(2024, 3, 15, tzinfo=UTC))
+    assert later.date == datetime(2024, 4, 2, tzinfo=UTC)
+    assert [m.content for m in first.messages] == [
+        "First question?",
+        "First answer ->-> 1,1",
+        "Second question",
+        "Second answer",
+    ]
+
+
+def test_probes_follow_the_abilities_with_one_gold_field(conversation: Conversation) -> None:
+    assert len(conversation.probes) == 20
+    assert [p.ability for p in conversation.probes[::2]] == list(ABILITIES)
+    first = conversation.probes[0]
+    assert first.id == "7-abstention-1"
+    assert first.gold == "abstention gold 1"
+    assert first.rubric == ("abstention nugget 1a", "abstention nugget 1b")
+    assert first.sources == (1, 3, 5)
+    assert conversation.probes[1].sources == (2,)
+    for probe in conversation.probes:
+        assert probe.gold == f"{probe.ability} gold {probe.id[-1]}"
+
+
+def test_a_batch_needs_exactly_one_date() -> None:
+    with pytest.raises(ValueError, match="dates"):
+        parse_conversation(row(chat=[[turn(0, "user", "q"), turn(1, "assistant", "a")]]))
+    two = [[turn(0, "user", "q", "March-01-2024"), turn(1, "assistant", "a", "March-02-2024")]]
+    with pytest.raises(ValueError, match="dates"):
+        parse_conversation(row(chat=two))
+
+
+def test_a_probe_needs_its_gold_field() -> None:
+    found = probes()
+    del found["summarization"][0]["ideal_summary"]
+    with pytest.raises(ValueError, match="no ideal_summary"):
+        parse_conversation({**row(), "probing_questions": repr(found)})
+
+
+def test_windows_hold_whole_pairs_under_the_limit(conversation: Conversation) -> None:
+    pair = len("user: First question?\n\nassistant: First answer ->-> 1,1")
+    windows = conversation.windows(limit=pair + 5)
+    assert [(w.batch, w.number, w.key) for w in windows] == [
+        (1, 1, "window-01-0001"),
+        (1, 2, "window-01-0002"),
+        (2, 1, "window-02-0001"),
+    ]
+    assert windows[0].text == "user: First question?\n\nassistant: First answer ->-> 1,1"
+    assert windows[0].messages == (0, 1)
+    assert windows[1].messages == (2, 3)
+    assert windows[2].date == datetime(2024, 4, 2, tzinfo=UTC)
+    whole = conversation.windows()
+    assert len(whole) == 2
+    assert whole[0].text.count("user: ") == 2
+
+
+def test_a_long_pair_is_cut_into_pieces() -> None:
+    long = "\n\n".join(["x" * 30] * 4)
+    chat = [[turn(0, "user", "q", "March-01-2024"), turn(1, "assistant", long)]]
+    windows = parse_conversation(row(chat=chat)).windows(limit=70)
+    assert len(windows) > 1
+    assert all(len(w.text) <= 70 for w in windows)
+    assert all(w.messages == (0, 1) for w in windows)
+    assert "".join(w.text for w in windows).count("x") == 120
+
+
+def test_cut_splits_at_blank_lines_and_then_hard() -> None:
+    assert cut("aa\n\nbb\n\ncc", 6) == ["aa\n\nbb", "cc"]
+    assert cut("abcdefgh", 3) == ["abc", "def", "gh"]
+    assert all(len(piece) <= 5 for piece in cut("a\n\n" + "b" * 12 + "\n\nc", 5))
+
+
+def test_load_checks_the_hash_and_the_ids(tmp_path: Path) -> None:
+    import pyarrow
+    import pyarrow.parquet
+
+    path = tmp_path / "beam.parquet"
+    pyarrow.parquet.write_table(pyarrow.Table.from_pylist([row("1"), row("2")]), path)
+    digest = hashlib.sha256(path.read_bytes()).hexdigest()
+    assert [c.id for c in load(path, digest)] == ["1", "2"]
+    with pytest.raises(ValueError, match="SHA-256"):
+        load(path, "0" * 64)
+    pyarrow.parquet.write_table(pyarrow.Table.from_pylist([row("1"), row("1")]), path)
+    with pytest.raises(ValueError, match="repeats a conversation id"):
+        load(path, None)
+
+
+def test_counts(conversation: Conversation) -> None:
+    found = counts([conversation])
+    assert found["conversations"] == 1
+    assert found["questions"] == 20
+    assert found["nuggets"] == 40
+    assert found["windows"] == 2
+    assert found["long_pairs"] == 0
+    assert found["domains"] == ["Coding"]

From 64fc8e1311a2a48d7f217776a2513ee93dce52ae Mon Sep 17 00:00:00 2001
From: Rohan Dsouza <rohan@hotdata.dev>
Date: Mon, 26 Oct 2026 14:23:50 +0530
Subject: [PATCH 2/8] Phase 12A task 2: draw BEAM samples A and B from the 100K
 split

Each sample is 5 conversations from 5 different domains, drawn with a
fixed seed, and sample B leaves out the conversations of sample A.
Sample A is conversations 2, 6, 12, 17, 18 (444 windows), and sample B
is 1, 4, 7, 14, 19 (386 windows).
---
 bench/beam.py            | 86 +++++++++++++++++++++++++++++++++++++++-
 bench/sample-beam-a.json | 27 +++++++++++++
 bench/sample-beam-b.json | 27 +++++++++++++
 bench/tests/test_beam.py | 22 ++++++++++
 4 files changed, 160 insertions(+), 2 deletions(-)
 create mode 100644 bench/sample-beam-a.json
 create mode 100644 bench/sample-beam-b.json

diff --git a/bench/beam.py b/bench/beam.py
index 93eb607..de276f9 100644
--- a/bench/beam.py
+++ b/bench/beam.py
@@ -4,12 +4,16 @@
 probing questions, 2 for each of the 10 abilities. Print the counts of a split with:
 
     uv run --group bench python -m bench.beam --split 1M
+
+With `--split 100K --samples`, it also writes the files of samples A and B.
 """
 
 from __future__ import annotations
 
 import argparse
 import ast
+import json
+import random
 import re
 import statistics
 import urllib.request
@@ -21,6 +25,8 @@
 
 from bench.data import DATA_DIR, sha256
 
+HERE = Path(__file__).resolve().parent
+
 DATA_REVISION = "3205395e897e7318c7b094ef4e6047b9b82dbb03"
 DATA_URL = f"https://huggingface.co/datasets/Mohammadta/BEAM/resolve/{DATA_REVISION}/data/"
 SPLITS = {
@@ -60,6 +66,8 @@
 }
 """The field of each ability that holds the gold answer."""
 WINDOW_CHARS = 10_000
+SAMPLE_SPLIT = "100K"
+SAMPLE_SIZE = 5
 DATE_FORMAT = "%B-%d-%Y"
 _MARKER = re.compile(r"\s*->->\s*\d+\s*,\s*\d+\s*$")
 _PARAGRAPH = "\n\n"
@@ -337,6 +345,69 @@ def load_split(split: str, directory: Path = DATA_DIR) -> list[Conversation]:
     return load(download(split, directory), SPLITS[split][1])
 
 
+@dataclass(frozen=True)
+class Sample:
+    """A fixed sample of SAMPLE_SPLIT: its file, its seed, and the samples it leaves out."""
+
+    file: str
+    seed: int
+    excludes: tuple[str, ...] = ()
+
+
+SAMPLES = {
+    "a": Sample("bench/sample-beam-a.json", 20261026),
+    "b": Sample("bench/sample-beam-b.json", 20261027, excludes=("a",)),
+}
+
+
+def choose_sample(
+    conversations: Sequence[Conversation],
+    seed: int,
+    exclude: Sequence[str] = (),
+    size: int = SAMPLE_SIZE,
+) -> list[Conversation]:
+    """Return `size` conversations from `size` different domains, drawn with `seed`.
+
+    The draw first picks the domains, then one conversation of each, and leaves out the
+    ids in `exclude`. The result is sorted by id. Raises ValueError if fewer than `size`
+    domains are left.
+    """
+    rng = random.Random(seed)
+    left = [c for c in conversations if c.id not in set(exclude)]
+    domains = sorted({c.domain for c in left})
+    if len(domains) < size:
+        raise ValueError(f"only {len(domains)} domains are left, {size} needed")
+    chosen = []
+    for domain in rng.sample(domains, size):
+        chosen.append(rng.choice(sorted((c for c in left if c.domain == domain), key=_order)))
+    return sorted(chosen, key=_order)
+
+
+def _order(conversation: Conversation) -> tuple[int, str]:
+    """Sort by the conversation id as a number, then as text."""
+    return (int(conversation.id), "") if conversation.id.isdigit() else (0, conversation.id)
+
+
+def sample_ids(
+    name: str, conversations: Sequence[Conversation], root: Path = HERE.parent
+) -> list[str]:
+    """Return the conversation ids of sample `name`, and write its file first if it is missing.
+
+    The file lists each conversation with its domain and its window count. The draw of a
+    sample leaves out every id of the samples that it excludes.
+    """
+    sample = SAMPLES[name]
+    path = root / sample.file
+    if not path.exists():
+        exclude = [i for other in sample.excludes for i in sample_ids(other, conversations, root)]
+        chosen = choose_sample(conversations, sample.seed, exclude)
+        entries = [
+            {"conversation": c.id, "domain": c.domain, "windows": len(c.windows())} for c in chosen
+        ]
+        path.write_text(json.dumps(entries, indent=1) + "\n")
+    return [entry["conversation"] for entry in json.loads(path.read_text())]
+
+
 def counts(conversations: Sequence[Conversation], limit: int = WINDOW_CHARS) -> dict[str, Any]:
     """Return the counts of `conversations`: questions, nuggets, characters, and windows."""
     windows = [conversation.windows(limit) for conversation in conversations]
@@ -373,12 +444,23 @@ def counts(conversations: Sequence[Conversation], limit: int = WINDOW_CHARS) ->
 
 
 def main(argv: Sequence[str] | None = None) -> None:
-    """Print the counts of one split."""
+    """Print the counts of one split, and write the sample files with `--samples`."""
     parser = argparse.ArgumentParser(description=__doc__.splitlines()[0] if __doc__ else "")
     parser.add_argument("--split", choices=sorted(SPLITS), required=True)
+    parser.add_argument(
+        "--samples",
+        action="store_true",
+        help=f"write the sample files from the {SAMPLE_SPLIT} split if they do not exist",
+    )
     args = parser.parse_args(argv)
-    for name, value in counts(load_split(args.split)).items():
+    conversations = load_split(args.split)
+    for name, value in counts(conversations).items():
         print(f"{name}: {value}")
+    if args.samples:
+        if args.split != SAMPLE_SPLIT:
+            parser.error(f"--samples needs --split {SAMPLE_SPLIT}")
+        for sample in SAMPLES:
+            print(f"sample {sample}: {', '.join(sample_ids(sample, conversations))}")
 
 
 if __name__ == "__main__":
diff --git a/bench/sample-beam-a.json b/bench/sample-beam-a.json
new file mode 100644
index 0000000..1b7e741
--- /dev/null
+++ b/bench/sample-beam-a.json
@@ -0,0 +1,27 @@
+[
+ {
+  "conversation": "2",
+  "domain": "Coding",
+  "windows": 69
+ },
+ {
+  "conversation": "6",
+  "domain": "Writing Assistant & Learning",
+  "windows": 80
+ },
+ {
+  "conversation": "12",
+  "domain": "Philosophical or Ethical Discussion",
+  "windows": 128
+ },
+ {
+  "conversation": "17",
+  "domain": "Lifestyle",
+  "windows": 68
+ },
+ {
+  "conversation": "18",
+  "domain": "Therapy and Emotional Support",
+  "windows": 99
+ }
+]
diff --git a/bench/sample-beam-b.json b/bench/sample-beam-b.json
new file mode 100644
index 0000000..347bd28
--- /dev/null
+++ b/bench/sample-beam-b.json
@@ -0,0 +1,27 @@
+[
+ {
+  "conversation": "1",
+  "domain": "Coding",
+  "windows": 74
+ },
+ {
+  "conversation": "4",
+  "domain": "Math",
+  "windows": 76
+ },
+ {
+  "conversation": "7",
+  "domain": "Writing Assistant & Learning",
+  "windows": 74
+ },
+ {
+  "conversation": "14",
+  "domain": "Asking Recommendation",
+  "windows": 58
+ },
+ {
+  "conversation": "19",
+  "domain": "Legal and Administrative",
+  "windows": 104
+ }
+]
diff --git a/bench/tests/test_beam.py b/bench/tests/test_beam.py
index 961d953..da454fd 100644
--- a/bench/tests/test_beam.py
+++ b/bench/tests/test_beam.py
@@ -9,10 +9,12 @@
     ABILITIES,
     GOLD_FIELDS,
     Conversation,
+    choose_sample,
     counts,
     cut,
     load,
     parse_conversation,
+    sample_ids,
     strip_marker,
 )
 
@@ -175,3 +177,23 @@ def test_counts(conversation: Conversation) -> None:
     assert found["windows"] == 2
     assert found["long_pairs"] == 0
     assert found["domains"] == ["Coding"]
+
+
+def test_a_sample_takes_one_conversation_from_each_of_five_domains(tmp_path: Path) -> None:
+    domains = ["A", "A", "B", "B", "C", "C", "D", "D", "E", "E", "F", "F"]
+    conversations = [
+        parse_conversation({**row(str(i)), "conversation_seed": {"category": domain, "id": i}})
+        for i, domain in enumerate(domains, start=1)
+    ]
+    (tmp_path / "bench").mkdir()
+    a = sample_ids("a", conversations, tmp_path)
+    b = sample_ids("b", conversations, tmp_path)
+    by_id = {c.id: c for c in conversations}
+    assert len(a) == len(b) == 5
+    assert len({by_id[i].domain for i in a}) == 5
+    assert len({by_id[i].domain for i in b}) == 5
+    assert not set(a) & set(b)
+    assert a == sample_ids("a", conversations[::-1], tmp_path)
+    assert choose_sample(conversations, 1) == choose_sample(conversations[::-1], 1)
+    with pytest.raises(ValueError, match="domains are left"):
+        choose_sample(conversations[:8], 1)

From d6e4fe56db5d129dca4b2afd6a94948c7fa3a04b Mon Sep 17 00:00:00 2001
From: Rohan Dsouza <rohan@hotdata.dev>
Date: Mon, 26 Oct 2026 14:28:28 +0530
Subject: [PATCH 3/8] Phase 12A task 3: score BEAM as the official code does

bench/beam_score.py scores a response to a BEAM probe as the official
code at commit b2da22e does: one judge call for each rubric nugget with
the unified judge prompt and its JSON parse, and for event ordering the
line match and Kendall tau-b, reported as tau_norm. The official answer
and judge prompts and the MIT license are in bench/beam_prompts/.

The test runs a copy of the official functions beside ours on fixed and
random replies with a fake model, and needs equal scores.

The model calls can now send a system message, which the official match
call uses. The cache key of a call without one is unchanged. The bench
group gains scipy and json-repair.
---
 bench/beam_prompts/LICENSE          |  21 ++
 bench/beam_prompts/answer.txt       |  18 ++
 bench/beam_prompts/judge.txt        |  69 +++++++
 bench/beam_prompts/match_system.txt |   6 +
 bench/beam_prompts/match_user.txt   |   4 +
 bench/beam_score.py                 | 206 ++++++++++++++++++++
 bench/models.py                     |  48 +++--
 bench/tests/test_beam_score.py      | 291 ++++++++++++++++++++++++++++
 bench/tests/test_e2e.py             |  42 +++-
 pyproject.toml                      |  10 +-
 uv.lock                             | 177 +++++++++++++++++
 11 files changed, 877 insertions(+), 15 deletions(-)
 create mode 100644 bench/beam_prompts/LICENSE
 create mode 100644 bench/beam_prompts/answer.txt
 create mode 100644 bench/beam_prompts/judge.txt
 create mode 100644 bench/beam_prompts/match_system.txt
 create mode 100644 bench/beam_prompts/match_user.txt
 create mode 100644 bench/beam_score.py
 create mode 100644 bench/tests/test_beam_score.py

diff --git a/bench/beam_prompts/LICENSE b/bench/beam_prompts/LICENSE
new file mode 100644
index 0000000..3f9b9d0
--- /dev/null
+++ b/bench/beam_prompts/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Mohammad Tavakoli
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/bench/beam_prompts/answer.txt b/bench/beam_prompts/answer.txt
new file mode 100644
index 0000000..0797b7f
--- /dev/null
+++ b/bench/beam_prompts/answer.txt
@@ -0,0 +1,18 @@
+
+You are an assistant that MUST answer questions using ONLY the information provided in the context below. 
+
+STRICT INSTRUCTIONS:
+1. Answer ONLY based on the provided context
+2. Do NOT use your internal knowledge
+
+CONTEXT:
+<context>
+
+QUESTION:
+<question>
+
+ANSWER REQUIREMENTS:
+- Be direct and concise
+- Only output the answer to the question without any explanation 
+
+RESPONSE:
diff --git a/bench/beam_prompts/judge.txt b/bench/beam_prompts/judge.txt
new file mode 100644
index 0000000..0938076
--- /dev/null
+++ b/bench/beam_prompts/judge.txt
@@ -0,0 +1,69 @@
+
+You are an expert evaluator tasked with judging whether the LLM's response demonstrates compliance with the specified RUBRIC CRITERION.
+
+## EVALUATION INPUTS
+- RUBRIC CRITERION (what to check): <rubric_item>
+- RESPONSE TO EVALUATE: <llm_response>
+
+## EVALUATION RUBRIC:
+The rubric defines a specific requirement, constraint, or expected behavior that the LLM response should demonstrate. 
+
+**IMPORTANT**: Pay careful attention to whether the rubric specifies:
+- **Positive requirements** (things the response SHOULD include/do)
+- **Negative constraints** (things the response SHOULD NOT include/do, often indicated by "no", "not", "avoid", "absent")
+
+## RESPONSIVENESS REQUIREMENT
+A compliant response must be **on-topic** and attempt to answer it.
+- If the response does not address the QUESTION, score **0.0** and stop.
+- For negative constraints, both must hold: (a) the response is responsive to the QUESTION, and (b) the prohibited element is absent.
+
+## SEMANTIC TOLERANCE RULES:
+Judge by meaning, not exact wording.
+- Accept **paraphrases** and **synonyms** that preserve intent.
+- **Case/punctuation/whitespace** differences must be ignored.
+- **Numbers/currencies/dates** may appear in equivalent forms (e.g., “$68,000”, “68k”, “68,000 USD”, or “sixty-eight thousand dollars”). Treat them as equal when numerically equivalent.
+- If the rubric expects a number or duration, prefer **normalized comparison** (extract and compare values) over string matching.
+
+## STYLE NEUTRALITY (prevents style contamination):
+Ignore tone, politeness, length, and flourish unless the rubric explicitly requires a format/structure (e.g., “itemized list”, “no citations”, “one sentence”).
+- Do **not** penalize hedging, voice, or verbosity if content satisfies the rubric.
+- Only evaluate format when the rubric **explicitly** mandates it.
+
+## SCORING SCALE:
+- **1.0 (Complete Compliance)**: Fully complies with the rubric criterion.
+  - Positive: required element present, accurate, properly executed (allowing semantic equivalents).
+  - Negative: prohibited element **absent** AND response is **responsive**.
+  
+- **0.5 (Partial Compliance)**: Partially complies.
+  - Positive: element present but minor inaccuracies/incomplete execution.
+  - Negative: generally responsive and mostly avoids the prohibited element but with minor/edge violations.
+  
+- **0.0 (No Compliance)**: Fails to comply.
+  - Positive: required element missing or incorrect.
+  - Negative: prohibited element present **or** response is non-responsive/evasive even if the element is absent.
+
+## EVALUATION INSTRUCTIONS:
+1. **Understand the Requirement**: Determine if the rubric is asking for something to be present (positive) or absent (negative/constraint).
+
+2. **Parse Compound Statements**: If the rubric contains multiple elements connected by "and" or commas, evaluate whether:
+   - **All elements** must be present for full compliance (1.0)
+   - **Some elements** present indicates partial compliance (0.5)
+   - **No elements** present indicates no compliance (0.0)
+   
+3. **Check Compliance**: 
+   - For positive requirements: Look for the presence and quality of the required element
+   - For negative constraints: Look for the absence of the prohibited element
+
+4. **Assign Score**: Based on compliance with the specific rubric criterion according to the scoring scale above.
+
+5. **Provide Reasoning**: Explain whether the rubric criterion was satisfied and justify the score.
+
+## OUTPUT FORMAT:
+Return your evaluation in JSON format with two fields:
+
+{
+   "score": [your score: 1.0, 0.5, or 0.0],
+   "reason": "[detailed explanation of whether the rubric criterion was satisfied and why this justified the assigned score]"
+}
+
+NOTE: ONLY output the json object, without any explanation before or after that
diff --git a/bench/beam_prompts/match_system.txt b/bench/beam_prompts/match_system.txt
new file mode 100644
index 0000000..a5c9784
--- /dev/null
+++ b/bench/beam_prompts/match_system.txt
@@ -0,0 +1,6 @@
+
+            You are a binary classifier.
+            If the TWO snippets describe the SAME event/fact, reply **YES**
+            Otherwise reply **NO**. No extra words.
+            DO NOT provide any exaplanation.
+        
\ No newline at end of file
diff --git a/bench/beam_prompts/match_user.txt b/bench/beam_prompts/match_user.txt
new file mode 100644
index 0000000..fb95413
--- /dev/null
+++ b/bench/beam_prompts/match_user.txt
@@ -0,0 +1,4 @@
+First snippet: {first} 
+
+                       Second snippet: {second}
+                    
\ No newline at end of file
diff --git a/bench/beam_score.py b/bench/beam_score.py
new file mode 100644
index 0000000..29ec619
--- /dev/null
+++ b/bench/beam_score.py
@@ -0,0 +1,206 @@
+"""The BEAM answer prompt and score, as the official code at commit b2da22e gives them.
+
+The prompts in `bench/beam_prompts/` are copied from github.com/mohammadtavakoli78/BEAM at
+commit b2da22e, under the MIT license in `bench/beam_prompts/LICENSE`. The functions here
+follow `src/evaluation/compute_metrics.py` and `src/evaluation/report_results.py` there:
+
+- Each nugget of the rubric gets one judge call with the unified judge prompt, which sees
+  the nugget and the response but not the question. The reply is parsed as JSON, and its
+  `score` is 0, 0.5, or 1. The nugget score of a question is the mean.
+- Event ordering also splits the response on newlines, matches each line to a nugget with
+  one binary call for each unmatched nugget, and computes Kendall tau-b over the matched
+  lists. The report scores event ordering by `tau_norm`, (tau-b + 1) / 2, alone.
+- The aggregate is the mean over the 10 abilities.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import re
+import statistics
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from json_repair import repair_json
+from scipy.stats import kendalltau
+
+from bench.beam import ABILITIES, Probe
+from bench.models import Reply
+
+BEAM_COMMIT = "b2da22e"
+PROMPTS = Path(__file__).resolve().parent / "beam_prompts"
+ANSWER_PROMPT = (PROMPTS / "answer.txt").read_text()
+JUDGE_PROMPT = (PROMPTS / "judge.txt").read_text()
+MATCH_SYSTEM = (PROMPTS / "match_system.txt").read_text()
+MATCH_USER = (PROMPTS / "match_user.txt").read_text()
+JUDGE = "gpt-4.1-mini-2025-04-14"
+JUDGE_MAX_TOKENS = 1000
+MATCH_MAX_TOKENS = 50
+EVENT_ORDERING = "event_ordering"
+Models = Callable[..., Reply]
+
+
+@dataclass(frozen=True)
+class Score:
+    """The score of one response to one probe.
+
+    `score` is the official score: `tau_norm` for event ordering, else `nugget`. `nugget`
+    is the mean of `nuggets`, the judge score of each nugget. For event ordering, `f1` is
+    the F1 of the line match, and `tau_undefined` is True if Kendall tau-b was not defined,
+    which the official code leaves as NaN and this score counts as 0.
+    """
+
+    score: float
+    nugget: float
+    nuggets: tuple[float, ...]
+    tau_norm: float | None = None
+    f1: float | None = None
+    tau_undefined: bool = False
+
+
+def answer_prompt(context: str, question: str) -> str:
+    """Return the official RAG answer prompt for `question` over `context`."""
+    return ANSWER_PROMPT.replace("<context>", context).replace("<question>", question)
+
+
+def judge_prompt(item: str, response: str) -> str:
+    """Return the official judge prompt for one rubric item and one response."""
+    return JUDGE_PROMPT.replace("<rubric_item>", item).replace("<llm_response>", response)
+
+
+def parse_json_response(response: str) -> Any:
+    """Return the JSON in `response`, as the official `parse_json_response` finds it.
+
+    Raises ValueError if no JSON is found.
+    """
+    response = response.strip()
+    if response.startswith("```"):
+        match = re.search(r"```(?:json)?\s*(\[.*\]|\{.*\})\s*```", response, re.DOTALL)
+        if match:
+            response = match.group(1).strip()
+    try:
+        return json.loads(response)
+    except json.JSONDecodeError:
+        pass
+    match = re.search(r"(\{.*?\}|\[.*?\])", response, re.DOTALL)
+    if match:
+        try:
+            return json.loads(match.group(1))
+        except Exception as error:
+            raise ValueError(f"Found possible JSON but failed to parse it: {error}") from error
+    raise ValueError("No valid JSON found in response.")
+
+
+def judge_reply(text: str) -> float:
+    """Return the score of one judge reply, with the official parse and its repair fallback."""
+    try:
+        parsed = parse_json_response(text.strip())
+    except ValueError:
+        parsed = json.loads(repair_json(text.strip()))
+    return float(parsed["score"])
+
+
+def nugget_scores(models: Models, rubric: Sequence[str], response: str) -> tuple[float, ...]:
+    """Return the judge score of each rubric item for `response`, one call each."""
+    return tuple(
+        judge_reply(models(JUDGE, judge_prompt(item, response), JUDGE_MAX_TOKENS).text)
+        for item in rubric
+    )
+
+
+def same_event(models: Models, first: str, second: str) -> bool:
+    """Return True if the binary classifier says the two snippets name the same event."""
+    prompt = MATCH_USER.replace("{first}", first, 1).replace("{second}", second, 1)
+    reply = models(JUDGE, prompt, MATCH_MAX_TOKENS, None, system=MATCH_SYSTEM)
+    return "yes" in reply.text.lower()
+
+
+def align(
+    models: Models, reference: Sequence[str], system: Sequence[str]
+) -> tuple[list[str], list[str]]:
+    """Return `reference` and `system` with each line replaced by its first unused match.
+
+    Each line of `system` is compared with each reference item that no earlier line
+    matched, in order, until one matches. A line with no match stays as it is.
+    """
+    used: set[int] = set()
+    out = []
+    for line in system:
+        matched = None
+        for index, item in enumerate(reference):
+            if index not in used and same_event(models, item, line):
+                matched = index
+                break
+        if matched is None:
+            out.append(line)
+        else:
+            out.append(reference[matched])
+            used.add(matched)
+    return list(reference), out
+
+
+def order_score(reference: Sequence[str], system: Sequence[str]) -> dict[str, float]:
+    """Return the precision, recall, F1, `tau_norm`, and final score of two aligned lists.
+
+    `tau_norm` is NaN if Kendall tau-b is not defined, as in the official code.
+    """
+    tp = len(set(reference) & set(system))
+    fp = len([x for x in system if x not in reference])
+    fn = len([x for x in reference if x not in system])
+    precision = tp / (tp + fp) if tp + fp else 0
+    recall = tp / (tp + fn) if tp + fn else 0
+    f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0
+    union = list(dict.fromkeys([*reference, *system]))
+    tie_rank = len(union) + 1
+
+    def to_rank(seq: Sequence[str]) -> list[int]:
+        ranks = {item: i + 1 for i, item in enumerate(seq)}
+        return [ranks.get(item, tie_rank) for item in union]
+
+    tau_b = kendalltau(to_rank(reference), to_rank(system), variant="b", method="auto")[0]
+    tau_norm = (float(tau_b) + 1) / 2
+    return {
+        "precision": precision,
+        "recall": recall,
+        "f1": f1,
+        "tau_norm": tau_norm,
+        "final_score": tau_norm * f1,
+    }
+
+
+def score(models: Models, probe: Probe, response: str) -> Score:
+    """Return the official score of `response` to `probe`."""
+    nuggets = nugget_scores(models, probe.rubric, response)
+    nugget = sum(nuggets) / len(nuggets)
+    if probe.ability != EVENT_ORDERING:
+        return Score(nugget, nugget, nuggets)
+    reference, system = align(models, probe.rubric, response.split("\n"))
+    order = order_score(reference, system)
+    undefined = math.isnan(order["tau_norm"])
+    tau_norm = 0.0 if undefined else order["tau_norm"]
+    return Score(tau_norm, nugget, nuggets, tau_norm, order["f1"], undefined)
+
+
+def summarize(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
+    """Return the scores of `rows`, each with `ability`, `domain`, and `score`.
+
+    `mean` is the mean over the abilities that `rows` hold, the aggregate of the official
+    report. `questions` is the mean over every row.
+    """
+    by_ability = {
+        ability: statistics.mean(r["score"] for r in rows if r["ability"] == ability)
+        for ability in ABILITIES
+        if any(r["ability"] == ability for r in rows)
+    }
+    domains = sorted({r["domain"] for r in rows})
+    by_domain = {d: statistics.mean(r["score"] for r in rows if r["domain"] == d) for d in domains}
+    return {
+        "mean": statistics.mean(by_ability.values()) if by_ability else 0.0,
+        "questions": statistics.mean(r["score"] for r in rows) if rows else 0.0,
+        "count": len(rows),
+        "by_ability": by_ability,
+        "by_domain": by_domain,
+    }
diff --git a/bench/models.py b/bench/models.py
index 7848589..172585c 100644
--- a/bench/models.py
+++ b/bench/models.py
@@ -43,17 +43,24 @@ def cost(self, model: str) -> float:
 
 
 class Complete(Protocol):
-    """Sends one prompt to one model and returns the reply."""
+    """Sends one prompt to one model, after `system` as a system message if given, and returns
+    the reply."""
 
     def __call__(
-        self, model: str, prompt: str, max_tokens: int, schema: dict[str, Any] | None
+        self,
+        model: str,
+        prompt: str,
+        max_tokens: int,
+        schema: dict[str, Any] | None,
+        system: str | None = None,
     ) -> Reply: ...
 
 
 class CachedModels:
     """Calls `complete` for each prompt that the cache does not hold, and keeps the reply.
 
-    The cache key is the SHA-256 of the model, the token limit, the schema, and the prompt.
+    The cache key is the SHA-256 of the model, the token limit, the schema, and the prompt,
+    and of the system message if there is one.
     `spent` sums the price of the replies that came from a model call in this process, and
     `calls` counts those calls.
     """
@@ -67,15 +74,21 @@ def __init__(self, complete: Complete, path: Path) -> None:
         self._db.execute("CREATE TABLE IF NOT EXISTS replies (key TEXT PRIMARY KEY, reply TEXT)")
 
     def __call__(
-        self, model: str, prompt: str, max_tokens: int, schema: dict[str, Any] | None = None
+        self,
+        model: str,
+        prompt: str,
+        max_tokens: int,
+        schema: dict[str, Any] | None = None,
+        system: str | None = None,
     ) -> Reply:
-        key = hashlib.sha256(
-            json.dumps([model, max_tokens, schema, prompt], sort_keys=True).encode()
-        ).hexdigest()
+        parts: list[Any] = [model, max_tokens, schema, prompt]
+        if system is not None:
+            parts.append(system)
+        key = hashlib.sha256(json.dumps(parts, sort_keys=True).encode()).hexdigest()
         row = self._db.execute("SELECT reply FROM replies WHERE key = ?", (key,)).fetchone()
         if row is not None:
             return Reply(**json.loads(row[0]))
-        reply = self._complete(model, prompt, max_tokens, schema)
+        reply = self._complete(model, prompt, max_tokens, schema, system)
         self.calls += 1
         self.spent += reply.cost(model)
         with self._db:
@@ -98,7 +111,12 @@ class Offline:
     """
 
     def __call__(
-        self, model: str, prompt: str, max_tokens: int, schema: dict[str, Any] | None
+        self,
+        model: str,
+        prompt: str,
+        max_tokens: int,
+        schema: dict[str, Any] | None,
+        system: str | None = None,
     ) -> Reply:
         raise CacheMiss(f"the cache holds no reply of {model} to this prompt: {prompt[:80]!r}")
 
@@ -123,8 +141,16 @@ def __init__(self) -> None:
         )
 
     def __call__(
-        self, model: str, prompt: str, max_tokens: int, schema: dict[str, Any] | None
+        self,
+        model: str,
+        prompt: str,
+        max_tokens: int,
+        schema: dict[str, Any] | None,
+        system: str | None = None,
     ) -> Reply:
+        messages: list[Any] = [{"role": "user", "content": prompt}]
+        if system is not None:
+            messages.insert(0, {"role": "system", "content": system})
         extra: dict[str, Any] = {}
         if schema is not None:
             extra["response_format"] = {
@@ -133,7 +159,7 @@ def __call__(
             }
         completion = self._openai.chat.completions.create(
             model=model,
-            messages=[{"role": "user", "content": prompt}],
+            messages=messages,
             n=1,
             temperature=0,
             max_tokens=max_tokens,
diff --git a/bench/tests/test_beam_score.py b/bench/tests/test_beam_score.py
new file mode 100644
index 0000000..d9983f5
--- /dev/null
+++ b/bench/tests/test_beam_score.py
@@ -0,0 +1,291 @@
+import hashlib
+import json
+import math
+import random
+import re
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+import pytest
+from json_repair import repair_json
+from scipy.stats import kendalltau
+
+from bench import beam_score
+from bench.beam import Probe
+from bench.beam_score import (
+    JUDGE,
+    MATCH_SYSTEM,
+    align,
+    answer_prompt,
+    judge_prompt,
+    judge_reply,
+    order_score,
+    parse_json_response,
+    score,
+    summarize,
+)
+from bench.models import Reply
+
+PROMPT_SHA256 = {
+    "answer.txt": "ed89b98d1024432a46bd297345961cf89d2429746daec37befb50db46e0c914a",
+    "judge.txt": "593373c642a288a7b590577d8a8fc92c3f9a2b70e2f64ad6e59a040a6c56b7f5",
+    "match_system.txt": "0863c78c1b27d36c97bb7bfd2d4d6b88ef69b093e92bb4719e665527722413e2",
+    "match_user.txt": "8686f5ce5009350f1d04fccc06fc23a234cfcd956603c5d54bb2161c2d2c6045",
+}
+
+
+# The reference below is copied from github.com/mohammadtavakoli78/BEAM at commit b2da22e,
+# src/evaluation/compute_metrics.py, under the MIT license in bench/beam_prompts/LICENSE.
+# Only the type hints and the line breaks differ.
+def official_parse_json_response(response: str) -> Any:
+    response = response.strip()
+    if response.startswith("```"):
+        match = re.search(r"```(?:json)?\s*(\[.*\]|\{.*\})\s*```", response, re.DOTALL)
+        if match:
+            response = match.group(1).strip()
+    try:
+        return json.loads(response)
+    except json.JSONDecodeError:
+        pass
+    match = re.search(r"(\{.*?\}|\[.*?\])", response, re.DOTALL)
+    if match:
+        json_part = match.group(1)
+        try:
+            return json.loads(json_part)
+        except Exception as e:
+            raise ValueError(f"Found possible JSON but failed to parse it: {e}") from e
+    raise ValueError("No valid JSON found in response.")
+
+
+def official_llm_equivalence(first_paragraph: str, second_paragraph: str, llm: Any) -> bool:
+    system_msg = {
+        "role": "system",
+        "content": """
+            You are a binary classifier.
+            If the TWO snippets describe the SAME event/fact, reply **YES**
+            Otherwise reply **NO**. No extra words.
+            DO NOT provide any exaplanation.
+        """,
+    }
+    user_msg = {
+        "role": "user",
+        "content": f"""First snippet: {first_paragraph} \n
+                       Second snippet: {second_paragraph}
+                    """,
+    }
+    messages = [system_msg, user_msg]
+    response: str = llm.invoke(messages).content.lower()
+    return "yes" in response
+
+
+def official_align_with_llm(
+    reference: list[str], system: list[str], llm: Any
+) -> tuple[list[str], list[str]]:
+    used: set[int] = set()
+    system_out = []
+    for s in system:
+        matched_index = None
+        for index, r in enumerate(reference):
+            if index in used:
+                continue
+            if official_llm_equivalence(first_paragraph=r, second_paragraph=s, llm=llm):
+                matched_index = index
+                break
+        if matched_index is not None:
+            system_out.append(reference[matched_index])
+            used.add(matched_index)
+        else:
+            system_out.append(s)
+    return reference, system_out
+
+
+def official_event_ordering_score(
+    reference_list: list[str], system_list: list[str], llm: Any
+) -> dict[str, float]:
+    reference_canon, system_canon = official_align_with_llm(reference_list, system_list, llm)
+    tp = len(set(reference_canon) & set(system_canon))
+    fp = len([x for x in system_canon if x not in reference_canon])
+    fn = len([x for x in reference_canon if x not in system_canon])
+    precision = tp / (tp + fp) if tp + fp else 0
+    recall = tp / (tp + fn) if tp + fn else 0
+    f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0
+    union = list(dict.fromkeys(reference_canon + system_canon))
+    tie_rank = len(union) + 1
+
+    def to_rank(seq: list[str]) -> list[int]:
+        r = {item: i + 1 for i, item in enumerate(seq)}
+        return [r.get(u, tie_rank) for u in union]
+
+    tau_b, p_value = kendalltau(
+        to_rank(reference_canon), to_rank(system_canon), variant="b", method="auto"
+    )
+    tau_b_norm = (tau_b + 1) / 2 if tau_b is not None else 0
+    final_score = tau_b_norm * f1
+    return dict(
+        precision=precision, recall=recall, f1=f1, tau_norm=tau_b_norm, final_score=final_score
+    )
+
+
+def official_nugget_score(rubric: list[str], llm_response: str, llm: Any) -> float:
+    score = 0.0
+    for item in rubric:
+        prompt = beam_score.JUDGE_PROMPT.replace("<rubric_item>", item).replace(
+            "<llm_response>", llm_response
+        )
+        response = llm.invoke(prompt).content.strip()
+        try:
+            response = official_parse_json_response(response=response)
+        except:  # noqa: E722
+            response = json.loads(repair_json(response))
+        score += float(response["score"])
+    return score / len(rubric)
+
+
+# End of the copied reference.
+
+
+class Content:
+    def __init__(self, content: str) -> None:
+        self.content = content
+
+
+class Oracle:
+    """A fake model that answers a match call by whether both snippets name the same event.
+
+    An event is a word `e<n>`. A judge call gets the next reply of `judge`.
+    """
+
+    def __init__(self, judge: Sequence[str] = ()) -> None:
+        self.judge = list(judge)
+        self.matches: list[tuple[str | None, str]] = []
+
+    def answer(self, system: str | None, prompt: str) -> str:
+        if system is None:
+            return self.judge.pop(0)
+        self.matches.append((system, prompt))
+        first, second = prompt.split("Second snippet:", 1)
+        left = set(re.findall(r"e\d+", first))
+        right = set(re.findall(r"e\d+", second))
+        return "YES" if left and left == right else "NO"
+
+    def invoke(self, prompt: str | list[dict[str, str]]) -> Content:
+        if isinstance(prompt, str):
+            return Content(self.answer(None, prompt))
+        return Content(self.answer(prompt[0]["content"], prompt[1]["content"]))
+
+    def __call__(
+        self,
+        model: str,
+        prompt: str,
+        max_tokens: int,
+        schema: dict[str, Any] | None = None,
+        system: str | None = None,
+    ) -> Reply:
+        assert model == JUDGE
+        return Reply(self.answer(system, prompt), 1, 1)
+
+
+def probe(ability: str, rubric: Sequence[str]) -> Probe:
+    return Probe("1-x-1", "1", ability, "q?", "gold", tuple(rubric), "easy", ())
+
+
+def test_the_prompts_are_the_official_files() -> None:
+    for name, digest in PROMPT_SHA256.items():
+        path = Path(beam_score.PROMPTS) / name
+        assert hashlib.sha256(path.read_bytes()).hexdigest() == digest, name
+    assert "MIT License" in (Path(beam_score.PROMPTS) / "LICENSE").read_text()
+
+
+def test_the_prompts_fill_their_slots() -> None:
+    prompt = answer_prompt("the block", "the question?")
+    assert "CONTEXT:\nthe block\n" in prompt
+    assert "QUESTION:\nthe question?\n" in prompt
+    judged = judge_prompt("LLM response should state: 7 days", "It is 7 days.")
+    assert "- RUBRIC CRITERION (what to check): LLM response should state: 7 days" in judged
+    assert "- RESPONSE TO EVALUATE: It is 7 days." in judged
+    assert "<" + "rubric_item>" not in judged
+
+
+REPLIES = [
+    '{"score": 1.0, "reason": "ok"}',
+    '```json\n{"score": 0.5, "reason": "partial"}\n```',
+    'Here is my verdict: {"score": 0.0, "reason": "missing"} done',
+    '{"score": 0.5, "reason": "a reason with an unclosed quote}',
+    '  {"score": 1, "reason": "spaces"}  ',
+]
+
+
+@pytest.mark.parametrize("text", REPLIES)
+def test_the_judge_parse_matches_the_official_one(text: str) -> None:
+    try:
+        expected = float(official_parse_json_response(text)["score"])
+    except ValueError:
+        expected = float(json.loads(repair_json(text))["score"])
+    assert judge_reply(text) == expected
+
+
+def test_parse_raises_without_json() -> None:
+    with pytest.raises(ValueError, match="No valid JSON"):
+        parse_json_response("no json here")
+
+
+def test_a_nugget_question_scores_the_mean_as_the_official_code() -> None:
+    rubric = ["n1", "n2", "n3", "n4", "n5"]
+    found = score(Oracle(REPLIES), probe("temporal_reasoning", rubric), "resp")
+    assert found.nuggets == (1.0, 0.5, 0.0, 0.5, 1.0)
+    assert found.score == found.nugget == official_nugget_score(rubric, "resp", Oracle(REPLIES))
+    assert found.tau_norm is None
+
+
+def random_case(rng: random.Random) -> tuple[list[str], str]:
+    events = [f"e{n}" for n in range(rng.randint(2, 8))]
+    reference = [f"Event {e} happened" for e in events]
+    shown = rng.sample(events, rng.randint(0, len(events)))
+    lines = [f"- then {e}" for e in shown]
+    for _ in range(rng.randint(0, 3)):
+        lines.insert(rng.randint(0, len(lines)), rng.choice(["", "Here is the order:", "e99"]))
+    if rng.random() < 0.2 and lines:
+        lines.append(lines[0])
+    return reference, "\n".join(lines)
+
+
+def test_event_ordering_matches_the_official_code_on_many_replies() -> None:
+    rng = random.Random(20261026)
+    for _ in range(300):
+        reference, response = random_case(rng)
+        ours, theirs = Oracle(), Oracle()
+        reference_out, system_out = align(ours, reference, response.split("\n"))
+        expected = official_event_ordering_score(reference, response.split("\n"), theirs)
+        found = order_score(reference_out, system_out)
+        assert ours.matches == theirs.matches
+        for key, value in expected.items():
+            if math.isnan(value):
+                assert math.isnan(found[key])
+            else:
+                assert found[key] == pytest.approx(value), (key, reference, response)
+
+
+def test_event_ordering_scores_tau_norm_and_keeps_the_nugget_mean() -> None:
+    rubric = ["Event e1 happened", "Event e2 happened", "Event e3 happened"]
+    response = "Order:\n- e2\n- e1\n- e3"
+    model = Oracle(['{"score": 1}', '{"score": 1}', '{"score": 0}'])
+    found = score(model, probe("event_ordering", rubric), response)
+    expected = official_event_ordering_score(rubric, response.split("\n"), Oracle())
+    assert found.score == found.tau_norm == pytest.approx(expected["tau_norm"])
+    assert found.f1 == pytest.approx(expected["f1"])
+    assert found.nugget == pytest.approx(2 / 3)
+    assert all(system == MATCH_SYSTEM for system, _ in model.matches)
+
+
+def test_summarize_takes_the_mean_over_the_abilities() -> None:
+    rows = [
+        {"ability": "abstention", "domain": "Coding", "score": 1.0},
+        {"ability": "abstention", "domain": "Math", "score": 0.0},
+        {"ability": "summarization", "domain": "Coding", "score": 1.0},
+    ]
+    found = summarize(rows)
+    assert found["by_ability"] == {"abstention": 0.5, "summarization": 1.0}
+    assert found["mean"] == 0.75
+    assert found["questions"] == pytest.approx(2 / 3)
+    assert found["by_domain"] == {"Coding": 1.0, "Math": 0.0}
diff --git a/bench/tests/test_e2e.py b/bench/tests/test_e2e.py
index 5a32c9c..6b265a7 100644
--- a/bench/tests/test_e2e.py
+++ b/bench/tests/test_e2e.py
@@ -1,5 +1,7 @@
+import hashlib
 import json
 import os
+import sqlite3
 import subprocess
 import uuid
 from collections.abc import Sequence
@@ -267,7 +269,13 @@ def truncate(self, text: str, limit: int) -> str:
 def test_cached_models_call_once(tmp_path: Path) -> None:
     calls: list[str] = []
 
-    def complete(model: str, prompt: str, max_tokens: int, schema: dict[str, Any] | None) -> Reply:
+    def complete(
+        model: str,
+        prompt: str,
+        max_tokens: int,
+        schema: dict[str, Any] | None,
+        system: str | None = None,
+    ) -> Reply:
         calls.append(prompt)
         return Reply("yes", 1_000_000, 0)
 
@@ -279,6 +287,30 @@ def complete(model: str, prompt: str, max_tokens: int, schema: dict[str, Any] |
     assert models.spent == pytest.approx(5.0)
 
 
+def test_a_system_message_is_passed_and_keys_the_cache(tmp_path: Path) -> None:
+    systems: list[str | None] = []
+
+    def complete(
+        model: str,
+        prompt: str,
+        max_tokens: int,
+        schema: dict[str, Any] | None,
+        system: str | None = None,
+    ) -> Reply:
+        systems.append(system)
+        return Reply(system or "none", 1, 1)
+
+    path = tmp_path / "replies.sqlite"
+    models = CachedModels(complete, path)
+    assert models(READER, "p", 10).text == "none"
+    assert models(READER, "p", 10, system="s").text == "s"
+    assert models(READER, "p", 10, system="s").text == "s"
+    assert systems == [None, "s"]
+    legacy = hashlib.sha256(json.dumps([READER, 10, None, "p"]).encode()).hexdigest()
+    keys = [row[0] for row in sqlite3.connect(path).execute("SELECT key FROM replies")]
+    assert legacy in keys
+
+
 def test_shards_deal_every_question_once(questions: dict[str, Question]) -> None:
     base = questions["q_one"]
     pool = [replace(base, id=f"q{n}") for n in range(10)]
@@ -368,7 +400,13 @@ def stopped(model: str, prompt: str, max_tokens: int, schema: Any = None) -> Rep
 
 
 def test_cached_models_keep_a_stopped_reply(tmp_path: Path) -> None:
-    def complete(model: str, prompt: str, max_tokens: int, schema: dict[str, Any] | None) -> Reply:
+    def complete(
+        model: str,
+        prompt: str,
+        max_tokens: int,
+        schema: dict[str, Any] | None,
+        system: str | None = None,
+    ) -> Reply:
         return Reply("{", 1, max_tokens, stopped=True)
 
     path = tmp_path / "replies.sqlite"
diff --git a/pyproject.toml b/pyproject.toml
index 2d1f5fa..9470e06 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -53,7 +53,13 @@ dev = [
     "ruff",
     { include-group = "bench" },
 ]
-bench = ["hotmemory[hotdata,openai]", "numpy", "tiktoken"]
+bench = [
+    "hotmemory[hotdata,openai]",
+    "json-repair>=0.64.2",
+    "numpy",
+    "scipy>=1.17.1",
+    "tiktoken",
+]
 
 [tool.ruff]
 line-length = 100
@@ -68,7 +74,7 @@ strict = true
 files = ["src", "tests", "skills/hotmemory/scripts", "bench"]
 
 [[tool.mypy.overrides]]
-module = ["pyarrow", "pyarrow.*"]
+module = ["pyarrow", "pyarrow.*", "scipy", "scipy.*"]
 ignore_missing_imports = true
 
 [tool.pytest.ini_options]
diff --git a/uv.lock b/uv.lock
index 52e06b9..3cbcc26 100644
--- a/uv.lock
+++ b/uv.lock
@@ -322,18 +322,24 @@ openai = [
 [package.dev-dependencies]
 bench = [
     { name = "hotmemory", extra = ["hotdata", "openai"] },
+    { name = "json-repair" },
     { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
     { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
+    { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
+    { name = "scipy", version = "1.18.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
     { name = "tiktoken" },
 ]
 dev = [
     { name = "hotmemory", extra = ["hotdata", "openai"] },
+    { name = "json-repair" },
     { name = "mypy" },
     { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
     { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
     { name = "pytest" },
     { name = "pytest-socket" },
     { name = "ruff" },
+    { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
+    { name = "scipy", version = "1.18.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
     { name = "tiktoken" },
 ]
 
@@ -348,16 +354,20 @@ provides-extras = ["hotdata", "openai"]
 [package.metadata.requires-dev]
 bench = [
     { name = "hotmemory", extras = ["hotdata", "openai"] },
+    { name = "json-repair", specifier = ">=0.64.2" },
     { name = "numpy" },
+    { name = "scipy", specifier = ">=1.17.1" },
     { name = "tiktoken" },
 ]
 dev = [
     { name = "hotmemory", extras = ["hotdata", "openai"] },
+    { name = "json-repair", specifier = ">=0.64.2" },
     { name = "mypy" },
     { name = "numpy" },
     { name = "pytest" },
     { name = "pytest-socket" },
     { name = "ruff" },
+    { name = "scipy", specifier = ">=1.17.1" },
     { name = "tiktoken" },
 ]
 
@@ -531,6 +541,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e1/8b/d76219ebdbcf3d4209d9d21a0810db4c8d0a6f88e3ee87d30bdea4e90d30/jiter-0.17.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2c0bf24c72fd0491405dce5d40194f2070e9021ce648c1a1d46234b93d848ff", size = 307147, upload-time = "2026-09-12T15:14:12.897Z" },
 ]
 
+[[package]]
+name = "json-repair"
+version = "0.64.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/7c/b2/52bdd0fca981b289a4d9966366e1dfe50fac315f2c87440fa70726bdcb69/json_repair-0.64.2.tar.gz", hash = "sha256:3f8e76631edab40e7e0c9b859e64c06992a39488b4bbb8285f0e9c3e61d684e9", size = 54227, upload-time = "2026-10-23T05:35:23.447Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9a/79/91b83df91d03b9647ee759f286900957f0d113f49a606e28c8a43d39365b/json_repair-0.64.2-py3-none-any.whl", hash = "sha256:1dec7868c05abff0c52e91dc896e01f28fda6ba1c9f2a9ede18ab520dde637e1", size = 52688, upload-time = "2026-10-23T05:35:21.978Z" },
+]
+
 [[package]]
 name = "librt"
 version = "0.16.0"
@@ -1368,6 +1387,164 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/fc/05/c16957eb287c3fc062e032619a25868d93d408a844b725bcf514f7a378ff/ruff-0.16.10-py3-none-win_arm64.whl", hash = "sha256:25a65fe998c4e6861ec079ada5826a2fc605e6cbccbe9dcd7fac1f54e791621b", size = 10366440, upload-time = "2026-10-01T18:03:19.04Z" },
 ]
 
+[[package]]
+name = "scipy"
+version = "1.17.1"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version < '3.12' and sys_platform == 'win32'",
+    "python_full_version < '3.12' and sys_platform == 'emscripten'",
+    "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'",
+]
+dependencies = [
+    { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/df/75/b4ce781849931fef6fd529afa6b63711d5a733065722d0c3e2724af9e40a/scipy-1.17.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec", size = 31613675, upload-time = "2026-02-23T00:16:00.13Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/58/bccc2861b305abdd1b8663d6130c0b3d7cc22e8d86663edbc8401bfd40d4/scipy-1.17.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696", size = 28162057, upload-time = "2026-02-23T00:16:09.456Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/ee/18146b7757ed4976276b9c9819108adbc73c5aad636e5353e20746b73069/scipy-1.17.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee", size = 20334032, upload-time = "2026-02-23T00:16:17.358Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/e6/cef1cf3557f0c54954198554a10016b6a03b2ec9e22a4e1df734936bd99c/scipy-1.17.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd", size = 22709533, upload-time = "2026-02-23T00:16:25.791Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/60/8804678875fc59362b0fb759ab3ecce1f09c10a735680318ac30da8cd76b/scipy-1.17.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c", size = 33062057, upload-time = "2026-02-23T00:16:36.931Z" },
+    { url = "https://files.pythonhosted.org/packages/09/7d/af933f0f6e0767995b4e2d705a0665e454d1c19402aa7e895de3951ebb04/scipy-1.17.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4", size = 35349300, upload-time = "2026-02-23T00:16:49.108Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/3d/7ccbbdcbb54c8fdc20d3b6930137c782a163fa626f0aef920349873421ba/scipy-1.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444", size = 35127333, upload-time = "2026-02-23T00:17:01.293Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/19/f926cb11c42b15ba08e3a71e376d816ac08614f769b4f47e06c3580c836a/scipy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082", size = 37741314, upload-time = "2026-02-23T00:17:12.576Z" },
+    { url = "https://files.pythonhosted.org/packages/95/da/0d1df507cf574b3f224ccc3d45244c9a1d732c81dcb26b1e8a766ae271a8/scipy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff", size = 36607512, upload-time = "2026-02-23T00:17:23.424Z" },
+    { url = "https://files.pythonhosted.org/packages/68/7f/bdd79ceaad24b671543ffe0ef61ed8e659440eb683b66f033454dcee90eb/scipy-1.17.1-cp311-cp311-win_arm64.whl", hash = "sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d", size = 24599248, upload-time = "2026-02-23T00:17:34.561Z" },
+    { url = "https://files.pythonhosted.org/packages/35/48/b992b488d6f299dbe3f11a20b24d3dda3d46f1a635ede1c46b5b17a7b163/scipy-1.17.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8", size = 31610954, upload-time = "2026-02-23T00:17:49.855Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/02/cf107b01494c19dc100f1d0b7ac3cc08666e96ba2d64db7626066cee895e/scipy-1.17.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76", size = 28172662, upload-time = "2026-02-23T00:18:01.64Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/a9/599c28631bad314d219cf9ffd40e985b24d603fc8a2f4ccc5ae8419a535b/scipy-1.17.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086", size = 20344366, upload-time = "2026-02-23T00:18:12.015Z" },
+    { url = "https://files.pythonhosted.org/packages/35/f5/906eda513271c8deb5af284e5ef0206d17a96239af79f9fa0aebfe0e36b4/scipy-1.17.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b", size = 22704017, upload-time = "2026-02-23T00:18:21.502Z" },
+    { url = "https://files.pythonhosted.org/packages/da/34/16f10e3042d2f1d6b66e0428308ab52224b6a23049cb2f5c1756f713815f/scipy-1.17.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21", size = 32927842, upload-time = "2026-02-23T00:18:35.367Z" },
+    { url = "https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458", size = 35235890, upload-time = "2026-02-23T00:18:49.188Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/5c/9d7f4c88bea6e0d5a4f1bc0506a53a00e9fcb198de372bfe4d3652cef482/scipy-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb", size = 35003557, upload-time = "2026-02-23T00:18:54.74Z" },
+    { url = "https://files.pythonhosted.org/packages/65/94/7698add8f276dbab7a9de9fb6b0e02fc13ee61d51c7c3f85ac28b65e1239/scipy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea", size = 37625856, upload-time = "2026-02-23T00:19:00.307Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/84/dc08d77fbf3d87d3ee27f6a0c6dcce1de5829a64f2eae85a0ecc1f0daa73/scipy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87", size = 36549682, upload-time = "2026-02-23T00:19:07.67Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/98/fe9ae9ffb3b54b62559f52dedaebe204b408db8109a8c66fdd04869e6424/scipy-1.17.1-cp312-cp312-win_arm64.whl", hash = "sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3", size = 24547340, upload-time = "2026-02-23T00:19:12.024Z" },
+    { url = "https://files.pythonhosted.org/packages/76/27/07ee1b57b65e92645f219b37148a7e7928b82e2b5dbeccecb4dff7c64f0b/scipy-1.17.1-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c", size = 31590199, upload-time = "2026-02-23T00:19:17.192Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/ae/db19f8ab842e9b724bf5dbb7db29302a91f1e55bc4d04b1025d6d605a2c5/scipy-1.17.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f", size = 28154001, upload-time = "2026-02-23T00:19:22.241Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/58/3ce96251560107b381cbd6e8413c483bbb1228a6b919fa8652b0d4090e7f/scipy-1.17.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d", size = 20325719, upload-time = "2026-02-23T00:19:26.329Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/83/15087d945e0e4d48ce2377498abf5ad171ae013232ae31d06f336e64c999/scipy-1.17.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4b400bdc6f79fa02a4d86640310dde87a21fba0c979efff5248908c6f15fad1b", size = 22683595, upload-time = "2026-02-23T00:19:30.304Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/e0/e58fbde4a1a594c8be8114eb4aac1a55bcd6587047efc18a61eb1f5c0d30/scipy-1.17.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6", size = 32896429, upload-time = "2026-02-23T00:19:35.536Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/5f/f17563f28ff03c7b6799c50d01d5d856a1d55f2676f537ca8d28c7f627cd/scipy-1.17.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464", size = 35203952, upload-time = "2026-02-23T00:19:42.259Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/a5/9afd17de24f657fdfe4df9a3f1ea049b39aef7c06000c13db1530d81ccca/scipy-1.17.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950", size = 34979063, upload-time = "2026-02-23T00:19:47.547Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/13/88b1d2384b424bf7c924f2038c1c409f8d88bb2a8d49d097861dd64a57b2/scipy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369", size = 37598449, upload-time = "2026-02-23T00:19:53.238Z" },
+    { url = "https://files.pythonhosted.org/packages/35/e5/d6d0e51fc888f692a35134336866341c08655d92614f492c6860dc45bb2c/scipy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:37425bc9175607b0268f493d79a292c39f9d001a357bebb6b88fdfaff13f6448", size = 36510943, upload-time = "2026-02-23T00:20:50.89Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/fd/3be73c564e2a01e690e19cc618811540ba5354c67c8680dce3281123fb79/scipy-1.17.1-cp313-cp313-win_arm64.whl", hash = "sha256:5cf36e801231b6a2059bf354720274b7558746f3b1a4efb43fcf557ccd484a87", size = 24545621, upload-time = "2026-02-23T00:20:55.871Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/6b/17787db8b8114933a66f9dcc479a8272e4b4da75fe03b0c282f7b0ade8cd/scipy-1.17.1-cp313-cp313t-macosx_10_14_x86_64.whl", hash = "sha256:d59c30000a16d8edc7e64152e30220bfbd724c9bbb08368c054e24c651314f0a", size = 31936708, upload-time = "2026-02-23T00:19:58.694Z" },
+    { url = "https://files.pythonhosted.org/packages/38/2e/524405c2b6392765ab1e2b722a41d5da33dc5c7b7278184a8ad29b6cb206/scipy-1.17.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0", size = 28570135, upload-time = "2026-02-23T00:20:03.934Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/c3/5bd7199f4ea8556c0c8e39f04ccb014ac37d1468e6cfa6a95c6b3562b76e/scipy-1.17.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:2ceb2d3e01c5f1d83c4189737a42d9cb2fc38a6eeed225e7515eef71ad301dce", size = 20741977, upload-time = "2026-02-23T00:20:07.935Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/b8/8ccd9b766ad14c78386599708eb745f6b44f08400a5fd0ade7cf89b6fc93/scipy-1.17.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:844e165636711ef41f80b4103ed234181646b98a53c8f05da12ca5ca289134f6", size = 23029601, upload-time = "2026-02-23T00:20:12.161Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/a0/3cb6f4d2fb3e17428ad2880333cac878909ad1a89f678527b5328b93c1d4/scipy-1.17.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e", size = 33019667, upload-time = "2026-02-23T00:20:17.208Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/c3/2d834a5ac7bf3a0c806ad1508efc02dda3c8c61472a56132d7894c312dea/scipy-1.17.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475", size = 35264159, upload-time = "2026-02-23T00:20:23.087Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/77/d3ed4becfdbd217c52062fafe35a72388d1bd82c2d0ba5ca19d6fcc93e11/scipy-1.17.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50", size = 35102771, upload-time = "2026-02-23T00:20:28.636Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/12/d19da97efde68ca1ee5538bb261d5d2c062f0c055575128f11a2730e3ac1/scipy-1.17.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca", size = 37665910, upload-time = "2026-02-23T00:20:34.743Z" },
+    { url = "https://files.pythonhosted.org/packages/06/1c/1172a88d507a4baaf72c5a09bb6c018fe2ae0ab622e5830b703a46cc9e44/scipy-1.17.1-cp313-cp313t-win_amd64.whl", hash = "sha256:e30bdeaa5deed6bc27b4cc490823cd0347d7dae09119b8803ae576ea0ce52e4c", size = 36562980, upload-time = "2026-02-23T00:20:40.575Z" },
+    { url = "https://files.pythonhosted.org/packages/70/b0/eb757336e5a76dfa7911f63252e3b7d1de00935d7705cf772db5b45ec238/scipy-1.17.1-cp313-cp313t-win_arm64.whl", hash = "sha256:a720477885a9d2411f94a93d16f9d89bad0f28ca23c3f8daa521e2dcc3f44d49", size = 24856543, upload-time = "2026-02-23T00:20:45.313Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/83/333afb452af6f0fd70414dc04f898647ee1423979ce02efa75c3b0f2c28e/scipy-1.17.1-cp314-cp314-macosx_10_14_x86_64.whl", hash = "sha256:a48a72c77a310327f6a3a920092fa2b8fd03d7deaa60f093038f22d98e096717", size = 31584510, upload-time = "2026-02-23T00:21:01.015Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/a6/d05a85fd51daeb2e4ea71d102f15b34fedca8e931af02594193ae4fd25f7/scipy-1.17.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:45abad819184f07240d8a696117a7aacd39787af9e0b719d00285549ed19a1e9", size = 28170131, upload-time = "2026-02-23T00:21:05.888Z" },
+    { url = "https://files.pythonhosted.org/packages/db/7b/8624a203326675d7746a254083a187398090a179335b2e4a20e2ddc46e83/scipy-1.17.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:3fd1fcdab3ea951b610dc4cef356d416d5802991e7e32b5254828d342f7b7e0b", size = 20342032, upload-time = "2026-02-23T00:21:09.904Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/35/2c342897c00775d688d8ff3987aced3426858fd89d5a0e26e020b660b301/scipy-1.17.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:7bdf2da170b67fdf10bca777614b1c7d96ae3ca5794fd9587dce41eb2966e866", size = 22678766, upload-time = "2026-02-23T00:21:14.313Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/f2/7cdb8eb308a1a6ae1e19f945913c82c23c0c442a462a46480ce487fdc0ac/scipy-1.17.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350", size = 32957007, upload-time = "2026-02-23T00:21:19.663Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/2e/7eea398450457ecb54e18e9d10110993fa65561c4f3add5e8eccd2b9cd41/scipy-1.17.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118", size = 35221333, upload-time = "2026-02-23T00:21:25.278Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/77/5b8509d03b77f093a0d52e606d3c4f79e8b06d1d38c441dacb1e26cacf46/scipy-1.17.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068", size = 35042066, upload-time = "2026-02-23T00:21:31.358Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/df/18f80fb99df40b4070328d5ae5c596f2f00fffb50167e31439e932f29e7d/scipy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118", size = 37612763, upload-time = "2026-02-23T00:21:37.247Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/39/f0e8ea762a764a9dc52aa7dabcfad51a354819de1f0d4652b6a1122424d6/scipy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:3877ac408e14da24a6196de0ddcace62092bfc12a83823e92e49e40747e52c19", size = 37290984, upload-time = "2026-02-23T00:22:35.023Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/56/fe201e3b0f93d1a8bcf75d3379affd228a63d7e2d80ab45467a74b494947/scipy-1.17.1-cp314-cp314-win_arm64.whl", hash = "sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293", size = 25192877, upload-time = "2026-02-23T00:22:39.798Z" },
+    { url = "https://files.pythonhosted.org/packages/96/ad/f8c414e121f82e02d76f310f16db9899c4fcde36710329502a6b2a3c0392/scipy-1.17.1-cp314-cp314t-macosx_10_14_x86_64.whl", hash = "sha256:1cc682cea2ae55524432f3cdff9e9a3be743d52a7443d0cba9017c23c87ae2f6", size = 31949750, upload-time = "2026-02-23T00:21:42.289Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/b0/c741e8865d61b67c81e255f4f0a832846c064e426636cd7de84e74d209be/scipy-1.17.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:2040ad4d1795a0ae89bfc7e8429677f365d45aa9fd5e4587cf1ea737f927b4a1", size = 28585858, upload-time = "2026-02-23T00:21:47.706Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/1b/3985219c6177866628fa7c2595bfd23f193ceebbe472c98a08824b9466ff/scipy-1.17.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:131f5aaea57602008f9822e2115029b55d4b5f7c070287699fe45c661d051e39", size = 20757723, upload-time = "2026-02-23T00:21:52.039Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/19/2a04aa25050d656d6f7b9e7b685cc83d6957fb101665bfd9369ca6534563/scipy-1.17.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:9cdc1a2fcfd5c52cfb3045feb399f7b3ce822abdde3a193a6b9a60b3cb5854ca", size = 23043098, upload-time = "2026-02-23T00:21:56.185Z" },
+    { url = "https://files.pythonhosted.org/packages/86/f1/3383beb9b5d0dbddd030335bf8a8b32d4317185efe495374f134d8be6cce/scipy-1.17.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad", size = 33030397, upload-time = "2026-02-23T00:22:01.404Z" },
+    { url = "https://files.pythonhosted.org/packages/41/68/8f21e8a65a5a03f25a79165ec9d2b28c00e66dc80546cf5eb803aeeff35b/scipy-1.17.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a", size = 35281163, upload-time = "2026-02-23T00:22:07.024Z" },
+    { url = "https://files.pythonhosted.org/packages/84/8d/c8a5e19479554007a5632ed7529e665c315ae7492b4f946b0deb39870e39/scipy-1.17.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4", size = 35116291, upload-time = "2026-02-23T00:22:12.585Z" },
+    { url = "https://files.pythonhosted.org/packages/52/52/e57eceff0e342a1f50e274264ed47497b59e6a4e3118808ee58ddda7b74a/scipy-1.17.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2", size = 37682317, upload-time = "2026-02-23T00:22:18.513Z" },
+    { url = "https://files.pythonhosted.org/packages/11/2f/b29eafe4a3fbc3d6de9662b36e028d5f039e72d345e05c250e121a230dd4/scipy-1.17.1-cp314-cp314t-win_amd64.whl", hash = "sha256:eb092099205ef62cd1782b006658db09e2fed75bffcae7cc0d44052d8aa0f484", size = 37345327, upload-time = "2026-02-23T00:22:24.442Z" },
+    { url = "https://files.pythonhosted.org/packages/07/39/338d9219c4e87f3e708f18857ecd24d22a0c3094752393319553096b98af/scipy-1.17.1-cp314-cp314t-win_arm64.whl", hash = "sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21", size = 25489165, upload-time = "2026-02-23T00:22:29.563Z" },
+]
+
+[[package]]
+name = "scipy"
+version = "1.18.1"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version >= '3.15' and sys_platform == 'win32'",
+    "python_full_version >= '3.15' and sys_platform == 'emscripten'",
+    "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'",
+    "python_full_version == '3.14.*' and sys_platform == 'win32'",
+    "python_full_version == '3.14.*' and sys_platform == 'emscripten'",
+    "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
+    "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'",
+    "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'",
+    "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
+]
+dependencies = [
+    { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/7e/74/66de6258867beb2ef08f35f9f2ac017a52cacd5081714d239ff1a442d458/scipy-1.18.1.tar.gz", hash = "sha256:52c4b7422442aba924d03ad4019852b08a92e64ea187b933135687bfe2747307", size = 30781235, upload-time = "2026-08-21T23:28:50.599Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/18/f7/240c110c08693826b4513a52f5717d62ec7c7af72f2920821247c03b17b3/scipy-1.18.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:457fd7a2a8edeb044ab6ffbc0aa03ff6cd18491356e5e0c834d76ce621b916d1", size = 31111061, upload-time = "2026-08-21T23:23:44.522Z" },
+    { url = "https://files.pythonhosted.org/packages/05/4a/78c6285577c375e7cf27277ea8ee6961224327f1e1a0c44af5f17f23635c/scipy-1.18.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:e708533e8b2ae2497d65346538a7dcc92814410b25b81432eac66de0f2af8265", size = 28733332, upload-time = "2026-08-21T23:23:50.015Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/f6/a5b82f8abbe14d134691b8b903696f701d25a081353a29dc655c364d9e62/scipy-1.18.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:7bbf207c4453ce1ad2e00b17313852b33310b83090c2311bdaf97f93c0380d12", size = 20475078, upload-time = "2026-08-21T23:23:54.138Z" },
+    { url = "https://files.pythonhosted.org/packages/23/22/0858a0bbd6b3e825ceb8cd9baf9eaf3b2f2b1d77727eb6be40500bcdc92f/scipy-1.18.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:78c0665edead396b1abb4897c41a5c1d9bf090c8a637a4c20a61678e0a264e66", size = 23108904, upload-time = "2026-08-21T23:23:57.824Z" },
+    { url = "https://files.pythonhosted.org/packages/75/9a/2e71719f31eaefe0e3a1706c4a1ded94e664bfd95ffca2b219a671faee01/scipy-1.18.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c085faa2cfa879c5141df483f836f4d691045a078224a670fa570fa01612d89", size = 34025113, upload-time = "2026-08-21T23:24:02.209Z" },
+    { url = "https://files.pythonhosted.org/packages/df/64/ff35eb9e54894cf471ff4716abd3c81eb0a0626869217ce3e6ba4ccf17d7/scipy-1.18.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f55fa87b6c612ecd6b058f167c53231b1d14e412efe361d3d6e38b3631c73218", size = 35344199, upload-time = "2026-08-21T23:24:07.844Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/af/c5538be1792f7034c12c7db6ee67cace58253c7b87b122d68253eaf5de89/scipy-1.18.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c35d74ce0e193ff740c2f2be2ac913ddc232fe6c1ff40b26cfecb9c670c63314", size = 35639587, upload-time = "2026-08-21T23:24:13.05Z" },
+    { url = "https://files.pythonhosted.org/packages/91/4c/075e4f66471bac101141ac739e9e135549be1bae584571bd03a530c056e1/scipy-1.18.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d2924a03db38dc2e848bca2fe9f077dafb891480b91a00a0963a8cf86dfc31c1", size = 37480330, upload-time = "2026-08-21T23:24:19.608Z" },
+    { url = "https://files.pythonhosted.org/packages/39/e7/979fd14e75008623df31ba70d6bb144700f68feadcea042021c06a05bf82/scipy-1.18.1-cp312-cp312-win_amd64.whl", hash = "sha256:5e4d44984abc0020154ea81b247adeddcc3ac5527b975ff798bd1ba0adc513c2", size = 36658278, upload-time = "2026-08-21T23:24:25.463Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/0b/e1525354ff9d7d5feb6d1b31af6d14072e5c91e9607b421fa1ec889660b3/scipy-1.18.1-cp312-cp312-win_arm64.whl", hash = "sha256:d65d448389b8436493abcf629cc94ad0cf32aecaf06e1acca1de53cc795f2f12", size = 24400588, upload-time = "2026-08-21T23:24:30.579Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/55/4540ee0f9c42a9ad7109d0d1a8cc70de54c3572b01c6693a2b1c70e90ceb/scipy-1.18.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:3ab3523da44749156e1f68b464dc56af11ae4cbc5c739a49d05f32b982eca9f3", size = 31089958, upload-time = "2026-08-21T23:24:35.8Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/f5/769f36d14922b8071a43e95d24d18b6bdafad10d7f5cf647867e1ac052bc/scipy-1.18.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e6fb6a55cc0ba97b59a1f288fb86dc6fce8bdfc0fffcbfd015e3a954bf2a2d93", size = 28715106, upload-time = "2026-08-21T23:24:40.775Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/d7/21d890274f75ea37a8209d5519e72da3da90302e3b9fb8397a0918386a62/scipy-1.18.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:ea324d9dd34c38bfb9bec8ca4d1b407db97dbb74029f566b8e322b1b6fe56fe6", size = 20456846, upload-time = "2026-08-21T23:24:45.066Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/01/798430ecea2e78ec7c02663d5f71c007bb6abeca931080debd40d7fa55ea/scipy-1.18.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:75b00eb8fb802090aa903f4ea1c7f5a584779f967361e68b7e98e531cc2d7174", size = 23087986, upload-time = "2026-08-21T23:24:49.539Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/5f/4634e9d35c68496e4e34cb6946eafab044458e6cedab42b40b6588e475b6/scipy-1.18.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d416b16cccfd70fbf62400e84d0bb2f4e6af519a45557f1692c749b37f14b315", size = 33998146, upload-time = "2026-08-21T23:24:54.714Z" },
+    { url = "https://files.pythonhosted.org/packages/41/48/6450ed9243315322bbc19ac57b9b70d66a20bf1d38d124c96bc4bf6af9ea/scipy-1.18.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fdaf5ea890a6183d0565f51a61799d67081bd5b1cf03c5f4b3fd3732108625c9", size = 35312578, upload-time = "2026-08-21T23:25:00.44Z" },
+    { url = "https://files.pythonhosted.org/packages/00/bd/bf5a4be6a3525676499f6dff307991739ff6fdcad1481b1aeb6745339f58/scipy-1.18.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c825cef2f49e46753726a7181a8e199804a912b29519ada542c6ebc654951899", size = 35612621, upload-time = "2026-08-21T23:25:06.144Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/4e/3c45c33e00a77996c4b1cb707929f833ba7b1d522ee29f882512c330676d/scipy-1.18.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e3b417bf8c2c7c16e8f58ad91db17783ec911ac16e7b50eb6eab6e809b4f5b07", size = 37457323, upload-time = "2026-08-21T23:25:12.483Z" },
+    { url = "https://files.pythonhosted.org/packages/93/0e/e0348fbc0dbab65c114cf78957e7dfeb49f8e8b556b4d930cc12ff195e18/scipy-1.18.1-cp313-cp313-win_amd64.whl", hash = "sha256:559ed65f60c1af5a03f3912605a1b5114f522c7c32fb23c3376ae8f03219fe28", size = 36622841, upload-time = "2026-08-21T23:25:18.722Z" },
+    { url = "https://files.pythonhosted.org/packages/50/a8/6a77f5f267c555108f0a864b6db714363dab567a8266422a79a385f9232b/scipy-1.18.1-cp313-cp313-win_arm64.whl", hash = "sha256:cd479fc04dd9401e3b4f49e76518768ef99c4f517a98c284eb091fd725719adf", size = 24399315, upload-time = "2026-08-21T23:25:23.458Z" },
+    { url = "https://files.pythonhosted.org/packages/06/d5/d8eb4e280ddb56a4ab2c6f02ee49b56b23f6e977cf0802fd6d68dbef14f5/scipy-1.18.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:83de5453a7799afc9048b4616bd085cef126e36412f0ea2f6370c36a2a3a51e7", size = 31090936, upload-time = "2026-08-21T23:25:28.686Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/49/59ea385dc3a62ff498ddf3cfff7c2b41b0f9f9d3c4122b3f1dcb6d6327fe/scipy-1.18.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:9554bcc6d715ee87a633a3cc8e7703c6628b100dd29cb8a2efc4c0533c7ff729", size = 28725221, upload-time = "2026-08-21T23:25:33.244Z" },
+    { url = "https://files.pythonhosted.org/packages/70/e8/6b0c288c50942d78193696c9f15f9a0874f5178aa0ddf40f83d9924b3e8d/scipy-1.18.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:011413b7426b75012840e35649e00fe0a2c3bae89fed433876e3a99251572efc", size = 20466839, upload-time = "2026-08-21T23:25:37.516Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/e0/54fd3793c729e3b936782f181b59cbb1205bf250ab605a16cb1ba61cdd5e/scipy-1.18.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:88f0e784020649f88ea48c9f5ddfa403bf9205820667c0914740b392035afb82", size = 23089121, upload-time = "2026-08-21T23:25:42.019Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/56/030af62bea3cf878e0028515dff78c123b01633606a879b63f42d2db99cc/scipy-1.18.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d3ab0e8c69a17dd3559eab8cbb88f258e285c94d572c2719033f90f83290c89", size = 34053851, upload-time = "2026-08-21T23:25:47.998Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/89/2a844506d49651e9aa1af6ef95b6bd8031cb1d5a4375edec6155037e04cf/scipy-1.18.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ac0333bdf38309aa3dcbe7e3fa7ea29e7a2c37c6ea306a757b700ded8e4596ad", size = 35329183, upload-time = "2026-08-21T23:25:53.522Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/56/c7370c3640e92ac9613cbf26cb3f729f9b12ddf1727b55b94b53b24d6f48/scipy-1.18.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:911de823097db8b63f034299d12662db93344e6ffa0b881cbb57748974b70168", size = 35672551, upload-time = "2026-08-21T23:25:59.387Z" },
+    { url = "https://files.pythonhosted.org/packages/24/16/ec8536f351421f8bf60a1120930638f83790f4710b8230446aca3d6159d4/scipy-1.18.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:95298364e251be3e60249facbeeca03631d3bb7584f85879516ec55ac717b81f", size = 37469416, upload-time = "2026-08-21T23:26:05.432Z" },
+    { url = "https://files.pythonhosted.org/packages/52/94/d73da0d28f16c45bb9b0a5691b91610b0275c5ef0eb5e43c87cf2dc1bf31/scipy-1.18.1-cp314-cp314-win_amd64.whl", hash = "sha256:78a0d7c918e74a232394117160e7e3db503377572a45bcef8826e4ab8a35feba", size = 37362755, upload-time = "2026-08-21T23:26:11.366Z" },
+    { url = "https://files.pythonhosted.org/packages/89/25/e996e4dc74e10e227b1e14db5eaf6608bb6dd33884a64851c38f18dd4249/scipy-1.18.1-cp314-cp314-win_arm64.whl", hash = "sha256:cbf38d043c1aa4ab306e1ada6ab6eddacc3322a20b7af1b30bc93254b366fe09", size = 25036090, upload-time = "2026-08-21T23:26:15.887Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/c9/c00213f92309d753b48903e6a451b87eb52ff5b7a16e789d1568bbf221c4/scipy-1.18.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0fcb3c93519f27bb4f0c4b0f7802cdcaca7fcf93267b75edda2e9f4e8a55cbd7", size = 31485550, upload-time = "2026-08-21T23:26:20.776Z" },
+    { url = "https://files.pythonhosted.org/packages/74/b2/e3067c487982d4eeab2938928529410370c06fea84a4d3f4925e7d96647d/scipy-1.18.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:ddef79fb382df40104a19bb7151b3b23e57c1778fcf857c71ceecd9bd264513f", size = 29174642, upload-time = "2026-08-21T23:26:25.395Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/ab/374c9fe2d1ec014e576c781a4b5d8e1ba340e8f6b4638c16f711d2b194f0/scipy-1.18.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:0e82073ecc7acc6436fac4b31674109c7e1d3e596789767eda01258a8c9e8123", size = 20916357, upload-time = "2026-08-21T23:26:30.112Z" },
+    { url = "https://files.pythonhosted.org/packages/90/38/223915c88a17317cafbf8ca2a42b11c265a9fb1e804aa665544132b5fe8a/scipy-1.18.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:8bcf3c1ba5d6456e2effd30fcbd3459b044d683fcdac79a2e6830f0bdf7de487", size = 23482611, upload-time = "2026-08-21T23:26:34.846Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/d1/db0948da8ca57a80b36520ef0a768b967d99f3af65f4b6f1bf6362ad4dd4/scipy-1.18.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cfbf154f2ba187f2ed6cce2639efff7d105f1140573642c0161615b6d91d6a87", size = 34143202, upload-time = "2026-08-21T23:26:40.4Z" },
+    { url = "https://files.pythonhosted.org/packages/87/53/39d046cc7574ed6acacb6bd5723e220107ece80bff12faaf3efc4ddeede4/scipy-1.18.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1d33a7836f7ddc1993427966a0823468ec41bcbdb1a9f9942d1d7e57f803ba3", size = 35380876, upload-time = "2026-08-21T23:26:46.1Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/da/32e0e799d875a85ca57d9bde6c78148afcc0e38276df683d95854eadc8c3/scipy-1.18.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f4b8bc363b6d65ee2152bec57568e3c52639bb34c46057b09857a307ed5e21d", size = 35770885, upload-time = "2026-08-21T23:26:51.533Z" },
+    { url = "https://files.pythonhosted.org/packages/88/2e/f97a666d362fee68b18f41c9c30ed502ca5c98b549749bfcb52a8b74d1eb/scipy-1.18.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:11c423f1049c5755ad4409af52a9ada1cff96fe9b50795d4af3619f292901239", size = 37525424, upload-time = "2026-08-21T23:26:56.751Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/d5/a9e765a84654ebba8479a1fd1b059ced1af72b168a3b2a3a46540ea38d20/scipy-1.18.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c24acac1e18912761c4700239bbc1fd32f615af690f1584d49b35859be51324d", size = 37416961, upload-time = "2026-08-21T23:27:01.546Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/16/e79e0d1c63ef698879d85439d37e9fb434e3b804e506a6991038d086ebd9/scipy-1.18.1-cp314-cp314t-win_arm64.whl", hash = "sha256:9f2897bf7737392ad0d5213ea7b6add72a4edf5679b3153106aeb88b6507b3b9", size = 25331848, upload-time = "2026-08-21T23:27:05.884Z" },
+    { url = "https://files.pythonhosted.org/packages/be/4f/1bd37c883b67163e2ca1f60977a399500e6879c15defecac62831c8d078d/scipy-1.18.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:eb0dfcf4e28a99c12c999744a2ff67c9b06200e20401c7c88186e33552a46331", size = 31091484, upload-time = "2026-08-21T23:27:11.051Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/c5/ba929d7feb9b2332f96827c12e0e924b61973b59b4dea383b603372c65ce/scipy-1.18.1-cp315-cp315-macosx_12_0_arm64.whl", hash = "sha256:30f464bee641fa8e282577c7dce027308403213c6ca8270bba73285c91024bc5", size = 28725057, upload-time = "2026-08-21T23:27:15.9Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/19/68f1c50f609d955d230e66d25d02bd3e1e167ec540232135354fb9a4b9e3/scipy-1.18.1-cp315-cp315-macosx_14_0_arm64.whl", hash = "sha256:1bca3b943fc2567ea49cd02c99abde49da4d5178ec46f624bd8255cda8755beb", size = 20466734, upload-time = "2026-08-21T23:27:20.044Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/6d/319fa29b73d1802fa80b32a6eaf3f5be456ef81526da2716a9493bcb5501/scipy-1.18.1-cp315-cp315-macosx_14_0_x86_64.whl", hash = "sha256:c9d18a33309122074ea483dd92dd444189166b8b2ec429fe9ed5ac73c7a0aa23", size = 23089664, upload-time = "2026-08-21T23:27:24.345Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/db/30992f9b51a63de671daf3888ffd18378b6cb9ec9f2c972264238ffa7fd6/scipy-1.18.1-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:82f201b4c878551d48558337aab270d3c6cca5507b8737c8d8a608d234cccde0", size = 34054035, upload-time = "2026-08-21T23:27:29.409Z" },
+    { url = "https://files.pythonhosted.org/packages/91/d4/bf3e735dc0b9d5a8ff45079d2540e17d3aff7a2f0048dd8f552ffd031d2b/scipy-1.18.1-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ac49ea97594532dd44b7136094d35f5440fa06e6d9c6384a74c01764df388c5", size = 35333883, upload-time = "2026-08-21T23:27:34.293Z" },
+    { url = "https://files.pythonhosted.org/packages/19/93/12d78ce9f871fe945fca588d32644e6e63f553c2a35c564d73f3b22a3313/scipy-1.18.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:ceb30a00ce7c92d459819443d29ca486d882b83fb6738bdcbb2a1cce94ac5daa", size = 35673124, upload-time = "2026-08-21T23:27:39.059Z" },
+    { url = "https://files.pythonhosted.org/packages/70/cd/886219313a1012a48e6ae0ec4f302c837151beb92e1ff0d709ef8fdfc488/scipy-1.18.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f29633129f9fa7e88a3f0fca835de2d030bfc9643f7799e1a0c46cee24d38fc7", size = 37470753, upload-time = "2026-08-21T23:27:44.435Z" },
+    { url = "https://files.pythonhosted.org/packages/17/6c/a776888ce618bee54fbde26172f0f46ac1da70d27b63861797fe78e1904b/scipy-1.18.1-cp315-cp315-win_amd64.whl", hash = "sha256:92c14f5bdbfb6216315ce33e78080474082de8b3830122ba97809bfbe65f75c0", size = 37361483, upload-time = "2026-08-21T23:27:49.334Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/09/97b651691322ebee97999b017ffc18a15a0b815103844c97e8da9d469731/scipy-1.18.1-cp315-cp315-win_arm64.whl", hash = "sha256:e402cf31eb68f453dbb2d36fc6d722b33f24a55d68b2ae1d92fa6305ca71c298", size = 25035883, upload-time = "2026-08-21T23:27:53.596Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/0f/9ec20467bbabd0d44e2a77d0fd3d124f884b4d67df92af82c91d2d6a486f/scipy-1.18.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:2a0b02f9fc46f8520330c23d45e6560db7e3a0d927232139427637f98943e11d", size = 31474926, upload-time = "2026-08-21T23:27:57.993Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/58/dcb79161e56efbedc50079fcd2f5fe427a0ebb53022eb476aa73c015ad8f/scipy-1.18.1-cp315-cp315t-macosx_12_0_arm64.whl", hash = "sha256:1d73131e358976663dd969e1fb4ed1404b815cd977eaaedc3b3a133ba2d81c35", size = 29164940, upload-time = "2026-08-21T23:28:03.062Z" },
+    { url = "https://files.pythonhosted.org/packages/71/d3/1eeea80c817fcb8ef7bd4a05a58824977a0e57a375cfc3d7ea7c911c01ad/scipy-1.18.1-cp315-cp315t-macosx_14_0_arm64.whl", hash = "sha256:bff0b729edd992766136b34e39cc76bc2fad905aa58897ee72a9cd000a6d8443", size = 20906742, upload-time = "2026-08-21T23:28:07.642Z" },
+    { url = "https://files.pythonhosted.org/packages/54/46/e59350428b6099301a20128108c995e2eb175a43f383af9a346e38824f9b/scipy-1.18.1-cp315-cp315t-macosx_14_0_x86_64.whl", hash = "sha256:10ac20c69d880f77f375db44c22e3e6a644f9fefa291d4cd2fb9790a89fc99fd", size = 23472183, upload-time = "2026-08-21T23:28:12.109Z" },
+    { url = "https://files.pythonhosted.org/packages/89/31/cc91623fa98f0621766a0f0aaaadb2c66de74a7ea7e3837164f6e4354260/scipy-1.18.1-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:33a834464fdabc0f26a45508df31b3cc5d028e04dbf6c5ed398541418e0a12fe", size = 34130796, upload-time = "2026-08-21T23:28:17.906Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/3e/8572ef536957ddb8aa81bb4090d9e25f257e3b4e05d97deb54319deb8a3a/scipy-1.18.1-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49023963c193dacee096301452f223ee24d86ec5807f8df93c0f7221d119e305", size = 35374253, upload-time = "2026-08-21T23:28:23.732Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/c6/59fdeffb4f1435299f93d9dc8140b43ad2916e6cfc944be6c3041fcec86d/scipy-1.18.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:d84a09d0dad90ba6525d8ac1c2334b33e64bf3ccfe9e841f02feb867a22681e4", size = 35758543, upload-time = "2026-08-21T23:28:29.431Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/d9/135be205d9de8783193aff9cc3bf483a03a38e4b29432c954e8cb66ac14e/scipy-1.18.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:179ce34a8d0fe273d8883ba59e17e052247d08973dfcb743ca52bb1cce2d60b0", size = 37521946, upload-time = "2026-08-21T23:28:35.245Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/a2/5b7d5270621ab7cfa3f7766067bf95dc360b5efb6394694e8143b4156e2b/scipy-1.18.1-cp315-cp315t-win_amd64.whl", hash = "sha256:5632e3ae3d09197c446310cd5187de63e28448ce22f0f67b2b93d97503c0c230", size = 37408295, upload-time = "2026-08-21T23:28:40.724Z" },
+    { url = "https://files.pythonhosted.org/packages/63/ad/741c19fcb66755ff953daf9243af8480e4bf3d7fbe57583c178c7d2b6b51/scipy-1.18.1-cp315-cp315t-win_arm64.whl", hash = "sha256:eda632a7981f69730d6281f451db9c1c370993a2c0d7ddb43e2a809a2862b83a", size = 25319710, upload-time = "2026-08-21T23:28:45.713Z" },
+]
+
 [[package]]
 name = "six"
 version = "1.17.0"

From 0a720ae87f09cd1787b9510cee182c70277e57be Mon Sep 17 00:00:00 2001
From: Rohan Dsouza <rohan@hotdata.dev>
Date: Mon, 26 Oct 2026 14:33:50 +0530
Subject: [PATCH 4/8] Phase 12A task 4: run BEAM end to end over one kept
 database per conversation

python -m bench.beam_e2e captures each conversation once into a local
database that keeps its memory, then for each probe recalls once for
each arm (memory, and chunks as the RAG baseline), reads the block with
each reader (gpt-4.1-nano and gpt-4.1-mini) and the official answer
prompt, and scores each answer with bench/beam_score.py.

Workers take the conversation with the most windows first. The reads
and judge calls of a conversation run in a thread pool, with one reply
cache connection for each thread. A rows file holds a capture row for
each conversation and a row for each probe, so --resume skips both, and
a stopped capture goes on over its kept database. --reuse reads with no
capture. The runner deletes no database, returns each error as text,
and stops a conversation that spends twice its estimate.

Each probe row records which windows hold its source turns and the
share of them that each block shows. bench/models.py gains the prices
of the gpt-4.1 models and the embedder, read 2026-10-26.
---
 bench/beam_e2e.py            | 836 +++++++++++++++++++++++++++++++++++
 bench/beam_score.py          |   4 +-
 bench/models.py              |  13 +-
 bench/tests/test_beam_e2e.py | 210 +++++++++
 4 files changed, 1059 insertions(+), 4 deletions(-)
 create mode 100644 bench/beam_e2e.py
 create mode 100644 bench/tests/test_beam_e2e.py

diff --git a/bench/beam_e2e.py b/bench/beam_e2e.py
new file mode 100644
index 0000000..00b9eee
--- /dev/null
+++ b/bench/beam_e2e.py
@@ -0,0 +1,836 @@
+"""The BEAM end-to-end score: capture each conversation once, then recall, read, and judge.
+
+Run it from the root of the repository, with the local stack up. Without `--run`, it prints
+the cost estimate and the expected time, and stops. With `--replay`, it calls no model:
+
+    uv run --group bench --env-file .env python -m bench.beam_e2e --split 100K --sample smoke
+    uv run --group bench --env-file .env python -m bench.beam_e2e --split 100K --sample a --run
+    uv run --group bench --env-file .env python -m bench.beam_e2e --split 1M --sample all \\
+        --workers 8 --run
+
+Each conversation gets one local database that keeps its memory after the run, named
+`<kept name>-<conversation id>`. A worker takes one conversation at a time, the one with the
+most windows first. It captures each window in order with the `detail` extractor, then, for
+each probe, recalls once for each arm, reads the block with each reader, and judges each
+answer with the official BEAM scoring. The reads and the judge calls of a conversation run
+in a thread pool.
+
+It appends one row for each captured conversation and one row for each finished probe to a
+rows file. `--resume` skips the finished probes and the captured conversations. A
+conversation that stopped during capture keeps its database, and its capture runs again:
+`capture` skips each window whose chunks are stored, with no extractor call. `--reuse`
+reads from the kept databases with no capture. The runner deletes no database.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import heapq
+import json
+import multiprocessing
+import statistics
+import threading
+import time
+import uuid
+from collections.abc import Callable, Mapping, Sequence
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
+from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from bench import embed
+from bench.beam import (
+    DATA_REVISION,
+    SAMPLE_SPLIT,
+    SAMPLES,
+    SPLITS,
+    WINDOW_CHARS,
+    Conversation,
+    Probe,
+    Window,
+    load_split,
+    sample_ids,
+)
+from bench.beam_score import BEAM_COMMIT, EVENT_ORDERING, JUDGE, JUDGE_MAX_TOKENS, answer_prompt
+from bench.beam_score import score as score_probe
+from bench.beam_score import summarize as summarize_scores
+from bench.e2e import (
+    COST_LIMIT,
+    Config,
+    WriteTimer,
+    append_row,
+    existing_databases,
+    extractor,
+    kept_name,
+    library_commit,
+    local_store,
+)
+from bench.embed import CACHE_DIR
+from bench.models import (
+    BEAM_MINI,
+    BEAM_NANO,
+    EMBEDDING_PRICE,
+    EXTRACTOR_MAX_TOKENS,
+    PRICES,
+    CachedModels,
+    Complete,
+    Reply,
+)
+from bench.retrieval import RESULTS_DIR, _commit
+from hotmemory import Filter, Memory, Store
+from hotmemory.contract import render_episode
+
+ARMS = ("memory", "chunks")
+READERS = {"nano": BEAM_NANO, "mini": BEAM_MINI}
+READER_MAX_TOKENS = 2000
+CONFIG = Config(
+    k=30,
+    budget=16_000,
+    extractor="detail",
+    session_episodes=True,
+    episodes="search",
+    chunk_chars=1000,
+)
+"""The phase 10 configuration of the memory arm."""
+NAMESPACE = ("bench", "beam")
+ACTOR = "beam-extractor"
+JUDGE_THREADS = 8
+SMOKE = "smoke"
+ALL = "all"
+SPEND_FACTOR = 2.0
+WINDOW_KEY_CHARS = len("window-01-0001")
+
+
+@dataclass(frozen=True)
+class Rates:
+    """The rates of the estimate. `docs/internal/plan.md` gives the derivation of each."""
+
+    extract_window: float = 0.0005
+    window_seconds: float = 8.3
+    embed_tokens_per_char: float = 2.1 / 4.4
+    reader_in: int = 3850
+    reader_out: int = 80
+    judge_in: int = 1200
+    judge_out: int = 80
+    match_calls: int = 85
+    match_in: int = 150
+    match_out: int = 2
+    call_seconds: float = 0.7
+
+
+@dataclass(frozen=True)
+class Estimate:
+    """The expected cost of a run in US dollars, by part, and its expected time in seconds."""
+
+    parts: dict[str, float]
+    by_conversation: dict[str, float]
+    seconds: float
+
+    @property
+    def total(self) -> float:
+        return sum(self.parts.values())
+
+
+def estimate(
+    conversations: Sequence[Conversation],
+    arms: Sequence[str],
+    readers: Sequence[str],
+    workers: int,
+    rates: Rates | None = None,
+    capture: bool = True,
+) -> Estimate:
+    """Return the expected cost and time of a run over `conversations`.
+
+    The time is the longest worker when each worker takes the conversation with the most
+    windows first. A conversation takes its windows times `window_seconds` for capture,
+    and its model calls of the read phase times `call_seconds`, over JUDGE_THREADS threads.
+    """
+    rates = rates or Rates()
+    judge_in, judge_out = PRICES[JUDGE]
+    nugget_call = (rates.judge_in * judge_in + rates.judge_out * judge_out) / 1e6
+    match_call = (rates.match_in * judge_in + rates.match_out * judge_out) / 1e6
+    parts = dict.fromkeys(("extraction", "embeddings", "readers", "judge"), 0.0)
+    by_conversation: dict[str, float] = {}
+    seconds: list[float] = []
+    combos = len(arms) * len(readers)
+    for conversation in conversations:
+        windows = len(conversation.windows()) if capture else 0
+        extraction = windows * rates.extract_window
+        embeddings = (
+            conversation.chars * rates.embed_tokens_per_char * EMBEDDING_PRICE / 1e6
+            if capture
+            else 0.0
+        )
+        reading = 0.0
+        for reader in readers:
+            price_in, price_out = PRICES[READERS[reader]]
+            call = (rates.reader_in * price_in + rates.reader_out * price_out) / 1e6
+            reading += call * len(arms) * len(conversation.probes)
+        nuggets = sum(len(probe.rubric) for probe in conversation.probes)
+        orderings = sum(1 for probe in conversation.probes if probe.ability == EVENT_ORDERING)
+        judging = combos * (nuggets * nugget_call + orderings * rates.match_calls * match_call)
+        parts["extraction"] += extraction
+        parts["embeddings"] += embeddings
+        parts["readers"] += reading
+        parts["judge"] += judging
+        by_conversation[conversation.id] = extraction + embeddings + reading + judging
+        calls = combos * (len(conversation.probes) + nuggets + orderings * rates.match_calls)
+        seconds.append(windows * rates.window_seconds + calls * rates.call_seconds / JUDGE_THREADS)
+    return Estimate(parts, by_conversation, longest_worker(seconds, workers))
+
+
+def longest_worker(seconds: Sequence[float], workers: int) -> float:
+    """Return the time of the busiest worker when each takes the longest job left first."""
+    loads = [0.0] * max(1, workers)
+    for job in sorted(seconds, reverse=True):
+        heapq.heapreplace(loads, loads[0] + job)
+    return max(loads)
+
+
+class SpendLimit(RuntimeError):
+    """A conversation spent more than its limit."""
+
+
+class ThreadModels:
+    """A `CachedModels` for each thread, over one cache file, with a spend limit.
+
+    `spent` and `calls` sum every thread. A call raises SpendLimit once `spent` passes
+    `limit`.
+    """
+
+    def __init__(self, complete: Complete, path: Path, limit: float = float("inf")) -> None:
+        self._complete = complete
+        self._path = path
+        self._limit = limit
+        self._local = threading.local()
+        self._lock = threading.Lock()
+        self._all: list[CachedModels] = []
+
+    def __call__(
+        self,
+        model: str,
+        prompt: str,
+        max_tokens: int,
+        schema: dict[str, Any] | None = None,
+        system: str | None = None,
+    ) -> Reply:
+        models: CachedModels | None = getattr(self._local, "models", None)
+        if models is None:
+            models = CachedModels(self._complete, self._path)
+            self._local.models = models
+            with self._lock:
+                self._all.append(models)
+        reply = models(model, prompt, max_tokens, schema, system)
+        if self.spent > self._limit:
+            raise SpendLimit(f"spent ${self.spent:.2f}, over the limit of ${self._limit:.2f}")
+        return reply
+
+    @property
+    def spent(self) -> float:
+        with self._lock:
+            return sum(models.spent for models in self._all)
+
+    @property
+    def calls(self) -> int:
+        with self._lock:
+            return sum(models.calls for models in self._all)
+
+    def close(self) -> None:
+        with self._lock:
+            for models in self._all:
+                models.close()
+
+
+@dataclass(frozen=True)
+class Block:
+    """The block of one arm for one probe, and what it shows."""
+
+    text: str
+    episodes: tuple[str, ...]
+    facts: int
+
+
+def memory_block(store: Store, namespace: Sequence[str], question: str) -> Block:
+    """Return the block of the memory arm: `recall` with the configuration of CONFIG."""
+    records, text = Memory(store).recall(
+        question, [namespace], budget=CONFIG.budget, k=CONFIG.k, episodes="search"
+    )
+    episodes = tuple(record.key for record in records if record.kind == "episode")
+    return Block(text, episodes, len(records) - len(episodes))
+
+
+def chunks_block(store: Store, namespace: Sequence[str], question: str) -> Block:
+    """Return the block of the chunks arm: the top `k` chunks for `question`, in the budget.
+
+    The chunks are rendered as `recall` renders episodes, and the block stops before the
+    first chunk that would pass the budget.
+    """
+    hits = store.search(question, [namespace], Filter(kind="episode"), k=CONFIG.k)
+    parts: list[str] = []
+    shown: list[str] = []
+    for hit in hits:
+        more = [*parts, render_episode(hit.record)]
+        if len("\n".join(more)) > CONFIG.budget:
+            break
+        parts, shown = more, [*shown, hit.record.key]
+    return Block("\n".join(parts), tuple(shown), 0)
+
+
+BLOCKS: dict[str, Callable[[Store, Sequence[str], str], Block]] = {
+    "memory": memory_block,
+    "chunks": chunks_block,
+}
+
+
+def source_windows(windows: Sequence[Window], probe: Probe) -> set[str]:
+    """Return the keys of the windows that hold a source turn of `probe`."""
+    sources = set(probe.sources)
+    return {window.key for window in windows if sources & set(window.messages)}
+
+
+def coverage(found: set[str], block: Block) -> float | None:
+    """Return the share of the windows in `found` that the block shows a chunk of."""
+    if not found:
+        return None
+    shown = {key[:WINDOW_KEY_CHARS] for key in block.episodes}
+    return len(found & shown) / len(found)
+
+
+@dataclass(frozen=True)
+class Job:
+    """What every worker of one run shares."""
+
+    url: str
+    arms: tuple[str, ...]
+    readers: tuple[str, ...]
+    replay: bool
+    kept: str
+    rows_file: Path
+
+
+@dataclass
+class Done:
+    """What one conversation did: its finished probes, what it cost, and its failure."""
+
+    conversation: str
+    probes: int = 0
+    spent: float = 0.0
+    calls: int = 0
+    embedder_misses: int = 0
+    failure: str | None = None
+    seconds: float = 0.0
+    capture: dict[str, Any] = field(default_factory=dict)
+
+
+def capture_conversation(
+    store: Store, models: ThreadModels, windows: Sequence[Window], namespace: Sequence[str]
+) -> dict[str, Any]:
+    """Capture each window in order, and return what the capture did and how long it took."""
+    memory = Memory(store)
+    timer = WriteTimer(store)
+    at_limit: list[str] = []
+    seconds: list[float] = []
+    written = 0
+    for window in windows:
+        started = time.perf_counter()
+        ids = memory.capture(
+            window.text,
+            namespace,
+            extractor(models, CONFIG.variant, "", at_limit),
+            actor=ACTOR,
+            observed_at=window.date,
+            episode=window.key,
+            chunk_chars=CONFIG.chunk_chars,
+        )
+        seconds.append(time.perf_counter() - started)
+        written += len(ids)
+    ordered = sorted(seconds)
+    return {
+        "windows": len(windows),
+        "windows_at_limit": len(at_limit),
+        "facts_written": written,
+        "window_s": {
+            "p50": round(statistics.median(ordered), 3) if ordered else None,
+            "p95": round(ordered[int(len(ordered) * 0.95)], 3) if ordered else None,
+            "highest": round(ordered[-1], 3) if ordered else None,
+            "sum": round(sum(ordered), 1),
+        },
+        "writes": timer.take(),
+    }
+
+
+def table_rows(store: Store) -> dict[str, int]:
+    """Return the row count of each record table of a HotdataStore."""
+    from hotmemory.hotdata import CUE_TABLE, RECORD_TABLES
+
+    client, database = store.client, store.database  # type: ignore[attr-defined]
+    counts = {}
+    for table in (*RECORD_TABLES, CUE_TABLE):
+        result = client.execute_sql(
+            f"SELECT COUNT(*) FROM default.public.{table}", database=database
+        )
+        counts[table] = int(result.rows[0][0])
+    return counts
+
+
+def answer_and_score(
+    models: ThreadModels, probe: Probe, block: Block, reader: str
+) -> dict[str, Any]:
+    """Read `block` with `reader`, judge the answer, and return the cell of the row."""
+    reply = models(READERS[reader], answer_prompt(block.text, probe.question), READER_MAX_TOKENS)
+    found = score_probe(models, probe, reply.text)
+    return {
+        "answer": reply.text,
+        "reader_input_tokens": reply.input_tokens,
+        "reader_output_tokens": reply.output_tokens,
+        "score": found.score,
+        "nugget": found.nugget,
+        "nuggets": list(found.nuggets),
+        "tau_norm": found.tau_norm,
+        "f1": found.f1,
+        "tau_undefined": found.tau_undefined,
+    }
+
+
+def run_on_store(
+    job: Job,
+    conversation: Conversation,
+    store: Store,
+    models: ThreadModels,
+    capture: bool,
+    finished: frozenset[str],
+    count_rows: Callable[[Store], dict[str, int]] | None = None,
+) -> Done:
+    """Capture `conversation` into `store` if `capture`, then run each probe not in `finished`.
+
+    Appends a capture row and each probe row to `job.rows_file`. With `count_rows`, the
+    capture row holds the row count of each table after the capture.
+    """
+    started = time.perf_counter()
+    done = Done(conversation.id)
+    namespace = (*NAMESPACE, conversation.id)
+    windows = conversation.windows()
+    if capture:
+        done.capture = capture_conversation(store, models, windows, namespace)
+        if count_rows is not None:
+            done.capture["rows"] = count_rows(store)
+        done.capture["seconds"] = round(time.perf_counter() - started, 1)
+        append_row(
+            job.rows_file, {"kind": "capture", "conversation": conversation.id, **done.capture}
+        )
+        print(
+            f"{conversation.id} captured {len(windows)} windows, "
+            f"{done.capture['windows_at_limit']} at the limit, "
+            f"rows {done.capture.get('rows')}, spent ${models.spent:.2f}",
+            flush=True,
+        )
+    probes = [probe for probe in conversation.probes if probe.id not in finished]
+    blocks = {
+        (probe.id, arm): BLOCKS[arm](store, namespace, probe.question)
+        for probe in probes
+        for arm in job.arms
+    }
+    tasks = [(probe, arm, reader) for probe in probes for arm in job.arms for reader in job.readers]
+    cells: dict[tuple[str, str, str], dict[str, Any]] = {}
+    with ThreadPoolExecutor(JUDGE_THREADS) as pool:
+        futures = {
+            pool.submit(answer_and_score, models, probe, blocks[(probe.id, arm)], reader): (
+                probe.id,
+                arm,
+                reader,
+            )
+            for probe, arm, reader in tasks
+        }
+        for future in as_completed(futures):
+            cells[futures[future]] = future.result()
+    for probe in probes:
+        append_row(job.rows_file, probe_row(conversation, probe, windows, blocks, cells, job))
+        done.probes += 1
+    print(f"{conversation.id} finished {len(probes)} probes, spent ${models.spent:.2f}", flush=True)
+    return done
+
+
+def probe_row(
+    conversation: Conversation,
+    probe: Probe,
+    windows: Sequence[Window],
+    blocks: Mapping[tuple[str, str], Block],
+    cells: Mapping[tuple[str, str, str], dict[str, Any]],
+    job: Job,
+) -> dict[str, Any]:
+    """Return the row of the result file for one probe."""
+    found = source_windows(windows, probe)
+    arms = {}
+    for arm in job.arms:
+        block = blocks[(probe.id, arm)]
+        arms[arm] = {
+            "context_chars": len(block.text),
+            "block_sha256": hashlib.sha256(block.text.encode()).hexdigest(),
+            "facts_in_block": block.facts,
+            "episodes_in_block": list(block.episodes),
+            "source_windows": sorted(found),
+            "source_coverage": coverage(found, block),
+        }
+    return {
+        "kind": "probe",
+        "id": probe.id,
+        "conversation": conversation.id,
+        "domain": conversation.domain,
+        "ability": probe.ability,
+        "arms": arms,
+        "combos": {
+            f"{arm}/{reader}": cells[(probe.id, arm, reader)]
+            for arm in job.arms
+            for reader in job.readers
+        },
+    }
+
+
+def run_conversation(
+    job: Job,
+    conversation: Conversation,
+    limit: float,
+    capture: bool,
+    finished: frozenset[str],
+) -> Done:
+    """Run `conversation` over its kept local database, as `run_on_store` does.
+
+    Returns every error as the text of `failure`, so that no exception crosses the process
+    pool. The database is kept in every case.
+    """
+    from bench.models import Offline, Remote
+
+    started = time.perf_counter()
+    done = Done(conversation.id)
+    models = ThreadModels(
+        Offline() if job.replay else Remote(), CACHE_DIR / "replies.sqlite", limit
+    )
+    embedder = embed.openai_embedder()
+    try:
+        name = f"{job.kept}-{conversation.id}"
+        with local_store(job.url, name, embedder, "never") as (stores, _):
+            store = stores[CONFIG.recall_with]
+            done = run_on_store(job, conversation, store, models, capture, finished, table_rows)
+    except Exception as error:
+        done.failure = f"{conversation.id}: {type(error).__name__}: {error}"
+        print(f"{conversation.id} failed: {done.failure}", flush=True)
+    finally:
+        done.spent, done.calls = models.spent, models.calls
+        done.embedder_misses = embedder.misses
+        done.seconds = round(time.perf_counter() - started, 1)
+        models.close()
+        embedder.close()
+    return done
+
+
+def read_rows(path: Path) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]:
+    """Return the header, the capture rows, and the probe rows of a rows file.
+
+    A last line that is not complete JSON is left out.
+    """
+    lines = path.read_text().splitlines()
+    header: dict[str, Any] = json.loads(lines[0])["header"]
+    captures: list[dict[str, Any]] = []
+    probes: list[dict[str, Any]] = []
+    for line in lines[1:]:
+        try:
+            row = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        (captures if row["kind"] == "capture" else probes).append(row)
+    return header, captures, probes
+
+
+def summarize(
+    rows: Sequence[Mapping[str, Any]], arms: Sequence[str], readers: Sequence[str]
+) -> dict[str, Any]:
+    """Return the scores of each arm and reader over `rows`, the probe rows of a run."""
+    return {
+        f"{arm}/{reader}": summarize_scores(
+            [
+                {
+                    "ability": row["ability"],
+                    "domain": row["domain"],
+                    "score": row["combos"][f"{arm}/{reader}"]["score"],
+                }
+                for row in rows
+            ]
+        )
+        for arm in arms
+        for reader in readers
+    }
+
+
+def choose(split: str, sample: str, conversations: Sequence[Conversation]) -> list[Conversation]:
+    """Return the conversations of `sample` in `split`, in the order of the split.
+
+    The smoke sample is the conversation of sample A with the fewest windows.
+    """
+    if sample == ALL:
+        return list(conversations)
+    if split != SAMPLE_SPLIT:
+        raise ValueError(f"sample {sample} needs --split {SAMPLE_SPLIT}")
+    ids = set(sample_ids("a" if sample == SMOKE else sample, conversations))
+    chosen = [conversation for conversation in conversations if conversation.id in ids]
+    if sample == SMOKE:
+        chosen = [min(chosen, key=lambda c: len(c.windows()))]
+    return chosen
+
+
+def main(argv: Sequence[str] | None = None) -> Path | None:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0] if __doc__ else "")
+    parser.add_argument("--split", choices=sorted(SPLITS), default=SAMPLE_SPLIT)
+    parser.add_argument("--sample", choices=(*SAMPLES, SMOKE, ALL), default=SMOKE)
+    parser.add_argument("--arms", default=",".join(ARMS), help="a comma-separated list of arms")
+    parser.add_argument(
+        "--readers", default=",".join(READERS), help="a comma-separated list of readers"
+    )
+    parser.add_argument(
+        "--run", action="store_true", help="call the models; without it, only estimate"
+    )
+    parser.add_argument(
+        "--replay", action="store_true", help="call no model: use cached replies, fail at a miss"
+    )
+    parser.add_argument(
+        "--reuse", action="store_true", help="read from the kept databases, with no capture"
+    )
+    parser.add_argument(
+        "--resume",
+        action="store_true",
+        help="skip what the rows file of an earlier run of this command holds",
+    )
+    parser.add_argument("--workers", type=int, default=1, help="conversations at a time")
+    parser.add_argument("--limit", type=int, default=None, help="the first N conversations")
+    parser.add_argument(
+        "--extract-rate",
+        type=float,
+        default=Rates.extract_window,
+        help="US dollars of extraction for each window, for the estimate",
+    )
+    parser.add_argument(
+        "--window-seconds",
+        type=float,
+        default=Rates.window_seconds,
+        help="worker seconds for each window, for the estimate",
+    )
+    parser.add_argument("--tag", default="", help="a suffix for the name of the result file")
+    parser.add_argument("--url", default="http://localhost:3000")
+    args = parser.parse_args(argv)
+    arms = tuple(arm for arm in args.arms.split(",") if arm)
+    readers = tuple(reader for reader in args.readers.split(",") if reader)
+    if not arms or not set(arms) <= set(ARMS):
+        parser.error(f"arms must be among {', '.join(ARMS)}")
+    if not readers or not set(readers) <= set(READERS):
+        parser.error(f"readers must be among {', '.join(READERS)}")
+    if args.workers < 1:
+        parser.error("workers must be at least 1")
+    try:
+        conversations = choose(args.split, args.sample, load_split(args.split))
+    except ValueError as error:
+        parser.error(str(error))
+    conversations = conversations[: args.limit]
+
+    rates = Rates(extract_window=args.extract_rate, window_seconds=args.window_seconds)
+    expected = estimate(conversations, arms, readers, args.workers, rates, not args.reuse)
+    for part, cost in expected.parts.items():
+        print(f"estimate {part}: ${cost:.2f}")
+    print(
+        f"estimate in total, {len(conversations)} conversations, "
+        f"{sum(len(c.probes) for c in conversations)} probes: ${expected.total:.2f}, "
+        f"and the limit is ${COST_LIMIT:.0f}"
+    )
+    print(
+        f"expected time with {args.workers} workers: {expected.seconds / 60:.0f} minutes; "
+        f"a conversation stops at {SPEND_FACTOR:g} times its estimate"
+    )
+    if expected.total > COST_LIMIT:
+        print("the estimate is over the limit: stop")
+        return None
+    if not args.run and not args.replay:
+        print("no model was called; pass --run to run, or --replay to run from the cache")
+        return None
+
+    try:
+        kept = kept_name(f"beam-{args.split.lower()}", CONFIG, library_commit())
+    except RuntimeError as error:
+        parser.error(str(error))
+    name = f"beam-{args.split}-{args.sample}"
+    name += f"-{args.tag}" if args.tag else ""
+    name += f"-first{args.limit}" if args.limit is not None else ""
+    path = RESULTS_DIR / f"{name}.json"
+    rows_file = RESULTS_DIR / f"{name}.rows.jsonl"
+    header = {
+        "split": args.split,
+        "sample": args.sample,
+        "limit": args.limit,
+        "arms": list(arms),
+        "readers": list(readers),
+        "config": asdict(CONFIG),
+        "kept": kept,
+        "reuse": args.reuse,
+    }
+    captured: set[str] = set()
+    finished: set[str] = set()
+    if args.resume:
+        if not rows_file.exists():
+            parser.error(f"--resume found no rows file {rows_file}")
+        found_header, earlier_captures, earlier_probes = read_rows(rows_file)
+        if found_header != header:
+            parser.error(f"the rows file {rows_file} is of another run: {found_header}")
+        captured = {row["conversation"] for row in earlier_captures}
+        finished = {row["id"] for row in earlier_probes}
+    elif rows_file.exists():
+        parser.error(f"the rows file {rows_file} exists: pass --resume, or delete it")
+    left = [c for c in conversations if any(p.id not in finished for p in c.probes)]
+    if args.resume:
+        print(f"resuming: {len(finished)} probes finished, {len(left)} conversations left")
+
+    names = [f"{kept}-{conversation.id}" for conversation in conversations]
+    found = existing_databases(args.url, names)
+    if args.reuse:
+        missing = sorted(set(names) - set(found))
+        if missing:
+            parser.error(f"{len(missing)} kept databases do not exist, for example {missing[0]}")
+    elif found and not args.resume:
+        parser.error(
+            f"{len(found)} kept databases exist already, for example {found[0]}: "
+            "pass --reuse to read from them, --resume to go on, or delete them"
+        )
+    print(f"{'reusing' if args.reuse else 'keeping'} the memory as {kept}-<conversation id>")
+
+    commit = _commit()
+    started = datetime.now(UTC)
+    run_id = f"{started:%Y%m%d%H%M%S}-{uuid.uuid4().hex[:6]}"
+    clock_start = time.perf_counter()
+    RESULTS_DIR.mkdir(exist_ok=True)
+    if not args.resume:
+        rows_file.write_text(json.dumps({"header": header}) + "\n")
+    job = Job(args.url, arms, readers, args.replay, kept, rows_file)
+    order = sorted(left, key=lambda c: len(c.windows()), reverse=True)
+    frozen = frozenset(finished)
+    calls = [
+        (
+            job,
+            conversation,
+            SPEND_FACTOR * expected.by_conversation[conversation.id],
+            not args.reuse and conversation.id not in captured,
+            frozen,
+        )
+        for conversation in order
+    ]
+    workers = min(args.workers, len(calls)) if calls else 0
+    if workers <= 1:
+        done = [run_conversation(*call) for call in calls]
+    else:
+        context = multiprocessing.get_context("spawn")
+        with ProcessPoolExecutor(workers, mp_context=context) as pool:
+            done = list(pool.map(run_conversation, *zip(*calls, strict=True)))
+    spent = sum(found_done.spent for found_done in done)
+    failed = [found_done.failure for found_done in done if found_done.failure]
+    if failed:
+        print(f"{len(failed)} conversations failed; spent ${spent:.2f}")
+        for failure in failed:
+            print(f"  {failure}")
+        print(f"the finished rows are in {rows_file}; run the command again with --resume")
+        return None
+
+    _, captures, probe_rows = read_rows(rows_file)
+    by_id = {row["id"]: row for row in probe_rows}
+    rows = [by_id[probe.id] for conversation in conversations for probe in conversation.probes]
+    capture_of = {row["conversation"]: row for row in captures}
+    summaries = summarize(rows, arms, readers)
+    data_name, data_sha256 = SPLITS[args.split]
+    result_file = {
+        "config": {
+            "benchmark": "BEAM",
+            "split": args.split,
+            "data_file": data_name,
+            "data_revision": DATA_REVISION,
+            "data_sha256": data_sha256,
+            "sample": (
+                {"name": ALL, "conversations": len(conversations)}
+                if args.sample == ALL
+                else {
+                    "name": args.sample,
+                    "file": SAMPLES["a" if args.sample == SMOKE else args.sample].file,
+                    "seed": SAMPLES["a" if args.sample == SMOKE else args.sample].seed,
+                    "conversations": [c.id for c in conversations],
+                }
+            ),
+            "readers": {
+                reader: {
+                    "model": READERS[reader],
+                    "temperature": 0,
+                    "max_tokens": READER_MAX_TOKENS,
+                    "prompt": f"answer_generation_for_rag, BEAM {BEAM_COMMIT}",
+                }
+                for reader in readers
+            },
+            "judge": {
+                "model": JUDGE,
+                "temperature": 0,
+                "max_tokens": JUDGE_MAX_TOKENS,
+                "prompt": f"unified_llm_judge_base_prompt, BEAM {BEAM_COMMIT}",
+                "event_ordering": "tau_norm of the official line match",
+                "question_shown": False,
+            },
+            "extractor": {
+                "variant": CONFIG.extractor,
+                "model": CONFIG.variant.model,
+                "max_tokens": EXTRACTOR_MAX_TOKENS,
+            },
+            "embedder": {"model": embed.MODEL, "dimensions": embed.DIMENSIONS},
+            "driver": "hotdata",
+            "location": "local",
+            "capture": {
+                "window_chars": WINDOW_CHARS,
+                "chunk_chars": CONFIG.chunk_chars,
+                "kept": kept,
+                "reused": args.reuse,
+            },
+            "recall": {
+                "ranking": CONFIG.recall_with,
+                "k": CONFIG.k,
+                "budget": CONFIG.budget,
+                "episodes": CONFIG.episodes,
+                "as_of": None,
+            },
+            "arms": {
+                "memory": "recall of facts, then chunks found by search, in the budget",
+                "chunks": f"the top {CONFIG.k} chunks by vector search, in the budget",
+            },
+            "conversations": len(conversations),
+            "probes": len(rows),
+            "estimate_usd": round(expected.total, 2),
+            "estimate_s": round(expected.seconds),
+            "replay": args.replay,
+            "workers": workers,
+            "spent_usd": round(spent, 2),
+            "resumed_probes": len(finished),
+            "model_calls": sum(found_done.calls for found_done in done),
+            "embedder_misses": sum(found_done.embedder_misses for found_done in done),
+            "windows_at_limit": sum(row["windows_at_limit"] for row in captures),
+            "hotmemory_commit": commit,
+            "run_id": run_id,
+            "started_at": started.isoformat(),
+            "duration_s": round(time.perf_counter() - clock_start, 1),
+        },
+        "summary": summaries,
+        "captures": [capture_of[c.id] for c in conversations if c.id in capture_of],
+        "probes": rows,
+    }
+    path.write_text(json.dumps(result_file, indent=1) + "\n")
+    rows_file.unlink()
+    for combo, found_summary in summaries.items():
+        print(
+            f"{combo}: mean over abilities {found_summary['mean']:.3f}, "
+            f"over questions {found_summary['questions']:.3f}"
+        )
+    print(f"spent ${spent:.2f}; wrote {path}")
+    return path
+
+
+if __name__ == "__main__":
+    main()
diff --git a/bench/beam_score.py b/bench/beam_score.py
index 29ec619..1adb7a2 100644
--- a/bench/beam_score.py
+++ b/bench/beam_score.py
@@ -28,7 +28,7 @@
 from scipy.stats import kendalltau
 
 from bench.beam import ABILITIES, Probe
-from bench.models import Reply
+from bench.models import BEAM_MINI, Reply
 
 BEAM_COMMIT = "b2da22e"
 PROMPTS = Path(__file__).resolve().parent / "beam_prompts"
@@ -36,7 +36,7 @@
 JUDGE_PROMPT = (PROMPTS / "judge.txt").read_text()
 MATCH_SYSTEM = (PROMPTS / "match_system.txt").read_text()
 MATCH_USER = (PROMPTS / "match_user.txt").read_text()
-JUDGE = "gpt-4.1-mini-2025-04-14"
+JUDGE = BEAM_MINI
 JUDGE_MAX_TOKENS = 1000
 MATCH_MAX_TOKENS = 50
 EVENT_ORDERING = "event_ordering"
diff --git a/bench/models.py b/bench/models.py
index 172585c..1068485 100644
--- a/bench/models.py
+++ b/bench/models.py
@@ -16,11 +16,20 @@
 READER_MAX_TOKENS = 800
 JUDGE_MAX_TOKENS = 10
 EXTRACTOR_MAX_TOKENS = 4000
+BEAM_NANO = "gpt-4.1-nano-2025-04-14"
+BEAM_MINI = "gpt-4.1-mini-2025-04-14"
 PRICES = {
     READER: (2.50, 10.00),
     EXTRACTOR: (0.15, 0.60),
+    BEAM_NANO: (0.10, 0.40),
+    BEAM_MINI: (0.40, 1.60),
 }
-"""US dollars for one million input tokens and one million output tokens, read on 2026-10-10."""
+"""US dollars for one million input tokens and one million output tokens.
+
+The first two were read on 2026-10-10, and the two `gpt-4.1` models on 2026-10-26.
+"""
+EMBEDDING_PRICE = 0.02
+"""US dollars for one million tokens of `text-embedding-3-small`, read on 2026-10-26."""
 
 
 @dataclass(frozen=True)
@@ -70,7 +79,7 @@ def __init__(self, complete: Complete, path: Path) -> None:
         self.calls = 0
         self._complete = complete
         path.parent.mkdir(parents=True, exist_ok=True)
-        self._db = sqlite3.connect(path, timeout=60)
+        self._db = sqlite3.connect(path, timeout=60, check_same_thread=False)
         self._db.execute("CREATE TABLE IF NOT EXISTS replies (key TEXT PRIMARY KEY, reply TEXT)")
 
     def __call__(
diff --git a/bench/tests/test_beam_e2e.py b/bench/tests/test_beam_e2e.py
new file mode 100644
index 0000000..db26556
--- /dev/null
+++ b/bench/tests/test_beam_e2e.py
@@ -0,0 +1,210 @@
+import json
+import threading
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from bench.beam import Conversation, parse_conversation
+from bench.beam_e2e import (
+    CONFIG,
+    NAMESPACE,
+    Block,
+    Job,
+    SpendLimit,
+    ThreadModels,
+    capture_conversation,
+    choose,
+    chunks_block,
+    coverage,
+    estimate,
+    longest_worker,
+    read_rows,
+    run_on_store,
+    source_windows,
+    summarize,
+)
+from bench.beam_score import JUDGE, MATCH_SYSTEM
+from bench.models import BEAM_NANO, EXTRACTOR, Reply
+from bench.tests.test_beam import row
+from hotmemory import Memory, MemoryStore
+from tests.conftest import FakeClock, fake_embedder
+
+
+class FakeModels:
+    """Facts from each user line, a fixed reader answer, and full marks from the judge."""
+
+    def __init__(self) -> None:
+        self.calls: list[str] = []
+        self.lock = threading.Lock()
+
+    def __call__(
+        self,
+        model: str,
+        prompt: str,
+        max_tokens: int,
+        schema: dict[str, Any] | None = None,
+        system: str | None = None,
+    ) -> Reply:
+        with self.lock:
+            self.calls.append(model)
+        if model == EXTRACTOR:
+            session = prompt.split("Session:\n", 1)[1]
+            facts = [
+                {"subject": "user", "content": line.removeprefix("user: ")}
+                for line in session.split("\n\n")
+                if line.startswith("user: ")
+            ]
+            return Reply(json.dumps({"facts": facts}), 100, 10)
+        if model == JUDGE and system == MATCH_SYSTEM:
+            return Reply("NO", 10, 1)
+        if model == JUDGE:
+            return Reply('{"score": 1.0, "reason": "ok"}', 100, 10)
+        return Reply("The answer.", 1000, 5)
+
+
+@pytest.fixture
+def conversation() -> Conversation:
+    return parse_conversation(row())
+
+
+def job(tmp_path: Path) -> Job:
+    rows_file = tmp_path / "rows.jsonl"
+    rows_file.write_text(json.dumps({"header": {"run": 1}}) + "\n")
+    return Job("", ("memory", "chunks"), ("nano", "mini"), False, "kept", rows_file)
+
+
+def store() -> MemoryStore:
+    return MemoryStore(embedder=fake_embedder, clock=FakeClock())
+
+
+def test_a_conversation_is_captured_once_and_every_probe_is_scored(
+    tmp_path: Path, conversation: Conversation
+) -> None:
+    fake = FakeModels()
+    models = ThreadModels(fake, tmp_path / "replies.sqlite")
+    found = job(tmp_path)
+    memory = store()
+    done = run_on_store(found, conversation, memory, models, True, frozenset())
+    assert done.probes == 20
+    assert done.capture["windows"] == 2
+    assert done.capture["windows_at_limit"] == 0
+    assert done.capture["facts_written"] == 3
+    header, captures, probes = read_rows(found.rows_file)
+    assert header == {"run": 1}
+    assert [c["conversation"] for c in captures] == ["7"]
+    assert len(probes) == 20
+    first = probes[0]
+    assert set(first["combos"]) == {"memory/nano", "memory/mini", "chunks/nano", "chunks/mini"}
+    assert first["arms"]["memory"]["facts_in_block"] == 3
+    assert first["arms"]["chunks"]["facts_in_block"] == 0
+    assert first["arms"]["chunks"]["episodes_in_block"]
+    assert first["arms"]["memory"]["source_windows"] == ["window-01-0001", "window-02-0001"]
+    nugget_rows = [p for p in probes if p["ability"] != "event_ordering"]
+    assert all(c["score"] == 1.0 for p in nugget_rows for c in p["combos"].values())
+    ordering = [p for p in probes if p["ability"] == "event_ordering"]
+    assert all(c["tau_norm"] is not None for p in ordering for c in p["combos"].values())
+    assert fake.calls.count(EXTRACTOR) == 2
+    assert fake.calls.count(BEAM_NANO) == 40
+    summaries = summarize(probes, ("memory", "chunks"), ("nano", "mini"))
+    assert summaries["memory/nano"]["count"] == 20
+    assert summaries["memory/nano"]["by_domain"].keys() == {"Coding"}
+    models.close()
+
+
+def test_a_resumed_conversation_runs_only_the_probes_left(
+    tmp_path: Path, conversation: Conversation
+) -> None:
+    fake = FakeModels()
+    models = ThreadModels(fake, tmp_path / "replies.sqlite")
+    memory = store()
+    capture_conversation(memory, models, conversation.windows(), (*NAMESPACE, "7"))
+    calls = len(fake.calls)
+    finished = frozenset(probe.id for probe in conversation.probes[:18])
+    found = job(tmp_path)
+    done = run_on_store(found, conversation, memory, models, False, finished)
+    assert done.probes == 2
+    assert done.capture == {}
+    assert EXTRACTOR not in fake.calls[calls:]
+    _, captures, probes = read_rows(found.rows_file)
+    assert captures == []
+    assert [p["id"] for p in probes] == [p.id for p in conversation.probes[18:]]
+    models.close()
+
+
+def test_a_second_capture_skips_the_stored_windows(
+    tmp_path: Path, conversation: Conversation
+) -> None:
+    fake = FakeModels()
+    models = ThreadModels(fake, tmp_path / "replies.sqlite")
+    memory = store()
+    namespace = (*NAMESPACE, "7")
+    capture_conversation(memory, models, conversation.windows(), namespace)
+    again = capture_conversation(memory, models, conversation.windows(), namespace)
+    assert again["facts_written"] == 0
+    assert fake.calls.count(EXTRACTOR) == 2
+    models.close()
+
+
+def test_thread_models_sum_every_thread_and_stop_at_the_limit(tmp_path: Path) -> None:
+    models = ThreadModels(FakeModels(), tmp_path / "replies.sqlite", limit=1.0)
+    threads = [threading.Thread(target=models, args=(BEAM_NANO, f"p{n}", 10)) for n in range(4)]
+    for thread in threads:
+        thread.start()
+    for thread in threads:
+        thread.join()
+    assert models.calls == 4
+    assert models.spent == pytest.approx(4 * (1000 * 0.10 + 5 * 0.40) / 1e6)
+    big = ThreadModels(lambda *_, **__: Reply("x", 10_000_000, 0), tmp_path / "r2.sqlite", 0.5)
+    with pytest.raises(SpendLimit):
+        big(BEAM_NANO, "p", 10)
+    models.close()
+    big.close()
+
+
+def test_the_chunks_block_keeps_to_the_budget(tmp_path: Path, conversation: Conversation) -> None:
+    memory = store()
+    namespace = (*NAMESPACE, "7")
+    Memory(memory).capture(
+        "\n\n".join(["word " * 190] * 25),
+        namespace,
+        lambda text, at, known: [],
+        episode="window-01-0001",
+        chunk_chars=CONFIG.chunk_chars,
+    )
+    block = chunks_block(memory, namespace, "word")
+    assert 0 < len(block.text) <= CONFIG.budget
+    assert len(block.episodes) < 25
+    assert all(key.startswith("window-01-0001-") for key in block.episodes)
+
+
+def test_coverage_counts_the_source_windows_in_the_block(conversation: Conversation) -> None:
+    windows = conversation.windows(limit=60)
+    probe = conversation.probes[0]
+    found = source_windows(windows, probe)
+    assert found == {"window-01-0001", "window-01-0002", "window-02-0001"}
+    block = Block("", ("window-01-0001-0001", "window-01-0001-0002", "window-02-0001-0001"), 0)
+    assert coverage(found, block) == pytest.approx(2 / 3)
+    assert coverage(set(), block) is None
+
+
+def test_the_estimate_counts_each_part(conversation: Conversation) -> None:
+    both = estimate([conversation], ("memory", "chunks"), ("nano", "mini"), 1)
+    assert both.parts["extraction"] == pytest.approx(2 * 0.0005)
+    assert both.parts["readers"] > 0 and both.parts["judge"] > 0
+    assert both.total == pytest.approx(sum(both.by_conversation.values()))
+    reused = estimate([conversation], ("memory",), ("nano",), 1, capture=False)
+    assert reused.parts["extraction"] == reused.parts["embeddings"] == 0
+    assert reused.parts["judge"] == pytest.approx(both.parts["judge"] / 4)
+
+
+def test_the_longest_worker_takes_the_longest_jobs_first() -> None:
+    assert longest_worker([3, 5, 3, 4], 2) == 8
+    assert longest_worker([3, 5, 3, 4], 1) == 15
+    assert longest_worker([], 4) == 0
+
+
+def test_a_sample_needs_the_sample_split(conversation: Conversation) -> None:
+    assert choose("1M", "all", [conversation]) == [conversation]
+    with pytest.raises(ValueError, match="needs --split 100K"):
+        choose("1M", "a", [conversation])

From bfb43f4f079d0de8cc43bbe6513db37f1841d6cc Mon Sep 17 00:00:00 2001
From: Rohan Dsouza <rohan@hotdata.dev>
Date: Mon, 26 Oct 2026 14:40:02 +0530
Subject: [PATCH 5/8] Phase 12A task 5: document the BEAM runner in
 bench/README.md

The new section gives the commands, the options, the kept database
name, the two arms, the official scoring, and how a run resumes.
---
 bench/README.md | 70 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 69 insertions(+), 1 deletion(-)

diff --git a/bench/README.md b/bench/README.md
index 9c08e6f..22bdab6 100644
--- a/bench/README.md
+++ b/bench/README.md
@@ -1,7 +1,8 @@
 # The benchmark runner
 
 This folder runs hotmemory on LongMemEval-S, a benchmark of 500 questions over long chat
-histories. It is not part of the hotmemory package. [docs/benchmarks.md](../docs/benchmarks.md)
+histories, and on BEAM, a benchmark of long conversations in many domains. It is not part
+of the hotmemory package. [docs/benchmarks.md](../docs/benchmarks.md)
 reports the numbers, and each number names the result file that it comes from.
 
 ## Before a run
@@ -188,6 +189,73 @@ questions that the rows file holds. It refuses a rows file of another configurat
 run without `--resume` refuses to start over a rows file. When every question has
 finished, the runner writes the result file and deletes the rows file.
 
+## BEAM
+
+BEAM (arXiv 2510.27246) holds conversations of about 100,000 to 1 million tokens, each
+with 20 probing questions, 2 for each of 10 abilities. `bench/beam.py` reads the splits
+`100K` and `1M` of the Hugging Face dataset `Mohammadta/BEAM` at a pinned revision, and
+checks the SHA-256 of each file. The data license is CC BY-SA 4.0, so the files stay in
+`bench/data/`, which git ignores. This command downloads a split and prints its counts:
+
+```sh
+uv run --group bench python -m bench.beam --split 1M
+```
+
+The loader cuts each batch of a conversation into windows of whole turn pairs, at most
+10,000 characters each, with the date of the batch. A turn pair longer than that is cut
+at blank lines. It strips the index markers, such as `->-> 1,1`, from the user turns.
+Samples A and B are 5 conversations each of the `100K` split, from 5 different domains,
+in `bench/sample-beam-a.json` and `bench/sample-beam-b.json`. The smoke sample is the
+conversation of sample A with the fewest windows.
+
+This command prints the cost estimate and the expected time of a run, and calls no model:
+
+```sh
+uv run --group bench --env-file .env python -m bench.beam_e2e --split 100K --sample a --workers 5
+```
+
+Add `--run` to run it. The options are these:
+
+| Option | Values | Default |
+|---|---|---|
+| `--split` | `100K` or `1M` | `100K` |
+| `--sample` | `smoke`, `a`, or `b` of the `100K` split, or `all` | `smoke` |
+| `--arms` | `memory`, `chunks`, or both, comma-separated | both |
+| `--readers` | `nano` (`gpt-4.1-nano`), `mini` (`gpt-4.1-mini`), or both | both |
+| `--workers` | conversations at a time, each in its own process | 1 |
+| `--reuse` | read from the kept databases, with no `capture` | off |
+| `--resume` | go on from the rows file of an earlier run | off |
+| `--replay` | call no model, and fail a conversation at a reply that the cache does not hold | off |
+| `--extract-rate`, `--window-seconds` | the rates of the estimate | $0.0005 and 8.3 s |
+| `--limit`, `--tag`, `--url` | as for `bench.e2e` | |
+
+Each conversation runs over one local database, named
+`hotmemory-kept-beam-<split>-<extractor><digest>-chunks1000-vector-<commit>-<conversation id>`.
+The database keeps the memory after the run, and the runner never deletes it. A worker
+takes the conversation with the most windows first. It captures each window in order with
+the `detail` extractor, as chunks of 1,000 characters. Then, for each probe, it builds one
+block of at most 16,000 characters for each arm:
+
+- `memory` calls `recall` with `k` 30 and `episodes="search"`.
+- `chunks` searches the chunks alone by vector, `k` 30, as the RAG baseline.
+
+Each reader answers from each block with the official RAG answer prompt of BEAM. The
+scoring follows the official code at commit `b2da22e`: one `gpt-4.1-mini` judge call for
+each nugget of the rubric, and for event ordering, the line match and Kendall tau-b. The
+official judge prompt does not see the question (upstream issue
+mohammadtavakoli78/BEAM#15). The prompts are copied in `bench/beam_prompts/` under their
+MIT license. The score of a split is the mean over the 10 abilities.
+
+The rows file holds one row for each captured conversation and one row for each finished
+probe. If a conversation fails, the others go on, and the runner writes no result file.
+Run the same command again with `--resume`. It skips the finished probes and the captured
+conversations, and a conversation that stopped during capture goes on over its kept
+database. `capture` skips each window whose chunks are stored, with no extractor call. A
+conversation stops when it spends twice its estimate. The result file is
+`bench/results/beam-<split>-<sample>[-<tag>].json`. Each probe row records the windows
+that hold its source turns, and the share of them that each block shows, in
+`source_coverage`.
+
 ## The checks of the results
 
 This command compares the vector rankings of two retrieval result files, as the driver

From a6e823cfda10bbcb50abb8c9c24a8f706cd0c38e Mon Sep 17 00:00:00 2001
From: Rohan Dsouza <rohan@hotdata.dev>
Date: Mon, 26 Oct 2026 17:17:13 +0530
Subject: [PATCH 6/8] Phase 12A: read BEAM with gpt-4.1-mini alone

The API no longer serves gpt-4.1-nano. Both the alias and the snapshot
gpt-4.1-nano-2025-04-14 return 404 "deprecated". The runner now has one
reader, gpt-4.1-mini, the model of the judge.
---
 bench/README.md              |  2 +-
 bench/beam_e2e.py            |  3 +--
 bench/models.py              |  4 +---
 bench/tests/test_beam_e2e.py | 31 +++++++++++++++++--------------
 4 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/bench/README.md b/bench/README.md
index 22bdab6..9a53cd7 100644
--- a/bench/README.md
+++ b/bench/README.md
@@ -221,7 +221,7 @@ Add `--run` to run it. The options are these:
 | `--split` | `100K` or `1M` | `100K` |
 | `--sample` | `smoke`, `a`, or `b` of the `100K` split, or `all` | `smoke` |
 | `--arms` | `memory`, `chunks`, or both, comma-separated | both |
-| `--readers` | `nano` (`gpt-4.1-nano`), `mini` (`gpt-4.1-mini`), or both | both |
+| `--readers` | `mini` (`gpt-4.1-mini`), the one reader | `mini` |
 | `--workers` | conversations at a time, each in its own process | 1 |
 | `--reuse` | read from the kept databases, with no `capture` | off |
 | `--resume` | go on from the rows file of an earlier run | off |
diff --git a/bench/beam_e2e.py b/bench/beam_e2e.py
index 00b9eee..44a19fc 100644
--- a/bench/beam_e2e.py
+++ b/bench/beam_e2e.py
@@ -70,7 +70,6 @@
 from bench.embed import CACHE_DIR
 from bench.models import (
     BEAM_MINI,
-    BEAM_NANO,
     EMBEDDING_PRICE,
     EXTRACTOR_MAX_TOKENS,
     PRICES,
@@ -83,7 +82,7 @@
 from hotmemory.contract import render_episode
 
 ARMS = ("memory", "chunks")
-READERS = {"nano": BEAM_NANO, "mini": BEAM_MINI}
+READERS = {"mini": BEAM_MINI}
 READER_MAX_TOKENS = 2000
 CONFIG = Config(
     k=30,
diff --git a/bench/models.py b/bench/models.py
index 1068485..0f6fdbc 100644
--- a/bench/models.py
+++ b/bench/models.py
@@ -16,17 +16,15 @@
 READER_MAX_TOKENS = 800
 JUDGE_MAX_TOKENS = 10
 EXTRACTOR_MAX_TOKENS = 4000
-BEAM_NANO = "gpt-4.1-nano-2025-04-14"
 BEAM_MINI = "gpt-4.1-mini-2025-04-14"
 PRICES = {
     READER: (2.50, 10.00),
     EXTRACTOR: (0.15, 0.60),
-    BEAM_NANO: (0.10, 0.40),
     BEAM_MINI: (0.40, 1.60),
 }
 """US dollars for one million input tokens and one million output tokens.
 
-The first two were read on 2026-10-10, and the two `gpt-4.1` models on 2026-10-26.
+The first two were read on 2026-10-10, and `gpt-4.1-mini` on 2026-10-26.
 """
 EMBEDDING_PRICE = 0.02
 """US dollars for one million tokens of `text-embedding-3-small`, read on 2026-10-26."""
diff --git a/bench/tests/test_beam_e2e.py b/bench/tests/test_beam_e2e.py
index db26556..ef755e6 100644
--- a/bench/tests/test_beam_e2e.py
+++ b/bench/tests/test_beam_e2e.py
@@ -25,7 +25,7 @@
     summarize,
 )
 from bench.beam_score import JUDGE, MATCH_SYSTEM
-from bench.models import BEAM_NANO, EXTRACTOR, Reply
+from bench.models import BEAM_MINI, EXTRACTOR, Reply
 from bench.tests.test_beam import row
 from hotmemory import Memory, MemoryStore
 from tests.conftest import FakeClock, fake_embedder
@@ -36,6 +36,7 @@ class FakeModels:
 
     def __init__(self) -> None:
         self.calls: list[str] = []
+        self.answers = 0
         self.lock = threading.Lock()
 
     def __call__(
@@ -58,8 +59,10 @@ def __call__(
             return Reply(json.dumps({"facts": facts}), 100, 10)
         if model == JUDGE and system == MATCH_SYSTEM:
             return Reply("NO", 10, 1)
-        if model == JUDGE:
+        if model == JUDGE and "RUBRIC CRITERION" in prompt:
             return Reply('{"score": 1.0, "reason": "ok"}', 100, 10)
+        with self.lock:
+            self.answers += 1
         return Reply("The answer.", 1000, 5)
 
 
@@ -71,7 +74,7 @@ def conversation() -> Conversation:
 def job(tmp_path: Path) -> Job:
     rows_file = tmp_path / "rows.jsonl"
     rows_file.write_text(json.dumps({"header": {"run": 1}}) + "\n")
-    return Job("", ("memory", "chunks"), ("nano", "mini"), False, "kept", rows_file)
+    return Job("", ("memory", "chunks"), ("mini",), False, "kept", rows_file)
 
 
 def store() -> MemoryStore:
@@ -95,7 +98,7 @@ def test_a_conversation_is_captured_once_and_every_probe_is_scored(
     assert [c["conversation"] for c in captures] == ["7"]
     assert len(probes) == 20
     first = probes[0]
-    assert set(first["combos"]) == {"memory/nano", "memory/mini", "chunks/nano", "chunks/mini"}
+    assert set(first["combos"]) == {"memory/mini", "chunks/mini"}
     assert first["arms"]["memory"]["facts_in_block"] == 3
     assert first["arms"]["chunks"]["facts_in_block"] == 0
     assert first["arms"]["chunks"]["episodes_in_block"]
@@ -105,10 +108,10 @@ def test_a_conversation_is_captured_once_and_every_probe_is_scored(
     ordering = [p for p in probes if p["ability"] == "event_ordering"]
     assert all(c["tau_norm"] is not None for p in ordering for c in p["combos"].values())
     assert fake.calls.count(EXTRACTOR) == 2
-    assert fake.calls.count(BEAM_NANO) == 40
-    summaries = summarize(probes, ("memory", "chunks"), ("nano", "mini"))
-    assert summaries["memory/nano"]["count"] == 20
-    assert summaries["memory/nano"]["by_domain"].keys() == {"Coding"}
+    assert fake.answers == 40
+    summaries = summarize(probes, ("memory", "chunks"), ("mini",))
+    assert summaries["memory/mini"]["count"] == 20
+    assert summaries["memory/mini"]["by_domain"].keys() == {"Coding"}
     models.close()
 
 
@@ -148,16 +151,16 @@ def test_a_second_capture_skips_the_stored_windows(
 
 def test_thread_models_sum_every_thread_and_stop_at_the_limit(tmp_path: Path) -> None:
     models = ThreadModels(FakeModels(), tmp_path / "replies.sqlite", limit=1.0)
-    threads = [threading.Thread(target=models, args=(BEAM_NANO, f"p{n}", 10)) for n in range(4)]
+    threads = [threading.Thread(target=models, args=(BEAM_MINI, f"p{n}", 10)) for n in range(4)]
     for thread in threads:
         thread.start()
     for thread in threads:
         thread.join()
     assert models.calls == 4
-    assert models.spent == pytest.approx(4 * (1000 * 0.10 + 5 * 0.40) / 1e6)
+    assert models.spent == pytest.approx(4 * (1000 * 0.40 + 5 * 1.60) / 1e6)
     big = ThreadModels(lambda *_, **__: Reply("x", 10_000_000, 0), tmp_path / "r2.sqlite", 0.5)
     with pytest.raises(SpendLimit):
-        big(BEAM_NANO, "p", 10)
+        big(BEAM_MINI, "p", 10)
     models.close()
     big.close()
 
@@ -189,13 +192,13 @@ def test_coverage_counts_the_source_windows_in_the_block(conversation: Conversat
 
 
 def test_the_estimate_counts_each_part(conversation: Conversation) -> None:
-    both = estimate([conversation], ("memory", "chunks"), ("nano", "mini"), 1)
+    both = estimate([conversation], ("memory", "chunks"), ("mini",), 1)
     assert both.parts["extraction"] == pytest.approx(2 * 0.0005)
     assert both.parts["readers"] > 0 and both.parts["judge"] > 0
     assert both.total == pytest.approx(sum(both.by_conversation.values()))
-    reused = estimate([conversation], ("memory",), ("nano",), 1, capture=False)
+    reused = estimate([conversation], ("memory",), ("mini",), 1, capture=False)
     assert reused.parts["extraction"] == reused.parts["embeddings"] == 0
-    assert reused.parts["judge"] == pytest.approx(both.parts["judge"] / 4)
+    assert reused.parts["judge"] == pytest.approx(both.parts["judge"] / 2)
 
 
 def test_the_longest_worker_takes_the_longest_jobs_first() -> None:

From 845047a1530f7d8a022da87a5bf1d519582a2aea Mon Sep 17 00:00:00 2001
From: Rohan Dsouza <rohan@hotdata.dev>
Date: Mon, 26 Oct 2026 18:36:55 +0530
Subject: [PATCH 7/8] Install json-repair and scipy for the bench tests in make
 floor

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index db49845..e765669 100644
--- a/Makefile
+++ b/Makefile
@@ -18,6 +18,7 @@ floor:
 		--python-version 3.11 --quiet --output-file "$$requirements" && \
 	uv run --isolated --no-project --python 3.11 --with-requirements "$$requirements" \
 		--with-editable . --with pytest --with pytest-socket --with tiktoken \
+		--with json-repair --with scipy \
 		pytest --disable-socket -q -p no:cacheprovider
 
 integration:

From 85da9a82c8539fa7d87f4328a6b36990e11aeb04 Mon Sep 17 00:00:00 2001
From: Rohan Dsouza <rohan@hotdata.dev>
Date: Mon, 26 Oct 2026 18:36:55 +0530
Subject: [PATCH 8/8] Phase 12A: make no model call after the spend limit

ThreadModels checked the limit only after a call returned, so each queued
reader task made one more paid call before it raised.
---
 bench/beam_e2e.py            |  8 ++++++--
 bench/tests/test_beam_e2e.py | 11 ++++++++++-
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/bench/beam_e2e.py b/bench/beam_e2e.py
index 44a19fc..99e61ba 100644
--- a/bench/beam_e2e.py
+++ b/bench/beam_e2e.py
@@ -196,7 +196,7 @@ class ThreadModels:
     """A `CachedModels` for each thread, over one cache file, with a spend limit.
 
     `spent` and `calls` sum every thread. A call raises SpendLimit once `spent` passes
-    `limit`.
+    `limit`, and makes no model call after that.
     """
 
     def __init__(self, complete: Complete, path: Path, limit: float = float("inf")) -> None:
@@ -221,10 +221,14 @@ def __call__(
             self._local.models = models
             with self._lock:
                 self._all.append(models)
+        self._check()
         reply = models(model, prompt, max_tokens, schema, system)
+        self._check()
+        return reply
+
+    def _check(self) -> None:
         if self.spent > self._limit:
             raise SpendLimit(f"spent ${self.spent:.2f}, over the limit of ${self._limit:.2f}")
-        return reply
 
     @property
     def spent(self) -> float:
diff --git a/bench/tests/test_beam_e2e.py b/bench/tests/test_beam_e2e.py
index ef755e6..669c7a5 100644
--- a/bench/tests/test_beam_e2e.py
+++ b/bench/tests/test_beam_e2e.py
@@ -158,9 +158,18 @@ def test_thread_models_sum_every_thread_and_stop_at_the_limit(tmp_path: Path) ->
         thread.join()
     assert models.calls == 4
     assert models.spent == pytest.approx(4 * (1000 * 0.40 + 5 * 1.60) / 1e6)
-    big = ThreadModels(lambda *_, **__: Reply("x", 10_000_000, 0), tmp_path / "r2.sqlite", 0.5)
+    paid: list[str] = []
+
+    def costly(model: str, prompt: str, *_: Any, **__: Any) -> Reply:
+        paid.append(prompt)
+        return Reply("x", 10_000_000, 0)
+
+    big = ThreadModels(costly, tmp_path / "r2.sqlite", 0.5)
     with pytest.raises(SpendLimit):
         big(BEAM_MINI, "p", 10)
+    with pytest.raises(SpendLimit):
+        big(BEAM_MINI, "q", 10)
+    assert paid == ["p"]
     models.close()
     big.close()