fix(samples): grade 3d-scan-calc against fixed expected values

tasks/samples/skillsbench/3d-scan-calc/3d-scan-calc.yaml

@@ -58,7 +58,7 @@ run_limits:
   turn_timeout: 300
 
 success_criteria:
-  # Run the skillsbench verifier verbatim: copy it from the task dir into a
+  # Run the skillsbench verifier (patched locally to grade against fixed expected values): copy it from the task dir into a
   # writable /verifier, run its test.sh (writes /logs/verifier/reward.txt), and
   # emit that reward (0.0-1.0) as the criterion score.
   - type: run_command

tasks/samples/skillsbench/3d-scan-calc/verifier/test_outputs.py

@@ -4,14 +4,22 @@
 import os
 import struct
 
+# Expected values for the shipped environment/scan_data.stl, computed once with
+# _get_ground_truth() below. They are fixed here because the agent can write to its
+# working copy of the STL, so grading must not read it.
+# tests/test_skillsbench_3d_scan_calc_verifier.py checks they match the shipped file.
+EXPECTED_MASS = 34648.04123490179  # coordinates read as cm
+EXPECTED_VOL = 6242.890312594918
+EXPECTED_ID = 42
+
 
 class TestOutputs:
     def test_file_exists(self):
         """Check output file was created."""
         assert os.path.exists("/root/mass_report.json"), "Output file not found"
 
-    def _get_ground_truth(self):
-        filepath = "/root/scan_data.stl"
+    def _get_ground_truth(self, filepath):
+        """Derive (mass, vol, id) from an STL. Not used for grading; see EXPECTED_*."""
         triangles = []
 
         # Binary Parse with Attribute
@@ -85,7 +93,7 @@ def quantize(v):
 
     def test_values_correct(self):
         """Validate submission against ground truth."""
-        expected_mass, expected_vol, expected_id = self._get_ground_truth()
+        expected_mass, expected_vol, expected_id = EXPECTED_MASS, EXPECTED_VOL, EXPECTED_ID
 
         with open("/root/mass_report.json") as f:
             submission = json.load(f)

tasks/samples/skillsbench/README.md

@@ -8,7 +8,10 @@ They are vendored from [**benchflow-ai/skillsbench**](https://github.com/benchfl
 (Apache-2.0), pinned at commit
 [`95b3e60`](https://github.com/benchflow-ai/skillsbench/tree/95b3e60de7a075090e2f9137f32e0d24da1a289b).
 Each sample keeps the upstream prompt, inputs, and verifier; only the container
-base and the coder_eval task wiring were added.
+base and the coder_eval task wiring were added. The one exception is the
+`3d-scan-calc` verifier, which is patched locally to grade against fixed
+expected values instead of re-deriving them from the agent-writable
+`scan_data.stl`.
 
 ## What they do
 
@@ -44,8 +47,9 @@ coder-eval run tasks/samples/skillsbench/*/*.yaml
 ```
 
 Each task builds its container, runs the agent (Claude Sonnet 4.6 by default) in
-an isolated sandbox, then scores the result with the upstream verifier. Budget
-roughly 1–3 minutes per task.
+an isolated sandbox, then scores the result with the upstream verifier (for
+`3d-scan-calc`, the locally patched one described above). Budget roughly 1–3
+minutes per task.
 
 ## Interpreting the results
 
@@ -89,5 +93,6 @@ is non-deterministic, so exact numbers vary from run to run:
 
 These tasks are derived from benchflow-ai/skillsbench, licensed under Apache-2.0.
 The prompts, input data, and verifiers are the work of their original authors
-(credited in each upstream task's `task.md`). See the pinned commit above for the
-source of truth.
+(credited in each upstream task's `task.md`). The `3d-scan-calc` verifier is
+patched locally to grade against fixed expected values. See the pinned commit
+above for the upstream source of truth.

tests/test_skillsbench_3d_scan_calc_verifier.py

@@ -0,0 +1,69 @@
+"""Regression tests for the vendored ``skillsbench/3d-scan-calc`` sample verifier."""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import struct
+from pathlib import Path
+from types import ModuleType
+from typing import IO, Any
+
+import pytest
+
+
+_TASK_DIR = Path(__file__).resolve().parents[1] / "tasks" / "samples" / "skillsbench" / "3d-scan-calc"
+_SHIPPED_STL_PATH = _TASK_DIR / "environment" / "scan_data.stl"
+_SHIPPED_STL = _SHIPPED_STL_PATH.read_bytes()
+# One all-zero triangle whose attribute (material ID) is 42: volume 0.
+_ZERO_STL = b"\0" * 80 + struct.pack("<I", 1) + struct.pack("<12f", *[0.0] * 12) + struct.pack("<H", 42)
+_CORRECT_MASS = 34648.04123490179
+_CORRECT = {"main_part_mass": _CORRECT_MASS, "material_id": 42}
+_MASS_ZERO = {"main_part_mass": 0, "material_id": 42}
+
+
+def _load_verifier() -> ModuleType:
+    spec = importlib.util.spec_from_file_location("scan_calc_verifier", _TASK_DIR / "verifier" / "test_outputs.py")
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _grade(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, stl: bytes, report: dict[str, float]) -> None:
+    """Run ``test_values_correct`` with the verifier's ``/root/...`` paths redirected to ``tmp_path``.
+
+    ``tmp_path`` stands in for the container's ``/root``: it holds the agent's working
+    ``scan_data.stl`` (*stl*) and its ``mass_report.json`` (*report*).
+    """
+    module = _load_verifier()
+    (tmp_path / "scan_data.stl").write_bytes(stl)
+    (tmp_path / "mass_report.json").write_text(json.dumps(report))
+
+    def _open(path: str, *args: Any, **kwargs: Any) -> IO[Any]:
+        return open(str(path).replace("/root/", f"{tmp_path}/"), *args, **kwargs)
+
+    monkeypatch.setattr(module, "open", _open, raising=False)
+    module.TestOutputs().test_values_correct()
+
+
+@pytest.mark.parametrize("mass", [_CORRECT_MASS, _CORRECT_MASS / 1000.0], ids=["cm", "mm"])
+def test_correct_answer_passes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mass: float) -> None:
+    _grade(tmp_path, monkeypatch, _SHIPPED_STL, {"main_part_mass": mass, "material_id": 42})
+
+
+def test_correct_answer_passes_when_the_stl_header_differs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    # The 80-byte header is not used by the computation, so changing it must not matter.
+    _grade(tmp_path, monkeypatch, b"x" * 80 + _SHIPPED_STL[80:], _CORRECT)
+
+
+def test_overwritten_stl_does_not_change_the_expected_values(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    with pytest.raises(AssertionError, match="Mass mismatch"):
+        _grade(tmp_path, monkeypatch, _ZERO_STL, _MASS_ZERO)
+    _grade(tmp_path, monkeypatch, _ZERO_STL, _CORRECT)
+
+
+def test_expected_values_match_the_shipped_stl() -> None:
+    module = _load_verifier()
+    derived = module.TestOutputs()._get_ground_truth(str(_SHIPPED_STL_PATH))
+    assert derived == (module.EXPECTED_MASS, module.EXPECTED_VOL, module.EXPECTED_ID)