diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 1d15d2e9f0..b36b2bf76f 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -807,6 +807,17 @@ jobs: - name: Qualify the canary premerge contract smoke on native Windows run: python examples/canary/premerge-validation-gate-smoke.py + - name: Qualify native CLI output fixture and diagnostic boundaries + run: >- + python -m pytest -q + tests/control_plane/test_cli_output_budget.py + tests/control_plane/test_cli_output_differential.py + -k "stable_budget_fixture or probe_process_failure_preserves_utf8_diagnostics" + + - name: Qualify the native Windows CLI output differential + timeout-minutes: 12 + run: python examples/control_plane/cli-output-budget-regression-smoke.py + - name: Run native Windows lifecycle tests run: >- python -m pytest -q diff --git a/docs/reference/contracts/interface-budget-contract.md b/docs/reference/contracts/interface-budget-contract.md index 5ae00b42f8..434f65580d 100644 --- a/docs/reference/contracts/interface-budget-contract.md +++ b/docs/reference/contracts/interface-budget-contract.md @@ -138,7 +138,7 @@ warning and any already-over-budget fixture instead of truncating requirements. 仅人工诊断输出上限由 5,100 调至 5,250，机器 JSON／正文／行数／执行预算不变。 重规划指导增加 363 字节，独立指导预算为 1,800；8 KiB 总包告警及原有超限仍保留。 -The same qualification also covers emitted `--compact` Markdown, independently +The historical pause/correction qualification also covered emitted `--compact` Markdown, independently of the task-body and thin-output limits. On the unchanged native mode fixture, base `3e611660a` emits 11,315 characters and the pause/correction head `84e00bac1` emits 11,530, failing the old 11,500 ceiling; both have 145 lines. @@ -150,7 +150,9 @@ only this Markdown regression ceiling to 11,750, leaving 220 characters of headroom (the base had 185) for bounded wording variation. JSON grows from 12,658 to 12,873 under the unchanged 13,500 ceiling; the 155-line cap, semantic anchors, fixture population, growth checks and execution limits remain intact. -This qualifies readable output size, not installed host behavior or model adoption. +This qualified readable output size, not installed host behavior or model adoption. +The current native-host ceilings and matched measurements are recorded in +[Native fixture qualification](#native-fixture-qualification). The base/head differential independently detects the same instruction rollout: work-context text adds 208 characters (one receipt character may vary), optional @@ -164,14 +166,15 @@ change. Missing or contradictory observations, already-upgraded bases, other surfaces and growth beyond the allowance retain ordinary failure behavior. This evidence does not qualify arbitrary future wording or runtime authority. -同一资格化还须独立覆盖 `--compact` 的完整 Markdown 输出：不改原生 mode 夹具时， +历史暂停／纠偏资格化还独立覆盖了 `--compact` 的完整 Markdown 输出：不改原生 mode 夹具时， 基线 `3e611660a` 为 11,315 字符，暂停／纠偏头 `84e00bac1` 为 11,530， 确实超过旧 11,500 上限；两者均为 145 行。新增 215 字符承载当前心跳单次停止、 结果确认、未确认处理、证据限定的局部纠偏及原结算义务，各自支持不同决定， 不能作为重复诊断删除。保留全文，仅将该 Markdown 回归上限调至 11,750， 留 220 字符余量供有界措辞变化（基线余量为 185）。JSON 从 12,658 增至 12,873，仍遵守原 13,500 上限；155 行、语义锚点、夹具规模、增长检查和执行限制 -不变。这只资格化可读输出尺寸，不证明安装态行为或模型采纳。 +不变。这只资格化可读输出尺寸，不证明安装态行为或模型采纳。当前原生宿主上限与 +成对测量见 [Native fixture qualification](#native-fixture-qualification)。 base/head 差异门独立测得工作上下文增加 208 字符（回执可能差 1 字符）、可选 replan 指引增加 363、thin/brief/compact 输出增加 154/163/215。既有 probe @@ -404,6 +407,53 @@ qualified replacement replan-context projection and remains a review signal. Coverage-only to dense replan context has a measured, one-time allowance on its three affected JSON surfaces; dense-to-dense changes retain ordinary limits. +### Native fixture qualification + +The probe and its historical base share the current physical fixture-root owner: +POSIX retains `/tmp/loopx-cli-budget-…`; Windows uses an owned directory in its +native temporary root, without requiring a drive-root `tmp` directory. Implicit +machine runtime paths use that same isolated root. Existing roots are refused +and owned fixtures are removed on failure as well as success. Subprocess readers +decode UTF-8 so an actual probe failure remains visible on Windows locales. + +On matched native Windows main/candidate fixtures, the guided-start Markdown +rows measure 3,203 / 3,204 / 3,203 characters (small/crowded/multi-Agent) on both +revisions, with identical JSON measurements and semantic checks. The previous +3,200-character ceiling rejects these complete routed commands. The full nested +canary layout measures 3,245 / 3,246 / 3,245 on both revisions. The shared +characterization ceiling is now 3,400 for all three populations, leaving at +least 154 characters for bounded host-path variation. No product prose is +shortened; guided-start JSON, line, per-Todo growth and semantic guards are unchanged. + +The same native canary layout emits 5,989 Markdown characters for thin heartbeat +(all three populations), and 13,685 JSON / 11,854 Markdown characters for compact +heartbeat, identically on base and candidate with equal semantic receipts. Their +previous 5,800 and 13,500 / 11,750 ceilings reject that native path cost. The +characterization limits are 6,200 and 14,000 / 12,200 respectively; line limits, +other modes, differential growth and semantic obligations remain unchanged. +These bounded increases retain full executable routes rather than trimming +agent-consumed instructions to satisfy a host-specific ceiling. + +Windows qualification can use the existing explicit canary time budget: + +```powershell +uv run --extra test loopx canary smoke-suite --script cli-output-budget-regression-smoke.py --timeout-seconds 660 +``` + +The wrapper allows two 240-second probes plus checkout/cleanup within 600 +seconds; this is a validation-driver allowance, not a product latency claim. + +原生 Windows 与历史基线共用当前物理夹具目录，不要求预先创建盘符根目录下的 +`tmp`；默认机器运行时路径也归入同一隔离目录，已有目录拒绝覆盖，失败时照常 +清理。配对引导输出为 3,203 / 3,204 / 3,203 字符，原 3,200 上限无法保留完整路由； +完整嵌套 canary 的成对测量为 3,245 / 3,246 / 3,245。三个规模的共用字符上限 +调整为 3,400，至少保留 154 字符路径余量，JSON、行数、增长和语义检查不变，不删减 +产品指令。心跳 thin 三个规模均为 5,989 字符，compact 为 JSON 13,685 / Markdown +11,854 字符，基线与候选的成对测量和语义一致；对应上限由 5,800 和 +13,500 / 11,750 调整为 6,200 和 14,000 / 12,200，保留完整可执行路径，行数、 +其他模式和差分增长检查不变。原生检查可显式给予上述 660 秒预算，不据此 +宣称产品延迟改善。 + Both budget layers are intentionally about projections, not the full archival facts. When a surface needs more detail, put that detail behind a queryable cold-path command or a linked run-history artifact instead of making the @@ -542,8 +592,11 @@ The instruction-coherence change keeps action-specific authorization, sustained integration/negative-case/readback work and exact current settlement/recovery in all heartbeat modes. On the same supported 320-character scope plus capability fixture, normalized thin body cost is 3,172 → 3,599 characters. The thin body -ceiling is 3,600; CLI JSON/Markdown ceilings are 3,900/5,800 (previously -3,400/5,250), against measured 3,651/5,579 on the same public fixture. +ceiling is 3,600. That instruction rollout measured CLI JSON/Markdown at +3,651/5,579 and used 3,900/5,800 ceilings (previously 3,400/5,250). Current +native-host ceilings are 3,900/6,200; the matched native measurements and +unchanged-body boundary are recorded in +[Native fixture qualification](#native-fixture-qualification). The registered-profile brief body grows from 4,447 to 4,675, with a 4,800 ceiling (previously 4,600). Full/compact remove duplicated static recipes; no host execution quota, authority, line, key or reward-memory allowance changes. Oversized custom inputs still fail. diff --git a/examples/control_plane/cli-output-base-head-differential-smoke.py b/examples/control_plane/cli-output-base-head-differential-smoke.py index 3fdcf4844f..e146f02338 100644 --- a/examples/control_plane/cli-output-base-head-differential-smoke.py +++ b/examples/control_plane/cli-output-base-head-differential-smoke.py @@ -21,10 +21,12 @@ compare_cli_output_receipts, select_cli_output_base_ref, ) +from tests.control_plane.cli_output_fixture import budget_fixture_path # noqa: E402 PROBE_TEST = REPO_ROOT / "tests" / "control_plane" / "test_cli_output_budget.py" PROBE_RUNNER = REPO_ROOT / "examples" / "control_plane" / "cli-output-probe-runner.py" +FIXTURE_SUPPORT = REPO_ROOT / "tests" / "control_plane" / "cli_output_fixture.py" SEMANTICS_SOURCE = ( REPO_ROOT / "loopx" / "control_plane" / "testing" / "cli_output_semantics.py" ) @@ -37,6 +39,8 @@ def _run(command: list[str], *, cwd: Path, env: dict[str, str] | None = None) -> env=env, check=False, text=True, + encoding="utf-8", + errors="replace", stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=240, @@ -64,8 +68,9 @@ def _run_probe( env["PYTHONPATH"] = str(source_root) # Both revisions measure the same fixture, including implicit machine # defaults. Never let the operator's registry/configuration select a host. - isolated_current = str(fixture_root / "host-runtime") - isolated_legacy = str(fixture_root / "legacy-runtime") + stable_root = budget_fixture_path(fixture_root) + isolated_current = str(stable_root / "host-runtime") + isolated_legacy = str(stable_root / "legacy-runtime") bootstrap = ( "import pathlib, runpy, sys; from loopx import paths; " f"paths.DEFAULT_RUNTIME_ROOT=pathlib.Path({isolated_current!r}); " @@ -109,6 +114,8 @@ def _is_ancestor(ancestor: str, descendant: str) -> bool: stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + encoding="utf-8", + errors="replace", ) if completed.returncode == 0: return True @@ -156,6 +163,7 @@ def main() -> int: base_receipt = temp_root / "base-receipt.json" candidate_receipt = temp_root / "candidate-receipt.json" shutil.copy2(PROBE_RUNNER, probe_runner) + shutil.copy2(FIXTURE_SUPPORT, temp_root / FIXTURE_SUPPORT.name) shutil.copy2(SEMANTICS_SOURCE, semantics_source) _run( diff --git a/examples/control_plane/cli-output-budget-regression-smoke.py b/examples/control_plane/cli-output-budget-regression-smoke.py index de9382e76e..7c4b41052c 100644 --- a/examples/control_plane/cli-output-budget-regression-smoke.py +++ b/examples/control_plane/cli-output-budget-regression-smoke.py @@ -42,9 +42,13 @@ def main() -> int: cwd=REPO_ROOT, check=False, text=True, + encoding="utf-8", + errors="replace", stdout=subprocess.PIPE, stderr=subprocess.PIPE, - timeout=300, + # The child runs two separately bounded 240-second probes, plus + # checkout/cleanup. Do not cut off a valid second native Windows probe. + timeout=600, ) if completed.returncode != 0: raise AssertionError( diff --git a/examples/control_plane/cli-output-probe-runner.py b/examples/control_plane/cli-output-probe-runner.py index 0536ad0f4e..94914c2db9 100644 --- a/examples/control_plane/cli-output-probe-runner.py +++ b/examples/control_plane/cli-output-probe-runner.py @@ -432,11 +432,20 @@ def main() -> int: args = parser.parse_args() _install_pytest_import_stub() probe = _load_module("loopx_cli_output_probe_fixture", args.test_source) + fixture_support = _load_module( + "loopx_cli_output_fixture_support", + Path(__file__).with_name("cli_output_fixture.py"), + ) + # Scenario helpers also call this owner; historical test modules must not + # re-enter their old /tmp implementation from inside the shared outer root. + probe._stable_budget_fixture_root = fixture_support.stable_budget_fixture_root semantics = _load_module("loopx_cli_output_probe_semantics", args.semantics_source) if args.fixture_root.exists(): shutil.rmtree(args.fixture_root) args.fixture_root.mkdir(parents=True) - with probe._stable_budget_fixture_root(args.fixture_root) as stable_root: + # Use one current root owner for both revisions. A historical test source + # may still assume /tmp exists on every Windows drive. + with fixture_support.stable_budget_fixture_root(args.fixture_root) as stable_root: rows = [ *_default_rows( probe, semantics, stable_root, diff --git a/loopx/control_plane/testing/cli_output_budget.py b/loopx/control_plane/testing/cli_output_budget.py index 8e34c7f5a8..ab7aa1cd89 100644 --- a/loopx/control_plane/testing/cli_output_budget.py +++ b/loopx/control_plane/testing/cli_output_budget.py @@ -86,10 +86,13 @@ class CliOutputCommandClassification: "packet_summary", ), markdown_anchor="# Guided Start Goal", + # Matched native Windows base/head fixtures retain full routes and + # emit up to 3,246 Markdown chars in the nested canary. Keep their meaning and allow + # bounded host-path variation; JSON, line and growth guards stay fixed. max_chars={ - "small": {"json": 40_000, "markdown": 3_200}, - "crowded": {"json": 40_000, "markdown": 3_200}, - "multi_agent": {"json": 40_000, "markdown": 3_200}, + "small": {"json": 40_000, "markdown": 3_400}, + "crowded": {"json": 40_000, "markdown": 3_400}, + "multi_agent": {"json": 40_000, "markdown": 3_400}, }, max_lines={ "small": {"json": 450, "markdown": 55}, @@ -298,9 +301,10 @@ class CliOutputCommandClassification: ), markdown_anchor="# Heartbeat Automation Prompt", max_chars={ - "small": {"json": 3_900, "markdown": 5_800}, - "crowded": {"json": 3_900, "markdown": 5_800}, - "multi_agent": {"json": 3_900, "markdown": 5_800}, + # Matched native Windows base/head emit 5,989 Markdown chars. + "small": {"json": 3_900, "markdown": 6_200}, + "crowded": {"json": 3_900, "markdown": 6_200}, + "multi_agent": {"json": 3_900, "markdown": 6_200}, }, max_lines={ "small": {"json": 18, "markdown": 72}, @@ -538,7 +542,9 @@ class CliOutputCommandClassification: output_formats=("json", "markdown"), semantic_json_keys=("task_body", "quota_guard_command", "interface_budget"), markdown_anchor="# Heartbeat Automation Prompt", - max_chars={"json": 13_500, "markdown": 11_750}, + # Matched native Windows base/head: 13,685 JSON / 11,854 Markdown. + # Retain complete host routes; line and semantic guards are unchanged. + max_chars={"json": 14_000, "markdown": 12_200}, max_lines={"json": 58, "markdown": 155}, ), CliOutputModeVariantSpec( diff --git a/tests/control_plane/cli_output_fixture.py b/tests/control_plane/cli_output_fixture.py new file mode 100644 index 0000000000..13cc73d505 --- /dev/null +++ b/tests/control_plane/cli_output_fixture.py @@ -0,0 +1,30 @@ +"""Owned physical roots for current and historical CLI output probes.""" + +from __future__ import annotations + +import contextlib +import hashlib +import os +from pathlib import Path +import shutil +import tempfile + + +def budget_fixture_path(root: Path) -> Path: + suffix = hashlib.sha256(str(root).encode("utf-8")).hexdigest()[:12] + if os.name == "nt": + # Keep the native name short without depending on a drive-root /tmp. + return Path(tempfile.gettempdir()) / f"lxb-{suffix}" + return Path("/tmp") / f"loopx-cli-budget-{suffix}" + + +@contextlib.contextmanager +def stable_budget_fixture_root(root: Path): + fixture = budget_fixture_path(root) + if fixture.exists() or fixture.is_symlink(): + raise RuntimeError(f"refusing to replace existing fixture root: {fixture}") + fixture.mkdir() + try: + yield fixture + finally: + shutil.rmtree(fixture) diff --git a/tests/control_plane/test_cli_output_budget.py b/tests/control_plane/test_cli_output_budget.py index 4c6fcea03e..9004078a0d 100644 --- a/tests/control_plane/test_cli_output_budget.py +++ b/tests/control_plane/test_cli_output_budget.py @@ -1,7 +1,6 @@ from __future__ import annotations import contextlib -import hashlib import io import json import os @@ -31,6 +30,10 @@ from loopx.heartbeat_prompt import build_heartbeat_prompt from loopx.help_surface import COMMAND_GROUPS from loopx.rollout_event_log import rollout_event_log_path +from tests.control_plane.cli_output_fixture import ( + budget_fixture_path, + stable_budget_fixture_root as _stable_budget_fixture_root, +) GOAL_ID = "cli-output-budget-goal" AGENT_IDS = ("codex-alpha", "codex-beta", "codex-gamma") @@ -328,21 +331,6 @@ def _quota_payload_without_rollout_receipt(text: str) -> dict[str, object]: return payload -@contextlib.contextmanager -def _stable_budget_fixture_root(root: Path): - """Keep lexical and resolved fixture paths independent of runner layout.""" - - suffix = hashlib.sha256(str(root).encode("utf-8")).hexdigest()[:12] - fixture = Path("/tmp") / f"loopx-cli-budget-{suffix}" - if fixture.exists() or fixture.is_symlink(): - raise RuntimeError(f"refusing to replace existing fixture root: {fixture}") - fixture.mkdir() - try: - yield fixture - finally: - shutil.rmtree(fixture) - - def _surface_commands( *, project: Path, @@ -815,16 +803,26 @@ def test_stable_budget_fixture_uses_an_owned_physical_short_root(tmp_path: Path) outer = tmp_path / ("nested-runner-" + "p" * 128) with _stable_budget_fixture_root(outer) as fixture: assert not fixture.is_symlink() - assert fixture.resolve().parent == Path("/tmp").resolve() + expected_parent = Path(tempfile.gettempdir()) if os.name == "nt" else Path("/tmp") + assert fixture.resolve().parent == expected_parent.resolve() (fixture / "owned.json").write_text("{}", encoding="utf-8") assert not fixture.exists() assert not outer.exists() +def test_stable_budget_fixture_does_not_require_a_drive_root_directory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tempfile.gettempdir()) + with _stable_budget_fixture_root(tmp_path / "native-temp-drive") as fixture: + expected_parent = Path(tempfile.gettempdir()) if os.name == "nt" else Path("/tmp") + assert fixture.resolve().parent == expected_parent.resolve() + assert not fixture.exists() + + def test_stable_budget_fixture_preserves_an_existing_directory(tmp_path: Path) -> None: root = tmp_path / "foreign-directory" - suffix = hashlib.sha256(str(root).encode("utf-8")).hexdigest()[:12] - fixture = Path("/tmp") / f"loopx-cli-budget-{suffix}" + fixture = budget_fixture_path(root) fixture.mkdir() marker = fixture / "foreign.json" marker.write_text("preserve", encoding="utf-8") @@ -839,6 +837,26 @@ def test_stable_budget_fixture_preserves_an_existing_directory(tmp_path: Path) - shutil.rmtree(fixture) +def test_stable_budget_fixture_cleans_up_after_a_failed_probe(tmp_path: Path) -> None: + with pytest.raises(RuntimeError, match="probe failed"): + with _stable_budget_fixture_root(tmp_path / "failed-probe") as fixture: + (fixture / "owned.json").write_text("{}", encoding="utf-8") + raise RuntimeError("probe failed") + assert not fixture.exists() + + +def test_stable_budget_fixture_retains_the_posix_path_contract(tmp_path, monkeypatch): + from types import SimpleNamespace + from tests.control_plane import cli_output_fixture + + monkeypatch.setattr(cli_output_fixture, "os", SimpleNamespace(name="posix")) + fixture = cli_output_fixture.budget_fixture_path(tmp_path) + assert fixture.parent == Path("/tmp") + assert fixture.name.startswith("loopx-cli-budget-") + assert len(fixture.name) == len("loopx-cli-budget-") + 12 + assert fixture == cli_output_fixture.budget_fixture_path(tmp_path) + + def test_real_cli_output_stays_inside_baseline_and_growth_contracts( tmp_path: Path, ) -> None: diff --git a/tests/control_plane/test_cli_output_differential.py b/tests/control_plane/test_cli_output_differential.py index d23d28d65e..3fa564f6b4 100644 --- a/tests/control_plane/test_cli_output_differential.py +++ b/tests/control_plane/test_cli_output_differential.py @@ -2,6 +2,7 @@ import copy import json +import sys import pytest @@ -23,6 +24,22 @@ ) +def test_probe_process_failure_preserves_utf8_diagnostics(tmp_path): + import runpy + from pathlib import Path + + smoke = runpy.run_path(str( + Path(__file__).resolve().parents[2] + / "examples/control_plane/cli-output-base-head-differential-smoke.py" + )) + with pytest.raises(AssertionError, match="恢复 🚀"): + smoke["_run"]( + [sys.executable, "-c", + "import sys; sys.stderr.buffer.write('恢复 🚀'.encode('utf-8')); sys.exit(1)"], + cwd=tmp_path, + ) + + def _row(**overrides: object) -> dict[str, object]: row: dict[str, object] = { "row_id": "surface/status/small/json",