[Update] Complete release manifest acceptance coverage

.github/workflows/flow-quality.yml

@@ -75,6 +75,9 @@ jobs:
             METHODOLOGY_REVISION="${GITHUB_SHA}" \
             release-manifest release-manifest-validate
 
+      - name: Validate complete fixture release evidence contents
+        run: EXPECT_CLEAN_RELEASE=true tests/check_fixture_release_manifest.sh
+
       - name: Upload fixture reports
         if: always()
         uses: actions/upload-artifact@v4

docs/development.md

@@ -200,6 +200,11 @@ revision handling. The workflow runs it both natively and in the pinned
 container job. Extend this fixture whenever the release schema or policy
 changes.
 
+`tests/check_fixture_release_manifest.sh` checks the complete portable fixture
+after real tool execution. It requires every canonical flow decision, portable
+tool identity, representative design input, both coverage classes, and the
+human-readable summary to be present in the generated release artifact.
+
 When editing workflow YAML or shell scripts, run `ci/check_flow_quality.sh`. It
 uses pinned ShellCheck and actionlint versions installed by
 `ci/install_ci_tools.sh`.

tests/check_fixture_release_manifest.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+flow_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+manifest_path="${flow_root}/tests/fixture-module/reports/release_manifest/native/manifest.json"
+summary_path="${flow_root}/tests/fixture-module/reports/release_manifest/native/summary.txt"
+status_path="${flow_root}/tests/fixture-module/reports/release_manifest/native/status.txt"
+
+test "$(<"${status_path}")" = PASS
+grep -Fq "MOSAIC release evidence" "${summary_path}"
+grep -Fq "Execution context: native" "${summary_path}"
+
+python3 - "${manifest_path}" "${EXPECT_CLEAN_RELEASE:-false}" <<'PY'
+import json
+import pathlib
+import re
+import sys
+
+manifest = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))
+expect_clean = sys.argv[2] == "true"
+deterministic = manifest["deterministic"]
+identity = deterministic["identity"]
+
+assert manifest["schema"] == "mosaic-release-evidence-v1"
+assert identity["module"] == "flow_fixture"
+assert identity["profile"] == "default"
+assert identity["execution_context"] == "native"
+assert re.fullmatch(r"[0-9a-f]{40}", identity["module_revision"])
+assert re.fullmatch(r"[0-9a-f]{40}", identity["methodology_revision"])
+
+expected_pass = {
+    "eqy_equivalence",
+    "pyuvm_open_source",
+    "slang_elaboration",
+    "symbiyosys_formal",
+    "verible_format",
+    "verible_lint",
+    "verilator_lint",
+    "verilator_sim",
+    "yosys_synthesis",
+}
+flow_statuses = {entry["id"]: entry["status"] for entry in deterministic["flows"]}
+assert set(flow_statuses) == {
+    "eqy_equivalence",
+    "openroad",
+    "pyuvm_commercial",
+    "pyuvm_open_source",
+    "sg_cdc",
+    "sg_dft",
+    "slang_elaboration",
+    "symbiyosys_formal",
+    "synopsys_primepower",
+    "synopsys_primetime",
+    "synopsys_synthesis",
+    "vc_cdc",
+    "vc_lint",
+    "vc_lp",
+    "vcs_sim",
+    "verible_format",
+    "verible_lint",
+    "verilator_lint",
+    "verilator_sim",
+    "yosys_synthesis",
+}
+assert {
+    name for name, status in flow_statuses.items() if status == "PASS"
+} == expected_pass
+assert all(
+    status == "SKIP"
+    for name, status in flow_statuses.items()
+    if name not in expected_pass
+)
+
+expected_tools = {
+    "cocotb",
+    "eqy",
+    "python",
+    "pyuvm",
+    "slang",
+    "symbiyosys",
+    "verible-verilog-format",
+    "verible-verilog-lint",
+    "verilator",
+    "yosys",
+}
+tools = deterministic["tools"]
+assert {entry["name"] for entry in tools} == expected_tools
+assert all(entry["version"] for entry in tools)
+assert all(entry["source"]["sha256"] for entry in tools if "source" in entry)
+
+input_paths = {entry["path"] for entry in deterministic["inputs"]}
+assert {
+    "config/design.mk",
+    "config/flows.mk",
+    "config/formal.sby",
+    "config/formal_cover.sby",
+    "config/equivalence.eqy",
+    "config/openroad.mk",
+    "config/verible.rules",
+    "config/verible_waivers.txt",
+    "config/verilator_waivers.vlt",
+    "constraints/timing.sdc",
+    "filelists/assertions.f",
+    "filelists/coverage.f",
+    "filelists/properties.f",
+    "filelists/rtl.f",
+    "filelists/tb.f",
+    "rtl/flow_fixture.sv",
+    "verif/pyuvm/test_flow_fixture.py",
+} <= input_paths
+assert all(
+    re.fullmatch(r"[0-9a-f]{64}", entry["sha256"])
+    for entry in deterministic["inputs"]
+)
+assert not any(
+    "__pycache__" in path or path.endswith(".pyc") for path in input_paths
+)
+
+coverage = deterministic["evidence"]["coverage"]
+assert {(entry["producer"], entry["kind"]) for entry in coverage} >= {
+    ("pyuvm_open_source", "pyuvm_functional"),
+    ("pyuvm_open_source", "systemverilog_native_report"),
+    ("verilator_sim", "systemverilog_native_report"),
+}
+if expect_clean:
+    assert manifest["volatile"]["source_tree"] == {
+        "methodology_dirty": False,
+        "module_dirty": False,
+    }
+PY
+
+echo "Complete fixture release manifest is valid"

tests/test_release_manifest.sh

@@ -155,6 +155,8 @@ expect_failure() {
 
 generate_manifest "${output_dir}" >/dev/null
 python3 "${release_tool}" validate --manifest "${output_dir}/manifest.json" >/dev/null
+test "$(<"${output_dir}/status.txt")" = PASS
+grep -Fq "MOSAIC release evidence" "${output_dir}/summary.txt"
 
 python3 - "${output_dir}/manifest.json" "${fixture_context}" <<'PY'
 import json
@@ -245,6 +247,22 @@ expect_failure "Required flow verible_lint is missing status evidence" \
 printf 'PASS\n' >"${report_dir}/verible_lint/status.txt"
 
 mkdir -p "${report_dir}/openroad"
+printf 'SKIP\n' >"${report_dir}/openroad/status.txt"
+generate_manifest "${output_dir}" >/dev/null
+python3 - "${output_dir}/manifest.json" <<'PY'
+import json
+import pathlib
+import sys
+
+manifest = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))
+openroad = next(
+    flow for flow in manifest["deterministic"]["flows"]
+    if flow["id"] == "openroad"
+)
+assert openroad["status"] == "SKIP"
+assert openroad["status_evidence"]["sha256"]
+PY
+
 printf 'PASS\n' >"${report_dir}/openroad/status.txt"
 expect_failure "Disabled flow openroad has status PASS" \
   generate_manifest "${output_dir}"
@@ -255,6 +273,21 @@ expect_failure "Supplemental gate qualification is missing status evidence" \
   generate_manifest "${output_dir}"
 printf 'PASS\n' >"${report_dir}/qualification/status.txt"
 
+printf 'SKIP\n' >"${report_dir}/verible_lint/status.txt"
+expect_failure "Required flow verible_lint has status SKIP" \
+  generate_manifest "${output_dir}"
+printf 'PASS\n' >"${report_dir}/verible_lint/status.txt"
+
+printf 'FAIL\n' >"${report_dir}/qualification/status.txt"
+expect_failure "Supplemental gate qualification has status FAIL" \
+  generate_manifest "${output_dir}"
+printf 'PASS\n' >"${report_dir}/qualification/status.txt"
+
+rm "${report_dir}/qualification/summary.json"
+expect_failure "Required release evidence is missing" \
+  generate_manifest "${output_dir}"
+printf '{"review":"complete"}\n' >"${report_dir}/qualification/summary.json"
+
 expect_failure "module revision must contain exactly 40 hex digits" \
   generate_manifest "${output_dir}" MODULE_REVISION=bad-revision
 expect_failure "Required release input is missing" \