diff --git a/.github/workflows/flow-quality.yml b/.github/workflows/flow-quality.yml index 978b4ed..7a28ea0 100644 --- a/.github/workflows/flow-quality.yml +++ b/.github/workflows/flow-quality.yml @@ -75,6 +75,9 @@ jobs: METHODOLOGY_REVISION="${GITHUB_SHA}" \ release-manifest release-manifest-validate + - name: Validate complete fixture release evidence contents + run: EXPECT_CLEAN_RELEASE=true tests/check_fixture_release_manifest.sh + - name: Upload fixture reports if: always() uses: actions/upload-artifact@v4 diff --git a/docs/development.md b/docs/development.md index ffb9e54..f7838a5 100644 --- a/docs/development.md +++ b/docs/development.md @@ -200,6 +200,11 @@ revision handling. The workflow runs it both natively and in the pinned container job. Extend this fixture whenever the release schema or policy changes. +`tests/check_fixture_release_manifest.sh` checks the complete portable fixture +after real tool execution. It requires every canonical flow decision, portable +tool identity, representative design input, both coverage classes, and the +human-readable summary to be present in the generated release artifact. + When editing workflow YAML or shell scripts, run `ci/check_flow_quality.sh`. It uses pinned ShellCheck and actionlint versions installed by `ci/install_ci_tools.sh`. diff --git a/tests/check_fixture_release_manifest.sh b/tests/check_fixture_release_manifest.sh new file mode 100755 index 0000000..c946157 --- /dev/null +++ b/tests/check_fixture_release_manifest.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +set -euo pipefail + +flow_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +manifest_path="${flow_root}/tests/fixture-module/reports/release_manifest/native/manifest.json" +summary_path="${flow_root}/tests/fixture-module/reports/release_manifest/native/summary.txt" +status_path="${flow_root}/tests/fixture-module/reports/release_manifest/native/status.txt" + +test "$(<"${status_path}")" = PASS +grep -Fq "MOSAIC release evidence" "${summary_path}" +grep -Fq "Execution context: native" "${summary_path}" + +python3 - "${manifest_path}" "${EXPECT_CLEAN_RELEASE:-false}" <<'PY' +import json +import pathlib +import re +import sys + +manifest = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) +expect_clean = sys.argv[2] == "true" +deterministic = manifest["deterministic"] +identity = deterministic["identity"] + +assert manifest["schema"] == "mosaic-release-evidence-v1" +assert identity["module"] == "flow_fixture" +assert identity["profile"] == "default" +assert identity["execution_context"] == "native" +assert re.fullmatch(r"[0-9a-f]{40}", identity["module_revision"]) +assert re.fullmatch(r"[0-9a-f]{40}", identity["methodology_revision"]) + +expected_pass = { + "eqy_equivalence", + "pyuvm_open_source", + "slang_elaboration", + "symbiyosys_formal", + "verible_format", + "verible_lint", + "verilator_lint", + "verilator_sim", + "yosys_synthesis", +} +flow_statuses = {entry["id"]: entry["status"] for entry in deterministic["flows"]} +assert set(flow_statuses) == { + "eqy_equivalence", + "openroad", + "pyuvm_commercial", + "pyuvm_open_source", + "sg_cdc", + "sg_dft", + "slang_elaboration", + "symbiyosys_formal", + "synopsys_primepower", + "synopsys_primetime", + "synopsys_synthesis", + "vc_cdc", + "vc_lint", + "vc_lp", + "vcs_sim", + "verible_format", + "verible_lint", + "verilator_lint", + "verilator_sim", + "yosys_synthesis", +} +assert { + name for name, status in flow_statuses.items() if status == "PASS" +} == expected_pass +assert all( + status == "SKIP" + for name, status in flow_statuses.items() + if name not in expected_pass +) + +expected_tools = { + "cocotb", + "eqy", + "python", + "pyuvm", + "slang", + "symbiyosys", + "verible-verilog-format", + "verible-verilog-lint", + "verilator", + "yosys", +} +tools = deterministic["tools"] +assert {entry["name"] for entry in tools} == expected_tools +assert all(entry["version"] for entry in tools) +assert all(entry["source"]["sha256"] for entry in tools if "source" in entry) + +input_paths = {entry["path"] for entry in deterministic["inputs"]} +assert { + "config/design.mk", + "config/flows.mk", + "config/formal.sby", + "config/formal_cover.sby", + "config/equivalence.eqy", + "config/openroad.mk", + "config/verible.rules", + "config/verible_waivers.txt", + "config/verilator_waivers.vlt", + "constraints/timing.sdc", + "filelists/assertions.f", + "filelists/coverage.f", + "filelists/properties.f", + "filelists/rtl.f", + "filelists/tb.f", + "rtl/flow_fixture.sv", + "verif/pyuvm/test_flow_fixture.py", +} <= input_paths +assert all( + re.fullmatch(r"[0-9a-f]{64}", entry["sha256"]) + for entry in deterministic["inputs"] +) +assert not any( + "__pycache__" in path or path.endswith(".pyc") for path in input_paths +) + +coverage = deterministic["evidence"]["coverage"] +assert {(entry["producer"], entry["kind"]) for entry in coverage} >= { + ("pyuvm_open_source", "pyuvm_functional"), + ("pyuvm_open_source", "systemverilog_native_report"), + ("verilator_sim", "systemverilog_native_report"), +} +if expect_clean: + assert manifest["volatile"]["source_tree"] == { + "methodology_dirty": False, + "module_dirty": False, + } +PY + +echo "Complete fixture release manifest is valid" diff --git a/tests/test_release_manifest.sh b/tests/test_release_manifest.sh index 6eb68cb..e65608b 100755 --- a/tests/test_release_manifest.sh +++ b/tests/test_release_manifest.sh @@ -155,6 +155,8 @@ expect_failure() { generate_manifest "${output_dir}" >/dev/null python3 "${release_tool}" validate --manifest "${output_dir}/manifest.json" >/dev/null +test "$(<"${output_dir}/status.txt")" = PASS +grep -Fq "MOSAIC release evidence" "${output_dir}/summary.txt" python3 - "${output_dir}/manifest.json" "${fixture_context}" <<'PY' import json @@ -245,6 +247,22 @@ expect_failure "Required flow verible_lint is missing status evidence" \ printf 'PASS\n' >"${report_dir}/verible_lint/status.txt" mkdir -p "${report_dir}/openroad" +printf 'SKIP\n' >"${report_dir}/openroad/status.txt" +generate_manifest "${output_dir}" >/dev/null +python3 - "${output_dir}/manifest.json" <<'PY' +import json +import pathlib +import sys + +manifest = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) +openroad = next( + flow for flow in manifest["deterministic"]["flows"] + if flow["id"] == "openroad" +) +assert openroad["status"] == "SKIP" +assert openroad["status_evidence"]["sha256"] +PY + printf 'PASS\n' >"${report_dir}/openroad/status.txt" expect_failure "Disabled flow openroad has status PASS" \ generate_manifest "${output_dir}" @@ -255,6 +273,21 @@ expect_failure "Supplemental gate qualification is missing status evidence" \ generate_manifest "${output_dir}" printf 'PASS\n' >"${report_dir}/qualification/status.txt" +printf 'SKIP\n' >"${report_dir}/verible_lint/status.txt" +expect_failure "Required flow verible_lint has status SKIP" \ + generate_manifest "${output_dir}" +printf 'PASS\n' >"${report_dir}/verible_lint/status.txt" + +printf 'FAIL\n' >"${report_dir}/qualification/status.txt" +expect_failure "Supplemental gate qualification has status FAIL" \ + generate_manifest "${output_dir}" +printf 'PASS\n' >"${report_dir}/qualification/status.txt" + +rm "${report_dir}/qualification/summary.json" +expect_failure "Required release evidence is missing" \ + generate_manifest "${output_dir}" +printf '{"review":"complete"}\n' >"${report_dir}/qualification/summary.json" + expect_failure "module revision must contain exactly 40 hex digits" \ generate_manifest "${output_dir}" MODULE_REVISION=bad-revision expect_failure "Required release input is missing" \