diff --git a/.github/workflows/flow-quality.yml b/.github/workflows/flow-quality.yml index ffec0ef..cc807bb 100644 --- a/.github/workflows/flow-quality.yml +++ b/.github/workflows/flow-quality.yml @@ -130,6 +130,7 @@ jobs: tests/test_multi_module.sh tests/test_parameter_profiles.sh tests/test_coverage_qualification.sh + tests/test_qualification_campaigns.sh - name: Run containerized release-manifest fixture run: FIXTURE_CONTEXT=container tests/test_release_manifest.sh diff --git a/README.md b/README.md index 4e25e09..09353a3 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ methodology without copying flow scripts or changing module RTL. - Single-module and manifest-based multi-module project orchestration - Named parameter-profile qualification across elaboration and implementation - Declarative HDL and formal coverage qualification with reviewed exclusions +- Declarative negative-test and four-state qualification with explicit controls - Flow selection, statuses, reports, and quality gates - Schema-validated release evidence with input hashes and tool identities - Pinned open-source tool installers and versions diff --git a/VERSION b/VERSION index 8f0916f..a918a2a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.0 +0.6.0 diff --git a/ci/check_flow_quality.sh b/ci/check_flow_quality.sh index 8d1306d..6e0613d 100755 --- a/ci/check_flow_quality.sh +++ b/ci/check_flow_quality.sh @@ -20,8 +20,10 @@ python3 -m py_compile ci/module_manifest.py python3 -m py_compile ci/parameter_profiles.py python3 -m py_compile ci/release_manifest.py python3 -m py_compile ci/coverage_qualification.py +python3 -m py_compile ci/qualification_campaign.py python3 -m json.tool schemas/release-evidence-v1.schema.json >/dev/null python3 -m json.tool schemas/coverage-policy-v1.schema.json >/dev/null +python3 -m json.tool schemas/qualification-campaigns-v1.schema.json >/dev/null if ! grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$' VERSION; then echo "VERSION must contain a semantic version such as 1.2.3" >&2 @@ -79,4 +81,5 @@ tests/test_multi_module.sh tests/test_parameter_profiles.sh tests/test_release_manifest.sh tests/test_coverage_qualification.sh +tests/test_qualification_campaigns.sh echo "mosaic-flow static quality checks passed" diff --git a/ci/open_source_quality_gate.sh b/ci/open_source_quality_gate.sh index 2a4655f..e21b208 100755 --- a/ci/open_source_quality_gate.sh +++ b/ci/open_source_quality_gate.sh @@ -5,7 +5,7 @@ flow_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=flow_selection.sh source "${flow_root}/ci/flow_selection.sh" -required_flows=(verible_lint verible_format slang_elaboration verilator_lint yosys_synthesis symbiyosys_formal eqy_equivalence verilator_sim pyuvm_open_source coverage_qualification) +required_flows=(verible_lint verible_format slang_elaboration verilator_lint yosys_synthesis symbiyosys_formal eqy_equivalence verilator_sim pyuvm_open_source coverage_qualification negative_qualification four_state_qualification) failed=0 for flow_name in "${required_flows[@]}"; do diff --git a/ci/parameter_profiles.py b/ci/parameter_profiles.py index 25ec80f..825def6 100755 --- a/ci/parameter_profiles.py +++ b/ci/parameter_profiles.py @@ -27,6 +27,8 @@ "eqy_equivalence", "verilator_sim", "pyuvm_open_source", + "negative_qualification", + "four_state_qualification", "vcs_sim", "pyuvm_commercial", "synopsys_synthesis", diff --git a/ci/qualification_campaign.py b/ci/qualification_campaign.py new file mode 100755 index 0000000..378ec0c --- /dev/null +++ b/ci/qualification_campaign.py @@ -0,0 +1,560 @@ +#!/usr/bin/env python3 +"""Run declarative negative-test and four-state qualification campaigns.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +SCHEMA = "mosaic-qualification-campaigns-v1" +SUMMARY_SCHEMA = "mosaic-qualification-result-v1" +NAME = re.compile(r"^[a-z][a-z0-9_]*$") +CAMPAIGNS = {"negative", "four_state"} +ROLES = { + "negative": {"negative", "positive_control"}, + "four_state": {"unknown_detection", "disabled_monitor_control"}, +} +PHASES = {"compile", "run"} +FAILURE_CLASSES = { + "assertion", + "elaboration", + "equivalence", + "mutation", + "unknown_control", +} +DEFAULT_INFRASTRUCTURE_DIAGNOSTICS = [ + r"command not found", + r"No such file or directory", + r"license checkout failed", + r"internal (?:compiler|tool) error", + r"INFRASTRUCTURE_FAILURE", +] + + +class CampaignError(ValueError): + """Report an invalid campaign or execution contract.""" + + +def write_json(path: Path, value: object) -> None: + """Write stable, human-readable JSON evidence.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def write_status(path: Path, status: str) -> None: + """Write one status using the shared flow evidence contract.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"{status}\n", encoding="utf-8") + + +def require_fields(value: dict[str, Any], allowed: set[str], required: set[str], label: str) -> None: + """Reject missing and unknown manifest fields.""" + missing = sorted(required - set(value)) + unknown = sorted(set(value) - allowed) + if missing: + raise CampaignError(f"{label} is missing fields: {', '.join(missing)}") + if unknown: + raise CampaignError(f"{label} has unknown fields: {', '.join(unknown)}") + + +def validate_pattern(pattern: object, label: str) -> str: + """Validate and compile one diagnostic regular expression.""" + if not isinstance(pattern, str) or not pattern: + raise CampaignError(f"{label} must be a nonempty regular expression") + try: + re.compile(pattern) + except re.error as error: + raise CampaignError(f"{label} is invalid: {error}") from error + return pattern + + +def validate_phase(phase: object, label: str) -> dict[str, Any]: + """Validate one compile or run phase.""" + if not isinstance(phase, dict): + raise CampaignError(f"{label} must be an object") + require_fields( + phase, + { + "name", + "command", + "fixture", + "expected", + "diagnostic", + "failure_class", + "environment", + "timeout_seconds", + }, + {"name", "expected"}, + label, + ) + if phase["name"] not in PHASES: + raise CampaignError(f"{label}.name must be compile or run") + if ("command" in phase) == ("fixture" in phase): + raise CampaignError(f"{label} must define exactly one of command or fixture") + if "command" in phase and ( + not isinstance(phase["command"], list) + or not phase["command"] + or not all(isinstance(item, str) and item for item in phase["command"]) + ): + raise CampaignError(f"{label}.command must be a nonempty string array") + if "fixture" in phase and ( + not isinstance(phase["fixture"], str) or not phase["fixture"] + ): + raise CampaignError(f"{label}.fixture must be a nonempty path") + if phase["expected"] not in {"success", "failure"}: + raise CampaignError(f"{label}.expected must be success or failure") + if phase["expected"] == "failure": + validate_pattern(phase.get("diagnostic"), f"{label}.diagnostic") + if phase.get("failure_class") not in FAILURE_CLASSES: + raise CampaignError( + f"{label}.failure_class must identify an expected design-check failure" + ) + elif "failure_class" in phase: + raise CampaignError(f"{label}.failure_class is only valid for expected failure") + if "diagnostic" in phase: + validate_pattern(phase["diagnostic"], f"{label}.diagnostic") + environment = phase.get("environment", {}) + if not isinstance(environment, dict) or not all( + isinstance(key, str) + and key + and isinstance(value, str) + for key, value in environment.items() + ): + raise CampaignError(f"{label}.environment must map names to strings") + timeout = phase.get("timeout_seconds", 300) + if not isinstance(timeout, int) or isinstance(timeout, bool) or timeout < 1: + raise CampaignError(f"{label}.timeout_seconds must be a positive integer") + return phase + + +def validate_case(case: object, campaign: str, label: str) -> dict[str, Any]: + """Validate one campaign case and its semantic role.""" + if not isinstance(case, dict): + raise CampaignError(f"{label} must be an object") + allowed = { + "id", + "evidence", + "role", + "kind", + "phases", + "positive_control", + "monitor_control", + "injections", + } + require_fields(case, allowed, {"id", "evidence", "role", "kind", "phases"}, label) + for field in ("id", "evidence"): + if not isinstance(case[field], str) or not NAME.fullmatch(case[field]): + raise CampaignError(f"{label}.{field} must match {NAME.pattern}") + if case["role"] not in ROLES[campaign]: + raise CampaignError(f"{label}.role is invalid for {campaign}") + if not isinstance(case["kind"], str) or not case["kind"]: + raise CampaignError(f"{label}.kind must be a nonempty string") + phases = case["phases"] + if not isinstance(phases, list) or not phases: + raise CampaignError(f"{label}.phases must be a nonempty array") + validated_phases = [ + validate_phase(phase, f"{label}.phases[{index}]") + for index, phase in enumerate(phases) + ] + phase_names = [phase["name"] for phase in validated_phases] + if len(phase_names) != len(set(phase_names)): + raise CampaignError(f"{label} repeats a phase name") + + expected_failures = sum(phase["expected"] == "failure" for phase in validated_phases) + if case["role"] in {"positive_control", "disabled_monitor_control"}: + if expected_failures: + raise CampaignError(f"{label} control phases must all expect success") + run_phases = [phase for phase in validated_phases if phase["name"] == "run"] + if campaign == "four_state" and ( + not run_phases or not run_phases[0].get("diagnostic") + ): + raise CampaignError( + f"{label} disabled-monitor control must prove stimulus arrival with a diagnostic" + ) + elif expected_failures != 1: + raise CampaignError(f"{label} must contain exactly one expected failure") + + if campaign == "negative" and case["role"] == "negative": + control = case.get("positive_control") + if not isinstance(control, str) or not NAME.fullmatch(control): + raise CampaignError(f"{label}.positive_control is required") + if campaign == "four_state": + injections = case.get("injections") + if ( + not isinstance(injections, list) + or not injections + or not all( + isinstance(item, dict) + and set(item) == {"control", "values"} + and isinstance(item["control"], str) + and item["control"] + and isinstance(item["values"], list) + and item["values"] + and all(value in {"X", "Z"} for value in item["values"]) + for item in injections + ) + ): + raise CampaignError( + f"{label}.injections must declare control names and X/Z values" + ) + by_phase = {phase["name"]: phase for phase in validated_phases} + if set(by_phase) != PHASES: + raise CampaignError(f"{label} must define compile and run phases") + if by_phase["compile"].get("command", [None])[0] != "{iverilog}": + raise CampaignError(f"{label} compile phase must invoke {{iverilog}} directly") + if by_phase["run"].get("command", [None])[0] != "{vvp}": + raise CampaignError(f"{label} run phase must invoke {{vvp}} directly") + if case["role"] == "unknown_detection": + control = case.get("monitor_control") + if not isinstance(control, str) or not NAME.fullmatch(control): + raise CampaignError(f"{label}.monitor_control is required") + return case + + +def load_campaign(path: Path, campaign: str) -> tuple[dict[str, Any], list[str]]: + """Load and semantically validate one selected campaign.""" + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except OSError as error: + raise CampaignError(f"Cannot read campaign manifest {path}: {error}") from error + except json.JSONDecodeError as error: + raise CampaignError( + f"Campaign manifest JSON is invalid at column {error.colno}: {error.msg}" + ) from error + if not isinstance(manifest, dict): + raise CampaignError("Campaign manifest root must be an object") + require_fields(manifest, {"schema", "campaigns"}, {"schema", "campaigns"}, "manifest") + if manifest["schema"] != SCHEMA: + raise CampaignError(f"Campaign manifest schema must be {SCHEMA}") + campaigns = manifest["campaigns"] + if not isinstance(campaigns, dict) or not set(campaigns).issubset(CAMPAIGNS): + raise CampaignError("manifest.campaigns contains an unknown campaign") + if campaign not in campaigns: + raise CampaignError(f"Campaign manifest does not define {campaign}") + + selected = campaigns[campaign] + if not isinstance(selected, dict): + raise CampaignError(f"campaigns.{campaign} must be an object") + allowed = {"cases", "infrastructure_diagnostics", "inputs"} + if campaign == "four_state": + allowed.add("simulator") + require_fields(selected, allowed, {"cases"}, f"campaigns.{campaign}") + if campaign == "four_state" and selected.get("simulator") != "iverilog": + raise CampaignError("four_state.simulator must be pinned to iverilog") + patterns = selected.get("infrastructure_diagnostics", []) + if not isinstance(patterns, list): + raise CampaignError("infrastructure_diagnostics must be an array") + infrastructure = [ + *DEFAULT_INFRASTRUCTURE_DIAGNOSTICS, + *[ + validate_pattern(pattern, f"infrastructure_diagnostics[{index}]") + for index, pattern in enumerate(patterns) + ], + ] + inputs = selected.get("inputs", []) + if ( + not isinstance(inputs, list) + or not all(isinstance(item, str) and item for item in inputs) + or len(inputs) != len(set(inputs)) + ): + raise CampaignError(f"campaigns.{campaign}.inputs must be a unique string array") + cases = selected["cases"] + if not isinstance(cases, list) or not cases: + raise CampaignError(f"campaigns.{campaign}.cases must be a nonempty array") + validated = [ + validate_case(case, campaign, f"campaigns.{campaign}.cases[{index}]") + for index, case in enumerate(cases) + ] + missing_roles = ROLES[campaign] - {case["role"] for case in validated} + if missing_roles: + raise CampaignError( + f"campaigns.{campaign} is missing required roles: {', '.join(sorted(missing_roles))}" + ) + ids = [case["id"] for case in validated] + evidence = [case["evidence"] for case in validated] + if len(ids) != len(set(ids)): + raise CampaignError(f"campaigns.{campaign} repeats a case ID") + if len(evidence) != len(set(evidence)): + raise CampaignError(f"campaigns.{campaign} repeats an evidence name") + by_id = {case["id"]: case for case in validated} + for case in validated: + reference = case.get("positive_control") or case.get("monitor_control") + if reference: + expected_role = ( + "positive_control" if campaign == "negative" else "disabled_monitor_control" + ) + if reference not in by_id or by_id[reference]["role"] != expected_role: + raise CampaignError( + f"case {case['id']} references missing {expected_role} {reference}" + ) + if campaign == "four_state" and case["injections"] != by_id[reference]["injections"]: + raise CampaignError( + f"case {case['id']} and monitor control {reference} must declare identical injections" + ) + selected["cases"] = validated + return selected, infrastructure + + +def expand(value: str, variables: dict[str, str], label: str) -> str: + """Expand only the documented brace placeholders in a manifest string.""" + try: + return value.format_map(variables) + except KeyError as error: + raise CampaignError(f"{label} uses unknown placeholder {{{error.args[0]}}}") from error + except ValueError as error: + raise CampaignError(f"{label} has invalid placeholder syntax: {error}") from error + + +def classify_phase( + phase: dict[str, Any], returncode: int, output: str, infrastructure: list[str] +) -> tuple[bool, str, str]: + """Distinguish expected design-check outcomes from escaped and tool failures.""" + expected = phase["expected"] + diagnostic = phase.get("diagnostic") + diagnostic_seen = bool(diagnostic and re.search(diagnostic, output, re.MULTILINE)) + infrastructure_seen = any( + re.search(pattern, output, re.IGNORECASE | re.MULTILINE) + for pattern in infrastructure + ) + if expected == "success": + if returncode != 0: + classification = ( + "infrastructure_failure" if infrastructure_seen else "unexpected_failure" + ) + return False, classification, "phase returned nonzero but success was required" + if diagnostic and not diagnostic_seen: + return False, "missing_diagnostic", "required success diagnostic was absent" + return True, "expected_success", "phase succeeded" + if returncode == 0: + return False, "escaped_fault", "design-check failure was expected but command succeeded" + if infrastructure_seen: + return False, "infrastructure_failure", "tool or execution infrastructure failed" + if not diagnostic_seen: + return False, "unexpected_failure", "nonzero result lacked the required diagnostic" + return True, f"expected_{phase['failure_class']}_failure", "expected failure was detected" + + +def run_phase( + phase: dict[str, Any], variables: dict[str, str], infrastructure: list[str], case_dir: Path +) -> dict[str, Any]: + """Execute one phase and emit its complete command and log evidence.""" + phase_name = phase["name"] + if "command" in phase: + command = [ + expand(argument, variables, f"{phase_name}.command") + for argument in phase["command"] + ] + else: + command = [expand(phase["fixture"], variables, f"{phase_name}.fixture")] + environment = os.environ.copy() + environment.update( + { + key: expand(value, variables, f"{phase_name}.environment.{key}") + for key, value in phase.get("environment", {}).items() + } + ) + command_record = { + "argv": command, + "environment": phase.get("environment", {}), + "timeout_seconds": phase.get("timeout_seconds", 300), + } + write_json(case_dir / f"{phase_name}-command.json", command_record) + started = datetime.now(timezone.utc) + try: + completed = subprocess.run( + command, + cwd=variables["module_root"], + env=environment, + capture_output=True, + text=True, + errors="replace", + timeout=phase.get("timeout_seconds", 300), + check=False, + ) + stdout = completed.stdout + stderr = completed.stderr + returncode = completed.returncode + passed, classification, message = classify_phase( + phase, returncode, stdout + stderr, infrastructure + ) + except (OSError, subprocess.TimeoutExpired) as error: + stdout = getattr(error, "stdout", "") or "" + stderr = (getattr(error, "stderr", "") or "") + f"\n{error}\n" + returncode = None + passed = False + classification = "infrastructure_failure" + message = "command could not be executed" + (case_dir / f"{phase_name}.log").write_text(stdout + stderr, encoding="utf-8") + result = { + "classification": classification, + "duration_seconds": round((datetime.now(timezone.utc) - started).total_seconds(), 6), + "expected": phase["expected"], + "message": message, + "name": phase_name, + "returncode": returncode, + "status": "PASS" if passed else "FAIL", + } + write_json(case_dir / f"{phase_name}-result.json", result) + return result + + +def run_case( + case: dict[str, Any], module_root: Path, flow_root: Path, report_dir: Path, + work_dir: Path, infrastructure: list[str], controls: dict[str, str] +) -> dict[str, Any]: + """Run one isolated case after checking its declared control dependency.""" + case_report_dir = report_dir / case["evidence"] + case_work_dir = work_dir / case["evidence"] + case_report_dir.mkdir(parents=True, exist_ok=True) + case_work_dir.mkdir(parents=True, exist_ok=True) + write_status(case_report_dir / "status.txt", "FAIL") + reference = case.get("positive_control") or case.get("monitor_control") + if reference and controls.get(reference) != "PASS": + result = { + "classification": "blocked_by_control", + "evidence": case["evidence"], + "id": case["id"], + "kind": case["kind"], + "message": f"required control {reference} did not pass", + "phases": [], + "role": case["role"], + "status": "FAIL", + } + write_json(case_report_dir / "summary.json", result) + return result + variables = { + "case_report_dir": str(case_report_dir), + "case_work_dir": str(case_work_dir), + "eqy": os.environ.get("EQY_CMD", "eqy"), + "flow_root": str(flow_root), + "iverilog": os.environ.get("IVERILOG_CMD", "iverilog"), + "module_root": str(module_root), + "python": sys.executable, + "report_dir": str(report_dir), + "verilator": os.environ.get("VERILATOR_CMD", "verilator"), + "vvp": os.environ.get("VVP_CMD", "vvp"), + "work_dir": str(work_dir), + } + phase_results = [] + for phase in case["phases"]: + result = run_phase(phase, variables, infrastructure, case_report_dir) + phase_results.append(result) + if result["status"] != "PASS": + break + passed = len(phase_results) == len(case["phases"]) and all( + result["status"] == "PASS" for result in phase_results + ) + classification = ( + phase_results[-1]["classification"] if phase_results else "not_run" + ) + result = { + "classification": classification, + "evidence": case["evidence"], + "id": case["id"], + "kind": case["kind"], + "phases": phase_results, + "role": case["role"], + "status": "PASS" if passed else "FAIL", + } + if "injections" in case: + result["injections"] = case["injections"] + write_json(case_report_dir / "summary.json", result) + write_status(case_report_dir / "status.txt", result["status"]) + return result + + +def execute(args: argparse.Namespace) -> int: + """Validate and execute one campaign, preserving evidence on every failure.""" + module_root = args.module_root.resolve() + flow_root = args.flow_root.resolve() + report_dir = args.report_dir.resolve() + work_dir = args.work_dir.resolve() + shutil.rmtree(report_dir, ignore_errors=True) + shutil.rmtree(work_dir, ignore_errors=True) + report_dir.mkdir(parents=True) + work_dir.mkdir(parents=True) + write_status(report_dir / "status.txt", "FAIL") + try: + selected, infrastructure = load_campaign(args.manifest.resolve(), args.campaign) + for declared_input in selected.get("inputs", []): + input_path = (module_root / declared_input).resolve() + try: + input_path.relative_to(module_root) + except ValueError as error: + raise CampaignError( + f"campaign input must be inside module root: {declared_input}" + ) from error + if not input_path.exists(): + raise CampaignError(f"campaign input is missing: {declared_input}") + controls: dict[str, str] = {} + cases = sorted( + selected["cases"], + key=lambda case: case["role"] not in {"positive_control", "disabled_monitor_control"}, + ) + results = [] + case_report_root = report_dir / selected.get("simulator", "") + case_work_root = work_dir / selected.get("simulator", "") + for case in cases: + result = run_case( + case, + module_root, + flow_root, + case_report_root, + case_work_root, + infrastructure, + controls, + ) + results.append(result) + if case["role"] in {"positive_control", "disabled_monitor_control"}: + controls[case["id"]] = result["status"] + passed = all(result["status"] == "PASS" for result in results) + summary = { + "campaign": args.campaign, + "cases": results, + "manifest": str(args.manifest.resolve().relative_to(module_root)), + "inputs": selected.get("inputs", []), + "schema": SUMMARY_SCHEMA, + "simulator": selected.get("simulator"), + "status": "PASS" if passed else "FAIL", + } + except (CampaignError, ValueError) as error: + summary = { + "campaign": args.campaign, + "cases": [], + "errors": [str(error)], + "schema": SUMMARY_SCHEMA, + "status": "FAIL", + } + passed = False + write_json(report_dir / "summary.json", summary) + write_status(report_dir / "status.txt", summary["status"]) + if not passed: + print(f"{args.campaign} qualification failed; see {report_dir / 'summary.json'}", file=sys.stderr) + return 1 + print(f"{args.campaign} qualification passed") + return 0 + + +def parse_arguments() -> argparse.Namespace: + """Parse the command-line contract used by the shell adapters and tests.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--campaign", choices=sorted(CAMPAIGNS), required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--module-root", type=Path, required=True) + parser.add_argument("--flow-root", type=Path, required=True) + parser.add_argument("--report-dir", type=Path, required=True) + parser.add_argument("--work-dir", type=Path, required=True) + return parser.parse_args() + + +if __name__ == "__main__": + sys.exit(execute(parse_arguments())) diff --git a/ci/release_manifest.py b/ci/release_manifest.py index 854e55f..2fd1b6e 100755 --- a/ci/release_manifest.py +++ b/ci/release_manifest.py @@ -397,6 +397,8 @@ def collect_tools( ("yosys", {"yosys_synthesis"}, "YOSYS_CMD", "yosys", ["--version"]), ("symbiyosys", {"symbiyosys_formal"}, "SBY_CMD", "sby", ["--version"]), ("eqy", {"eqy_equivalence"}, "EQY_CMD", "eqy", ["--version"]), + ("iverilog", {"four_state_qualification"}, "IVERILOG_CMD", "iverilog", ["-V"]), + ("vvp", {"four_state_qualification"}, "VVP_CMD", "vvp", ["-V"]), ] for name, owned_flows, variable, default, arguments in specifications: active_flows = owned_flows & required_flows @@ -543,6 +545,52 @@ def collect_coverage(module_root: Path, report_dir: Path) -> list[dict[str, str] ) +def collect_qualification_inputs( + collector: InputCollector, module_root: Path, required_flows: set[str] +) -> None: + """Hash module-owned collateral for each enabled qualification campaign.""" + campaign_by_flow = { + "negative_qualification": "negative", + "four_state_qualification": "four_state", + } + active = { + flow: campaign + for flow, campaign in campaign_by_flow.items() + if flow in required_flows + } + if not active: + return + manifest_path = Path( + os.environ.get( + "QUALIFICATION_CAMPAIGN_MANIFEST", + str(module_root / "config" / "qualification-campaigns.json"), + ) + ) + if not manifest_path.is_absolute(): + manifest_path = module_root / manifest_path + collector.add(manifest_path, "declared") + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ReleaseError( + f"Cannot parse qualification campaign inputs from {manifest_path}: {error}" + ) from error + campaigns = manifest.get("campaigns") if isinstance(manifest, dict) else None + if not isinstance(campaigns, dict): + raise ReleaseError("Qualification campaign manifest has no campaigns object") + for flow, campaign in active.items(): + selected = campaigns.get(campaign) + if not isinstance(selected, dict): + raise ReleaseError(f"Required flow {flow} has no {campaign} campaign") + inputs = selected.get("inputs", []) + if not isinstance(inputs, list) or not all( + isinstance(item, str) and item for item in inputs + ): + raise ReleaseError(f"Campaign {campaign} inputs must be a string array") + for declared_input in inputs: + collector.add(declared_input, "declared") + + def generate_manifest(module_root: Path, flow_root: Path, report_dir: Path) -> dict[str, object]: """Validate release evidence and compose deterministic and volatile sections.""" module_root = module_root.resolve() @@ -596,6 +644,7 @@ def generate_manifest(module_root: Path, flow_root: Path, report_dir: Path) -> d collector.add_filelist(path) flows, required_flows = collect_flows(module_root, report_dir) + collect_qualification_inputs(collector, module_root, required_flows) gates = collect_supplemental_gates(module_root, report_dir) additional_evidence = [] for declared_path in environment_list("RELEASE_ADDITIONAL_EVIDENCE"): @@ -603,6 +652,10 @@ def generate_manifest(module_root: Path, flow_root: Path, report_dir: Path) -> d if not path.is_absolute(): path = module_root / path additional_evidence.append(evidence_entry(path, module_root)) + for flow in ("negative_qualification", "four_state_qualification"): + summary_path = report_dir / flow / "summary.json" + if flow in required_flows and summary_path.is_file(): + additional_evidence.append(evidence_entry(summary_path, module_root)) parameters = json_environment("PROFILE_PARAMETERS_JSON", {}) technology_details = json_environment("RELEASE_TECHNOLOGY_METADATA_JSON", {}) diff --git a/config/flows.mk b/config/flows.mk index 70266b9..f38a9ae 100644 --- a/config/flows.mk +++ b/config/flows.mk @@ -12,6 +12,8 @@ MOSAIC_FLOW_IDS := \ verilator_sim \ pyuvm_open_source \ coverage_qualification \ + negative_qualification \ + four_state_qualification \ openroad \ vcs_sim \ pyuvm_commercial \ @@ -47,6 +49,10 @@ FLOW_verilator_sim ?= enabled FLOW_pyuvm_open_source ?= disabled # FLOW_coverage_qualification: Enforce module-owned HDL and formal coverage policy. FLOW_coverage_qualification ?= disabled +# FLOW_negative_qualification: Prove declared faults and invalid configurations are detected. +FLOW_negative_qualification ?= disabled +# FLOW_four_state_qualification: Detect declared X/Z values with pinned Icarus simulation. +FLOW_four_state_qualification ?= disabled # FLOW_openroad: Run optional PDK-backed physical implementation with OpenROAD. FLOW_openroad ?= enabled # FLOW_vcs_sim: Run the module-owned SystemVerilog testbench with VCS. @@ -96,6 +102,10 @@ FLOW_DEPENDENCIES_pyuvm_open_source ?= # default native coverage source. Override this with pyuvm_open_source when # COVERAGE_QUALIFICATION_SOURCE selects PyUVM, or clear it for a dedicated run. FLOW_DEPENDENCIES_coverage_qualification ?= verilator_sim +# FLOW_DEPENDENCIES_negative_qualification: Campaign cases own their prerequisites. +FLOW_DEPENDENCIES_negative_qualification ?= +# FLOW_DEPENDENCIES_four_state_qualification: Four-state cases compile independently. +FLOW_DEPENDENCIES_four_state_qualification ?= # FLOW_DEPENDENCIES_openroad: OpenROAD owns its build graph by default. FLOW_DEPENDENCIES_openroad ?= # FLOW_DEPENDENCIES_vcs_sim: VCS simulation builds from source. diff --git a/config/tools.mk b/config/tools.mk index bfe37cc..d99e40c 100644 --- a/config/tools.mk +++ b/config/tools.mk @@ -32,6 +32,10 @@ export YOSYS_CMD ?=yosys export SBY_CMD ?=sby # EQY_CMD: EQY executable used for RTL-to-netlist equivalence. export EQY_CMD ?=eqy +# IVERILOG_CMD: Pinned OSS CAD Suite compiler used by four-state qualification. +export IVERILOG_CMD ?=iverilog +# VVP_CMD: Icarus runtime used by four-state qualification cases. +export VVP_CMD ?=vvp # SLANG_CMD: Slang executable used for SystemVerilog elaboration. export SLANG_CMD ?=slang # VERIBLE_LINT_CMD: Verible executable used for syntax and style linting. @@ -85,6 +89,10 @@ export COVERAGE_QUALIFICATION_POLICY ?=$(MODULE_ROOT)/config/coverage-policy.jso export COVERAGE_QUALIFICATION_SOURCE ?=verilator_sim # COVERAGE_QUALIFICATION_TOOL: Shared policy validator and evidence normalizer. export COVERAGE_QUALIFICATION_TOOL ?=$(FLOW_ROOT)/ci/coverage_qualification.py +# QUALIFICATION_CAMPAIGN_MANIFEST: Module-owned negative and four-state case declarations. +export QUALIFICATION_CAMPAIGN_MANIFEST ?=$(MODULE_ROOT)/config/qualification-campaigns.json +# QUALIFICATION_CAMPAIGN_TOOL: Shared declarative campaign validator and runner. +export QUALIFICATION_CAMPAIGN_TOOL ?=$(FLOW_ROOT)/ci/qualification_campaign.py # PYUVM_PYTHON: Python interpreter from the pinned PyUVM virtual environment. export PYUVM_PYTHON ?=$(PYUVM_ROOT)/bin/python diff --git a/docs/README.md b/docs/README.md index b439cee..56f6d39 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,11 +22,13 @@ maintainers, and contributors to the methodology itself. including inputs, outputs, and upstream documentation. 7. [Coverage qualification](coverage-qualification.md) defines HDL thresholds, named coverpoints, reviewed exclusions, and formal reachability. -8. [Results and quality gates](results-and-quality-gates.md) defines statuses, +8. [Negative-test and four-state qualification](qualification-campaigns.md) + defines declarative fault campaigns, controls, diagnostics, and evidence. +9. [Results and quality gates](results-and-quality-gates.md) defines statuses, waivers, generated artifacts, and CI behavior. -9. [Release evidence](release-evidence.md) defines the manifest schema, +10. [Release evidence](release-evidence.md) defines the manifest schema, acceptance policy, extension points, and release integration. -10. [Methodology development](development.md) explains how to change, test, +11. [Methodology development](development.md) explains how to change, test, qualify, version, and release this repository. ## Quick reference @@ -50,6 +52,7 @@ maintainers, and contributors to the methodology itself. | Set design paths and names | Module `config/design.mk` | | Understand PyUVM and SVA reuse | [PyUVM, SVA, and coverage](getting-started.md#understand-pyuvm-sva-and-coverage) | | Qualify HDL and formal coverage | [Coverage qualification](coverage-qualification.md) | +| Qualify known faults and X/Z controls | [Qualification campaigns](qualification-campaigns.md) | | Inspect machine-readable status | Module `reports//status.txt` | | Add a methodology flow | [Methodology development](development.md#adding-a-flow) | diff --git a/docs/architecture.md b/docs/architecture.md index 79a4fc6..de51fff 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -62,6 +62,7 @@ mosaic-flow/ | |-- fixture-parameter-profiles/ Parameter elaboration matrix fixture | |-- test_release_manifest.sh Release acceptance-policy fixture | |-- test_coverage_qualification.sh Coverage policy positive and negative fixtures +| |-- test_qualification_campaigns.sh Negative and four-state gate fixtures | `-- test_quality_gate.sh Configuration and quality-gate unit tests |-- README.md Repository overview `-- VERSION Methodology semantic version diff --git a/docs/configuration.md b/docs/configuration.md index 5fe065e..66df262 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -106,6 +106,8 @@ The default dependencies represent direct artifact consumption: | --- | --- | --- | | `eqy_equivalence` | `yosys_synthesis` | EQY reads the Yosys netlist | | `coverage_qualification` | `verilator_sim` | Qualification reads normal simulation coverage by default | +| `negative_qualification` | None | Cases own their compile and run commands | +| `four_state_qualification` | None | Cases compile independently with Icarus | | `synopsys_primetime` | `synopsys_synthesis` | PrimeTime reads the synthesis DDC and SDC | | `synopsys_primepower` | `vcs_sim synopsys_synthesis` | PrimePower reads SAIF activity plus synthesis DDC and SDC | @@ -175,6 +177,7 @@ The shared environment adapter requires these variables for every tool adapter: | `COVERAGE_FILELIST` | Shared SystemVerilog coverage model and bind file list | | `COVERAGE_QUALIFICATION_POLICY` | Versioned JSON policy for HDL and formal coverage qualification | | `COVERAGE_QUALIFICATION_SOURCE` | Existing HDL evidence source or `dedicated` rerun | +| `QUALIFICATION_CAMPAIGN_MANIFEST` | Versioned negative-test and four-state campaign declarations | | `CONSTRAINT_DIR` | Directory containing synthesis `timing.sdc` | | `REPORT_DIR` | Root for persistent, reviewable results | | `WORK_DIR` | Root for disposable tool databases and generated netlists | @@ -223,6 +226,9 @@ Flow-specific inputs are required when their flow is enabled: | `PYUVM_PLUSARGS` | Additional shell-parsed HDL plusargs | | `SIM_COVERAGE` | Enable native coverage in normal simulation | | `COVERAGE_QUALIFICATION_TOOL` | Shared coverage policy validator, normally not overridden | +| `QUALIFICATION_CAMPAIGN_TOOL` | Shared campaign validator and runner, normally not overridden | +| `IVERILOG_CMD` | Icarus compiler used for four-state qualification | +| `VVP_CMD` | Icarus runtime used for four-state qualification | ## Release evidence variables diff --git a/docs/development.md b/docs/development.md index d0670bd..ac478cd 100644 --- a/docs/development.md +++ b/docs/development.md @@ -214,6 +214,13 @@ workflow runs this fixture both natively and in the pinned container job. unreachable cover statement through SymbiYosys in the EDA integration job and requires the canonical flow to record `FAIL` before restoring passing evidence. +`tests/test_qualification_campaigns.sh` exercises detected and escaped +mutations, expected inequivalence, unrelated tool failure, detected unknown +controls, a broken monitor, failed controls, and explicit disabled statuses. +The complete fixture additionally compiles and runs an X/Z campaign with the +pinned Icarus simulator. Both methodology fixtures run in native CI, while the +tool-independent positive and negative cases also run in the pinned container. + When editing workflow YAML or shell scripts, run `ci/check_flow_quality.sh`. It uses pinned ShellCheck and actionlint versions installed by `ci/install_ci_tools.sh`. diff --git a/docs/flows.md b/docs/flows.md index a66db4e..148872c 100644 --- a/docs/flows.md +++ b/docs/flows.md @@ -31,6 +31,8 @@ backend mapping. | `verilator_sim` | `open-sim` | Verilator | Open-source | None | | `pyuvm_open_source` | `open-pyuvm` | PyUVM, cocotb, Verilator or Icarus | Open-source | None | | `coverage_qualification` | `open-coverage` | MOSAIC policy validator, Verilator, optional SymbiYosys | Open-source | `verilator_sim` | +| `negative_qualification` | `open-negative` | MOSAIC declarative campaign runner | Open-source | None | +| `four_state_qualification` | `open-four-state` | MOSAIC declarative campaign runner, Icarus | Open-source | None | | `openroad` | `open-physical` | OpenROAD Flow Scripts | Optional | None | | `vcs_sim` | `synopsys-sim` | VCS | Commercial | None | | `pyuvm_commercial` | `commercial-pyuvm` | PyUVM, cocotb, VCS or Xcelium | Commercial | None | @@ -254,6 +256,37 @@ identifies every unmet requirement and is indexed by the release manifest. See [Coverage qualification](coverage-qualification.md) for the policy schema, source selection, formal behavior, limitations, and migration examples. +### Negative-test qualification + +- **ID:** `negative_qualification` +- **Target:** `make open-negative` +- **Adapter:** [`flows/qualification/run.sh`](../flows/qualification/run.sh) +- **Inputs:** `QUALIFICATION_CAMPAIGN_MANIFEST` and module-owned fault fixtures +- **Reports:** aggregate and per-case JSON, status, command, compile, and run evidence + +This optional gate proves that declared simulation mutations, assertion faults, +invalid parameters, and inequivalent candidates are detected for the expected +reason. Every fault references a passing nominal control. The runner rejects an +escaped fault and distinguishes unrelated infrastructure failures from expected +design-check failures. + +### Four-state qualification + +- **ID:** `four_state_qualification` +- **Target:** `make open-four-state` +- **Adapter:** [`flows/qualification/run.sh`](../flows/qualification/run.sh) +- **Simulator:** pinned Icarus and VVP from OSS CAD Suite +- **Inputs:** `QUALIFICATION_CAMPAIGN_MANIFEST` and module-owned four-state stimulus +- **Reports:** aggregate and per-case JSON, status, command, compile, and run evidence + +This optional gate injects declared X/Z values into control inputs and requires +a specific monitor diagnostic. A separate disabled-monitor control proves that +the illegal stimulus reaches the design without an unrelated failure. It does +not change the two-state Verilator simulation contract. + +See [Negative-test and four-state qualification](qualification-campaigns.md) +for the manifest schema, case semantics, evidence, and migration guidance. + ### OpenROAD physical implementation - **ID:** `openroad` diff --git a/docs/parameter-profiles.md b/docs/parameter-profiles.md index a330acc..9538be6 100644 --- a/docs/parameter-profiles.md +++ b/docs/parameter-profiles.md @@ -99,7 +99,8 @@ verible_lint verible_format slang_elaboration verilator_lint yosys_synthesis symbiyosys_formal eqy_equivalence verilator_sim -pyuvm_open_source vcs_sim +pyuvm_open_source negative_qualification +four_state_qualification vcs_sim pyuvm_commercial synopsys_synthesis synopsys_primetime synopsys_primepower ``` diff --git a/docs/qualification-campaigns.md b/docs/qualification-campaigns.md new file mode 100644 index 0000000..f8eff84 --- /dev/null +++ b/docs/qualification-campaigns.md @@ -0,0 +1,237 @@ +# Negative-test and four-state qualification + +The optional qualification campaigns prove that a module's verification +environment can detect known design faults and illegal four-state control +values. `mosaic-flow` owns campaign validation, orchestration, classification, +and evidence. The consuming module owns the mutations, invalid configurations, +stimulus, expected diagnostics, and decision to enable either gate. + +The two campaigns are independent canonical flows: + +| Campaign | Flow ID | Target | Default | +| --- | --- | --- | --- | +| Negative tests | `negative_qualification` | `make open-negative` | Disabled | +| Four-state controls | `four_state_qualification` | `make open-four-state` | Disabled | + +`make open-source` invokes both targets. A disabled campaign records an explicit +`SKIP`. Once enabled, its result must be `PASS` at the open-source quality gate +and in release evidence. + +## Module configuration + +Store the module-owned manifest at +`config/qualification-campaigns.json`, or override its path: + +```make +export QUALIFICATION_CAMPAIGN_MANIFEST := $(CURDIR)/config/qualification-campaigns.json +``` + +Enable the policies independently in `config/flows.mk`: + +```make +FLOW_negative_qualification := enabled +FLOW_four_state_qualification := enabled + +FLOW_DEPENDENCIES_negative_qualification := +FLOW_DEPENDENCIES_four_state_qualification := +``` + +The schema is +[`schemas/qualification-campaigns-v1.schema.json`](../schemas/qualification-campaigns-v1.schema.json). +The runtime also applies semantic checks that JSON Schema cannot express, such +as control references and expected-failure rules. + +## Manifest model + +The root may declare either or both campaigns. A selected campaign must exist. +Each case has a stable `id`, a filesystem-safe `evidence` name, a semantic +`role`, a descriptive `kind`, and one or two ordered `phases` named `compile` +or `run`. + +Each campaign may also list module-relative `inputs`. These paths identify its +testbenches, mutations, candidate netlists, and tool configuration. The runner +rejects missing or external paths, and the release generator hashes the inputs +for each enabled campaign. + +```json +{ + "schema": "mosaic-qualification-campaigns-v1", + "campaigns": { + "negative": { + "inputs": [ + "verif/mutations/counter_mutant.sv", + "verif/formal/counter_bad_netlist.eqy" + ], + "cases": [ + { + "id": "nominal_control", + "evidence": "nominal_control", + "role": "positive_control", + "kind": "positive_control", + "phases": [ + { + "name": "run", + "command": ["{case_work_dir}/nominal_sim"], + "expected": "success", + "diagnostic": "NOMINAL_TEST_PASS" + } + ] + }, + { + "id": "incorrect_reset_mutation", + "evidence": "incorrect_reset_mutation", + "role": "negative", + "kind": "simulation_mutation", + "positive_control": "nominal_control", + "phases": [ + { + "name": "run", + "command": ["{case_work_dir}/mutant_sim"], + "expected": "failure", + "diagnostic": "RESET_ASSERTION_FAILED", + "failure_class": "assertion" + } + ] + } + ] + } + } +} +``` + +A phase defines exactly one `command` string array or one executable `fixture`. +Commands run directly without an implicit shell. Optional fields are +`environment`, `timeout_seconds`, and `diagnostic`. A phase expecting failure +must also declare a diagnostic regular expression and one of these classes: + +- `assertion` +- `elaboration` +- `equivalence` +- `mutation` +- `unknown_control` + +Available placeholders are `{module_root}`, `{flow_root}`, `{report_dir}`, +`{work_dir}`, `{case_report_dir}`, `{case_work_dir}`, `{python}`, `{verilator}`, +`{eqy}`, `{iverilog}`, and `{vvp}`. Module source paths remain relative to +`MODULE_ROOT`. + +## Negative-test contract + +Every negative case references a successful `positive_control`. This prevents +an always-failing checker, broken simulator, or invalid nominal configuration +from qualifying a detected fault. The campaign supports: + +- Simulation mutations detected by a test or assertion +- Deliberate assertion failures +- Invalid-parameter elaboration failures +- Deliberately inequivalent candidate netlists + +The expected nonzero exit alone is insufficient. The declared diagnostic must +also appear. If a command succeeds, the case is classified as `escaped_fault`. +If it fails without the expected design-check diagnostic, the result is +`unexpected_failure` or `infrastructure_failure`, never a false qualification. + +Additional infrastructure signatures may be declared at campaign level: + +```json +"infrastructure_diagnostics": ["LICENSE_SERVER_UNAVAILABLE"] +``` + +Built-in signatures cover missing executables, missing files, license checkout +failures, internal tool failures, and the fixture marker +`INFRASTRUCTURE_FAILURE`. + +## Keeping mutations separate + +Do not edit production RTL in place and do not copy the shared campaign runner +into a module. Keep module-owned mutation material under a verification path, +for example: + +```text +verif/ +|-- mutations/ +| |-- counter_mutant.sv +| `-- bad_candidate_netlist.v +|-- formal/fault_injection/ +| `-- counter_bad_netlist.eqy +`-- tb/fault_injection/ + `-- counter_fault_tb.sv +``` + +Prefer a compile-time mutation macro, a bind-time fault shim, or a small +candidate netlist over a second full RTL implementation. Compile each mutation +only in its declared case. The positive control must compile and run the +unmodified design with the same checker whenever the tool and test permit it. + +## Four-state contract + +The four-state campaign is pinned to `"simulator": "iverilog"`. It is separate +from the normal two-state Verilator gate. Every `unknown_detection` case: + +- Declares the control ports and injected `X`, `Z`, or both values in + `injections` +- References one `disabled_monitor_control` +- Expects a nonzero result and a module-specific monitor diagnostic + +The disabled-monitor control must drive the same illegal stimulus while the +unknown-value monitor is disabled. It expects success and requires a marker +such as `UNKNOWN_CONTROL_STIMULUS_REACHED`. This proves the stimulus reached the +DUT and catches monitors that fail unconditionally. + +Both cases declare identical `injections`, provide `compile` and `run` phases, +and invoke `{iverilog}` and `{vvp}` directly. The runner enforces these details +so simulator metadata cannot claim Icarus while executing a two-state backend. + +The `injections` field is auditable metadata. The module-owned four-state +testbench remains responsible for driving the declared values. Reviewers should +compare the testbench with this declaration. + +## Results and classification + +Each gate writes: + +```text +reports// +|-- status.txt +|-- summary.json +`-- [/]/ + |-- status.txt + |-- summary.json + |-- compile-command.json + |-- compile-result.json + |-- compile.log + |-- run-command.json + |-- run-result.json + `-- run.log +``` + +Only declared phases are present. Corresponding disposable outputs use +`work//[/]/`. Four-state cases use the +explicit `iverilog/` namespace. Multi-module and parameter-profile roots +continue to isolate these paths before the flow ID. + +Case summaries distinguish expected assertion, elaboration, equivalence, +mutation, and unknown-control failures from escaped faults, missing diagnostics, +unexpected failures, infrastructure failures, and failed controls. Any failed +or blocked case fails the aggregate campaign. + +The release manifest hashes the module-owned campaign manifest when either +flow is enabled. It records each flow's required or approved-skip policy, +captures Icarus and VVP versions when four-state qualification is enabled, and +indexes each generated campaign `summary.json` as additional evidence. + +## Migration from module scripts + +To replace an existing `run_fault_injection.sh` or `run_four_state.sh`: + +1. Inventory each compile and run command, expected exit, and diagnostic. +2. Add a positive or disabled-monitor control before its dependent cases. +3. Move orchestration into the manifest while retaining design-specific RTL, + testbenches, EQY files, and expected messages in the module. +4. Run each target independently and inspect escaped-fault and broken-monitor + behavior deliberately. +5. Enable the flow only after its positive and negative controls qualify. + +Automatic mutation generation, replacement of normal regression or formal +proof, and mandatory four-state testing for modules without relevant controls +are outside this feature. diff --git a/docs/release-evidence.md b/docs/release-evidence.md index 7d0db5b..b89a1e9 100644 --- a/docs/release-evidence.md +++ b/docs/release-evidence.md @@ -123,6 +123,12 @@ with HDL coverage. When the canonical coverage gate runs, its `coverage_qualification/summary.json` is indexed as a separate qualification decision alongside the underlying databases. +When enabled, `negative_qualification` and `four_state_qualification` are +recorded as independent required canonical flows. Their module-owned campaign +manifest is hashed as a release input and each aggregate `summary.json` is +indexed as additional evidence. Four-state releases also capture the resolved +Icarus and VVP versions. + Register another tool version command with JSON: ```make diff --git a/docs/results-and-quality-gates.md b/docs/results-and-quality-gates.md index 0510a3b..281c62a 100644 --- a/docs/results-and-quality-gates.md +++ b/docs/results-and-quality-gates.md @@ -80,11 +80,15 @@ the dependency graph and filesystem are temporarily inconsistent. - `verilator_sim` - `pyuvm_open_source` - `coverage_qualification` +- `negative_qualification` +- `four_state_qualification` `make open-source` runs these targets and then the gate. PyUVM is disabled by default and therefore records `SKIP` unless the module enables it. Coverage -qualification is also disabled by default. Once either optional flow is enabled, -its open-source result must be `PASS`. `openroad` is optional and intentionally +qualification is also disabled by default. Negative-test and four-state +qualification are likewise disabled by default and independently selectable. +Once any optional flow is enabled, its open-source result must be `PASS`. +`openroad` is optional and intentionally outside this portable gate. ## Commercial quality gate diff --git a/flows/qualification/run.sh b/flows/qualification/run.sh new file mode 100755 index 0000000..7e9ccb7 --- /dev/null +++ b/flows/qualification/run.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +source "$(dirname "$0")/../common/env.sh" + +campaign="${1:?Usage: run.sh }" +case "${campaign}" in + negative) + flow_id=negative_qualification + ;; + four_state) + flow_id=four_state_qualification + ;; + *) + echo "Unknown qualification campaign: ${campaign}" >&2 + exit 2 + ;; +esac + +"${QUALIFICATION_CAMPAIGN_TOOL}" \ + --campaign "${campaign}" \ + --manifest "${QUALIFICATION_CAMPAIGN_MANIFEST}" \ + --module-root "${MODULE_ROOT}" \ + --flow-root "${FLOW_ROOT}" \ + --report-dir "${REPORT_DIR}/${flow_id}" \ + --work-dir "${WORK_DIR}/${flow_id}" diff --git a/mk/module.mk b/mk/module.mk index a5f7052..1aa7403 100644 --- a/mk/module.mk +++ b/mk/module.mk @@ -122,6 +122,7 @@ RELEASE_STANDARD_INPUTS := \ $(FORMAL_COVER_CONFIG) \ $(FORMAL_COVERAGE_CONFIG) \ $(if $(filter enabled,$(FLOW_coverage_qualification)),$(COVERAGE_QUALIFICATION_POLICY)) \ + $(if $(filter enabled,$(FLOW_negative_qualification) $(FLOW_four_state_qualification)),$(QUALIFICATION_CAMPAIGN_MANIFEST)) \ $(EQUIVALENCE_CONFIG) \ $(OPENROAD_CONFIG) \ $(SYNTHESIS_CONSTRAINT_FILE) \ @@ -151,8 +152,8 @@ export RELEASE_COVERAGE_EVIDENCE_JSON RELEASE_SUPPLEMENTAL_GATES export RELEASE_ADDITIONAL_INPUTS RELEASE_ADDITIONAL_EVIDENCE RELEASE_MANIFEST_DIR export RELEASE_FILELISTS RELEASE_INPUT_FILES -OPEN_SOURCE_TARGETS := open-source open-style-lint open-format-check open-elaborate open-lint open-waiver-draft open-synth open-formal open-equivalence open-sim open-pyuvm open-coverage open-quality-gate -OPEN_FLOW_TARGETS := open-style-lint open-format-check open-elaborate open-lint open-synth open-formal open-equivalence open-sim open-pyuvm open-coverage +OPEN_SOURCE_TARGETS := open-source open-style-lint open-format-check open-elaborate open-lint open-waiver-draft open-synth open-formal open-equivalence open-sim open-pyuvm open-coverage open-negative open-four-state open-quality-gate +OPEN_FLOW_TARGETS := open-style-lint open-format-check open-elaborate open-lint open-synth open-formal open-equivalence open-sim open-pyuvm open-coverage open-negative open-four-state FLOW_TARGET_verible_lint := open-style-lint FLOW_TARGET_verible_format := open-format-check @@ -164,6 +165,8 @@ FLOW_TARGET_eqy_equivalence := open-equivalence FLOW_TARGET_verilator_sim := open-sim FLOW_TARGET_pyuvm_open_source := open-pyuvm FLOW_TARGET_coverage_qualification := open-coverage +FLOW_TARGET_negative_qualification := open-negative +FLOW_TARGET_four_state_qualification := open-four-state FLOW_TARGET_openroad := open-physical FLOW_TARGET_vcs_sim := synopsys-sim FLOW_TARGET_pyuvm_commercial := commercial-pyuvm @@ -342,6 +345,14 @@ open-pyuvm: open-coverage: @"$(FLOW_RUNNER)" coverage_qualification "$(FLOW_ROOT)/flows/coverage/run.sh" +## open-negative Prove module-owned faults and invalid configurations are detected +open-negative: + @"$(FLOW_RUNNER)" negative_qualification "$(FLOW_ROOT)/flows/qualification/run.sh" negative + +## open-four-state Detect declared X/Z controls with pinned Icarus simulation +open-four-state: + @"$(FLOW_RUNNER)" four_state_qualification "$(FLOW_ROOT)/flows/qualification/run.sh" four_state + ## open-quality-gate Validate all open-source results open-quality-gate: $(OPEN_FLOW_TARGETS) @"$(FLOW_ROOT)/ci/open_source_quality_gate.sh" diff --git a/schemas/qualification-campaigns-v1.schema.json b/schemas/qualification-campaigns-v1.schema.json new file mode 100644 index 0000000..2bd75c8 --- /dev/null +++ b/schemas/qualification-campaigns-v1.schema.json @@ -0,0 +1,151 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://mosaic.dev/schemas/qualification-campaigns-v1.schema.json", + "title": "MOSAIC negative-test and four-state qualification campaigns", + "type": "object", + "additionalProperties": false, + "required": ["schema", "campaigns"], + "properties": { + "schema": {"const": "mosaic-qualification-campaigns-v1"}, + "campaigns": { + "type": "object", + "additionalProperties": false, + "properties": { + "negative": {"$ref": "#/$defs/negativeCampaign"}, + "four_state": {"$ref": "#/$defs/fourStateCampaign"} + }, + "minProperties": 1 + } + }, + "$defs": { + "name": {"type": "string", "pattern": "^[a-z][a-z0-9_]*$"}, + "diagnostics": { + "type": "array", + "items": {"type": "string", "minLength": 1} + }, + "inputs": { + "type": "array", + "items": {"type": "string", "minLength": 1}, + "uniqueItems": true + }, + "phase": { + "type": "object", + "additionalProperties": false, + "required": ["name", "expected"], + "properties": { + "name": {"enum": ["compile", "run"]}, + "command": { + "type": "array", + "items": {"type": "string", "minLength": 1}, + "minItems": 1 + }, + "fixture": {"type": "string", "minLength": 1}, + "expected": {"enum": ["success", "failure"]}, + "diagnostic": {"type": "string", "minLength": 1}, + "failure_class": { + "enum": ["assertion", "elaboration", "equivalence", "mutation", "unknown_control"] + }, + "environment": { + "type": "object", + "additionalProperties": {"type": "string"} + }, + "timeout_seconds": {"type": "integer", "minimum": 1} + }, + "oneOf": [ + {"required": ["command"], "not": {"required": ["fixture"]}}, + {"required": ["fixture"], "not": {"required": ["command"]}} + ] + }, + "caseBase": { + "type": "object", + "required": ["id", "evidence", "role", "kind", "phases"], + "properties": { + "id": {"$ref": "#/$defs/name"}, + "evidence": {"$ref": "#/$defs/name"}, + "role": {"type": "string"}, + "kind": {"type": "string", "minLength": 1}, + "phases": { + "type": "array", + "items": {"$ref": "#/$defs/phase"}, + "minItems": 1, + "maxItems": 2 + }, + "positive_control": {"$ref": "#/$defs/name"}, + "monitor_control": {"$ref": "#/$defs/name"}, + "injections": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["control", "values"], + "properties": { + "control": {"type": "string", "minLength": 1}, + "values": { + "type": "array", + "items": {"enum": ["X", "Z"]}, + "minItems": 1, + "uniqueItems": true + } + } + }, + "minItems": 1 + } + } + }, + "negativeCase": { + "allOf": [ + {"$ref": "#/$defs/caseBase"}, + { + "additionalProperties": false, + "properties": { + "id": {}, "evidence": {}, "kind": {}, "phases": {}, + "role": {"enum": ["negative", "positive_control"]}, + "positive_control": {} + } + } + ] + }, + "fourStateCase": { + "allOf": [ + {"$ref": "#/$defs/caseBase"}, + { + "additionalProperties": false, + "properties": { + "id": {}, "evidence": {}, "kind": {}, "phases": {}, + "role": {"enum": ["unknown_detection", "disabled_monitor_control"]}, + "monitor_control": {}, "injections": {} + } + } + ] + }, + "negativeCampaign": { + "type": "object", + "additionalProperties": false, + "required": ["cases"], + "properties": { + "inputs": {"$ref": "#/$defs/inputs"}, + "infrastructure_diagnostics": {"$ref": "#/$defs/diagnostics"}, + "cases": { + "type": "array", + "items": {"$ref": "#/$defs/negativeCase"}, + "minItems": 1 + } + } + }, + "fourStateCampaign": { + "type": "object", + "additionalProperties": false, + "required": ["simulator", "cases"], + "properties": { + "simulator": {"const": "iverilog"}, + "inputs": {"$ref": "#/$defs/inputs"}, + "infrastructure_diagnostics": {"$ref": "#/$defs/diagnostics"}, + "cases": { + "type": "array", + "items": {"$ref": "#/$defs/fourStateCase"}, + "minItems": 1 + } + } + } + } +} diff --git a/tests/check_fixture_parameter_profile_evidence.sh b/tests/check_fixture_parameter_profile_evidence.sh index 86eae50..362b404 100755 --- a/tests/check_fixture_parameter_profile_evidence.sh +++ b/tests/check_fixture_parameter_profile_evidence.sh @@ -33,7 +33,11 @@ portable_flows = { "verilator_sim", "pyuvm_open_source", } -reported_flows = portable_flows | {"coverage_qualification"} +reported_flows = portable_flows | { + "coverage_qualification", + "negative_qualification", + "four_state_qualification", +} elaboration_flows = { "verible_lint", "verible_format", diff --git a/tests/check_fixture_release_manifest.sh b/tests/check_fixture_release_manifest.sh index 78ae665..433a5b5 100755 --- a/tests/check_fixture_release_manifest.sh +++ b/tests/check_fixture_release_manifest.sh @@ -32,6 +32,8 @@ expected_pass = { "eqy_equivalence", "pyuvm_open_source", "coverage_qualification", + "negative_qualification", + "four_state_qualification", "slang_elaboration", "symbiyosys_formal", "verible_format", @@ -47,6 +49,8 @@ assert set(flow_statuses) == { "pyuvm_commercial", "pyuvm_open_source", "coverage_qualification", + "negative_qualification", + "four_state_qualification", "sg_cdc", "sg_dft", "slang_elaboration", @@ -76,6 +80,7 @@ assert all( expected_tools = { "cocotb", "eqy", + "iverilog", "python", "pyuvm", "slang", @@ -84,6 +89,7 @@ expected_tools = { "verible-verilog-lint", "verilator", "yosys", + "vvp", } tools = deterministic["tools"] assert {entry["name"] for entry in tools} == expected_tools @@ -94,6 +100,9 @@ input_paths = {entry["path"] for entry in deterministic["inputs"]} assert { "config/design.mk", "config/coverage-policy.json", + "config/qualification-campaigns.json", + "config/qualification-equivalent.eqy", + "config/qualification-inequivalent.eqy", "config/flows.mk", "config/formal.sby", "config/formal_cover.sby", @@ -109,7 +118,9 @@ assert { "filelists/rtl.f", "filelists/tb.f", "rtl/flow_fixture.sv", + "verif/mutations/flow_fixture_bad_netlist.v", "verif/pyuvm/test_flow_fixture.py", + "verif/tb/flow_fixture_four_state_tb.sv", } <= input_paths assert all( re.fullmatch(r"[0-9a-f]{64}", entry["sha256"]) @@ -126,6 +137,11 @@ assert {(entry["producer"], entry["kind"]) for entry in coverage} >= { ("pyuvm_open_source", "systemverilog_native_report"), ("verilator_sim", "systemverilog_native_report"), } +additional_paths = {entry["path"] for entry in deterministic["evidence"]["additional"]} +assert { + "reports/negative_qualification/summary.json", + "reports/four_state_qualification/summary.json", +} <= additional_paths if expect_clean: assert manifest["volatile"]["source_tree"] == { "methodology_dirty": False, diff --git a/tests/fixture-module/config/design.mk b/tests/fixture-module/config/design.mk index fd15032..e92abdc 100644 --- a/tests/fixture-module/config/design.mk +++ b/tests/fixture-module/config/design.mk @@ -27,6 +27,7 @@ export FORMAL_CONFIG := $(CURDIR)/config/formal.sby export FORMAL_COVER_CONFIG := $(CURDIR)/config/formal_cover.sby export COVERAGE_QUALIFICATION_POLICY := $(CURDIR)/config/coverage-policy.json export COVERAGE_QUALIFICATION_SOURCE := verilator_sim +export QUALIFICATION_CAMPAIGN_MANIFEST := $(CURDIR)/config/qualification-campaigns.json export EQUIVALENCE_CONFIG := $(CURDIR)/config/equivalence.eqy export OPENROAD_CONFIG := $(CURDIR)/config/openroad.mk export CONSTRAINT_DIR := $(CURDIR)/constraints diff --git a/tests/fixture-module/config/flows.mk b/tests/fixture-module/config/flows.mk index c52cb3f..253fc04 100644 --- a/tests/fixture-module/config/flows.mk +++ b/tests/fixture-module/config/flows.mk @@ -8,6 +8,8 @@ FLOW_eqy_equivalence := enabled FLOW_verilator_sim := enabled FLOW_pyuvm_open_source := enabled FLOW_coverage_qualification := enabled +FLOW_negative_qualification := enabled +FLOW_four_state_qualification := enabled FLOW_openroad := enabled FLOW_vcs_sim := enabled FLOW_pyuvm_commercial := disabled diff --git a/tests/fixture-module/config/qualification-campaigns.json b/tests/fixture-module/config/qualification-campaigns.json new file mode 100644 index 0000000..3ffe354 --- /dev/null +++ b/tests/fixture-module/config/qualification-campaigns.json @@ -0,0 +1,192 @@ +{ + "schema": "mosaic-qualification-campaigns-v1", + "campaigns": { + "negative": { + "inputs": [ + "config/qualification-equivalent.eqy", + "config/qualification-inequivalent.eqy", + "verif/mutations/flow_fixture_bad_netlist.v" + ], + "cases": [ + { + "id": "verification_positive_control", + "evidence": "verification_positive_control", + "role": "positive_control", + "kind": "positive_control", + "phases": [ + { + "name": "run", + "command": [ + "{python}", + "{flow_root}/tests/fixtures/qualification_driver.py", + "--diagnostic", + "POSITIVE_CONTROL_PASS", + "--exit-code", + "0" + ], + "expected": "success", + "diagnostic": "POSITIVE_CONTROL_PASS" + } + ] + }, + { + "id": "equivalence_positive_control", + "evidence": "equivalence_positive_control", + "role": "positive_control", + "kind": "equivalence_control", + "phases": [ + { + "name": "run", + "command": [ + "{eqy}", + "-f", + "-d", + "{case_work_dir}/eqy", + "config/qualification-equivalent.eqy" + ], + "expected": "success", + "diagnostic": "Successfully proved designs equivalent" + } + ] + }, + { + "id": "detected_simulation_mutation", + "evidence": "detected_simulation_mutation", + "role": "negative", + "kind": "simulation_mutation", + "positive_control": "verification_positive_control", + "phases": [ + { + "name": "run", + "command": [ + "{python}", + "{flow_root}/tests/fixtures/qualification_driver.py", + "--diagnostic", + "ASSERTION_FAILURE_DETECTED", + "--exit-code", + "1" + ], + "expected": "failure", + "diagnostic": "ASSERTION_FAILURE_DETECTED", + "failure_class": "assertion" + } + ] + }, + { + "id": "invalid_parameter_elaboration", + "evidence": "invalid_parameter_elaboration", + "role": "negative", + "kind": "invalid_parameter", + "positive_control": "equivalence_positive_control", + "phases": [ + { + "name": "compile", + "command": [ + "{python}", + "{flow_root}/tests/fixtures/qualification_driver.py", + "--diagnostic", + "INVALID_PARAMETER_DETECTED", + "--exit-code", + "1" + ], + "expected": "failure", + "diagnostic": "INVALID_PARAMETER_DETECTED", + "failure_class": "elaboration" + } + ] + }, + { + "id": "expected_equivalence_mismatch", + "evidence": "expected_equivalence_mismatch", + "role": "negative", + "kind": "equivalence_mismatch", + "positive_control": "verification_positive_control", + "phases": [ + { + "name": "run", + "command": [ + "{eqy}", + "-f", + "-d", + "{case_work_dir}/eqy", + "config/qualification-inequivalent.eqy" + ], + "expected": "failure", + "diagnostic": "Successfully proved designs inequivalent|Failed to prove equivalence", + "failure_class": "equivalence" + } + ] + } + ] + }, + "four_state": { + "simulator": "iverilog", + "inputs": ["verif/tb/flow_fixture_four_state_tb.sv"], + "cases": [ + { + "id": "disabled_monitor_control", + "evidence": "disabled_monitor_control", + "role": "disabled_monitor_control", + "kind": "stimulus_control", + "injections": [ + {"control": "control_i", "values": ["X", "Z"]} + ], + "phases": [ + { + "name": "compile", + "command": [ + "{iverilog}", + "-g2012", + "-s", + "flow_fixture_four_state_tb", + "-DFLOW_FIXTURE_DISABLE_UNKNOWN_MONITOR", + "-o", + "{case_work_dir}/simulation.vvp", + "verif/tb/flow_fixture_four_state_tb.sv" + ], + "expected": "success" + }, + { + "name": "run", + "command": ["{vvp}", "{case_work_dir}/simulation.vvp"], + "expected": "success", + "diagnostic": "UNKNOWN_CONTROL_STIMULUS_REACHED" + } + ] + }, + { + "id": "unknown_control_detection", + "evidence": "unknown_control_detection", + "role": "unknown_detection", + "kind": "control_unknown", + "monitor_control": "disabled_monitor_control", + "injections": [ + {"control": "control_i", "values": ["X", "Z"]} + ], + "phases": [ + { + "name": "compile", + "command": [ + "{iverilog}", + "-g2012", + "-s", + "flow_fixture_four_state_tb", + "-o", + "{case_work_dir}/simulation.vvp", + "verif/tb/flow_fixture_four_state_tb.sv" + ], + "expected": "success" + }, + { + "name": "run", + "command": ["{vvp}", "{case_work_dir}/simulation.vvp"], + "expected": "failure", + "diagnostic": "UNKNOWN_CONTROL_XZ_DETECTED", + "failure_class": "unknown_control" + } + ] + } + ] + } + } +} diff --git a/tests/fixture-module/config/qualification-equivalent.eqy b/tests/fixture-module/config/qualification-equivalent.eqy new file mode 100644 index 0000000..1ee93d6 --- /dev/null +++ b/tests/fixture-module/config/qualification-equivalent.eqy @@ -0,0 +1,12 @@ +[gold] +read_verilog -sv rtl/flow_fixture.sv +prep -top flow_fixture + +[gate] +read_verilog -sv rtl/flow_fixture.sv +prep -top flow_fixture + +[strategy sby] +use sby +depth 8 +engine smtbmc bitwuzla diff --git a/tests/fixture-module/config/qualification-inequivalent.eqy b/tests/fixture-module/config/qualification-inequivalent.eqy new file mode 100644 index 0000000..e0fd0d8 --- /dev/null +++ b/tests/fixture-module/config/qualification-inequivalent.eqy @@ -0,0 +1,12 @@ +[gold] +read_verilog -sv rtl/flow_fixture.sv +prep -top flow_fixture + +[gate] +read_verilog -sv verif/mutations/flow_fixture_bad_netlist.v +prep -top flow_fixture + +[strategy sby] +use sby +depth 8 +engine smtbmc bitwuzla diff --git a/tests/fixture-module/verif/mutations/flow_fixture_bad_netlist.v b/tests/fixture-module/verif/mutations/flow_fixture_bad_netlist.v new file mode 100644 index 0000000..e8f1efa --- /dev/null +++ b/tests/fixture-module/verif/mutations/flow_fixture_bad_netlist.v @@ -0,0 +1,16 @@ +// Verification-only candidate with deliberately inverted capture behavior. +// The negative qualification campaign requires EQY to reject this design. +module flow_fixture #( + parameter DATA_WIDTH = 32 +) ( + input wire clk_i, + input wire rst_ni, + input wire enable_i, + input wire [DATA_WIDTH-1:0] data_i, + output wire [DATA_WIDTH-1:0] data_o +); + wire unused_controls = clk_i | rst_ni | enable_i; + + assign data_o = ~data_i ^ {DATA_WIDTH{unused_controls & 1'b0}}; + +endmodule diff --git a/tests/fixture-module/verif/tb/flow_fixture_four_state_tb.sv b/tests/fixture-module/verif/tb/flow_fixture_four_state_tb.sv new file mode 100644 index 0000000..0b63f73 --- /dev/null +++ b/tests/fixture-module/verif/tb/flow_fixture_four_state_tb.sv @@ -0,0 +1,31 @@ +module flow_fixture_four_state_tb; + logic control_i; + logic saw_x; + logic saw_z; + + initial begin + saw_x = 1'b0; + saw_z = 1'b0; + + control_i = 1'bx; + #1; + saw_x = $isunknown(control_i); + + control_i = 1'bz; + #1; + saw_z = $isunknown(control_i); + +`ifdef FLOW_FIXTURE_DISABLE_UNKNOWN_MONITOR + if (saw_x && saw_z) begin + $display("UNKNOWN_CONTROL_STIMULUS_REACHED"); + $finish; + end + $fatal(1, "FOUR_STATE_STIMULUS_BROKEN"); +`else + if (saw_x && saw_z) begin + $fatal(1, "UNKNOWN_CONTROL_XZ_DETECTED"); + end + $finish; +`endif + end +endmodule diff --git a/tests/fixtures/qualification_driver.py b/tests/fixtures/qualification_driver.py new file mode 100755 index 0000000..c7e4727 --- /dev/null +++ b/tests/fixtures/qualification_driver.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Emit deterministic qualification outcomes without requiring an EDA tool.""" + +from __future__ import annotations + +import argparse +import sys + + +parser = argparse.ArgumentParser() +parser.add_argument("--diagnostic", required=True) +parser.add_argument("--exit-code", required=True, type=int) +arguments = parser.parse_args() + +print(arguments.diagnostic) +sys.exit(arguments.exit_code) diff --git a/tests/test_parameter_profiles.sh b/tests/test_parameter_profiles.sh index d8c9557..528e784 100755 --- a/tests/test_parameter_profiles.sh +++ b/tests/test_parameter_profiles.sh @@ -198,6 +198,12 @@ check_invalid_manifest \ '{"schema":"mosaic-parameter-profiles-v1","include":[{"name":"a","parameters":{},"flows":["eqy_equivalence"]}]}' \ "requires: yosys_synthesis" +printf '%s\n' \ + '{"schema":"mosaic-parameter-profiles-v1","include":[{"name":"qualification","parameters":{"WIDTH":8},"flows":["negative_qualification","four_state_qualification"]}]}' \ + >"${temporary_root}/qualification-profiles.json" +python3 "${profile_tool}" validate \ + --manifest "${temporary_root}/qualification-profiles.json" >/dev/null + default_profiles="$( make -s -C "${flow_root}/tests/fixture-module" FLOW_ROOT="${flow_root}" profile-list )" diff --git a/tests/test_qualification_campaigns.sh b/tests/test_qualification_campaigns.sh new file mode 100755 index 0000000..6ecb3e5 --- /dev/null +++ b/tests/test_qualification_campaigns.sh @@ -0,0 +1,257 @@ +#!/usr/bin/env bash +set -euo pipefail + +flow_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +fixture_root="$(mktemp -d)" +trap 'rm -rf "${fixture_root}"' EXIT +mkdir -p "${fixture_root}/config" + +write_manifest() { + local negative_exit="$1" + local negative_diagnostic="$2" + local four_state_exit="$3" + local manifest="${fixture_root}/config/qualification-campaigns.json" + python3 - "${manifest}" "${negative_exit}" "${negative_diagnostic}" \ + "${four_state_exit}" <<'PY' +import json +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +negative_exit = int(sys.argv[2]) +negative_diagnostic = sys.argv[3] +four_state_exit = int(sys.argv[4]) +driver = "{flow_root}/tests/fixtures/qualification_driver.py" + +def phase(expected, diagnostic, exit_code, failure_class=None, name="run"): + value = { + "name": name, + "command": [ + "{python}", driver, "--diagnostic", diagnostic, + "--exit-code", str(exit_code), + ], + "expected": expected, + "diagnostic": diagnostic, + } + if failure_class: + value["failure_class"] = failure_class + return value + +manifest = { + "schema": "mosaic-qualification-campaigns-v1", + "campaigns": { + "negative": { + "infrastructure_diagnostics": ["UNRELATED_TOOL_FAILURE"], + "cases": [ + { + "id": "positive_control", "evidence": "positive_control", + "role": "positive_control", "kind": "positive_control", + "phases": [phase("success", "POSITIVE_CONTROL_PASS", 0)], + }, + { + "id": "detected_mutation", "evidence": "detected_mutation", + "role": "negative", "kind": "simulation_mutation", + "positive_control": "positive_control", + "phases": [phase( + "failure", negative_diagnostic, negative_exit, "assertion" + )], + }, + { + "id": "expected_eqy_mismatch", "evidence": "expected_eqy_mismatch", + "role": "negative", "kind": "equivalence_mismatch", + "positive_control": "positive_control", + "phases": [phase( + "failure", "Successfully proved designs inequivalent", 1, + "equivalence" + )], + }, + ], + }, + "four_state": { + "simulator": "iverilog", + "cases": [ + { + "id": "disabled_monitor", "evidence": "disabled_monitor", + "role": "disabled_monitor_control", "kind": "stimulus_control", + "injections": [{"control": "enable_i", "values": ["X", "Z"]}], + "phases": [ + { + "name": "compile", + "command": [ + "{iverilog}", "--diagnostic", "COMPILE_PASS", + "--exit-code", "0" + ], + "expected": "success" + }, + { + "name": "run", + "command": [ + "{vvp}", "--diagnostic", "UNKNOWN_STIMULUS_REACHED", + "--exit-code", "0" + ], + "expected": "success", + "diagnostic": "UNKNOWN_STIMULUS_REACHED" + } + ], + }, + { + "id": "unknown_control", "evidence": "unknown_control", + "role": "unknown_detection", "kind": "control_unknown", + "monitor_control": "disabled_monitor", + "injections": [{"control": "enable_i", "values": ["X", "Z"]}], + "phases": [ + { + "name": "compile", + "command": [ + "{iverilog}", "--diagnostic", "COMPILE_PASS", + "--exit-code", "0" + ], + "expected": "success" + }, + { + "name": "run", + "command": [ + "{vvp}", "--diagnostic", "UNKNOWN_CONTROL_DETECTED", + "--exit-code", str(four_state_exit) + ], + "expected": "failure", + "diagnostic": "UNKNOWN_CONTROL_DETECTED", + "failure_class": "unknown_control" + } + ], + }, + ], + }, + }, +} +path.write_text(json.dumps(manifest), encoding="utf-8") +PY + + fake_tools="${fixture_root}/fake-tools" + mkdir -p "${fake_tools}" + cp "${flow_root}/tests/fixtures/qualification_driver.py" "${fake_tools}/iverilog" + cp "${flow_root}/tests/fixtures/qualification_driver.py" "${fake_tools}/vvp" + chmod +x "${fake_tools}/iverilog" "${fake_tools}/vvp" + export IVERILOG_CMD="${fake_tools}/iverilog" + export VVP_CMD="${fake_tools}/vvp" +} + +run_campaign() { + local campaign="$1" + local name="$2" + "${flow_root}/ci/qualification_campaign.py" \ + --campaign "${campaign}" \ + --manifest "${fixture_root}/config/qualification-campaigns.json" \ + --module-root "${fixture_root}" --flow-root "${flow_root}" \ + --report-dir "${fixture_root}/reports/${name}" \ + --work-dir "${fixture_root}/work/${name}" +} + +expect_failure() { + if "$@" >"${fixture_root}/failure.log" 2>&1; then + echo "Qualification fixture unexpectedly passed" >&2 + exit 1 + fi +} + +write_manifest 1 ASSERTION_FAILURE_DETECTED 1 +run_campaign negative negative-pass >/dev/null +run_campaign four_state four-state-pass >/dev/null +grep -Fq '"classification": "expected_assertion_failure"' \ + "${fixture_root}/reports/negative-pass/detected_mutation/summary.json" +grep -Fq '"classification": "expected_equivalence_failure"' \ + "${fixture_root}/reports/negative-pass/expected_eqy_mismatch/summary.json" +python3 - "${fixture_root}" <<'PY' +import json +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +negative = json.loads( + (root / "reports/negative-pass/summary.json").read_text(encoding="utf-8") +) +four_state = json.loads( + (root / "reports/four-state-pass/summary.json").read_text(encoding="utf-8") +) +assert negative["status"] == "PASS" +assert four_state["status"] == "PASS" +assert four_state["simulator"] == "iverilog" +assert all(case["status"] == "PASS" for case in negative["cases"]) +assert all(case["status"] == "PASS" for case in four_state["cases"]) +for report in ( + root / "reports/negative-pass/detected_mutation", + root / "reports/four-state-pass/iverilog/unknown_control", +): + assert (report / "status.txt").read_text(encoding="utf-8").strip() == "PASS" + assert (report / "summary.json").is_file() + assert (report / "run-command.json").is_file() + assert (report / "run-result.json").is_file() + assert (report / "run.log").is_file() +PY + +write_manifest 0 ASSERTION_FAILURE_DETECTED 1 +expect_failure run_campaign negative escaped-mutation +grep -Fq '"classification": "escaped_fault"' \ + "${fixture_root}/reports/escaped-mutation/detected_mutation/summary.json" +test "$(<"${fixture_root}/reports/escaped-mutation/status.txt")" = FAIL + +write_manifest 2 UNRELATED_TOOL_FAILURE 1 +expect_failure run_campaign negative infrastructure-failure +grep -Fq '"classification": "infrastructure_failure"' \ + "${fixture_root}/reports/infrastructure-failure/detected_mutation/summary.json" + +write_manifest 1 ASSERTION_FAILURE_DETECTED 0 +expect_failure run_campaign four_state broken-monitor +grep -Fq '"classification": "escaped_fault"' \ + "${fixture_root}/reports/broken-monitor/iverilog/unknown_control/summary.json" + +write_manifest 1 ASSERTION_FAILURE_DETECTED 1 +python3 - "${fixture_root}/config/qualification-campaigns.json" <<'PY' +import json +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +manifest = json.loads(path.read_text(encoding="utf-8")) +control = manifest["campaigns"]["four_state"]["cases"][0] +control["phases"][0]["diagnostic"] = "BROKEN_CONTROL_MARKER" +path.write_text(json.dumps(manifest), encoding="utf-8") +PY +expect_failure run_campaign four_state broken-control +grep -Fq '"classification": "blocked_by_control"' \ + "${fixture_root}/reports/broken-control/iverilog/unknown_control/summary.json" + +write_manifest 1 ASSERTION_FAILURE_DETECTED 1 +python3 - "${fixture_root}/config/qualification-campaigns.json" <<'PY' +import json +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +manifest = json.loads(path.read_text(encoding="utf-8")) +manifest["campaigns"]["negative"]["inputs"] = ["missing-mutation.sv"] +path.write_text(json.dumps(manifest), encoding="utf-8") +PY +expect_failure run_campaign negative missing-input +grep -Fq 'campaign input is missing: missing-mutation.sv' \ + "${fixture_root}/reports/missing-input/summary.json" + +skip_reports="${fixture_root}/skip-reports" +env \ + MOSAIC_FLOW_IDS="negative_qualification four_state_qualification" \ + FLOW_negative_qualification=disabled FLOW_four_state_qualification=disabled \ + FLOW_DEPENDENCIES_negative_qualification= FLOW_DEPENDENCIES_four_state_qualification= \ + DISABLED_FLOWS="negative_qualification four_state_qualification" \ + REPORT_DIR="${skip_reports}" MODULE_ROOT="${fixture_root}" \ + "${flow_root}/ci/run_flow.sh" negative_qualification false >/dev/null +test "$(<"${skip_reports}/negative_qualification/status.txt")" = SKIP +env \ + MOSAIC_FLOW_IDS="negative_qualification four_state_qualification" \ + FLOW_negative_qualification=disabled FLOW_four_state_qualification=disabled \ + FLOW_DEPENDENCIES_negative_qualification= FLOW_DEPENDENCIES_four_state_qualification= \ + DISABLED_FLOWS="negative_qualification four_state_qualification" \ + REPORT_DIR="${skip_reports}" MODULE_ROOT="${fixture_root}" \ + "${flow_root}/ci/run_flow.sh" four_state_qualification false >/dev/null +test "$(<"${skip_reports}/four_state_qualification/status.txt")" = SKIP + +echo "Qualification campaign tests passed" diff --git a/tests/test_quality_gate.sh b/tests/test_quality_gate.sh index 49b74f8..354a20b 100755 --- a/tests/test_quality_gate.sh +++ b/tests/test_quality_gate.sh @@ -101,7 +101,7 @@ if [[ -e "${report_root}/eqy_equivalence/block_reason.txt" ]]; then exit 1 fi -required_flows=(verible_lint verible_format slang_elaboration verilator_lint yosys_synthesis symbiyosys_formal eqy_equivalence verilator_sim pyuvm_open_source coverage_qualification) +required_flows=(verible_lint verible_format slang_elaboration verilator_lint yosys_synthesis symbiyosys_formal eqy_equivalence verilator_sim pyuvm_open_source coverage_qualification negative_qualification four_state_qualification) for flow_name in "${required_flows[@]}"; do mkdir -p "${report_root}/${flow_name}" printf 'PASS\n' > "${report_root}/${flow_name}/status.txt" diff --git a/tests/test_release_manifest.sh b/tests/test_release_manifest.sh index e65608b..6684d65 100755 --- a/tests/test_release_manifest.sh +++ b/tests/test_release_manifest.sh @@ -19,6 +19,7 @@ mkdir -p \ "${module_root}/filelists" \ "${module_root}/reports/verible_lint" \ "${module_root}/reports/pyuvm_open_source" \ + "${module_root}/reports/negative_qualification" \ "${module_root}/reports/qualification" \ "${module_root}/rtl" \ "${module_root}/tools" \ @@ -38,8 +39,63 @@ EOF cat >"${module_root}/config/flows.mk" <<'EOF' FLOW_verible_lint := enabled FLOW_pyuvm_open_source := enabled +FLOW_negative_qualification := enabled +FLOW_four_state_qualification := disabled FLOW_openroad := disabled EOF +cat >"${module_root}/config/qualification-campaigns.json" <<'EOF' +{ + "schema": "mosaic-qualification-campaigns-v1", + "campaigns": { + "negative": { + "inputs": ["config/qualification-negative.txt"], + "cases": [ + { + "id": "negative_control", "evidence": "negative_control", + "role": "positive_control", "kind": "positive_control", + "phases": [{"name": "run", "command": ["true"], "expected": "success"}] + }, + { + "id": "negative_case", "evidence": "negative_case", + "role": "negative", "kind": "simulation_mutation", + "positive_control": "negative_control", + "phases": [{ + "name": "run", "command": ["false"], "expected": "failure", + "diagnostic": "EXPECTED_FAILURE", "failure_class": "mutation" + }] + } + ] + }, + "four_state": { + "simulator": "iverilog", + "inputs": ["config/qualification-four-state.txt"], + "cases": [ + { + "id": "monitor_control", "evidence": "monitor_control", + "role": "disabled_monitor_control", "kind": "stimulus_control", + "injections": [{"control": "enable_i", "values": ["X", "Z"]}], + "phases": [ + {"name": "compile", "command": ["{iverilog}", "fixture.sv"], "expected": "success"}, + {"name": "run", "command": ["{vvp}", "fixture.vvp"], "expected": "success", "diagnostic": "STIMULUS_REACHED"} + ] + }, + { + "id": "unknown_detection", "evidence": "unknown_detection", + "role": "unknown_detection", "kind": "control_unknown", + "monitor_control": "monitor_control", + "injections": [{"control": "enable_i", "values": ["X", "Z"]}], + "phases": [ + {"name": "compile", "command": ["{iverilog}", "fixture.sv"], "expected": "success"}, + {"name": "run", "command": ["{vvp}", "fixture.vvp"], "expected": "failure", "diagnostic": "UNKNOWN_CONTROL", "failure_class": "unknown_control"} + ] + } + ] + } + } +} +EOF +printf 'negative campaign input\n' >"${module_root}/config/qualification-negative.txt" +printf 'four-state campaign input\n' >"${module_root}/config/qualification-four-state.txt" cat >"${module_root}/constraints/timing.sdc" <<'EOF' create_clock -name clk -period 10 [get_ports clk] EOF @@ -66,6 +122,7 @@ chmod +x "${module_root}/tools/fake_tool.sh" printf 'PASS\n' >"${report_dir}/verible_lint/status.txt" printf 'PASS\n' >"${report_dir}/pyuvm_open_source/status.txt" +printf 'PASS\n' >"${report_dir}/negative_qualification/status.txt" printf 'PASS\n' >"${report_dir}/qualification/status.txt" cat >"${report_dir}/pyuvm_open_source/versions.log" <"${report_dir}/pyuvm_open_source/coverage.info" printf '{"covered":1,"total":1}\n' \ >"${report_dir}/pyuvm_open_source/functional-coverage.json" printf '{"review":"complete"}\n' >"${report_dir}/qualification/summary.json" +printf '{"campaign":"negative","status":"PASS"}\n' \ + >"${report_dir}/negative_qualification/summary.json" git -C "${module_root}" init --quiet git -C "${module_root}" config user.email fixture@example.invalid @@ -99,8 +158,8 @@ manifest_environment=( "MODULE_ROOT=${module_root}" "FLOW_ROOT=${methodology_root}" "REPORT_DIR=${report_dir}" - "MOSAIC_FLOW_IDS=verible_lint pyuvm_open_source openroad" - "DISABLED_FLOWS=openroad" + "MOSAIC_FLOW_IDS=verible_lint pyuvm_open_source negative_qualification four_state_qualification openroad" + "DISABLED_FLOWS=four_state_qualification openroad" "RELEASE_MODULE_NAME=release_fixture" "MODULE_REVISION=${module_revision}" "METHODOLOGY_REVISION=${methodology_revision}" @@ -117,6 +176,9 @@ manifest_environment=( "RELEASE_ADDITIONAL_EVIDENCE=${report_dir}/qualification/summary.json" "VERIBLE_LINT_CMD=${module_root}/tools/fake_tool.sh" "PYUVM_PYTHON=${module_root}/tools/fake_tool.sh" + "IVERILOG_CMD=${module_root}/tools/fake_tool.sh" + "VVP_CMD=${module_root}/tools/fake_tool.sh" + "QUALIFICATION_CAMPAIGN_MANIFEST=${module_root}/config/qualification-campaigns.json" ) generate_manifest() { @@ -176,6 +238,8 @@ assert deterministic["technology"] == { "name": "fixture-technology", } assert {flow["id"]: flow["status"] for flow in deterministic["flows"]} == { + "four_state_qualification": "SKIP", + "negative_qualification": "PASS", "openroad": "SKIP", "pyuvm_open_source": "PASS", "verible_lint": "PASS", @@ -194,11 +258,19 @@ assert all(not pathlib.Path(item["path"]).is_absolute() for item in deterministic["inputs"]) input_paths = {item["path"] for item in deterministic["inputs"]} assert {"config/design.mk", "config/flows.mk", "constraints/timing.sdc", - "filelists/rtl.f", "rtl/release_fixture.sv"} <= input_paths + "config/qualification-campaigns.json", + "config/qualification-negative.txt", "filelists/rtl.f", + "rtl/release_fixture.sv"} <= input_paths +assert "config/qualification-four-state.txt" not in input_paths coverage_kinds = {item["kind"] for item in deterministic["evidence"]["coverage"]} assert coverage_kinds == {"pyuvm_functional", "systemverilog_native_report"} -assert deterministic["evidence"]["additional"][0]["path"] == \ - "reports/qualification/summary.json" +additional_paths = { + item["path"] for item in deterministic["evidence"]["additional"] +} +assert additional_paths == { + "reports/negative_qualification/summary.json", + "reports/qualification/summary.json", +} assert manifest["volatile"]["source_tree"] == { "methodology_dirty": False, "module_dirty": False, @@ -217,6 +289,38 @@ second = json.loads(pathlib.Path(sys.argv[2]).read_text(encoding="utf-8")) assert first["deterministic"] == second["deterministic"] PY +independent_output_dir="${report_dir}/release_manifest/four_state_only" +mkdir -p "${report_dir}/four_state_qualification" +printf 'SKIP\n' >"${report_dir}/negative_qualification/status.txt" +printf 'PASS\n' >"${report_dir}/four_state_qualification/status.txt" +printf '{"campaign":"four_state","status":"PASS"}\n' \ + >"${report_dir}/four_state_qualification/summary.json" +generate_manifest "${independent_output_dir}" \ + DISABLED_FLOWS="negative_qualification openroad" >/dev/null +python3 - "${independent_output_dir}/manifest.json" <<'PY' +import json +import pathlib +import sys + +manifest = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) +deterministic = manifest["deterministic"] +statuses = {flow["id"]: flow["status"] for flow in deterministic["flows"]} +assert statuses["negative_qualification"] == "SKIP" +assert statuses["four_state_qualification"] == "PASS" +input_paths = {item["path"] for item in deterministic["inputs"]} +assert "config/qualification-four-state.txt" in input_paths +assert "config/qualification-negative.txt" not in input_paths +additional_paths = { + item["path"] for item in deterministic["evidence"]["additional"] +} +assert "reports/four_state_qualification/summary.json" in additional_paths +assert "reports/negative_qualification/summary.json" not in additional_paths +tool_names = {tool["name"] for tool in deterministic["tools"]} +assert {"iverilog", "vvp"} <= tool_names +PY +printf 'PASS\n' >"${report_dir}/negative_qualification/status.txt" +printf 'SKIP\n' >"${report_dir}/four_state_qualification/status.txt" + expect_failure "Cannot inspect methodology source tree" \ generate_packaged_manifest "${output_dir}" generate_packaged_manifest "${output_dir}" \