diff --git a/abevalflow/mcp/README.md b/abevalflow/mcp/README.md index be911dc3..c7723be3 100644 --- a/abevalflow/mcp/README.md +++ b/abevalflow/mcp/README.md @@ -1,11 +1,11 @@ # `abevalflow/mcp/` — MCP evaluation pipeline logic -All evaluation logic for the MCP-server evaluation pipeline (ADR Approach 4, -Phases 1 and 2). This package **reads** result JSON, applies policy, and -**decides** pass/fail. No shell-outs to external tools, no network calls. -Testable with JSON fixtures. +All evaluation logic for the MCP-server evaluation pipeline (ADR Approach 4: +static, conformance, and behavioral evaluations). This package **reads** result +JSON, applies policy, and **decides** pass/fail. No shell-outs to external tools, +no network calls. Testable with JSON fixtures. -## Structure (organized by role, phase-prefixed) +## Structure (organized by role and evaluation type) | Directory | Role | What it does | |-----------|------|-------------| @@ -14,20 +14,20 @@ Testable with JSON fixtures. ### `gates/` — all pass/fail gates -| File | Phase | What it does | -|------|-------|-------------| -| `phase1_secrets_gate.py` | 1 | `SecretsGate` — reads `secrets-scan.json`, counts HIGH findings | -| `phase1_no_user_code_gate.py` | 1 | `NoUserCodeGate` — reads `no-user-code-scan.json`, counts HIGH findings | -| `phase1_license_gate.py` | 1 | `LicenseGate` — reads `license-scan.json`, checks against allow-list | -| `phase2_conformance_gate.py` | 2 | `Phase2Gate` — reads any `-check.json`, maps three-state to `GateResult` | -| `__init__.py` | 1+2 | Orchestrators: `run_phase1()`, `run_phase2()`, gate lists | +| File | Evaluation Type | What it does | +|------|-----------------|-------------| +| `static_eval_secrets_gate.py` | Static | `SecretsGate` — reads `secrets-scan.json`, counts HIGH findings | +| `static_eval_no_user_code_gate.py` | Static | `NoUserCodeGate` — reads `no-user-code-scan.json`, counts HIGH findings | +| `static_eval_license_gate.py` | Static | `LicenseGate` — reads `license-scan.json`, checks against allow-list | +| `conformance_eval_gate.py` | Conformance | `ConformanceEvalGate` — reads any `-check.json`, maps three-state to `GateResult` | +| `__init__.py` | All | Orchestrators: `run_static_eval()`, `run_conformance_eval()`, gate lists | ### `checks/` — conformance probes and Compass-consumed checks -| File | Phase | What it does | -|------|-------|-------------| -| `phase2_conformance_checks.py` | 2 | 8 `check_*` functions + `probe_all()` — conformance checks against a running MCP server | -| `phase2_compass_checks.py` | 2 | `map_facts()` + `map_results()` — maps Compass facts/results to `CheckOutcome` | +| File | Evaluation Type | What it does | +|------|-----------------|-------------| +| `conformance_eval_checks.py` | Conformance | 8 `check_*` functions + `probe_all()` — conformance checks against a running MCP server | +| `conformance_eval_compass_checks.py` | Conformance | `map_facts()` + `map_results()` — maps Compass facts/results to `CheckOutcome` | ### Root modules @@ -40,31 +40,31 @@ Testable with JSON fixtures. ## Data flow ``` -Phase 1 (static — no running server) +Static Evaluation (build-time — no running server) ┌─────────────────────────────────────────────────────────────────┐ │ scripts/mcp/scanners/ │ -│ phase1_secrets_scan ──→ secrets-scan.json │ -│ phase1_no_user_code_scan ──→ no-user-code-scan.json │ -│ phase1_license_scan ──→ license-scan.json │ +│ static_eval_secrets_scan ──→ secrets-scan.json │ +│ static_eval_no_user_code_scan ──→ no-user-code-scan.json │ +│ static_eval_license_scan ──→ license-scan.json │ └──────────────────────────────────┬──────────────────────────────┘ │ JSON on disk ┌──────────────────────────────────▼──────────────────────────────┐ │ abevalflow/mcp/gates/ │ -│ Phase1SecretsGate ──→ GateResult (pass/fail) │ -│ Phase1NoUserCodeGate ──→ GateResult (pass/fail) │ -│ Phase1LicenseGate ──→ GateResult (pass/fail) │ +│ SecretsGate ──→ GateResult (pass/fail) │ +│ NoUserCodeGate ──→ GateResult (pass/fail) │ +│ LicenseGate ──→ GateResult (pass/fail) │ └─────────────────────────────────────────────────────────────────┘ -Phase 2 (live — against the running MCP server) +Conformance Evaluation (live — against the running MCP server) ┌─────────────────────────────────────────────────────────────────┐ │ scripts/mcp/probes/ │ -│ phase2_conformance_probe ──→ -check.json (per check) │ -│ phase2_compass_probe ──→ -check.json (consumed facts) │ +│ conformance_eval_probe ──→ -check.json (per check) │ +│ conformance_eval_compass_probe ──→ -check.json (facts)│ └──────────────────────────────────┬──────────────────────────────┘ │ JSON on disk ┌──────────────────────────────────▼──────────────────────────────┐ │ abevalflow/mcp/gates/ │ -│ Phase2ConformanceGate (×N) ──→ GateResult (pass/fail/not_evaluated) │ +│ ConformanceEvalGate (×N) ──→ GateResult (pass/fail/not_eval) │ └─────────────────────────────────────────────────────────────────┘ ``` @@ -74,9 +74,9 @@ This package contains the **what** (evaluation logic — reads JSON, decides). `scripts/mcp/` contains the **how** (CLI entry points — runs tools, writes JSON). The contract between them is JSON files on disk. -## Three-state model (Phase 2) +## Three-state model (Conformance Evaluation) -Every Phase 2 check produces one of three states: +Every conformance evaluation check produces one of three states: | State | Meaning | Effect on verdict | Effect on grade | |---|---|---|---| @@ -86,17 +86,17 @@ Every Phase 2 check produces one of three states: A server is never penalized for something it was never tested for. -## Adding a new check +## Adding a new conformance check -1. **Add the check function** in `checks/phase2_conformance_checks.py` — takes the +1. **Add the check function** in `checks/conformance_eval_checks.py` — takes the client and/or a response, returns a `CheckOutcome`. 2. **Wire it** in `probe_all()` (same file). -3. **Add the check name** to `PHASE2_CHECKS` in `gates/__init__.py`. +3. **Add the check name** to `CONFORMANCE_EVAL_CHECKS` in `gates/__init__.py`. 4. The gate picks it up automatically — it reads any `-check.json` file. No gate code changes needed. ## Testing All code in this package is testable with JSON fixtures — no running server, -no external tools. Tests are in `tests/test_mcp_phase1.py` and -`tests/test_mcp_phase2.py`. +no external tools. Tests are in `tests/test_mcp_static_eval.py` and +`tests/test_mcp_conformance_eval.py`. diff --git a/abevalflow/mcp/__init__.py b/abevalflow/mcp/__init__.py index a6c95317..7ef20aed 100644 --- a/abevalflow/mcp/__init__.py +++ b/abevalflow/mcp/__init__.py @@ -5,17 +5,17 @@ Isolated from the skill evaluation pipeline (see ADR: Evaluation Strategy for MCP Servers as Standalone Software Components, Approach 4). -Layout (organized by role, phase-prefixed): +Layout (organized by role and evaluation type): - ``types`` - core types: ``CheckOutcome``, ``STATUS_*`` - ``helpers`` - finding builders, severity normalization, check-result I/O - ``mcp_client`` - MCP protocol client (JSON-RPC over Streamable HTTP) -- ``gates/`` - all pass/fail gates (Phase 1 + Phase 2) -- ``checks/`` - conformance check functions + Compass mapping (Phase 2) +- ``gates/`` - all pass/fail gates (static + conformance evaluation) +- ``checks/`` - conformance check functions + Compass mapping The CLI entry points that invoke this logic and handle I/O (shell out to scanners, create an MCP client, read/write files) live in ``scripts/mcp/``. -Phases 1 and 2 are implemented here; Phase 3 (behavioral, mcpchecker) is out of -scope for now. +Static and conformance evaluations are implemented here; behavioral evaluation +(mcpchecker) is out of scope for now. """ diff --git a/abevalflow/mcp/checks/phase2_conformance_checks.py b/abevalflow/mcp/checks/conformance_eval_checks.py similarity index 98% rename from abevalflow/mcp/checks/phase2_conformance_checks.py rename to abevalflow/mcp/checks/conformance_eval_checks.py index cdb9f5b1..32bcf36f 100644 --- a/abevalflow/mcp/checks/phase2_conformance_checks.py +++ b/abevalflow/mcp/checks/conformance_eval_checks.py @@ -1,4 +1,4 @@ -"""Phase 2 live conformance check functions. +"""Conformance evaluation live check functions. Each check takes a ``MCPClient`` (and/or the ``tools/list`` response) and returns a ``CheckOutcome`` carrying a three-state result (pass / fail / not_evaluated). @@ -6,7 +6,7 @@ These are pure evaluation logic: the client handles I/O, and the checks analyze responses and return verdicts. The CLI entry point that creates the client and -writes results to disk lives in ``scripts/mcp/probes/phase2_conformance_probe.py``. +writes results to disk lives in ``scripts/mcp/probes/conformance_eval_probe.py``. """ from __future__ import annotations @@ -223,7 +223,7 @@ def check_schema_conformance(client: MCPClient, tools_resp: RpcResponse) -> Chec inputSchema is required by the MCP tool type, so a missing/non-object one is a fail. outputSchema is optional in the MCP spec, so it is validated only when present (its absence is not a violation). Whether actual tool RESPONSES conform - to the declared schema is behavioral and is left to Phase 3. + to the declared schema is behavioral and is left to behavioral evaluation. """ err = _tools_list_error(tools_resp) if err: @@ -283,7 +283,7 @@ def check_tool_annotations(client: MCPClient, tools_resp: RpcResponse) -> CheckO conformance violation: a server that declares none is not_evaluated here (nothing to verify), never failed. Only a *malformed* annotation (a hint field present with a non-boolean value) is a real fail. Honesty of the hints - is behavioral and is left to Phase 3. + is behavioral and is left to behavioral evaluation. """ err = _tools_list_error(tools_resp) if err: @@ -524,7 +524,7 @@ def check_openapi_conformance( def probe_all(server_url: str, *, timeout: float = 30.0, openapi_url: str | None = None) -> list[CheckOutcome]: - """Run every live Phase 2 check against ``server_url``. + """Run every live conformance evaluation check against ``server_url``. If the server is unreachable, all checks are reported not_evaluated (ADR: a readiness/reachability failure must never be scored as a fail). diff --git a/abevalflow/mcp/checks/phase2_compass_checks.py b/abevalflow/mcp/checks/conformance_eval_compass_checks.py similarity index 86% rename from abevalflow/mcp/checks/phase2_compass_checks.py rename to abevalflow/mcp/checks/conformance_eval_compass_checks.py index edde20c8..549d7f60 100644 --- a/abevalflow/mcp/checks/phase2_compass_checks.py +++ b/abevalflow/mcp/checks/conformance_eval_compass_checks.py @@ -1,9 +1,9 @@ -"""Phase 2 Compass-consumed check mapping. +"""Conformance evaluation Compass-consumed check mapping. -A few Phase 2 checks are already collected by Compass as real automated facts, so -the pipeline consumes them rather than re-probing (avoids duplicating what -Compass does well). This module maps those Compass SoundCheck facts into the same -three-state ``CheckOutcome`` the Phase 2 gates read. +A few conformance evaluation checks are already collected by Compass as real +automated facts, so the pipeline consumes them rather than re-probing (avoids +duplicating what Compass does well). This module maps those Compass SoundCheck +facts into the same three-state ``CheckOutcome`` the conformance evaluation gates read. Consumed checks and their source facts:: @@ -26,7 +26,7 @@ results from ``GET /api/soundcheck/results``. The CLI entry point that reads facts from a file or fetches live results from -the SoundCheck API lives in ``scripts/mcp/probes/phase2_compass_probe.py``. +the SoundCheck API lives in ``scripts/mcp/probes/conformance_eval_compass_probe.py``. """ from __future__ import annotations @@ -48,8 +48,8 @@ CHECK_TOOL_NAME = "mcp-tool-schema-check" CHECK_OAUTH = "mcp-oauth-scope-match" -# Maps a SoundCheck check ID to the Phase 2 check name our gates read. -_CHECK_ID_TO_PHASE2 = { +# Maps a SoundCheck check ID to the conformance evaluation check name our gates read. +_CHECK_ID_TO_CONFORMANCE_EVAL = { CHECK_TOOL_NAME: "tool-name-rules", CHECK_OAUTH: "oauth-catalog-match", } @@ -76,7 +76,7 @@ def _bool_outcome(check: str, value: Any, ok_reason: str, bad_reason: str) -> Ch def map_facts(facts: dict[str, Any]) -> list[CheckOutcome]: - """Map Compass facts to the consumed Phase 2 check outcomes.""" + """Map Compass facts to the consumed conformance evaluation check outcomes.""" outcomes: list[CheckOutcome] = [] outcomes.append( @@ -159,7 +159,7 @@ def map_facts(facts: dict[str, Any]) -> list[CheckOutcome]: def map_results(results: list[dict]) -> list[CheckOutcome]: - """Map SoundCheck API check results to Phase 2 check outcomes. + """Map SoundCheck API check results to conformance evaluation check outcomes. ``results`` is the ``results`` array from ``GET /api/soundcheck/results?entityRef=...&checks=...``. Each element has @@ -171,12 +171,12 @@ def map_results(results: list[dict]) -> list[CheckOutcome]: by_check_id = {r["checkId"]: r for r in results if isinstance(r, dict) and "checkId" in r} outcomes: list[CheckOutcome] = [] - for check_id, phase2_name in _CHECK_ID_TO_PHASE2.items(): + for check_id, check_name in _CHECK_ID_TO_CONFORMANCE_EVAL.items(): result = by_check_id.get(check_id) if result is None: outcomes.append( CheckOutcome( - phase2_name, + check_name, STATUS_NOT_EVALUATED, f"No Compass result for {check_id}; entity may not be registered yet.", source="compass", @@ -196,25 +196,23 @@ def map_results(results: list[dict]) -> list[CheckOutcome]: if state == "passed": outcomes.append( - CheckOutcome(phase2_name, STATUS_PASS, notes or f"Compass check {check_id} passed.", source="compass") + CheckOutcome(check_name, STATUS_PASS, notes or f"Compass check {check_id} passed.", source="compass") ) elif state == "failed": outcomes.append( CheckOutcome( - phase2_name, + check_name, STATUS_FAIL, notes or f"Compass check {check_id} failed.", source="compass", - findings=[ - make_finding(severity="high", message=notes or f"{check_id} failed", rule_id=phase2_name) - ], + findings=[make_finding(severity="high", message=notes or f"{check_id} failed", rule_id=check_name)], ) ) else: # "error", "not-applicable", or unknown state outcomes.append( CheckOutcome( - phase2_name, + check_name, STATUS_NOT_EVALUATED, f"Compass check {check_id} returned state '{state}': {notes}".strip(), source="compass", diff --git a/abevalflow/mcp/gates/__init__.py b/abevalflow/mcp/gates/__init__.py index 88751c0d..6e935424 100644 --- a/abevalflow/mcp/gates/__init__.py +++ b/abevalflow/mcp/gates/__init__.py @@ -1,12 +1,12 @@ """MCP evaluation gates — read result JSON, decide pass/fail. -All gates for both phases live here. Phase 1 gates subclass SecurityGate and -read normalized scan JSON. Phase 2 uses a single data-driven gate (Phase2Gate) -that reads three-state check result JSON. +All gates for all evaluation types live here. Static evaluation gates subclass +SecurityGate and read normalized scan JSON. Conformance evaluation uses a single +data-driven gate (ConformanceEvalGate) that reads three-state check result JSON. Orchestrator functions: -- ``run_phase1(reports_dir, policy)`` — evaluates all Phase 1 gates -- ``run_phase2(reports_dir, policy)`` — evaluates all Phase 2 gates +- ``run_static_eval(reports_dir, policy)`` — evaluates all static evaluation gates +- ``run_conformance_eval(reports_dir, policy)`` — evaluates all conformance evaluation gates """ from __future__ import annotations @@ -15,42 +15,42 @@ from abevalflow.gates.base import GateResult from abevalflow.gates.security.base import SecurityGate -from abevalflow.mcp.gates.phase1_license_gate import DEFAULT_ALLOWED_LICENSES, LicenseGate -from abevalflow.mcp.gates.phase1_no_user_code_gate import NoUserCodeGate -from abevalflow.mcp.gates.phase1_secrets_gate import SecretsGate -from abevalflow.mcp.gates.phase2_conformance_gate import Phase2Gate +from abevalflow.mcp.gates.conformance_eval_gate import ConformanceEvalGate +from abevalflow.mcp.gates.static_eval_license_gate import DEFAULT_ALLOWED_LICENSES, LicenseGate +from abevalflow.mcp.gates.static_eval_no_user_code_gate import NoUserCodeGate +from abevalflow.mcp.gates.static_eval_secrets_gate import SecretsGate from abevalflow.schemas import GatePolicy -# ---- Phase 1 --------------------------------------------------------------- +# ---- Static Evaluation ------------------------------------------------------ -# Ordered list of the Phase 1 gate classes, in execution order. -PHASE1_GATES: tuple[type[SecurityGate], ...] = ( +# Ordered list of the static evaluation gate classes, in execution order. +STATIC_EVAL_GATES: tuple[type[SecurityGate], ...] = ( SecretsGate, NoUserCodeGate, LicenseGate, ) -def get_phase1_gates() -> list[SecurityGate]: - """Instantiate all Phase 1 gates, in order.""" - return [gate_cls() for gate_cls in PHASE1_GATES] +def get_static_eval_gates() -> list[SecurityGate]: + """Instantiate all static evaluation gates, in order.""" + return [gate_cls() for gate_cls in STATIC_EVAL_GATES] -def run_phase1(reports_dir: Path, policy: GatePolicy) -> list[GateResult]: - """Evaluate all Phase 1 gates against the scan reports. +def run_static_eval(reports_dir: Path, policy: GatePolicy) -> list[GateResult]: + """Evaluate all static evaluation gates against the scan reports. Args: reports_dir: Path to ``reports/{submission-name}/`` holding the - normalized scan JSON files written by the Phase 1 scanners. + normalized scan JSON files written by the static evaluation scanners. policy: Gate policy controlling mode (disabled/warn/block) per gate. Returns: - One GateResult per Phase 1 gate, in :data:`PHASE1_GATES` order. + One GateResult per static evaluation gate, in :data:`STATIC_EVAL_GATES` order. """ - return [gate.evaluate(reports_dir, policy) for gate in get_phase1_gates()] + return [gate.evaluate(reports_dir, policy) for gate in get_static_eval_gates()] -# ---- Phase 2 --------------------------------------------------------------- +# ---- Conformance Evaluation ------------------------------------------------- # Live probe checks (need a running server). LIVE_CHECKS: tuple[str, ...] = ( @@ -70,31 +70,31 @@ def run_phase1(reports_dir: Path, policy: GatePolicy) -> list[GateResult]: "oauth-catalog-match", ) -PHASE2_CHECKS: tuple[str, ...] = LIVE_CHECKS + COMPASS_CHECKS +CONFORMANCE_EVAL_CHECKS: tuple[str, ...] = LIVE_CHECKS + COMPASS_CHECKS -def get_phase2_gates() -> list[Phase2Gate]: - """Instantiate one gate per Phase 2 check, in order.""" - return [Phase2Gate(check) for check in PHASE2_CHECKS] +def get_conformance_eval_gates() -> list[ConformanceEvalGate]: + """Instantiate one gate per conformance evaluation check, in order.""" + return [ConformanceEvalGate(check) for check in CONFORMANCE_EVAL_CHECKS] -def run_phase2(reports_dir: Path, policy: GatePolicy) -> list[GateResult]: - """Evaluate every Phase 2 gate against the check result files in ``reports_dir``.""" - return [gate.evaluate(reports_dir, policy) for gate in get_phase2_gates()] +def run_conformance_eval(reports_dir: Path, policy: GatePolicy) -> list[GateResult]: + """Evaluate every conformance evaluation gate against the check result files in ``reports_dir``.""" + return [gate.evaluate(reports_dir, policy) for gate in get_conformance_eval_gates()] __all__ = [ "COMPASS_CHECKS", + "CONFORMANCE_EVAL_CHECKS", + "ConformanceEvalGate", "DEFAULT_ALLOWED_LICENSES", "LIVE_CHECKS", "LicenseGate", "NoUserCodeGate", - "PHASE1_GATES", - "PHASE2_CHECKS", - "Phase2Gate", + "STATIC_EVAL_GATES", "SecretsGate", - "get_phase1_gates", - "get_phase2_gates", - "run_phase1", - "run_phase2", + "get_conformance_eval_gates", + "get_static_eval_gates", + "run_conformance_eval", + "run_static_eval", ] diff --git a/abevalflow/mcp/gates/phase2_conformance_gate.py b/abevalflow/mcp/gates/conformance_eval_gate.py similarity index 86% rename from abevalflow/mcp/gates/phase2_conformance_gate.py rename to abevalflow/mcp/gates/conformance_eval_gate.py index 28383307..af622001 100644 --- a/abevalflow/mcp/gates/phase2_conformance_gate.py +++ b/abevalflow/mcp/gates/conformance_eval_gate.py @@ -1,12 +1,12 @@ -"""Phase 2 gate: turn a three-state check result file into a GateResult. +"""Conformance evaluation gate: turn a three-state check result file into a GateResult. -Unlike the Phase 1 security gates (one class per gate, reusing -``SecurityGate.evaluate_scan_json``), Phase 2 has many small, uniform checks that -differ only by name and by a three-state status, so a single data-driven gate is -used instead of a subclass per check. +Unlike the static evaluation security gates (one class per gate, reusing +``SecurityGate.evaluate_scan_json``), conformance evaluation has many small, uniform +checks that differ only by name and by a three-state status, so a single data-driven +gate is used instead of a subclass per check. -Each check result file (written by scripts/mcp/probes/phase2_conformance_probe.py or phase2_compass_probe.py) -has the shape:: +Each check result file (written by scripts/mcp/probes/conformance_eval_probe.py or +conformance_eval_compass_probe.py) has the shape:: {"check": ..., "status": "pass"|"fail"|"not_evaluated", "source": ..., "reason": ..., "findings": [ ... ]} @@ -46,8 +46,8 @@ def _weighted_score(findings: list[Finding]) -> float: return sum(_SEVERITY_WEIGHT[f.severity] for f in findings) / len(findings) -class Phase2Gate: - """Data-driven gate for one Phase 2 check.""" +class ConformanceEvalGate: + """Data-driven gate for one conformance evaluation check.""" def __init__(self, check: str) -> None: self.check = check @@ -99,9 +99,9 @@ def evaluate(self, reports_dir: Path, policy: GatePolicy) -> GateResult: # A corrupt result file is a real pipeline defect, so log it loudly and # score 0.0 - but per the three-state model a "couldn't read" must not # block outside block mode (mirrors the fail branch below). This is a - # deliberate Phase 2 choice (warn never blocks, even on a corrupt file; + # deliberate conformance evaluation choice (warn never blocks, even on a corrupt file; # see test_corrupt_result_file_blocks_only_in_block_mode) and differs - # from Phase 1's security gates, which fail closed regardless of mode. + # from static evaluation's security gates, which fail closed regardless of mode. logger.error("Failed to read %s: %s", path, exc) return self._result( passed=mode != GateMode.BLOCK, diff --git a/abevalflow/mcp/gates/phase1_license_gate.py b/abevalflow/mcp/gates/static_eval_license_gate.py similarity index 91% rename from abevalflow/mcp/gates/phase1_license_gate.py rename to abevalflow/mcp/gates/static_eval_license_gate.py index fe984eba..1c4fdc04 100644 --- a/abevalflow/mcp/gates/phase1_license_gate.py +++ b/abevalflow/mcp/gates/static_eval_license_gate.py @@ -1,11 +1,11 @@ -"""Phase 1 license-compliance gate. +"""Static evaluation license-compliance gate. Reads license-scan.json produced by scripts/mcp/run_license_scan.py (a licensee run compared against the approved-license allow-list) and converts it into a standardized GateResult. Fails in block mode when the top-level declared license is missing or not in the allow-list (emitted as a HIGH finding by the scanner). -ADR Phase 1: "License compliance - verify the MCP server's top-level declared +ADR static evaluation: "License compliance - verify the MCP server's top-level declared license against a list of approved/supported licenses ... Scoped to the top-level declared license only; transitive dependency license scanning is out of scope." diff --git a/abevalflow/mcp/gates/phase1_no_user_code_gate.py b/abevalflow/mcp/gates/static_eval_no_user_code_gate.py similarity index 88% rename from abevalflow/mcp/gates/phase1_no_user_code_gate.py rename to abevalflow/mcp/gates/static_eval_no_user_code_gate.py index 20dcef30..cb68cf71 100644 --- a/abevalflow/mcp/gates/phase1_no_user_code_gate.py +++ b/abevalflow/mcp/gates/static_eval_no_user_code_gate.py @@ -1,4 +1,4 @@ -"""Phase 1 no-user-code-execution gate. +"""Static evaluation no-user-code-execution gate. Reads no-user-code-scan.json produced by scripts/mcp/run_no_user_code_scan.py (a normalized semgrep run) and converts it into a standardized GateResult. Fails @@ -6,7 +6,7 @@ present (e.g. eval/exec, os.system, subprocess shell=True, Function(), Runtime.exec). -ADR Phase 1: "Does not execute user-provided code - static analysis of the +ADR static evaluation: "Does not execute user-provided code - static analysis of the artifact for dynamic/arbitrary code execution." """ diff --git a/abevalflow/mcp/gates/phase1_secrets_gate.py b/abevalflow/mcp/gates/static_eval_secrets_gate.py similarity index 86% rename from abevalflow/mcp/gates/phase1_secrets_gate.py rename to abevalflow/mcp/gates/static_eval_secrets_gate.py index 529944d0..416011c5 100644 --- a/abevalflow/mcp/gates/phase1_secrets_gate.py +++ b/abevalflow/mcp/gates/static_eval_secrets_gate.py @@ -1,11 +1,11 @@ -"""Phase 1 secrets-management gate. +"""Static evaluation secrets-management gate. Reads secrets-scan.json produced by scripts/mcp/run_secrets_scan.py (a normalized gitleaks run) and converts it into a standardized GateResult. Fails in block mode when any HIGH/CRITICAL hardcoded-credential finding is present. -ADR Phase 1: "Secrets management - static scan of the artifact for hardcoded -credentials." +ADR static evaluation: "Secrets management - static scan of the artifact for +hardcoded credentials." """ from __future__ import annotations diff --git a/abevalflow/mcp/helpers.py b/abevalflow/mcp/helpers.py index f2131357..5b4eff7c 100644 --- a/abevalflow/mcp/helpers.py +++ b/abevalflow/mcp/helpers.py @@ -1,7 +1,7 @@ """Helper functions for the MCP evaluation pipeline. Finding builders, severity normalization, and check-result file I/O. Used by -both Phase 1 scanners (via ``make_finding``) and Phase 2 checks/probes (via +both static evaluation scanners (via ``make_finding``) and conformance evaluation checks/probes (via ``make_finding`` + ``write_check_result``). """ diff --git a/abevalflow/mcp/mcp_client.py b/abevalflow/mcp/mcp_client.py index 09bf186a..d3d7edd3 100644 --- a/abevalflow/mcp/mcp_client.py +++ b/abevalflow/mcp/mcp_client.py @@ -1,7 +1,7 @@ -"""Minimal MCP client over Streamable HTTP for Phase 2 conformance probing. +"""Minimal MCP client over Streamable HTTP for conformance evaluation probing. Hand-rolled JSON-RPC 2.0 over ``httpx`` (deliberately NOT the ``mcp`` SDK) so the -Phase 2 probe can: +conformance evaluation probe can: * send well-formed calls (initialize, tools/list, tools/call), and * send deliberately malformed / spec-violating payloads and inspect the raw @@ -34,7 +34,7 @@ DEFAULT_PROTOCOL_VERSION = "2025-06-18" ACCEPT_HEADER = "application/json, text/event-stream" -CLIENT_INFO = {"name": "mcp-phase2-probe", "version": "0.1"} +CLIENT_INFO = {"name": "mcp-conformance-eval-probe", "version": "0.1"} @dataclass diff --git a/abevalflow/mcp/types.py b/abevalflow/mcp/types.py index fa6189b9..213e0ba8 100644 --- a/abevalflow/mcp/types.py +++ b/abevalflow/mcp/types.py @@ -1,11 +1,11 @@ """Core types for the MCP evaluation pipeline. -Defines the data shapes both phases use: +Defines the data shapes all evaluations use: -- **Phase 2** check outcomes: ``CheckOutcome`` carries a three-state result - (pass / fail / not_evaluated) that the Phase 2 gate reads. +- **Conformance evaluation** check outcomes: ``CheckOutcome`` carries a three-state result + (pass / fail / not_evaluated) that the conformance evaluation gate reads. -Phase 2 uses the three-state model because a check that cannot produce a +Conformance evaluation uses the three-state model because a check that cannot produce a meaningful signal (server unreachable, no OpenAPI contract, a capability the server does not expose) must report ``not_evaluated`` rather than a false pass or fail (ADR: three-state reporting). @@ -20,7 +20,7 @@ from typing import Any # --------------------------------------------------------------------------- -# Phase 2 three-state model +# Conformance evaluation three-state model # --------------------------------------------------------------------------- STATUS_PASS = "pass" @@ -31,7 +31,7 @@ @dataclass class CheckOutcome: - """Result of a single Phase 2 check.""" + """Result of a single conformance evaluation check.""" check: str status: str diff --git a/containers/mcp-eval/Containerfile b/containers/mcp-eval/Containerfile index 152e29e0..dadfbeea 100644 --- a/containers/mcp-eval/Containerfile +++ b/containers/mcp-eval/Containerfile @@ -1,26 +1,28 @@ -# MCP server evaluation image for ABEvalFlow Phase 1 (static) + Phase 2 (conformance). +# MCP server evaluation image for ABEvalFlow. # -# Dedicated image so the two MCP Tekton tasks no longer clone the pipeline repo or -# install tools at runtime. Nothing else pulls this image, so changes here cannot -# affect the shared eval-base pipelines. +# Single image for the full MCP evaluation pipeline: static analysis, +# deterministic conformance testing, and behavioral evaluation (mcpchecker). # # Bakes: # - the MCP eval code and its full import closure (abevalflow.mcp / gates / # schemas / observability, and scripts.mcp) importable via PYTHONPATH=/opt # - the scanners/probes it shells out to: gitleaks, semgrep, licensee # - the minimal Python deps the code imports: pydantic, pyyaml, httpx +# - mcpchecker binary + Node.js (copied from mcpchecker-runner) +# - default behavioral task suite (suites/mcp/default) +# - mcpchecker result aggregation script +# - envsubst for runtime MCP_URL injection # # Build from repo root (COPY needs repo-root context): # podman build -t quay.io/ecosystem-appeng/abevalflow-mcp-eval:0.1 \ # -f containers/mcp-eval/Containerfile . -# Base tracks :latest (same as Dockerfile.base / agent-eval-harness) so UBI9 CVE -# patches keep flowing. The tools and Python deps that affect runtime behavior are -# pinned below, which is where version drift would actually break the eval code. +# ── Stage 1: mcpchecker-runner (source for mcpchecker binary + Node.js) ── +FROM quay.io/ecosystem-appeng/mcpchecker-runner:latest AS mcpchecker-runner + +# ── Stage 2: Runtime image ── FROM registry.access.redhat.com/ubi9/python-312:latest -# Pinned tool + Python dep versions (avoids the version drift called out in review; -# e.g. an unpinned pydantic v3 would break the schemas the eval code relies on). ARG GITLEAKS_VERSION=8.30.0 ARG SEMGREP_VERSION=1.179.0 ARG LICENSEE_VERSION=9.18.0 @@ -30,6 +32,10 @@ ARG HTTPX_VERSION=0.28.1 USER 0 +# Copy mcpchecker binary from mcpchecker-runner +COPY --from=mcpchecker-runner /usr/local/bin/mcpchecker /usr/local/bin/mcpchecker +RUN chmod +x /usr/local/bin/mcpchecker + # MCP eval code — only the verified import closure (abevalflow.mcp pulls in # gates, schemas, observability; scripts.mcp are the Tekton entry points). COPY abevalflow/__init__.py /opt/abevalflow/__init__.py @@ -40,12 +46,24 @@ COPY abevalflow/observability /opt/abevalflow/observability COPY scripts/__init__.py /opt/scripts/__init__.py COPY scripts/mcp /opt/scripts/mcp +# Behavioral eval: aggregation script + task suites +COPY abevalflow/mcpchecker_report.py /opt/abevalflow/mcpchecker_report.py +COPY abevalflow/report.py /opt/abevalflow/report.py +COPY scripts/aggregate_mcpchecker.py /opt/scripts/aggregate_mcpchecker.py +COPY suites/mcp/default /opt/suites/mcp/default + +# Kubernetes Job templates for conformance and behavioral evaluation +COPY templates/mcp-conformance-eval-job.yaml /opt/templates/mcp-conformance-eval-job.yaml +COPY templates/mcp-behavioral-eval-job.yaml /opt/templates/mcp-behavioral-eval-job.yaml + ENV PYTHONPATH=/opt -# Scanners/probes (shelled out to, not imported) + minimal Python deps (imported). +# Node.js (mcpchecker MCP protocol support), envsubst (runtime config injection), +# Python deps. # licensee's native dep (rugged -> libgit2) needs a build toolchain to compile; # install it, build the gem, then drop the toolchain to keep the image lean. -RUN curl -LsSf \ +RUN dnf install -y nodejs npm gettext --nodocs \ + && curl -LsSf \ "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" \ | tar -xz -C /usr/local/bin gitleaks \ && pip install --no-cache-dir \ diff --git a/containers/mcpchecker/Containerfile b/containers/mcpchecker/Containerfile index b0cd3d77..e5b080f2 100644 --- a/containers/mcpchecker/Containerfile +++ b/containers/mcpchecker/Containerfile @@ -9,7 +9,7 @@ # Push: podman push quay.io/rhecosystemappeng/mcpchecker-runner:latest # Stage 1: Build mcpchecker from source -FROM golang:1.22-alpine AS builder +FROM golang:1.26-alpine AS builder RUN apk add --no-cache git diff --git a/pipeline/integration/Makefile b/pipeline/integration/Makefile index 0fa5fc42..7bf4e295 100644 --- a/pipeline/integration/Makefile +++ b/pipeline/integration/Makefile @@ -3,7 +3,8 @@ VERSION ?= 0.1 TASKS_DIR := ../tasks/konflux TASKS = parse-snapshot prepare test red-team evaluate analyze-scorecard store emit-result \ - create-namespace deploy-mcp wait-ready teardown-namespace + create-namespace deploy-mcp wait-ready teardown-namespace \ + mcp-static-eval mcp-conformance-eval mcp-behavioral-eval check-label .PHONY: bundles login list clean help diff --git a/pipeline/integration/konflux-mcp-eval-pipelinerun.yaml b/pipeline/integration/konflux-mcp-eval-pipelinerun.yaml index ed54a8c9..e3fcc1fb 100644 --- a/pipeline/integration/konflux-mcp-eval-pipelinerun.yaml +++ b/pipeline/integration/konflux-mcp-eval-pipelinerun.yaml @@ -8,7 +8,9 @@ # the pipeline tasks to reach the target cluster. # # Pipeline stages: -# parse-snapshot → fetch-mcp-repo → create-namespace → deploy-mcp → wait-ready +# parse-snapshot → fetch-mcp-repo → mcp-static-eval +# parse-snapshot → create-namespace → deploy-mcp → wait-ready +# → mcp-conformance-eval → mcp-behavioral-eval # finally: teardown-namespace # # PREREQUISITES: @@ -18,7 +20,11 @@ # bound to the ServiceAccount referenced in the kubeconfig # 3. (Optional) Credentials Secret on the target cluster if the MCP # server requires backend auth — see CREDENTIALS_SECRET param -# 4. VPN connectivity between Konflux cluster and target cluster +# 4. Secrets "mcp-eval-llm-agent" and "mcp-eval-llm-judge" in the +# default namespace on the TARGET cluster, each containing api-key +# and endpoint for the respective LLM model (behavioral +# evaluation). +# 5. VPN connectivity between Konflux cluster and target cluster apiVersion: tekton.dev/v1 kind: PipelineRun metadata: @@ -83,6 +89,39 @@ spec: Name of the image pull Secret in the Konflux namespace to copy into the ephemeral namespace for registry authentication. + # === MCP Behavioral evaluation === + - name: JOB_TIMEOUT + type: string + default: "1800" + description: Timeout in seconds for the mcpchecker Job + - name: PASS_THRESHOLD + type: string + default: "0.7" + description: Minimum overall score (0.0–1.0) for a pass recommendation + - name: LLM_AGENT_SECRET + type: string + default: "mcp-eval-llm-agent" + description: >- + Name of the Secret on the target cluster (in the default + namespace) containing the agent LLM api-key, endpoint, and model. + - name: LLM_JUDGE_SECRET + type: string + default: "mcp-eval-llm-judge" + description: >- + Name of the Secret on the target cluster (in the default + namespace) containing the judge LLM model name. Credentials + come from the agent secret. + - name: MCP_EVAL_IMAGE + type: string + default: "quay.io/ecosystem-appeng/abevalflow-mcp-eval:0.1" + description: >- + MCP eval image with mcpchecker binary, task suite files, and + aggregation scripts baked in. + - name: MCPCHECKER_LABEL + type: string + default: "mcpchecker" + description: PR label that enables the MCP behavioral evaluation task + workspaces: - name: shared-workspace - name: target-kubeconfig @@ -134,7 +173,32 @@ spec: subPath: _mcp-server-repo # ================================================================ - # Stage 3: Create ephemeral namespace on target cluster + # Stage 3: MCP Static Evaluation + # ================================================================ + - name: mcp-static-eval + runAfter: [fetch-mcp-repo] + taskRef: + resolver: bundles + params: + - name: name + value: mcp-static-eval + - name: bundle + value: quay.io/ecosystem-appeng/abevalflow-task-mcp-static-eval:0.1 + - name: kind + value: task + params: + - name: mcp-eval-image + value: $(params.MCP_EVAL_IMAGE) + - name: source-subdir + value: "_mcp-server-repo" + - name: mode + value: "block" + workspaces: + - name: source + workspace: shared-workspace + + # ================================================================ + # Stage 4: Create ephemeral namespace on target cluster # ================================================================ - name: create-namespace runAfter: [parse-snapshot] @@ -157,7 +221,7 @@ spec: workspace: target-kubeconfig # ================================================================ - # Stage 4: Deploy MCP server into ephemeral namespace + # Stage 5: Deploy MCP server into ephemeral namespace # ================================================================ - name: deploy-mcp runAfter: [fetch-mcp-repo, create-namespace] @@ -198,7 +262,7 @@ spec: workspace: target-kubeconfig # ================================================================ - # Stage 5: Wait for MCP server readiness + # Stage 6: Wait for MCP server readiness # ================================================================ - name: wait-ready runAfter: [deploy-mcp] @@ -220,6 +284,106 @@ spec: - name: target-kubeconfig workspace: target-kubeconfig + # ================================================================ + # Stage 7: MCP Conformance Evaluation + # ================================================================ + - name: mcp-conformance-eval + runAfter: [wait-ready] + taskRef: + resolver: bundles + params: + - name: name + value: mcp-conformance-eval + - name: bundle + value: quay.io/ecosystem-appeng/abevalflow-task-mcp-conformance-eval:0.1 + - name: kind + value: task + params: + - name: mcp-eval-image + value: $(params.MCP_EVAL_IMAGE) + - name: namespace-name + value: $(tasks.create-namespace.results.namespace-name) + - name: service-name + value: $(params.SERVICE_NAME) + - name: service-port + value: $(params.SERVICE_PORT) + - name: mode + value: "block" + - name: timeout + value: "30" + workspaces: + - name: source + workspace: shared-workspace + - name: target-kubeconfig + workspace: target-kubeconfig + + # ================================================================ + # Stage 7b: Check whether the gating PR label is present + # ================================================================ + - name: check-label + runAfter: [parse-snapshot] + taskRef: + resolver: bundles + params: + - name: name + value: check-label + - name: bundle + value: quay.io/ecosystem-appeng/abevalflow-task-check-label:0.1 + - name: kind + value: task + params: + - name: git-url + value: $(tasks.parse-snapshot.results.git-url) + - name: git-revision + value: $(tasks.parse-snapshot.results.git-revision) + - name: target-label + value: $(params.MCPCHECKER_LABEL) + + # ================================================================ + # Stage 8: MCP Behavioral Evaluation (mcpchecker) + # ================================================================ + - name: mcp-behavioral-eval + runAfter: [wait-ready, check-label] + when: + - input: "$(tasks.check-label.results.has-label)" + operator: in + values: ["true"] + taskRef: + resolver: bundles + params: + - name: name + value: mcp-behavioral-eval + - name: bundle + value: quay.io/ecosystem-appeng/abevalflow-task-mcp-behavioral-eval:0.1 + - name: kind + value: task + params: + - name: namespace-name + value: $(tasks.create-namespace.results.namespace-name) + - name: service-name + value: $(params.SERVICE_NAME) + - name: service-port + value: $(params.SERVICE_PORT) + - name: job-timeout + value: $(params.JOB_TIMEOUT) + - name: pass-threshold + value: $(params.PASS_THRESHOLD) + - name: llm-agent-secret + value: $(params.LLM_AGENT_SECRET) + - name: llm-judge-secret + value: $(params.LLM_JUDGE_SECRET) + - name: mcp-eval-image + value: $(params.MCP_EVAL_IMAGE) + - name: pipeline-run-id + value: $(context.pipelineRun.name) + - name: submission-name + value: $(tasks.parse-snapshot.results.component-name) + workspaces: + - name: source + workspace: shared-workspace + - name: target-kubeconfig + workspace: target-kubeconfig + # ================================================================ # Finally: Teardown ephemeral namespace (always runs) # ================================================================ diff --git a/pipeline/tasks/konflux/check-label.yaml b/pipeline/tasks/konflux/check-label.yaml new file mode 100644 index 00000000..37e3bde4 --- /dev/null +++ b/pipeline/tasks/konflux/check-label.yaml @@ -0,0 +1,91 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: check-label + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + Checks whether a target PR label is present on the pull/merge request + associated with a given git commit. Supports both GitHub and GitLab. + Derives the API endpoint from the git-url host. Returns "true" if the + label is found, "false" otherwise. + params: + - name: git-url + type: string + description: Git repository URL (e.g. https://github.com/owner/repo) + - name: git-revision + type: string + description: Git commit SHA to look up associated PRs/MRs for + - name: target-label + type: string + default: "mcpchecker" + description: PR/MR label name to check for + results: + - name: has-label + description: "'true' if target-label appears on the PR/MR, 'false' otherwise" + steps: + - name: check + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + env: + - name: GIT_URL + value: $(params.git-url) + - name: REVISION + value: $(params.git-revision) + - name: TARGET_LABEL + value: $(params.target-label) + - name: GIT_TOKEN + valueFrom: + secretKeyRef: + name: git-secret + key: password + optional: true + script: | + #!/usr/bin/env bash + set -euo pipefail + + microdnf install -y jq --nodocs + + GIT_HOST=$(echo "$GIT_URL" | sed -E 's|https?://([^/]+)/.*|\1|') + REPO_PATH=$(echo "$GIT_URL" | sed -E 's|https?://[^/]+/||; s|\.git$||; s|/$||') + + echo "Checking labels for $REPO_PATH @ $REVISION (host: $GIT_HOST)" + echo "Looking for label: $TARGET_LABEL" + + if [ -z "$GIT_TOKEN" ]; then + echo "WARNING: GIT_TOKEN not available, returning false" + RESULT="false" + elif echo "$GIT_HOST" | grep -qi "github"; then + if [ "$GIT_HOST" = "github.com" ]; then + API_BASE="https://api.github.com" + else + API_BASE="https://${GIT_HOST}/api/v3" + fi + + RESPONSE=$(curl -sSfk \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GIT_TOKEN" \ + "${API_BASE}/repos/${REPO_PATH}/commits/${REVISION}/pulls") + + RESULT=$(echo "$RESPONSE" | jq -r --arg targetlabel "$TARGET_LABEL" \ + '[.[].labels[].name] | if any(. == $targetlabel) then "true" else "false" end') + + elif echo "$GIT_HOST" | grep -qi "gitlab"; then + ENCODED_PATH=$(echo "$REPO_PATH" | jq -Rr @uri) + + RESPONSE=$(curl -sSfk \ + -H "PRIVATE-TOKEN: $GIT_TOKEN" \ + "https://${GIT_HOST}/api/v4/projects/${ENCODED_PATH}/repository/commits/${REVISION}/merge_requests") + + RESULT=$(echo "$RESPONSE" | jq -r --arg targetlabel "$TARGET_LABEL" \ + '[.[].labels[]] | if any(. == $targetlabel) then "true" else "false" end') + + else + echo "ERROR: Unsupported Git platform: $GIT_HOST" + exit 1 + fi + + + echo "Result: $RESULT" + echo -n "$RESULT" > "$(results.has-label.path)" diff --git a/pipeline/tasks/konflux/create-namespace.yaml b/pipeline/tasks/konflux/create-namespace.yaml index 02989404..7470d370 100644 --- a/pipeline/tasks/konflux/create-namespace.yaml +++ b/pipeline/tasks/konflux/create-namespace.yaml @@ -7,10 +7,9 @@ metadata: app.kubernetes.io/component: konflux spec: description: >- - Creates an ephemeral, isolated namespace for MCP server evaluation - on a remote target cluster. Applies security guardrails: NetworkPolicy - (default deny ingress and egress with scoped DNS and backend allowances), - ResourceQuota, LimitRange, and restricted pod security standards. + Creates an ephemeral namespace for MCP server evaluation on a remote + target cluster and grants the pipeline ServiceAccount access through a + RoleBinding. Namespace security restrictions are currently disabled. The namespace is named mcp-eval- and is intended to be cleaned up by the teardown-namespace task in the finally block. params: @@ -21,11 +20,8 @@ spec: type: string default: "" description: >- - Comma-separated list of host:port pairs for backend endpoints - that the MCP server needs to reach. Hostnames are resolved to - all known IPs at pipeline time (NetworkPolicy requires IPs). - For stable endpoints, prefer providing IPs directly. - Example: "10.0.1.5:443,api.internal:5432" + Retained for pipeline compatibility. NetworkPolicy restrictions are + currently disabled and this parameter is not used. workspaces: - name: target-kubeconfig description: >- @@ -59,166 +55,28 @@ spec: app.kubernetes.io/part-of: abevalflow abevalflow/type: mcp-eval abevalflow/pipeline-run: "$LABEL_RUN_ID" - pod-security.kubernetes.io/enforce: restricted - pod-security.kubernetes.io/warn: restricted - pod-security.kubernetes.io/audit: restricted EOF echo "Namespace created" - # Default deny all ingress cat </dev/null | awk '{print $1}' | sort -u || echo "") - if [ -z "$IPS" ]; then - echo " WARNING: Could not resolve $HOST — skipping" - continue - fi - for IP in $IPS; do - EGRESS_RULES="${EGRESS_RULES} - - to: - - ipBlock: - cidr: ${IP}/32 - ports: - - protocol: TCP - port: ${PORT}" - echo " Allowing egress to $HOST ($IP):$PORT" - done - done - - if [ -n "$EGRESS_RULES" ]; then - cat <- + Behavioral evaluation of an MCP server using mcpchecker. + Runs mcpchecker as a Kubernetes Job on the target cluster so it + can reach the deployed MCP service directly. The generic default + task suite tests tool correctness, edge-case handling, and + sensitive data redaction — applicable to any MCP server. + Suite files and aggregation code are baked into the mcp-eval image. + params: + - name: namespace-name + type: string + description: Ephemeral namespace where the MCP server is deployed + - name: service-name + type: string + default: "mcp-server" + description: Kubernetes Service name of the deployed MCP server + - name: service-port + type: string + default: "8080" + description: Port on which the MCP server listens + - name: job-timeout + type: string + default: "1800" + description: Timeout in seconds for the mcpchecker Job + - name: pass-threshold + type: string + default: "0.7" + description: >- + Minimum overall score (0.0–1.0) for a pass recommendation. + Default 0.7 per ADR specification. + - name: llm-agent-secret + type: string + default: "mcp-eval-llm-agent" + description: >- + Name of the Secret on the target cluster (in the default namespace) + containing LLM credentials. Keys: openai-api-key, openai-base-url, + anthropic-api-key, anthropic-base-url (fill only the active + provider), and model-name (e.g. "openai:gpt-4o"). Copied into + the ephemeral namespace. + - name: llm-judge-secret + type: string + default: "mcp-eval-llm-judge" + description: >- + Name of the Secret on the target cluster (in the default namespace) + containing the judge LLM model name. Only the model key is used; + credentials come from the agent secret (mcpchecker shares provider + env vars between agent and judge). + - name: llm-ca-secret + type: string + default: "mcp-eval-llm-ca" + description: >- + Name of the Secret on the target cluster (in the default namespace) + containing the CA certificate bundle for internal LLM endpoints. + Key: ca.crt. Mounted into the mcpchecker container trust store. + Optional — if the secret does not exist, TLS uses default system CAs. + - name: mcp-eval-image + type: string + default: "quay.io/ecosystem-appeng/abevalflow-mcp-eval:0.1" + description: >- + MCP eval image with mcpchecker binary, task suite files, and + aggregation scripts baked in. + - name: job-template + type: string + default: "/opt/templates/mcp-behavioral-eval-job.yaml" + description: Path to the Kubernetes Job template for behavioral evaluation + - name: pipeline-run-id + type: string + description: Pipeline run ID for provenance tracking + - name: submission-name + type: string + description: Name of the MCP component being evaluated (from the Konflux snapshot) + workspaces: + - name: source + description: Shared workspace for storing results + - name: target-kubeconfig + description: >- + Kubeconfig for the target cluster. Must contain a file named + "kubeconfig". + results: + - name: eval-passed + description: Whether MCP behavioral evaluation passed (true/false/error). + - name: report-path + description: Workspace-relative path to the MCP behavioral evaluation reports directory. + steps: + # ── Step 1: Copy LLM secrets to ephemeral namespace ── + - name: setup + image: registry.redhat.io/openshift4/ose-cli:latest + env: + - name: KUBECONFIG + value: $(workspaces.target-kubeconfig.path)/kubeconfig + script: | + #!/usr/bin/env bash + set -euo pipefail + + NAMESPACE="$(params.namespace-name)" + + for SECRET_NAME in "$(params.llm-agent-secret)" "$(params.llm-judge-secret)" "$(params.llm-ca-secret)"; do + if [ -n "$SECRET_NAME" ]; then + oc get secret "$SECRET_NAME" -n default -o yaml \ + | sed '/^\s*namespace:/d; /^\s*uid:/d; /^\s*resourceVersion:/d; /^\s*creationTimestamp:/d' \ + | oc apply -n "$NAMESPACE" -f - || { + echo "WARNING: Could not copy secret '$SECRET_NAME'" + } + fi + done + + # ── Step 2: Prepare Job manifest from template ── + - name: prepare-job + image: $(params.mcp-eval-image) + imagePullPolicy: Always + env: + - name: NAMESPACE + value: $(params.namespace-name) + - name: JOB_NAME + value: mcpchecker-eval + - name: IMAGE + value: $(params.mcp-eval-image) + - name: JOB_TIMEOUT + value: $(params.job-timeout) + - name: MCP_URL + value: http://$(params.service-name):$(params.service-port) + - name: AGENT_SECRET + value: $(params.llm-agent-secret) + - name: JUDGE_SECRET + value: $(params.llm-judge-secret) + - name: CA_SECRET + value: $(params.llm-ca-secret) + - name: JOB_TEMPLATE + value: $(params.job-template) + script: | + #!/usr/bin/env bash + set -euo pipefail + + envsubst \ + '${JOB_NAME} ${NAMESPACE} ${JOB_TIMEOUT} ${IMAGE} ${MCP_URL} ${AGENT_SECRET} ${JUDGE_SECRET} ${CA_SECRET}' \ + < "$JOB_TEMPLATE" \ + > "$(workspaces.source.path)/behavioral-eval-job.yaml" + + echo "Generated Job manifest:" + cat "$(workspaces.source.path)/behavioral-eval-job.yaml" + + # ── Step 3: Apply Job and collect results ── + - name: run-mcpchecker-job + image: registry.redhat.io/openshift4/ose-cli:latest + env: + - name: KUBECONFIG + value: $(workspaces.target-kubeconfig.path)/kubeconfig + - name: NAMESPACE + value: $(params.namespace-name) + - name: JOB_NAME + value: mcpchecker-eval + - name: JOB_TIMEOUT + value: $(params.job-timeout) + script: | + #!/usr/bin/env bash + set -euo pipefail + + # Apply the Job + oc apply -f "$(workspaces.source.path)/behavioral-eval-job.yaml" --namespace="$NAMESPACE" + + # Wait for Job completion + TIMEOUT=$JOB_TIMEOUT + DEADLINE=$((SECONDS + TIMEOUT)) + + while true; do + STATUS=$(oc get job "$JOB_NAME" -n "$NAMESPACE" \ + -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' 2>/dev/null || true) + FAILED=$(oc get job "$JOB_NAME" -n "$NAMESPACE" \ + -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true) + + if [ "$STATUS" = "True" ]; then + echo "Job completed" + break + fi + if [ "$FAILED" = "True" ]; then + echo "WARNING: Job failed, collecting logs" + break + fi + if [ $SECONDS -ge $DEADLINE ]; then + echo "ERROR: Job timed out after ${TIMEOUT}s" + break + fi + sleep 10 + done + + # Collect Job output + RESULTS_DIR="$(workspaces.source.path)/results" + mkdir -p "$RESULTS_DIR" + + RAW_LOGS="$RESULTS_DIR/$JOB_NAME-logs.txt" + oc logs "job/$JOB_NAME" -n "$NAMESPACE" > "$RAW_LOGS" 2>&1 || { + echo '{"error": "Failed to collect mcpchecker logs", "results": []}' > "$RAW_LOGS" + } + + OUTPUT="$RESULTS_DIR/$JOB_NAME-output.json" + if grep -n '^{' "$RAW_LOGS" > /dev/null 2>&1; then + START_LINE=$(grep -n '^{' "$RAW_LOGS" | head -1 | cut -d: -f1) + tail -n +$START_LINE "$RAW_LOGS" > "$OUTPUT" + else + echo '{"error": "No JSON in mcpchecker output", "results": []}' > "$OUTPUT" + fi + + if ! python3 -c "import json; json.load(open('$OUTPUT'))" 2>/dev/null; then + echo '{"error": "Invalid JSON from mcpchecker", "results": []}' > "$OUTPUT" + fi + + # ── Step 4: Aggregate results and write Tekton results ── + - name: aggregate-results + image: $(params.mcp-eval-image) + imagePullPolicy: Always + env: + - name: JOB_NAME + value: mcpchecker-eval + script: | + #!/usr/bin/env bash + set -euo pipefail + + RESULTS_DIR="$(workspaces.source.path)/results" + REPORTS_DIR="$(workspaces.source.path)/reports" + OUTPUT_FILE="$RESULTS_DIR/$JOB_NAME-output.json" + REPORT_REL="reports/behavioral-eval-summary.json" + + if [ ! -f "$OUTPUT_FILE" ]; then + echo -n "error" > "$(results.eval-passed.path)" + echo -n "$REPORT_REL" > "$(results.report-path.path)" + exit 0 + fi + + HAS_ERROR=$(python3 -c " + import json + d = json.load(open('$OUTPUT_FILE')) + print('yes' if d.get('error') and not d.get('results') and not d.get('taskResults') else 'no') + " 2>/dev/null || echo "yes") + + if [ "$HAS_ERROR" = "yes" ]; then + echo -n "error" > "$(results.eval-passed.path)" + echo -n "$REPORT_REL" > "$(results.report-path.path)" + exit 0 + fi + + mkdir -p "$RESULTS_DIR" "$REPORTS_DIR" + + # Write detailed reports to results/ + python3 /opt/scripts/aggregate_mcpchecker.py \ + --output-json "$OUTPUT_FILE" \ + --submission-name "$(params.submission-name)" \ + --report-dir "$RESULTS_DIR" \ + --pipeline-run-id "$(params.pipeline-run-id)" \ + > /tmp/aggregate_output.json 2>/tmp/aggregate_stderr.log || { + echo '{"overall_score": 0, "passed_tasks": 0, "total_tasks": 0, "recommendation": "error"}' \ + > /tmp/aggregate_output.json + } + + # Generate unified behavioral-eval-summary.json directly in reports/ + python3 -m scripts.mcp.runners.behavioral_eval_summary \ + --results-dir "$RESULTS_DIR" \ + --report-dir "$REPORTS_DIR" || echo "WARNING: Failed to generate behavioral-eval-summary.json" + + RECOMMENDATION=$(python3 -c "import json; d=json.load(open('/tmp/aggregate_output.json')); print(d.get('recommendation', 'fail'))") + + if [ "$RECOMMENDATION" = "pass" ]; then + echo -n "true" > "$(results.eval-passed.path)" + else + echo -n "false" > "$(results.eval-passed.path)" + fi + echo -n "$REPORT_REL" > "$(results.report-path.path)" + echo "MCP behavioral evaluation result: $RECOMMENDATION" diff --git a/pipeline/tasks/konflux/mcp-conformance-eval.yaml b/pipeline/tasks/konflux/mcp-conformance-eval.yaml new file mode 100644 index 00000000..c0df6185 --- /dev/null +++ b/pipeline/tasks/konflux/mcp-conformance-eval.yaml @@ -0,0 +1,229 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: mcp-conformance-eval + labels: + app.kubernetes.io/name: abevalflow + app.kubernetes.io/component: konflux +spec: + description: >- + MCP conformance evaluation - deterministic contract/conformance (ADR Approach 4). + Black-box, no LLM. Probes the deployed, running MCP server over Streamable + HTTP for protocol/schema/HTTP conformance and operational limits, and + consumes the checks Compass already collects as real facts (tool-name rules, + OAuth match). Uses the three-state model + (pass / fail / not_evaluated): an unreachable server or a check that cannot + produce a meaningful signal is reported not_evaluated, never failed. + Runs as a Job on the target cluster so it can reach the MCP server via + internal cluster DNS. + params: + - name: mcp-eval-image + type: string + default: "quay.io/ecosystem-appeng/abevalflow-mcp-eval:0.1" + description: >- + MCP eval image with the probe dependencies (httpx, pydantic, pyyaml) and + the pipeline code (scripts.mcp, abevalflow.mcp) baked in at PYTHONPATH=/opt. + - name: namespace-name + type: string + description: Ephemeral namespace where the MCP server is deployed + - name: service-name + type: string + default: "mcp-server" + description: Kubernetes Service name of the deployed MCP server + - name: service-port + type: string + default: "8080" + description: Port on which the MCP server listens + - name: mode + type: string + default: "block" + description: Gate enforcement mode - block, warn, or disabled. + - name: timeout + type: string + default: "30" + description: Per-request HTTP timeout (seconds) for the probe. + - name: job-timeout + type: string + default: "300" + description: Timeout in seconds for the probe Job + - name: openapi-url + type: string + default: "" + description: >- + Explicit OpenAPI contract URL for the openapi-conformance check. Empty + probes well-known paths on the server root; a server publishing no + reachable contract fails the check (ADR baseline requirement). + - name: compass-secret + type: string + default: "mcp-eval-compass" + description: >- + Name of a Secret on the target cluster (in the default namespace) + containing Compass SoundCheck credentials. Keys: api-key, + compass-url, entity-ref. Copied into the ephemeral namespace. + If empty or the secret does not exist, Compass-consumed checks + report not_evaluated. + - name: job-template + type: string + default: "/opt/templates/mcp-conformance-eval-job.yaml" + description: Path to the Kubernetes Job template for the conformance probe + workspaces: + - name: source + description: Workspace for the reports directory. + - name: target-kubeconfig + description: >- + Kubeconfig for the target cluster. Must contain a file named + "kubeconfig". + results: + - name: conformance-eval-passed + description: Whether conformance evaluation passed (no check FAILED in block mode). + - name: report-path + description: Workspace-relative path to the conformance evaluation reports directory. + steps: + # ── Step 1: Prepare Job manifest from template ── + - name: prepare-job + image: $(params.mcp-eval-image) + imagePullPolicy: Always + env: + - name: NAMESPACE + value: $(params.namespace-name) + - name: JOB_NAME + value: conformance-eval-probe + - name: IMAGE + value: $(params.mcp-eval-image) + - name: JOB_TIMEOUT + value: $(params.job-timeout) + - name: SERVER_URL + value: http://$(params.service-name):$(params.service-port)/mcp + - name: TIMEOUT + value: $(params.timeout) + - name: OPENAPI_URL + value: $(params.openapi-url) + - name: COMPASS_SECRET + value: $(params.compass-secret) + - name: JOB_TEMPLATE + value: $(params.job-template) + script: | + #!/usr/bin/env bash + set -euo pipefail + + envsubst \ + '${JOB_NAME} ${NAMESPACE} ${JOB_TIMEOUT} ${IMAGE} ${SERVER_URL} ${TIMEOUT} ${OPENAPI_URL} ${COMPASS_SECRET}' \ + < "$JOB_TEMPLATE" \ + > "$(workspaces.source.path)/conformance-eval-job.yaml" + + echo "Generated Job manifest:" + cat "$(workspaces.source.path)/conformance-eval-job.yaml" + + # ── Step 2: Apply Job and collect results ── + - name: run-probe-job + image: registry.redhat.io/openshift4/ose-cli:latest + env: + - name: KUBECONFIG + value: $(workspaces.target-kubeconfig.path)/kubeconfig + - name: NAMESPACE + value: $(params.namespace-name) + - name: JOB_NAME + value: conformance-eval-probe + - name: JOB_TIMEOUT + value: $(params.job-timeout) + - name: COMPASS_SECRET + value: $(params.compass-secret) + script: | + #!/usr/bin/env bash + set -euo pipefail + + # Copy Compass secret into ephemeral namespace if it exists + if [ -n "$COMPASS_SECRET" ]; then + if oc get secret "$COMPASS_SECRET" -n default -o yaml \ + | sed '/^\s*namespace:/d; /^\s*uid:/d; /^\s*resourceVersion:/d; /^\s*creationTimestamp:/d' \ + | oc apply -n "$NAMESPACE" -f - 2>/dev/null; then + echo "Copied Compass secret '$COMPASS_SECRET' into $NAMESPACE" + else + echo "WARNING: Could not copy Compass secret '$COMPASS_SECRET'; consumed checks will report not_evaluated." + fi + fi + + # Apply the Job + oc apply -f "$(workspaces.source.path)/conformance-eval-job.yaml" --namespace="$NAMESPACE" + + RESULTS_DIR="$(workspaces.source.path)/results" + mkdir -p "$RESULTS_DIR" + find "$RESULTS_DIR" -maxdepth 1 -type f -name '*-check.json' -delete + + # Wait for the Job to finish. Logs remain available after the pod + # completes, so no exec or file copy is needed. + DEADLINE=$((SECONDS + JOB_TIMEOUT)) + JOB_STATE="running" + while true; do + COMPLETE=$(oc get job "$JOB_NAME" -n "$NAMESPACE" \ + -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' \ + 2>/dev/null || true) + FAILED=$(oc get job "$JOB_NAME" -n "$NAMESPACE" \ + -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' \ + 2>/dev/null || true) + + if [ "$COMPLETE" = "True" ]; then + JOB_STATE="complete" + echo "Probe Job completed" + break + fi + if [ "$FAILED" = "True" ]; then + JOB_STATE="failed" + echo "WARNING: Probe Job failed; collecting available logs" + break + fi + if [ "$SECONDS" -ge "$DEADLINE" ]; then + JOB_STATE="timed_out" + echo "ERROR: Probe Job timed out after ${JOB_TIMEOUT}s; collecting available logs" + break + fi + sleep 10 + done + + RAW_LOGS="$RESULTS_DIR/$JOB_NAME-logs.txt" + oc logs "job/$JOB_NAME" -n "$NAMESPACE" > "$RAW_LOGS" 2>&1 + + # Extract reports from logs: each ===REPORT_JSON:filename=== marker + # is followed by a single-line JSON object. + REPORT_COUNT=0 + while IFS= read -r marker; do + report="${marker#===REPORT_JSON:}" + report="${report%===}" + json=$(awk -v m="$marker" '$0 == m { getline; print; exit }' "$RAW_LOGS") + if [ -n "$json" ]; then + echo "$json" > "$RESULTS_DIR/$report" + REPORT_COUNT=$((REPORT_COUNT + 1)) + echo "Saved $report" + else + echo "WARNING: No JSON after marker for $report" + fi + done < <(grep -E '^===REPORT_JSON:.*===$' "$RAW_LOGS") + + oc delete job "$JOB_NAME" -n "$NAMESPACE" --ignore-not-found=true >/dev/null 2>&1 || true + + echo "Saved $REPORT_COUNT report(s) to $RESULTS_DIR" + ls -la "$RESULTS_DIR" + + # ── Step 3: Run gate evaluation ── + - name: gate + image: $(params.mcp-eval-image) + imagePullPolicy: Always + env: + - name: MODE + value: $(params.mode) + script: | + #!/usr/bin/env bash + set -euo pipefail + RESULTS_DIR="$(workspaces.source.path)/results" + REPORTS_DIR="$(workspaces.source.path)/reports" + mkdir -p "$RESULTS_DIR" "$REPORTS_DIR" + + bash /opt/scripts/mcp/gate_wrapper.sh \ + scripts.mcp.runners.conformance_eval_gate \ + "$RESULTS_DIR" \ + "$REPORTS_DIR" \ + "$MODE" \ + "$REPORTS_DIR/conformance-eval-summary.json" \ + "$(results.conformance-eval-passed.path)" \ + "reports/conformance-eval-summary.json" \ + "$(results.report-path.path)" diff --git a/pipeline/tasks/konflux/mcp-phase2-conformance.yaml b/pipeline/tasks/konflux/mcp-phase2-conformance.yaml deleted file mode 100644 index a3723b60..00000000 --- a/pipeline/tasks/konflux/mcp-phase2-conformance.yaml +++ /dev/null @@ -1,147 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: Task -metadata: - name: mcp-phase2-conformance - labels: - app.kubernetes.io/name: abevalflow - app.kubernetes.io/component: konflux -spec: - description: >- - MCP evaluation Phase 2 - deterministic contract/conformance (ADR Approach 4). - Black-box, no LLM. Probes the deployed, running MCP server over Streamable - HTTP for protocol/schema/HTTP conformance and operational limits, and - consumes the checks Compass already collects as real facts (tool-name rules, - OAuth match). Uses the three-state model - (pass / fail / not_evaluated): an unreachable server or a check that cannot - produce a meaningful signal is reported not_evaluated, never failed. Assumes - the server was already deployed and became ready in a prior pipeline step - (reached at :8080 by convention). - params: - - name: mcp-eval-image - type: string - default: "quay.io/ecosystem-appeng/abevalflow-mcp-eval:0.1" - description: >- - MCP eval image with the probe dependencies (httpx, pydantic, pyyaml) and - the pipeline code (scripts.mcp, abevalflow.mcp) baked in at PYTHONPATH=/opt. - - name: server-url - type: string - description: MCP endpoint URL, e.g. http://:8080/mcp - - name: submission-name - type: string - description: Name used for the report directory (reports//). - - name: mode - type: string - default: "block" - description: Gate enforcement mode - block, warn, or disabled. - - name: timeout - type: string - default: "30" - description: Per-request HTTP timeout (seconds) for the probe. - - name: openapi-url - type: string - default: "" - description: >- - Explicit OpenAPI contract URL for the openapi-conformance check. Empty - probes well-known paths on the server root; a server publishing no - reachable contract fails the check (ADR baseline requirement). - - name: compass-facts-file - type: string - default: "" - description: >- - Workspace-relative path to a Compass facts JSON for the consumed checks - (file mode). Ignored when compass-url + compass-entity-ref are set. - - name: compass-url - type: string - default: "" - description: >- - SoundCheck API base URL for live Compass result consumption, e.g. - https://compass.stage.redhat.com/api/soundcheck. Requires - compass-entity-ref and COMPASS_API_KEY in the pod environment. - - name: compass-entity-ref - type: string - default: "" - description: >- - Compass entity ref for the MCP server being evaluated, e.g. - mcpserver:redhat/my-server. Used with compass-url for live mode. - workspaces: - - name: source - description: Workspace for the reports directory. - results: - - name: phase2-passed - description: Whether Phase 2 passed (no check FAILED in block mode). - - name: report-path - description: Workspace-relative path to the Phase 2 reports directory. - steps: - - name: probe - image: $(params.mcp-eval-image) - # User-controlled params are bound to env vars (Tekton substitutes them as - # literal strings) instead of inlined into the script, so a value containing - # shell metacharacters cannot execute in the task pod. - env: - - name: SERVER_URL - value: $(params.server-url) - - name: SUBMISSION_NAME - value: $(params.submission-name) - - name: TIMEOUT - value: $(params.timeout) - - name: OPENAPI_URL - value: $(params.openapi-url) - - name: COMPASS_FACTS_FILE - value: $(params.compass-facts-file) - - name: COMPASS_URL - value: $(params.compass-url) - - name: COMPASS_ENTITY_REF - value: $(params.compass-entity-ref) - script: | - #!/usr/bin/env bash - set -euo pipefail - REPORTS_DIR="$(workspaces.source.path)/reports/$SUBMISSION_NAME" - mkdir -p "$REPORTS_DIR" - - echo "=== Phase 2 live probe against $SERVER_URL ===" - python -m scripts.mcp.probes.phase2_conformance_probe \ - --server-url "$SERVER_URL" \ - --reports-dir "$REPORTS_DIR" \ - --timeout "$TIMEOUT" \ - --openapi-url "$OPENAPI_URL" - - # Compass-consumed checks: live mode (API) takes precedence over file mode. - if [ -n "$COMPASS_URL" ] && [ -n "$COMPASS_ENTITY_REF" ]; then - echo "=== Phase 2 Compass consume (live) for $COMPASS_ENTITY_REF ===" - python -m scripts.mcp.probes.phase2_compass_probe \ - --compass-url "$COMPASS_URL" \ - --entity-ref "$COMPASS_ENTITY_REF" \ - --reports-dir "$REPORTS_DIR" - elif [ -n "$COMPASS_FACTS_FILE" ]; then - echo "=== Phase 2 Compass consume (file) from $COMPASS_FACTS_FILE ===" - python -m scripts.mcp.probes.phase2_compass_probe \ - --facts-file "$(workspaces.source.path)/$COMPASS_FACTS_FILE" \ - --reports-dir "$REPORTS_DIR" - else - echo "No Compass source given; consumed checks will report not_evaluated." - python -m scripts.mcp.probes.phase2_compass_probe \ - --reports-dir "$REPORTS_DIR" - fi - - - name: gate - image: $(params.mcp-eval-image) - env: - - name: SUBMISSION_NAME - value: $(params.submission-name) - - name: MODE - value: $(params.mode) - script: | - #!/usr/bin/env bash - set -euo pipefail - REPORTS_DIR="$(workspaces.source.path)/reports/$SUBMISSION_NAME" - - # Shared gate wrapper (baked in the image): runs the gate module, emits the - # Tekton results from the summary, and propagates the gate's exit code. - bash /opt/scripts/mcp/gate_wrapper.sh \ - scripts.mcp.runners.phase2_gate \ - "$REPORTS_DIR" \ - "$MODE" \ - "$REPORTS_DIR/phase2-summary.json" \ - "$(results.phase2-passed.path)" \ - "reports/$SUBMISSION_NAME" \ - "$(results.report-path.path)" diff --git a/pipeline/tasks/konflux/mcp-phase1-static.yaml b/pipeline/tasks/konflux/mcp-static-eval.yaml similarity index 59% rename from pipeline/tasks/konflux/mcp-phase1-static.yaml rename to pipeline/tasks/konflux/mcp-static-eval.yaml index babcbeef..19362969 100644 --- a/pipeline/tasks/konflux/mcp-phase1-static.yaml +++ b/pipeline/tasks/konflux/mcp-static-eval.yaml @@ -1,19 +1,18 @@ apiVersion: tekton.dev/v1 kind: Task metadata: - name: mcp-phase1-static + name: mcp-static-eval labels: app.kubernetes.io/name: abevalflow app.kubernetes.io/component: konflux spec: description: >- - MCP evaluation Phase 1 - static / build-time analysis (ADR Approach 4). + MCP static evaluation - build-time analysis (ADR Approach 4). Runs three language-agnostic static checks against the MCP server source, before deployment and without running the server: secrets management (gitleaks), no user-provided code execution (semgrep), and license compliance (licensee). Always executes, independent of runtime outcomes; - resolves to pass/fail only. Normalized scan JSON and per-gate GateResults - are written under reports//. + resolves to pass/fail only. params: - name: mcp-eval-image type: string @@ -27,9 +26,6 @@ spec: description: >- Path of the MCP server source within the workspace to scan (relative to the workspace root). Defaults to the workspace root. - - name: submission-name - type: string - description: Name used for the report directory (reports//). - name: mode type: string default: "block" @@ -44,29 +40,39 @@ spec: - name: source description: Workspace containing the MCP server source to evaluate. results: - - name: phase1-passed - description: Whether all Phase 1 gates passed (true/false). + - name: static-eval-passed + description: Whether all static evaluation gates passed (true/false). - name: report-path - description: Workspace-relative path to the Phase 1 reports directory. + description: Workspace-relative path to the static evaluation reports directory. steps: - name: scan image: $(params.mcp-eval-image) + imagePullPolicy: Always # User-controlled params are bound to env vars (Tekton substitutes them as # literal strings) instead of inlined into the script, so a value containing # shell metacharacters cannot execute in the task pod. env: - name: SOURCE_SUBDIR value: $(params.source-subdir) - - name: SUBMISSION_NAME - value: $(params.submission-name) - name: ALLOWED_LICENSES value: $(params.allowed-licenses) script: | #!/usr/bin/env bash set -euo pipefail SOURCE_DIR="$(workspaces.source.path)/$SOURCE_SUBDIR" - REPORTS_DIR="$(workspaces.source.path)/reports/$SUBMISSION_NAME" - mkdir -p "$REPORTS_DIR" + RESULTS_DIR="$(workspaces.source.path)/results" + mkdir -p "$RESULTS_DIR" + + # licensee (Rugged/libgit2) refuses to open a repo whose directory is not + # owned by the current UID. The repo was cloned by the pipeline's + # fetch-mcp-repo task, which runs in a different pod/UID than this step, + # so that ownership check always trips here even though we only need + # read access. Mark the workspace as a trusted ("safe") directory the + # same way `git config --global --add safe.directory` would - written + # directly since this image has no git CLI, only libgit2 bindings. + export HOME="${HOME:-/tmp}" + mkdir -p "$HOME" + printf '[safe]\n\tdirectory = *\n' >> "$HOME/.gitconfig" LICENSE_ARGS="" # shellcheck disable=SC2086 @@ -74,31 +80,33 @@ spec: LICENSE_ARGS="$LICENSE_ARGS --allow $spdx" done - echo "=== Phase 1 scanners on $SOURCE_DIR ===" - python -m scripts.mcp.scanners.phase1_secrets_scan "$SOURCE_DIR" --reports-dir "$REPORTS_DIR" - python -m scripts.mcp.scanners.phase1_no_user_code_scan "$SOURCE_DIR" --reports-dir "$REPORTS_DIR" + echo "=== Static evaluation scanners on $SOURCE_DIR ===" + python -m scripts.mcp.scanners.static_eval_secrets_scan "$SOURCE_DIR" --reports-dir "$RESULTS_DIR" + python -m scripts.mcp.scanners.static_eval_no_user_code_scan "$SOURCE_DIR" --reports-dir "$RESULTS_DIR" # shellcheck disable=SC2086 - python -m scripts.mcp.scanners.phase1_license_scan "$SOURCE_DIR" --reports-dir "$REPORTS_DIR" $LICENSE_ARGS + python -m scripts.mcp.scanners.static_eval_license_scan "$SOURCE_DIR" --reports-dir "$RESULTS_DIR" $LICENSE_ARGS - name: gate image: $(params.mcp-eval-image) + imagePullPolicy: Always env: - - name: SUBMISSION_NAME - value: $(params.submission-name) - name: MODE value: $(params.mode) script: | #!/usr/bin/env bash set -euo pipefail - REPORTS_DIR="$(workspaces.source.path)/reports/$SUBMISSION_NAME" + RESULTS_DIR="$(workspaces.source.path)/results" + REPORTS_DIR="$(workspaces.source.path)/reports" + mkdir -p "$RESULTS_DIR" "$REPORTS_DIR" # Shared gate wrapper (baked in the image): runs the gate module, emits the # Tekton results from the summary, and propagates the gate's exit code. bash /opt/scripts/mcp/gate_wrapper.sh \ - scripts.mcp.runners.phase1_gate \ + scripts.mcp.runners.static_eval_gate \ + "$RESULTS_DIR" \ "$REPORTS_DIR" \ "$MODE" \ - "$REPORTS_DIR/phase1-summary.json" \ - "$(results.phase1-passed.path)" \ - "reports/$SUBMISSION_NAME" \ + "$REPORTS_DIR/static-eval-summary.json" \ + "$(results.static-eval-passed.path)" \ + "reports/static-eval-summary.json" \ "$(results.report-path.path)" diff --git a/scripts/mcp/README.md b/scripts/mcp/README.md index 3448d57d..711fc6d4 100644 --- a/scripts/mcp/README.md +++ b/scripts/mcp/README.md @@ -5,30 +5,31 @@ Thin CLI entry points for the MCP-server evaluation pipeline. These are the step as `python -m scripts.mcp..`. They shell out to external tools, invoke evaluation logic from `abevalflow.mcp`, and write result JSON. -## Structure (organized by role, phase-prefixed) +## Structure (organized by role and evaluation type) | Directory | Role | What it does | |-----------|------|-------------| -| `scanners/` | **Scan** source code | Shell out to external tools (gitleaks, semgrep, licensee) and write normalized scan JSON. Phase 1 only — no running server needed. | -| `runners/` | **Run** gate evaluations | Thin CLIs that invoke the gate logic from `abevalflow.mcp.gates` and write summary JSON. Both phases. | -| `probes/` | **Probe** the live server | Connect to the running MCP server (or Compass API) and write per-check result JSON. Phase 2 only — server must be running. | +| `scanners/` | **Scan** source code | Shell out to external tools (gitleaks, semgrep, licensee) and write normalized scan JSON. Static evaluation only — no running server needed. | +| `runners/` | **Run** gate evaluations | Thin CLIs that invoke the gate logic from `abevalflow.mcp.gates` and write summary JSON. All evaluation types. | +| `probes/` | **Probe** the live server | Connect to the running MCP server (or Compass API) and write per-check result JSON. Conformance evaluation only — server must be running. | | `rules/` | **Bundled rulesets** | Offline semgrep rules for the no-user-code scan. Not a runner — consumed by the scanner. | -### `scanners/` — Phase 1 scanner runners +### `scanners/` — Static evaluation scanner runners | File | What it does | Emits | |------|-------------|-------| -| `phase1_secrets_scan.py` | Shells out to gitleaks | `secrets-scan.json` | -| `phase1_no_user_code_scan.py` | Shells out to semgrep (bundled offline ruleset) | `no-user-code-scan.json` | -| `phase1_license_scan.py` | Shells out to licensee (top-level license only) | `license-scan.json` | +| `static_eval_secrets_scan.py` | Shells out to gitleaks | `secrets-scan.json` | +| `static_eval_no_user_code_scan.py` | Shells out to semgrep (bundled offline ruleset) | `no-user-code-scan.json` | +| `static_eval_license_scan.py` | Shells out to licensee (top-level license only) | `license-scan.json` | | `helpers.py` | `run_tool()` + `write_scan()` — scanner I/O helpers | — | ### `runners/` — gate evaluation runners | File | What it does | Emits | |------|-------------|-------| -| `phase1_gate.py` | Calls `abevalflow.mcp.gates.run_phase1()` | `phase1-summary.json` | -| `phase2_gate.py` | Calls `abevalflow.mcp.gates.run_phase2()` | `phase2-summary.json` | +| `static_eval_gate.py` | Calls `abevalflow.mcp.gates.run_static_eval()` | `static-eval-summary.json` | +| `conformance_eval_gate.py` | Calls `abevalflow.mcp.gates.run_conformance_eval()` | `conformance-eval-summary.json` | +| `behavioral_eval_summary.py` | Maps mcpchecker results to unified summary | `behavioral-eval-summary.json` | ### `gate_wrapper.sh` — shared Tekton gate wrapper @@ -36,18 +37,18 @@ Lives at `scripts/mcp/gate_wrapper.sh` (flat, following repo convention for utility scripts). Both Tekton gate steps call it to run the gate module, emit Tekton results (`passed` flag + report path), and propagate the exit code. -### `probes/` — Phase 2 probe runners +### `probes/` — Conformance evaluation probe runners | File | What it does | Emits | |------|-------------|-------| -| `phase2_conformance_probe.py` | Creates MCPClient, calls `probe_all()` | per-check `-check.json` | -| `phase2_compass_probe.py` | Fetches Compass results (live API or file mode) | per-check `-check.json` | +| `conformance_eval_probe.py` | Creates MCPClient, calls `probe_all()` | per-check `-check.json` | +| `conformance_eval_compass_probe.py` | Fetches Compass results (live API or file mode) | per-check `-check.json` | ### `rules/` — bundled offline rulesets | File | Used by | |------|---------| -| `no_user_code.yml` | semgrep (Phase 1 no-user-code scan) | +| `no_user_code.yml` | semgrep (static evaluation no-user-code scan) | ## Two-package split @@ -60,19 +61,19 @@ The contract between them: **JSON files on disk.** Writers produce them, readers ## Running locally -`run_mcp_eval.sh` is the local dev/testing runner. It mirrors what the two -Tekton Tasks do, so a local run exercises the same code paths as a cluster run. +`run_mcp_eval.sh` is the local dev/testing runner. It mirrors what the Tekton +Tasks do, so a local run exercises the same code paths as a cluster run. ```bash -# Both phases — scan source + probe running server: +# Both evaluations — scan source + probe running server: bash scripts/mcp/run_mcp_eval.sh \ --repo /path/to/mcp-server-source \ --server-url http://localhost:8080/mcp -# Phase 1 only (no server needed): +# Static evaluation only (no server needed): bash scripts/mcp/run_mcp_eval.sh --repo /path/to/mcp-server-source -# Phase 2 only (server must be running): +# Conformance evaluation only (server must be running): bash scripts/mcp/run_mcp_eval.sh --server-url http://localhost:8080/mcp # With explicit gate mode and license allow-list: @@ -112,14 +113,15 @@ bash scripts/mcp/run_mcp_eval.sh \ ## On a cluster (Tekton) -Two Tekton Tasks run inside the eval container image +Three Tekton Tasks run inside the eval container image (`quay.io/ecosystem-appeng/abevalflow-mcp-eval:0.1`): -- `mcp-phase1-static` — scanners + phase 1 gate -- `mcp-phase2-conformance` — probe + compass + phase 2 gate +- `mcp-static-eval` — scanners + static evaluation gate +- `mcp-conformance-eval` — probe + compass + conformance evaluation gate +- `mcp-behavioral-eval` — mcpchecker + behavioral evaluation summary -Task YAMLs: `pipeline/tasks/konflux/mcp-phase1-static.yaml` and -`mcp-phase2-conformance.yaml`. +Task YAMLs: `pipeline/tasks/konflux/mcp-static-eval.yaml`, +`mcp-conformance-eval.yaml`, and `mcp-behavioral-eval.yaml`. ## Future repo split diff --git a/scripts/mcp/__init__.py b/scripts/mcp/__init__.py index 7450ad89..483348cc 100644 --- a/scripts/mcp/__init__.py +++ b/scripts/mcp/__init__.py @@ -8,8 +8,8 @@ Organized by role: -- ``scanners/`` - Phase 1 scanner runners (gitleaks, semgrep, licensee) +- ``scanners/`` - Static evaluation scanner runners (gitleaks, semgrep, licensee) - ``gates/`` - Gate evaluation runners + gate_wrapper.sh -- ``probes/`` - Phase 2 probe + Compass consume runners +- ``probes/`` - Conformance evaluation probe + Compass consume runners - ``rules/`` - Bundled offline rulesets (semgrep) """ diff --git a/scripts/mcp/gate_wrapper.sh b/scripts/mcp/gate_wrapper.sh index 0c7d2316..87075a6f 100644 --- a/scripts/mcp/gate_wrapper.sh +++ b/scripts/mcp/gate_wrapper.sh @@ -1,27 +1,28 @@ #!/usr/bin/env bash -# Shared Phase 1 / Phase 2 gate wrapper, baked into the MCP eval image. +# Shared gate wrapper for static/conformance evaluation, baked into the MCP eval image. # -# Both phase gate steps do the same thing: run the phase's gate module, emit the -# Tekton results (passed flag + report path) from the summary JSON, then exit +# All evaluation gate steps do the same thing: run the evaluation's gate module, emit +# the Tekton results (passed flag + report path) from the summary JSON, then exit # with the gate's own code so a failing gate fails the step. Keeping it here means -# the two tasks stay consistent when this behavior changes. +# the tasks stay consistent when this behavior changes. # # Usage: # gate_wrapper.sh \ # set -uo pipefail -GATE_MODULE="$1" # e.g. scripts.mcp.runners.phase1_gate -REPORTS_DIR="$2" # absolute reports dir for this submission/phase -MODE="$3" # block | warn | disabled -SUMMARY="$4" # phaseN-summary.json written by the gate module -PASSED_RESULT_PATH="$5" # Tekton result path for the passed flag -REPORT_REL="$6" # workspace-relative report path (result value) -REPORT_PATH_RESULT="$7" # Tekton result path for the report path +GATE_MODULE="$1" # e.g. scripts.mcp.runners.static_eval_gate +RESULTS_DIR="$2" # absolute results dir for this submission/evaluation (input scans) +REPORTS_DIR="$3" # absolute reports dir for this submission/evaluation (output summaries) +MODE="$4" # block | warn | disabled +SUMMARY="$5" # *-eval-summary.json written by the gate module +PASSED_RESULT_PATH="$6" # Tekton result path for the passed flag +REPORT_REL="$7" # workspace-relative report path (result value) +REPORT_PATH_RESULT="$8" # Tekton result path for the report path # Never abort before writing results: capture the gate's exit code, emit the # Tekton results, then propagate the code so the step fails if a gate failed. -python -m "$GATE_MODULE" --reports-dir "$REPORTS_DIR" --mode "$MODE" +python -m "$GATE_MODULE" --results-dir "$RESULTS_DIR" --reports-dir "$REPORTS_DIR" --mode "$MODE" GATE_EXIT=$? # Emit the passed flag from the summary. A missing summary or one without a diff --git a/scripts/mcp/probes/__init__.py b/scripts/mcp/probes/__init__.py index ddd3fe12..78c063bb 100644 --- a/scripts/mcp/probes/__init__.py +++ b/scripts/mcp/probes/__init__.py @@ -1 +1 @@ -"""Phase 2 probe CLI entry points — connect to servers, fetch Compass, write check JSON.""" +"""Conformance evaluation probe CLI entry points — connect to servers, fetch Compass, write check JSON.""" diff --git a/scripts/mcp/probes/phase2_compass_probe.py b/scripts/mcp/probes/conformance_eval_compass_probe.py similarity index 91% rename from scripts/mcp/probes/phase2_compass_probe.py rename to scripts/mcp/probes/conformance_eval_compass_probe.py index f7e99c9b..8b3fe03a 100644 --- a/scripts/mcp/probes/phase2_compass_probe.py +++ b/scripts/mcp/probes/conformance_eval_compass_probe.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""MCP Phase 2 Compass-consumed checks — CLI entry point. +"""MCP conformance evaluation Compass-consumed checks — CLI entry point. Two modes: @@ -12,15 +12,15 @@ If both are given, live mode takes precedence. If neither is given the consumed checks report not_evaluated. -Mapping logic lives in ``abevalflow.mcp.checks.phase2_compass_checks``; this module is the +Mapping logic lives in ``abevalflow.mcp.checks.conformance_eval_compass_checks``; this module is the thin CLI wrapper that handles I/O (file reads, HTTP calls) and writes results. Usage (file mode): - python -m scripts.mcp.probes.phase2_compass_probe \\ + python -m scripts.mcp.probes.conformance_eval_compass_probe \\ --facts-file facts.json --reports-dir Usage (live mode): - python -m scripts.mcp.probes.phase2_compass_probe \\ + python -m scripts.mcp.probes.conformance_eval_compass_probe \\ --compass-url https://compass.stage.redhat.com/api/soundcheck \\ --entity-ref mcpserver:redhat/my-server \\ --reports-dir @@ -42,7 +42,7 @@ import httpx -from abevalflow.mcp.checks.phase2_compass_checks import ( +from abevalflow.mcp.checks.conformance_eval_compass_checks import ( CHECK_OAUTH, CHECK_TOOL_NAME, map_facts, @@ -107,7 +107,7 @@ def _load_facts_file(path: Path) -> dict[str, Any]: def main() -> int: - parser = argparse.ArgumentParser(description="MCP Phase 2 Compass-consumed checks") + parser = argparse.ArgumentParser(description="MCP conformance evaluation Compass-consumed checks") parser.add_argument( "--reports-dir", type=Path, required=True, help="Directory to write -check.json files into" ) diff --git a/scripts/mcp/probes/phase2_conformance_probe.py b/scripts/mcp/probes/conformance_eval_probe.py similarity index 71% rename from scripts/mcp/probes/phase2_conformance_probe.py rename to scripts/mcp/probes/conformance_eval_probe.py index f714b1fe..1b8bb1ce 100644 --- a/scripts/mcp/probes/phase2_conformance_probe.py +++ b/scripts/mcp/probes/conformance_eval_probe.py @@ -1,19 +1,19 @@ #!/usr/bin/env python3 -"""MCP Phase 2 live conformance probe — CLI entry point. +"""MCP conformance evaluation live probe — CLI entry point. Connects to a running MCP server over Streamable HTTP and runs the deterministic (no-LLM) contract/conformance checks, writing one three-state result file per -check for the Phase 2 gates to read. +check for the conformance evaluation gates to read. -Check logic lives in ``abevalflow.mcp.checks.phase2_conformance_checks``; this module is the thin +Check logic lives in ``abevalflow.mcp.checks.conformance_eval_checks``; this module is the thin CLI wrapper that creates the client, invokes the checks, and writes results. Checks that Compass already collects as real automated facts (tool-name rules, OAuth match) are NOT probed here — they are consumed via -``scripts/mcp/probes/phase2_compass_probe.py`` instead. +``scripts/mcp/probes/conformance_eval_compass_probe.py`` instead. Usage: - python -m scripts.mcp.probes.phase2_conformance_probe --server-url http://:8080/mcp \ + python -m scripts.mcp.probes.conformance_eval_probe --server-url http://:8080/mcp \ --reports-dir [--timeout 30] [--openapi-url URL] """ @@ -25,7 +25,7 @@ import time from pathlib import Path -from abevalflow.mcp.checks.phase2_conformance_checks import probe_all +from abevalflow.mcp.checks.conformance_eval_checks import probe_all from abevalflow.mcp.helpers import write_check_result logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") @@ -33,7 +33,7 @@ def main() -> int: - parser = argparse.ArgumentParser(description="MCP Phase 2 live conformance probe") + parser = argparse.ArgumentParser(description="MCP conformance evaluation live probe") parser.add_argument("--server-url", required=True, help="MCP endpoint URL, e.g. http://:8080/mcp") parser.add_argument( "--reports-dir", type=Path, required=True, help="Directory to write -check.json files into" @@ -50,7 +50,7 @@ def main() -> int: outcomes = probe_all(args.server_url, timeout=args.timeout, openapi_url=args.openapi_url) for outcome in outcomes: write_check_result(args.reports_dir, outcome) - logger.info("Phase 2 probe complete: %d checks in %.1fs", len(outcomes), time.monotonic() - start) + logger.info("Conformance probe complete: %d checks in %.1fs", len(outcomes), time.monotonic() - start) return 0 diff --git a/scripts/mcp/rules/no_user_code.yml b/scripts/mcp/rules/no_user_code.yml index e254acd6..7690938e 100644 --- a/scripts/mcp/rules/no_user_code.yml +++ b/scripts/mcp/rules/no_user_code.yml @@ -1,11 +1,11 @@ -# Semgrep ruleset for MCP Phase 1 "does not execute user-provided code". +# Semgrep ruleset for MCP static evaluation "does not execute user-provided code". # # Flags dynamic / arbitrary code-execution sinks across the languages MCP # servers are commonly written in. Deliberately narrow (high-signal sinks only) # to keep false positives low and to run fully offline (no --config p/registry # network fetch). Extend per language as needed. # -# Severity ERROR -> mapped to HIGH by scripts/mcp/scanners/phase1_no_user_code_scan.py. +# Severity ERROR -> mapped to HIGH by scripts/mcp/scanners/static_eval_no_user_code_scan.py. rules: # ---- Python ------------------------------------------------------------- - id: python-dynamic-exec diff --git a/scripts/mcp/run_mcp_eval.sh b/scripts/mcp/run_mcp_eval.sh index fc2d62ae..2373280e 100644 --- a/scripts/mcp/run_mcp_eval.sh +++ b/scripts/mcp/run_mcp_eval.sh @@ -1,29 +1,29 @@ #!/usr/bin/env bash # -# run_mcp_eval.sh - Run the MCP-server evaluation Phase 1 + Phase 2 against any +# run_mcp_eval.sh - Run the MCP-server evaluation Static eval + Conformance eval against any # MCP server you point it at. This is a demo/dev driver (uncommitted): it mirrors -# what the two Tekton Tasks do (pipeline/tasks/konflux/mcp-phase1-static.yaml and -# mcp-phase2-conformance.yaml), so a local run and a cluster run exercise the same scanners, +# what the two Tekton Tasks do (pipeline/tasks/konflux/mcp-static-eval.yaml and +# mcp-conformance-eval.yaml), so a local run and a cluster run exercise the same scanners, # probe, and gates. # # Two execution modes: # # local (default) Run everything on this machine, no cluster needed. -# Phase 1: run the three static scanners (gitleaks / semgrep / -# licensee) over a repo, then the Phase 1 gate. -# Phase 2: probe a running MCP server over Streamable HTTP, -# optionally consume a Compass facts file, then the Phase 2 gate. +# Static eval: run the three static scanners (gitleaks / semgrep / +# licensee) over a repo, then the Static eval gate. +# Conformance eval: probe a running MCP server over Streamable HTTP, +# optionally consume a Compass facts file, then the Conformance eval gate. # # --oc Run on OpenShift/Konflux instead, by starting the real -# Tekton Tasks (mcp-phase1-static, mcp-phase2-conformance). +# Tekton Tasks (mcp-static-eval, mcp-conformance-eval). # This is the "pipeline demo, two phases only" path. It needs a # cluster you are logged in to (oc), tkn, the Tasks applied, and # a workspace PVC. Without those it prints a preflight report and # the exact commands it would run, then exits (nothing is # executed on a cluster you cannot reach). # -# Scope: Phase 1 (static) + Phase 2 (deterministic conformance) only, per the ADR -# (Approach 4). Phase 3 (behavioral) and pipeline wiring are out of scope here. +# Scope: Static eval (static) + Conformance eval (deterministic conformance) only, per the ADR +# (Approach 4). Behavioral evaluation and pipeline wiring are out of scope here. # # Examples: # # Local, both phases, against a repo dir and a running mock server: @@ -31,10 +31,10 @@ # --repo /tmp/generic-mock-mcp-server \ # --server-url http://localhost:8080/mcp # -# # Local, Phase 1 only (no server given): +# # Local, Static eval only (no server given): # bash scripts/mcp/run_mcp_eval.sh --repo https://github.com/some/mcp-server.git # -# # Local, Phase 2 only (no repo given), with Compass facts: +# # Local, Conformance eval only (no repo given), with Compass facts: # bash scripts/mcp/run_mcp_eval.sh --server-url http://localhost:8080/mcp \ # --compass-facts /tmp/facts.json # @@ -55,12 +55,12 @@ set -uo pipefail # --------------------------------------------------------------------------- # ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -REPO="" # Phase 1 target: local dir or git URL (empty => skip Phase 1) +REPO="" # Static eval target: local dir or git URL (empty => skip Static eval) SUBDIR="." # source subdir within the repo to scan -SERVER_URL="" # Phase 2 target: live MCP endpoint (empty => skip live probe) -COMPASS_FACTS="" # optional Compass facts JSON for the consumed Phase 2 checks +SERVER_URL="" # Conformance eval target: live MCP endpoint (empty => skip live probe) +COMPASS_FACTS="" # optional Compass facts JSON for the consumed Conformance eval checks MODE="block" # gate mode: block | warn | disabled -TIMEOUT="30" # Phase 2 per-request HTTP timeout (seconds) +TIMEOUT="30" # Conformance eval per-request HTTP timeout (seconds) OPENAPI_URL="" # explicit OpenAPI contract URL (empty => probe well-known paths) ALLOW="" # space-separated SPDX allow-list override for the license gate SUBMISSION="" # report dir name (defaults to the repo basename or "local") @@ -108,7 +108,7 @@ while [ $# -gt 0 ]; do done case "$MODE" in block|warn|disabled) ;; *) die "--mode must be block|warn|disabled (got '$MODE')" ;; esac -[ -n "$REPO" ] || [ -n "$SERVER_URL" ] || die "Give at least one of --repo (Phase 1) or --server-url (Phase 2). See --help." +[ -n "$REPO" ] || [ -n "$SERVER_URL" ] || die "Give at least one of --repo (Static eval) or --server-url (Conformance eval). See --help." # Report directory name mirrors the Tekton reports// layout. if [ -z "$SUBMISSION" ]; then @@ -120,17 +120,17 @@ fi # ========================================================================== # if [ "$RUN_OC" -eq 1 ]; then info "=== MCP eval on OpenShift/Konflux via Tekton (submission: $SUBMISSION, mode: $MODE) ===" - log "This runs the same two Tasks the pipeline would: mcp-phase1-static + mcp-phase2-conformance." + log "This runs the same two Tasks the pipeline would: mcp-static-eval + mcp-conformance-eval." log "" # Build the exact commands we would run, so they are visible whether or not the # cluster is reachable. tkn reads the params/workspaces defined in the Task YAML. - p1_cmd=(tkn task start mcp-phase1-static -n "$NAMESPACE" + p1_cmd=(tkn task start mcp-static-eval -n "$NAMESPACE" -p submission-name="$SUBMISSION" -p mode="$MODE" -p source-subdir="$SUBDIR") [ -n "$ALLOW" ] && p1_cmd+=(-p allowed-licenses="$ALLOW") p1_cmd+=(-w "name=source,claimName=$PVC" --showlog) - p2_cmd=(tkn task start mcp-phase2-conformance -n "$NAMESPACE" + p2_cmd=(tkn task start mcp-conformance-eval -n "$NAMESPACE" -p server-url="$SERVER_URL" -p submission-name="$SUBMISSION" -p mode="$MODE" -p timeout="$TIMEOUT") [ -n "$COMPASS_FACTS" ] && p2_cmd+=(-p compass-facts-file="$COMPASS_FACTS") p2_cmd+=(-w "name=source,claimName=$PVC" --showlog) @@ -148,11 +148,11 @@ if [ "$RUN_OC" -eq 1 ]; then printf '%s\n' "${out% }" } print_plan() { - log "${c_bold}Phase 1 (static):${c_off}"; print_cmd "${p1_cmd[@]}"; log "" + log "${c_bold}Static eval (static):${c_off}"; print_cmd "${p1_cmd[@]}"; log "" if [ -n "$SERVER_URL" ]; then - log "${c_bold}Phase 2 (conformance):${c_off}"; print_cmd "${p2_cmd[@]}" + log "${c_bold}Conformance eval (conformance):${c_off}"; print_cmd "${p2_cmd[@]}" else - log "${c_bold}Phase 2:${c_off} skipped (no --server-url)." + log "${c_bold}Conformance eval:${c_off} skipped (no --server-url)." fi } @@ -165,8 +165,8 @@ if [ "$RUN_OC" -eq 1 ]; then if command -v oc >/dev/null 2>&1; then oc whoami >/dev/null 2>&1 || problems+=("not logged in to a cluster (run: oc login ...)") if oc whoami >/dev/null 2>&1; then - oc get task mcp-phase1-static -n "$NAMESPACE" >/dev/null 2>&1 || problems+=("Task mcp-phase1-static not found in namespace $NAMESPACE (apply pipeline/tasks/konflux/mcp-phase1-static.yaml)") - oc get task mcp-phase2-conformance -n "$NAMESPACE" >/dev/null 2>&1 || problems+=("Task mcp-phase2-conformance not found in namespace $NAMESPACE (apply pipeline/tasks/konflux/mcp-phase2-conformance.yaml)") + oc get task mcp-static-eval -n "$NAMESPACE" >/dev/null 2>&1 || problems+=("Task mcp-static-eval not found in namespace $NAMESPACE (apply pipeline/tasks/konflux/mcp-static-eval.yaml)") + oc get task mcp-conformance-eval -n "$NAMESPACE" >/dev/null 2>&1 || problems+=("Task mcp-conformance-eval not found in namespace $NAMESPACE (apply pipeline/tasks/konflux/mcp-conformance-eval.yaml)") [ -n "$PVC" ] && { oc get pvc "$PVC" -n "$NAMESPACE" >/dev/null 2>&1 || problems+=("PVC '$PVC' not found in namespace $NAMESPACE"); } fi fi @@ -190,13 +190,13 @@ if [ "$RUN_OC" -eq 1 ]; then fi # Cluster is reachable and everything is in place: run for real. - info "Preflight passed. Starting Phase 1 Task..." + info "Preflight passed. Starting Static eval Task..." "${p1_cmd[@]}"; p1_rc=$? if [ -n "$SERVER_URL" ]; then - info "Starting Phase 2 Task..." + info "Starting Conformance eval Task..." "${p2_cmd[@]}"; p2_rc=$? else - p2_rc=0; warn "Phase 2 skipped (no --server-url)." + p2_rc=0; warn "Conformance eval skipped (no --server-url)." fi log "" info "=== Cluster run submitted. Inspect TaskRuns with: tkn taskrun list -n $NAMESPACE ===" @@ -220,15 +220,15 @@ pyrun() { ( cd "$ROOT" && "${PY[@]}" "$@" ); } if [ -z "$REPORTS_ROOT" ]; then REPORTS_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/mcp-eval-XXXXXX")" fi -P1_REPORTS="$REPORTS_ROOT/reports/$SUBMISSION/phase1" -P2_REPORTS="$REPORTS_ROOT/reports/$SUBMISSION/phase2" -mkdir -p "$P1_REPORTS" "$P2_REPORTS" +RESULTS_DIR="$REPORTS_ROOT/results" +REPORTS_DIR="$REPORTS_ROOT/reports" +mkdir -p "$RESULTS_DIR" "$REPORTS_DIR" info "=== MCP eval (local) - submission: $SUBMISSION, mode: $MODE ===" log "Reports: $REPORTS_ROOT" log "" -# --- Resolve the Phase 1 target: clone if it looks like a git URL. ----------- +# --- Resolve the Static eval target: clone if it looks like a git URL. ----------- CLONE_DIR="" cleanup() { [ -n "$CLONE_DIR" ] && rm -rf "$CLONE_DIR"; } trap cleanup EXIT @@ -238,7 +238,7 @@ if [ -n "$REPO" ]; then case "$REPO" in http://*|https://*|git@*|ssh://*) CLONE_DIR="$(mktemp -d "${TMPDIR:-/tmp}/mcp-src-XXXXXX")" - info "Cloning Phase 1 target: $REPO" + info "Cloning Static eval target: $REPO" if ! git clone --depth 1 "$REPO" "$CLONE_DIR" 2>&1 | tail -2; then die "git clone failed for $REPO" fi @@ -256,7 +256,7 @@ fi P1_RAN=0; P1_PASSED="" if [ -n "$TARGET" ]; then P1_RAN=1 - info "--- Phase 1 (static) on $TARGET ---" + info "--- Static eval (static) on $TARGET ---" have_gitleaks=0; have_semgrep=0; have_licensee=0 command -v gitleaks >/dev/null 2>&1 && have_gitleaks=1 || warn "gitleaks not found; the secrets scan will be skipped (its gate then depends on --mode: block => FAIL on the missing scan)." @@ -270,29 +270,29 @@ if [ -n "$TARGET" ]; then # secrets (gitleaks) if [ "$have_gitleaks" -eq 1 ]; then - pyrun -m scripts.mcp.scanners.phase1_secrets_scan "$TARGET" --reports-dir "$P1_REPORTS" 2>&1 | tail -1 + pyrun -m scripts.mcp.scanners.static_eval_secrets_scan "$TARGET" --reports-dir "$RESULTS_DIR" 2>&1 | tail -1 fi # no-user-code (semgrep) if [ "$have_semgrep" -eq 1 ]; then - pyrun -m scripts.mcp.scanners.phase1_no_user_code_scan "$TARGET" --reports-dir "$P1_REPORTS" 2>&1 | tail -1 + pyrun -m scripts.mcp.scanners.static_eval_no_user_code_scan "$TARGET" --reports-dir "$RESULTS_DIR" 2>&1 | tail -1 fi # license (licensee, or heuristic fallback that reuses the real evaluate_licenses) if [ "$have_licensee" -eq 1 ]; then # shellcheck disable=SC2086 lic_allow=""; for s in $ALLOW; do lic_allow="$lic_allow --allow $s"; done # shellcheck disable=SC2086 - pyrun -m scripts.mcp.scanners.phase1_license_scan "$TARGET" --reports-dir "$P1_REPORTS" $lic_allow 2>&1 | tail -1 + pyrun -m scripts.mcp.scanners.static_eval_license_scan "$TARGET" --reports-dir "$RESULTS_DIR" $lic_allow 2>&1 | tail -1 else - pyrun - "$P1_REPORTS" "$TARGET" "$ALLOW" <<'PY' + pyrun - "$RESULTS_DIR" "$TARGET" "$ALLOW" <<'PY' # Heuristic license fallback for when the licensee binary is absent. Uses the REAL # evaluate_licenses() + write_scan() so gate behavior matches production; only the # licensee subprocess is replaced by a conservative marker scan of the license # file. It never over-passes: an unrecognized or missing license => a finding. import sys, re from pathlib import Path -from scripts.mcp.scanners import phase1_license_scan as license_scan +from scripts.mcp.scanners import static_eval_license_scan as license_scan from scripts.mcp.scanners.helpers import write_scan -from abevalflow.mcp.gates.phase1_license_gate import DEFAULT_ALLOWED_LICENSES +from abevalflow.mcp.gates.static_eval_license_gate import DEFAULT_ALLOWED_LICENSES reports, target = Path(sys.argv[1]), Path(sys.argv[2]) allow_arg = sys.argv[3].split() if len(sys.argv) > 3 and sys.argv[3].strip() else [] @@ -325,58 +325,58 @@ print(f"INFO: license (heuristic, licensee absent): detected={lic}, {len(finding PY fi - # Phase 1 gate + # Static eval gate log "" - pyrun -m scripts.mcp.runners.phase1_gate --reports-dir "$P1_REPORTS" --mode "$MODE" + pyrun -m scripts.mcp.runners.static_eval_gate --results-dir "$RESULTS_DIR" --reports-dir "$REPORTS_DIR" --mode "$MODE" P1_GATE_RC=$? - P1_PASSED="$(pyrun -c "import json,sys;print('true' if json.load(open(sys.argv[1]))['passed'] else 'false')" "$P1_REPORTS/phase1-summary.json" 2>/dev/null)" - # Phase 1 grade = mean of the three gate scores. All three always run, so this + P1_PASSED="$(pyrun -c "import json,sys;print('true' if json.load(open(sys.argv[1]))['passed'] else 'false')" "$REPORTS_DIR/static-eval-summary.json" 2>/dev/null)" + # Static eval grade = mean of the three gate scores. All three always run, so this # is a real 0.00-1.00 grade, not just a pass/fail bit (ADR "Check Results"). - P1_GRADE="$(pyrun -c "import json,sys;g=json.load(open(sys.argv[1]))['gates'];s=[x['score'] for x in g];print(f'{sum(s)/len(s):.2f}')" "$P1_REPORTS/phase1-summary.json" 2>/dev/null)" - pyrun - "$P1_REPORTS/phase1-summary.json" <<'PY' 2>/dev/null || true + P1_GRADE="$(pyrun -c "import json,sys;g=json.load(open(sys.argv[1]))['gates'];s=[x['score'] for x in g];print(f'{sum(s)/len(s):.2f}')" "$REPORTS_DIR/static-eval-summary.json" 2>/dev/null)" + pyrun - "$REPORTS_DIR/static-eval-summary.json" <<'PY' 2>/dev/null || true import json, sys s = json.load(open(sys.argv[1])) -print(" Phase 1 overall:", "PASS" if s["passed"] else "FAIL") +print(" Static eval overall:", "PASS" if s["passed"] else "FAIL") for g in s["gates"]: print(f" {g['gate']:<20} passed={str(g['passed']):<5} score={g['score']:.2f} findings={g['findings']}") scores = [g["score"] for g in s["gates"]] print(f" grade: {sum(scores)/len(scores):.2f} [mean of {len(scores)} gate scores]") PY else - warn "No --repo given; skipping Phase 1." + warn "No --repo given; skipping Static eval." fi # =============================== PHASE 2 =================================== # log "" P2_PASSED="" -info "--- Phase 2 (deterministic conformance) ---" +info "--- Conformance eval (deterministic conformance) ---" if [ -n "$SERVER_URL" ]; then info "Live probe against $SERVER_URL (timeout ${TIMEOUT}s)" - pyrun -m scripts.mcp.probes.phase2_conformance_probe --server-url "$SERVER_URL" --reports-dir "$P2_REPORTS" --timeout "$TIMEOUT" --openapi-url "$OPENAPI_URL" 2>&1 | tail -3 + pyrun -m scripts.mcp.probes.conformance_eval_probe --server-url "$SERVER_URL" --reports-dir "$RESULTS_DIR" --timeout "$TIMEOUT" --openapi-url "$OPENAPI_URL" 2>&1 | tail -3 else warn "No --server-url given; skipping the live probe. Probed checks will report not_evaluated." fi if [ -n "$COMPASS_FACTS" ]; then [ -f "$COMPASS_FACTS" ] || die "--compass-facts file not found: $COMPASS_FACTS" info "Compass consume from $COMPASS_FACTS" - pyrun -m scripts.mcp.probes.phase2_compass_probe --facts-file "$COMPASS_FACTS" --reports-dir "$P2_REPORTS" 2>&1 | tail -1 + pyrun -m scripts.mcp.probes.conformance_eval_compass_probe --facts-file "$COMPASS_FACTS" --reports-dir "$RESULTS_DIR" 2>&1 | tail -1 else log "No --compass-facts given; Compass-consumed checks will report not_evaluated." fi log "" -pyrun -m scripts.mcp.runners.phase2_gate --reports-dir "$P2_REPORTS" --mode "$MODE" +pyrun -m scripts.mcp.runners.conformance_eval_gate --results-dir "$RESULTS_DIR" --reports-dir "$REPORTS_DIR" --mode "$MODE" P2_GATE_RC=$? -P2_PASSED="$(pyrun -c "import json,sys;print('true' if json.load(open(sys.argv[1]))['passed'] else 'false')" "$P2_REPORTS/phase2-summary.json" 2>/dev/null)" -# Phase 2 grade = mean score over EVALUATED checks only (status pass/fail). +P2_PASSED="$(pyrun -c "import json,sys;print('true' if json.load(open(sys.argv[1]))['passed'] else 'false')" "$REPORTS_DIR/conformance-eval-summary.json" 2>/dev/null)" +# Conformance eval grade = mean score over EVALUATED checks only (status pass/fail). # not_evaluated checks carry score 1.0 and must be excluded, or an unprobed server # would look perfect. "n/a" when nothing was evaluated (honest, not a free 1.00). -P2_GRADE="$(pyrun -c "import json,sys;ch=json.load(open(sys.argv[1]))['checks'];e=[x['score'] for x in ch if x['status'] in ('pass','fail')];print(f'{sum(e)/len(e):.2f}' if e else 'n/a')" "$P2_REPORTS/phase2-summary.json" 2>/dev/null)" -pyrun - "$P2_REPORTS/phase2-summary.json" <<'PY' 2>/dev/null || true +P2_GRADE="$(pyrun -c "import json,sys;ch=json.load(open(sys.argv[1]))['checks'];e=[x['score'] for x in ch if x['status'] in ('pass','fail')];print(f'{sum(e)/len(e):.2f}' if e else 'n/a')" "$REPORTS_DIR/conformance-eval-summary.json" 2>/dev/null)" +pyrun - "$REPORTS_DIR/conformance-eval-summary.json" <<'PY' 2>/dev/null || true import json, sys s = json.load(open(sys.argv[1])) c = s["counts"] -print(" Phase 2 overall:", "PASS" if s["passed"] else "FAIL", +print(" Conformance eval overall:", "PASS" if s["passed"] else "FAIL", f"(pass={c.get('pass',0)} fail={c.get('fail',0)} not_evaluated={c.get('not_evaluated',0)})") for ch in s["checks"]: print(f" {ch['check']:<22} {ch['status']:<14} score={ch['score']:.2f} findings={ch['findings']} [{ch['source']}]") @@ -388,8 +388,8 @@ PY # =============================== VERDICT =================================== # log "" info "=== Combined verdict (mode: $MODE) ===" -[ "$P1_RAN" -eq 1 ] && log " Phase 1: ${P1_PASSED:-unknown} grade=${P1_GRADE:-?} [mean of 3 gate scores]" || log " Phase 1: skipped" -log " Phase 2: ${P2_PASSED:-unknown} grade=${P2_GRADE:-?} [mean over evaluated checks]" +[ "$P1_RAN" -eq 1 ] && log " Static eval: ${P1_PASSED:-unknown} grade=${P1_GRADE:-?} [mean of 3 gate scores]" || log " Static eval: skipped" +log " Conformance eval: ${P2_PASSED:-unknown} grade=${P2_GRADE:-?} [mean over evaluated checks]" log " Reports: $REPORTS_ROOT" # Exit non-zero only when a gate actually failed (block mode). warn/disabled never diff --git a/scripts/mcp/runners/behavioral_eval_summary.py b/scripts/mcp/runners/behavioral_eval_summary.py new file mode 100644 index 00000000..0c09fc06 --- /dev/null +++ b/scripts/mcp/runners/behavioral_eval_summary.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Generate unified behavioral-eval-summary.json from mcpchecker-report.json. + +Reads the detailed MCPCheckerResult and writes a unified summary matching +the static/conformance eval format for cross-phase consistency. + +Usage: + python -m scripts.mcp.runners.behavioral_eval_summary --results-dir --report-dir +""" + +from __future__ import annotations + +import argparse +import json +import logging +import sys +from pathlib import Path + +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +logger = logging.getLogger(__name__) + +REPORT_FILENAME = "mcpchecker-report.json" +SUMMARY_FILENAME = "behavioral-eval-summary.json" + + +def map_mcpchecker_to_summary(result: dict) -> dict: + """Map MCPCheckerResult to unified behavioral-eval-summary.json format. + + Args: + result: Parsed mcpchecker-report.json dict + + Returns: + Unified summary dict matching static/conformance eval format + """ + checks = [] + for task in result.get("tasks", []): + # Map status to three-state model + status_map = { + "passed": "pass", + "failed": "fail", + "error": "not_evaluated", + "skipped": "not_evaluated", + } + task_status = task.get("status", "error") + status = status_map.get(task_status, "not_evaluated") + + # Score: use judge pass rate if available, else binary + judge_pass_rate = task.get("llm_judge_pass_rate") + if task_status == "passed": + score = judge_pass_rate if judge_pass_rate is not None else 1.0 + elif task_status == "failed": + score = judge_pass_rate if judge_pass_rate is not None else 0.0 + else: # error or skipped + score = 0.0 + + # Findings: count of failed judge checks + findings = sum(1 for j in task.get("llm_judge_results", []) if not j.get("passed", True)) + + # Message: error or summary + task_name = task.get("task_name", "unknown") + error_msg = task.get("error_message") + if error_msg: + message = error_msg + elif task_status == "passed": + message = f"All checks passed for {task_name}" + else: + message = f"{task_name} evaluation {task_status}" + + checks.append( + { + "check": task_name, + "status": status, + "passed": task_status == "passed", + "score": score, + "findings": findings, + "source": "mcpchecker", + "message": message, + } + ) + + return { + "phase": "behavioral-eval", + "passed": result.get("recommendation") == "pass", + "counts": { + "pass": result.get("passed_tasks", 0), + "fail": result.get("failed_tasks", 0), + "not_evaluated": result.get("error_tasks", 0) + result.get("skipped_tasks", 0), + }, + "checks": checks, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Generate behavioral evaluation unified summary") + parser.add_argument( + "--results-dir", + type=Path, + required=True, + help="Directory containing mcpchecker-report.json (detailed results)", + ) + parser.add_argument( + "--report-dir", + type=Path, + required=True, + help="Directory to write behavioral-eval-summary.json (summaries only)", + ) + args = parser.parse_args() + + if not args.results_dir.is_dir(): + logger.error("Results directory not found: %s", args.results_dir) + return 1 + + report_path = args.results_dir / REPORT_FILENAME + if not report_path.exists(): + logger.error("Report not found: %s", report_path) + return 1 + + # Read mcpchecker-report.json from results/ + try: + with open(report_path) as f: + result = json.load(f) + except Exception as e: + logger.error("Failed to read report: %s", e) + return 1 + + # Generate unified summary + summary = map_mcpchecker_to_summary(result) + + # Write behavioral-eval-summary.json to reports/ + args.report_dir.mkdir(parents=True, exist_ok=True) + summary_path = args.report_dir / SUMMARY_FILENAME + with open(summary_path, "w") as f: + json.dump(summary, f, indent=2) + + logger.info("Wrote unified summary to %s", summary_path) + logger.info( + "Behavioral evaluation overall: %s (pass=%d fail=%d not_evaluated=%d)", + "PASS" if summary["passed"] else "FAIL", + summary["counts"]["pass"], + summary["counts"]["fail"], + summary["counts"]["not_evaluated"], + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/mcp/runners/phase2_gate.py b/scripts/mcp/runners/conformance_eval_gate.py similarity index 56% rename from scripts/mcp/runners/phase2_gate.py rename to scripts/mcp/runners/conformance_eval_gate.py index f9b195d9..3604a821 100644 --- a/scripts/mcp/runners/phase2_gate.py +++ b/scripts/mcp/runners/conformance_eval_gate.py @@ -1,20 +1,19 @@ #!/usr/bin/env python3 -"""Run the MCP Phase 2 gates over the three-state check result files. +"""Run the MCP conformance evaluation gates over the three-state check result files. -Reads the ``-check.json`` files produced by the Phase 2 probe and the -Compass-consume step, evaluates each gate, and writes results into the reports -directory: +Reads the ``-check.json`` files produced by the conformance evaluation probe +and the Compass-consume step, evaluates each gate, and writes results: -- ``-result.json`` per gate (the full GateResult) -- ``phase2-summary.json`` (overall pass/fail + per-check summary with status) +- ``-result.json`` per gate (the full GateResult) → results directory +- ``conformance-eval-summary.json`` (overall pass/fail + per-check summary) → reports directory -Three-state aware: a ``not_evaluated`` check never fails the phase. The step +Three-state aware: a ``not_evaluated`` check never fails the evaluation. The step fails (non-zero exit) only when a check genuinely FAILED in block mode. Reporting to the Evaluation-results database is still handled by the pipeline layer later; this writes local JSON only. Usage: - python -m scripts.mcp.runners.phase2_gate --reports-dir [--mode block|warn] + python -m scripts.mcp.runners.conformance_eval_gate --results-dir --reports-dir [--mode block|warn] """ from __future__ import annotations @@ -25,19 +24,28 @@ import sys from pathlib import Path -from abevalflow.mcp.gates import run_phase2 +from abevalflow.mcp.gates import run_conformance_eval from abevalflow.schemas import GateMode, GatePolicy logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") logger = logging.getLogger(__name__) -SUMMARY_FILENAME = "phase2-summary.json" +SUMMARY_FILENAME = "conformance-eval-summary.json" def main() -> int: - parser = argparse.ArgumentParser(description="Run MCP Phase 2 gates") + parser = argparse.ArgumentParser(description="Run MCP conformance evaluation gates") parser.add_argument( - "--reports-dir", type=Path, required=True, help="Directory holding the Phase 2 check result files" + "--results-dir", + type=Path, + required=True, + help="Directory holding the conformance evaluation check result files (input)", + ) + parser.add_argument( + "--reports-dir", + type=Path, + required=True, + help="Directory for writing the summary JSON file (output)", ) parser.add_argument( "--mode", @@ -47,18 +55,20 @@ def main() -> int: ) args = parser.parse_args() - if not args.reports_dir.is_dir(): - logger.error("Not a directory: %s", args.reports_dir) + if not args.results_dir.is_dir(): + logger.error("Not a directory: %s", args.results_dir) return 1 + args.reports_dir.mkdir(parents=True, exist_ok=True) + policy = GatePolicy(default_mode=GateMode(args.mode)) - results = run_phase2(args.reports_dir, policy) + results = run_conformance_eval(args.results_dir, policy) summary: list[dict] = [] for result in results: check_key = result.get_policy_key() status = result.details.get("status", "unknown") - (args.reports_dir / f"{check_key}-result.json").write_text(result.model_dump_json(indent=2)) + (args.results_dir / f"{check_key}-result.json").write_text(result.model_dump_json(indent=2)) summary.append( { "check": check_key, @@ -79,10 +89,12 @@ def main() -> int: "not_evaluated": sum(1 for s in summary if s["status"] == "not_evaluated"), } (args.reports_dir / SUMMARY_FILENAME).write_text( - json.dumps({"phase": "phase2", "passed": overall_passed, "counts": counts, "checks": summary}, indent=2) + json.dumps( + {"phase": "conformance-eval", "passed": overall_passed, "counts": counts, "checks": summary}, indent=2 + ) ) - logger.info("Phase 2 overall: %s (%s)", "PASS" if overall_passed else "FAIL", counts) + logger.info("Conformance evaluation overall: %s (%s)", "PASS" if overall_passed else "FAIL", counts) # A gate is only not-passed when a real check FAILED in block mode # (not_evaluated and warn-mode failures keep passed=True), so this exits # non-zero exactly when the step should fail. diff --git a/scripts/mcp/runners/phase1_gate.py b/scripts/mcp/runners/static_eval_gate.py similarity index 56% rename from scripts/mcp/runners/phase1_gate.py rename to scripts/mcp/runners/static_eval_gate.py index 908d7a85..fa58c2ca 100644 --- a/scripts/mcp/runners/phase1_gate.py +++ b/scripts/mcp/runners/static_eval_gate.py @@ -1,17 +1,17 @@ #!/usr/bin/env python3 -"""Run the MCP Phase 1 gates over the normalized scan reports. +"""Run the MCP static evaluation gates over the normalized scan reports. -Reads the three scan JSON files produced by the Phase 1 scanners, evaluates each -gate, and writes the results back into the reports directory: +Reads the three scan JSON files produced by the static evaluation scanners, evaluates +each gate, and writes results: -- ``-result.json`` per gate (the full GateResult) -- ``phase1-summary.json`` (overall pass/fail + per-gate summary) +- ``-result.json`` per gate (the full GateResult) → results directory +- ``static-eval-summary.json`` (overall pass/fail + per-gate summary) → reports directory Reporting stub: the three-state Evaluation-results-database writer is added by the -pipeline layer later; Phase 1 checks are pass/fail only. +pipeline layer later; static evaluation checks are pass/fail only. Usage: - python -m scripts.mcp.runners.phase1_gate --reports-dir [--mode block|warn] + python -m scripts.mcp.runners.static_eval_gate --results-dir --reports-dir [--mode block|warn] """ from __future__ import annotations @@ -22,22 +22,28 @@ import sys from pathlib import Path -from abevalflow.mcp.gates import run_phase1 +from abevalflow.mcp.gates import run_static_eval from abevalflow.schemas import GateMode, GatePolicy logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") logger = logging.getLogger(__name__) -SUMMARY_FILENAME = "phase1-summary.json" +SUMMARY_FILENAME = "static-eval-summary.json" def main() -> int: - parser = argparse.ArgumentParser(description="Run MCP Phase 1 gates") + parser = argparse.ArgumentParser(description="Run MCP static evaluation gates") + parser.add_argument( + "--results-dir", + type=Path, + required=True, + help="Directory holding the static evaluation scan JSON files (input)", + ) parser.add_argument( "--reports-dir", type=Path, required=True, - help="Directory holding the Phase 1 scan JSON files", + help="Directory for writing the summary JSON file (output)", ) parser.add_argument( "--mode", @@ -47,17 +53,19 @@ def main() -> int: ) args = parser.parse_args() - if not args.reports_dir.is_dir(): - logger.error("Not a directory: %s", args.reports_dir) + if not args.results_dir.is_dir(): + logger.error("Not a directory: %s", args.results_dir) return 1 + args.reports_dir.mkdir(parents=True, exist_ok=True) + policy = GatePolicy(default_mode=GateMode(args.mode)) - results = run_phase1(args.reports_dir, policy) + results = run_static_eval(args.results_dir, policy) summary: list[dict] = [] for result in results: gate_key = result.get_policy_key() - (args.reports_dir / f"{gate_key}-result.json").write_text(result.model_dump_json(indent=2)) + (args.results_dir / f"{gate_key}-result.json").write_text(result.model_dump_json(indent=2)) summary.append( { "gate": gate_key, @@ -78,12 +86,12 @@ def main() -> int: overall_passed = all(r.passed for r in results) (args.reports_dir / SUMMARY_FILENAME).write_text( json.dumps( - {"phase": "phase1", "passed": overall_passed, "gates": summary}, + {"phase": "static-eval", "passed": overall_passed, "gates": summary}, indent=2, ) ) - logger.info("Phase 1 overall: %s", "PASS" if overall_passed else "FAIL") + logger.info("Static evaluation overall: %s", "PASS" if overall_passed else "FAIL") # Exit non-zero in block mode when any gate failed, so the Tekton step fails. if not overall_passed and args.mode == GateMode.BLOCK.value: return 1 diff --git a/scripts/mcp/scanners/__init__.py b/scripts/mcp/scanners/__init__.py index be296f4c..c77d3a3a 100644 --- a/scripts/mcp/scanners/__init__.py +++ b/scripts/mcp/scanners/__init__.py @@ -1 +1 @@ -"""Phase 1 scanner CLI entry points — shell out to external tools, write scan JSON.""" +"""Static evaluation scanner CLI entry points — shell out to external tools, write scan JSON.""" diff --git a/scripts/mcp/scanners/helpers.py b/scripts/mcp/scanners/helpers.py index b3684056..f1289ae3 100644 --- a/scripts/mcp/scanners/helpers.py +++ b/scripts/mcp/scanners/helpers.py @@ -1,10 +1,10 @@ -"""Scanner I/O helpers for MCP Phase 1. +"""Scanner I/O helpers for MCP static evaluation. Provides ``run_tool`` (shell out to a scanner CLI) and ``write_scan`` (write normalized scan JSON). Finding construction (``make_finding``) lives in ``abevalflow.mcp.types`` — scanners import it from there. -All Phase 1 scanners emit the same JSON shape so the Phase 1 gates (which reuse +All static evaluation scanners emit the same JSON shape so the static evaluation gates (which reuse ``SecurityGate.evaluate_scan_json``) can read them uniformly:: { @@ -39,7 +39,7 @@ def write_scan( """Write a normalized scan report to ``scan_path``. ``extra`` merges additional top-level metadata (e.g. a ``coverage`` block) - into the payload. The Phase 1 gates read only ``findings``, so extra keys are + into the payload. The static evaluation gates read only ``findings``, so extra keys are informational and never change pass/fail. The canonical keys always win, so a caller cannot accidentally clobber ``findings`` via ``extra``. """ diff --git a/scripts/mcp/scanners/phase1_license_scan.py b/scripts/mcp/scanners/static_eval_license_scan.py similarity index 91% rename from scripts/mcp/scanners/phase1_license_scan.py rename to scripts/mcp/scanners/static_eval_license_scan.py index 8155c1de..a444fc0a 100644 --- a/scripts/mcp/scanners/phase1_license_scan.py +++ b/scripts/mcp/scanners/static_eval_license_scan.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 -"""MCP Phase 1 license-compliance scan. +"""MCP static evaluation license-compliance scan. Runs licensee to detect the repository's TOP-LEVEL declared license (from the LICENSE file and/or package manifests) and checks it against an approved-license allow-list. Emits license-scan.json for LicenseGate. Transitive/dependency -license scanning is out of scope (ADR Phase 1). +license scanning is out of scope for static evaluation. Usage: - python -m scripts.mcp.scanners.phase1_license_scan --reports-dir \ + python -m scripts.mcp.scanners.static_eval_license_scan --reports-dir \ [--allow apache-2.0 --allow mit ...] """ @@ -19,7 +19,7 @@ import sys from pathlib import Path -from abevalflow.mcp.gates.phase1_license_gate import DEFAULT_ALLOWED_LICENSES +from abevalflow.mcp.gates.static_eval_license_gate import DEFAULT_ALLOWED_LICENSES from abevalflow.mcp.helpers import make_finding from scripts.mcp.scanners.helpers import run_tool, write_scan @@ -74,7 +74,7 @@ def evaluate_licenses( def main() -> int: - parser = argparse.ArgumentParser(description="MCP Phase 1 license scan (licensee)") + parser = argparse.ArgumentParser(description="MCP static evaluation license scan (licensee)") parser.add_argument("target_dir", type=Path, help="MCP server repo to scan") parser.add_argument( "--reports-dir", diff --git a/scripts/mcp/scanners/phase1_no_user_code_scan.py b/scripts/mcp/scanners/static_eval_no_user_code_scan.py similarity index 96% rename from scripts/mcp/scanners/phase1_no_user_code_scan.py rename to scripts/mcp/scanners/static_eval_no_user_code_scan.py index 956865e5..4b78b010 100644 --- a/scripts/mcp/scanners/phase1_no_user_code_scan.py +++ b/scripts/mcp/scanners/static_eval_no_user_code_scan.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""MCP Phase 1 "does not execute user-provided code" scan. +"""MCP static evaluation "does not execute user-provided code" scan. Runs semgrep with the bundled offline ruleset (rules/no_user_code.yml) over an MCP server repo and normalizes results into no-user-code-scan.json for @@ -15,7 +15,7 @@ high-severity finding - which fails a blocking gate - rather than being trusted. Usage: - python -m scripts.mcp.scanners.phase1_no_user_code_scan --reports-dir + python -m scripts.mcp.scanners.static_eval_no_user_code_scan --reports-dir """ from __future__ import annotations @@ -92,7 +92,7 @@ def normalize(semgrep_results: list[dict]) -> list[dict]: def main() -> int: - parser = argparse.ArgumentParser(description="MCP Phase 1 no-user-code-execution scan (semgrep)") + parser = argparse.ArgumentParser(description="MCP static evaluation no-user-code-execution scan (semgrep)") parser.add_argument("target_dir", type=Path, help="MCP server repo to scan") parser.add_argument( "--reports-dir", diff --git a/scripts/mcp/scanners/phase1_secrets_scan.py b/scripts/mcp/scanners/static_eval_secrets_scan.py similarity index 93% rename from scripts/mcp/scanners/phase1_secrets_scan.py rename to scripts/mcp/scanners/static_eval_secrets_scan.py index fced43c2..e1fa34e4 100644 --- a/scripts/mcp/scanners/phase1_secrets_scan.py +++ b/scripts/mcp/scanners/static_eval_secrets_scan.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -"""MCP Phase 1 secrets scan. +"""MCP static evaluation secrets scan. Runs gitleaks over an MCP server repository (static working-tree scan, no live verification) and normalizes its output into secrets-scan.json for SecretsGate. Usage: - python -m scripts.mcp.scanners.phase1_secrets_scan --reports-dir + python -m scripts.mcp.scanners.static_eval_secrets_scan --reports-dir """ from __future__ import annotations @@ -48,7 +48,7 @@ def normalize(gitleaks_findings: list[dict]) -> list[dict]: def main() -> int: - parser = argparse.ArgumentParser(description="MCP Phase 1 secrets scan (gitleaks)") + parser = argparse.ArgumentParser(description="MCP static evaluation secrets scan (gitleaks)") parser.add_argument("target_dir", type=Path, help="MCP server repo to scan") parser.add_argument( "--reports-dir", diff --git a/suites/mcp/default/eval.yaml b/suites/mcp/default/eval.yaml new file mode 100644 index 00000000..d16edb9d --- /dev/null +++ b/suites/mcp/default/eval.yaml @@ -0,0 +1,37 @@ +# Default MCP Behavioral Evaluation Configuration +# +# Generic task suite for MCP behavioral testing of any MCP server. +# These tests are server-agnostic: they discover tools dynamically +# and verify general behavioral properties (correctness, error handling, +# data redaction) without assuming specific tool names or schemas. +# +# Used by the mcp-behavioral-eval pipeline task. Model values are +# overridden by pipeline parameters at runtime via ${AGENT_LLM_MODEL} +# and ${JUDGE_LLM_MODEL} environment variables. + +kind: Eval +metadata: + name: mcp-behavioral-eval + description: >- + Generic behavioral evaluation for MCP servers. Covers tool correctness, + edge-case handling, and sensitive data redaction per the MCP Server + Evaluation specification. + +config: + agent: + type: builtin.llm-agent + model: ${AGENT_LLM_MODEL} + + llmJudge: + ref: + type: builtin.llm-agent + model: ${JUDGE_LLM_MODEL} + + mcpConfigFile: mcp-config.yaml + + defaultTaskTimeout: 300s + + taskSets: + - path: tasks/tool-correctness.yaml + - path: tasks/edge-case-handling.yaml + - path: tasks/sensitive-data-redaction.yaml diff --git a/suites/mcp/default/mcp-config.yaml b/suites/mcp/default/mcp-config.yaml new file mode 100644 index 00000000..07ba511f --- /dev/null +++ b/suites/mcp/default/mcp-config.yaml @@ -0,0 +1,11 @@ +# MCP Server Connection Configuration +# +# The MCP_URL environment variable is injected at runtime by the +# mcp-behavioral-eval pipeline task, constructed from the service +# name and port of the deployed MCP server. + +mcpServers: + mcp-server-under-test: + type: http + url: ${MCP_URL}/mcp + enableAllTools: true diff --git a/suites/mcp/default/tasks/edge-case-handling.yaml b/suites/mcp/default/tasks/edge-case-handling.yaml new file mode 100644 index 00000000..88bd8e82 --- /dev/null +++ b/suites/mcp/default/tasks/edge-case-handling.yaml @@ -0,0 +1,41 @@ +# MCP Behavioral Evaluation — Edge-Case Handling +# +# Send malformed and boundary inputs to the MCP server's tools +# and verify that the server handles them gracefully (returns an +# error message) rather than crashing or hanging. + +apiVersion: mcpchecker/v1alpha2 +kind: Task +metadata: + name: edge-case-handling + difficulty: medium + +spec: + prompt: + inline: | + You are testing an MCP server's edge-case handling. + + 1. List all available tools on the MCP server. + 2. Pick at least two tools to test. + 3. For each selected tool, attempt the following edge cases: + a. Call the tool with EMPTY or missing required arguments. + b. Call the tool with arguments of the WRONG TYPE + (e.g., pass a string where a number is expected). + c. If the tool accepts a string, send an extremely long value + (1000+ characters of repeated text). + 4. For each edge-case call, report: + - The tool name and the malformed input + - Whether the server returned an error message or failed silently + - Whether the server remained responsive after the call + + The server should return clear error responses for invalid inputs. + It must NOT crash, hang, or return a success status for bad input. + + verify: + - llmJudge: + contains: | + Server handles malformed input gracefully. + The agent: + 1. Tested at least two tools with invalid or boundary inputs + 2. Confirmed the server returned error messages (not crashes or hangs) for malformed inputs + 3. Confirmed the server remained responsive after receiving bad input diff --git a/suites/mcp/default/tasks/sensitive-data-redaction.yaml b/suites/mcp/default/tasks/sensitive-data-redaction.yaml new file mode 100644 index 00000000..d89048af --- /dev/null +++ b/suites/mcp/default/tasks/sensitive-data-redaction.yaml @@ -0,0 +1,41 @@ +# MCP Behavioral Evaluation — Sensitive Data Redaction +# +# Verify that MCP server responses do not leak sensitive data +# such as API keys, tokens, passwords, or internal credentials. + +apiVersion: mcpchecker/v1alpha2 +kind: Task +metadata: + name: sensitive-data-redaction + difficulty: medium + +spec: + prompt: + inline: | + You are testing an MCP server for sensitive data redaction. + + 1. List all available tools on the MCP server. + 2. Invoke several tools with valid inputs — pick tools that are + likely to return detailed information or configuration data. + 3. Carefully examine each response for signs of sensitive data + leakage. Look for: + - API keys or tokens (strings that look like "sk-...", "ghp_...", + "Bearer ...", base64-encoded credentials) + - Passwords or secrets in plaintext + - Internal hostnames, IP addresses, or connection strings that + include credentials + - Environment variable values that contain secrets + - Stack traces that expose internal file paths or configuration + 4. Report your findings: + - Which tools you tested + - Whether any response contained potentially sensitive data + - Specific examples of any leaked data patterns found + + verify: + - llmJudge: + contains: | + No sensitive data found in server responses. + The agent: + 1. Invoked at least one MCP tool and examined the response + 2. Checked for common sensitive data patterns (API keys, tokens, passwords, credentials) + 3. Confirmed no sensitive data was leaked diff --git a/suites/mcp/default/tasks/tool-correctness.yaml b/suites/mcp/default/tasks/tool-correctness.yaml new file mode 100644 index 00000000..467e40ab --- /dev/null +++ b/suites/mcp/default/tasks/tool-correctness.yaml @@ -0,0 +1,41 @@ +# MCP Behavioral Evaluation — Tool Correctness +# +# Run real tasks against the MCP server's tools and verify that +# the outputs are valid and conform to the declared schemas. +# This test is server-agnostic: it discovers available tools +# dynamically and invokes each with reasonable inputs. + +apiVersion: mcpchecker/v1alpha2 +kind: Task +metadata: + name: tool-correctness + difficulty: easy + +spec: + prompt: + inline: | + You are testing an MCP server for tool correctness. + + 1. List all available tools on the MCP server. + 2. For each tool, read its input schema carefully. + 3. Invoke each tool with valid, reasonable input arguments that + conform to its declared input schema. + 4. Report for each tool: + - The tool name + - The input you provided + - Whether the call succeeded or failed + - A summary of the response received + + Do not skip any tools. If a tool requires specific resource + identifiers you do not have, use plausible placeholder values + and note this in your report. + + verify: + - llmJudge: + contains: | + All discovered tools were invoked with valid inputs. + The agent successfully: + 1. Listed the available tools on the MCP server + 2. Attempted to invoke each discovered tool at least once + 3. Provided inputs that match each tool's declared input schema + 4. Reported the result of each tool invocation diff --git a/templates/mcp-behavioral-eval-job.yaml b/templates/mcp-behavioral-eval-job.yaml new file mode 100644 index 00000000..2eeaba5b --- /dev/null +++ b/templates/mcp-behavioral-eval-job.yaml @@ -0,0 +1,81 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: ${JOB_NAME} + namespace: ${NAMESPACE} +spec: + backoffLimit: 0 + activeDeadlineSeconds: ${JOB_TIMEOUT} + template: + spec: + restartPolicy: Never + volumes: + - name: internal-ca + secret: + secretName: ${CA_SECRET} + optional: true + containers: + - name: mcpchecker + image: ${IMAGE} + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + args: + - | + set -euo pipefail + + # Build combined CA bundle if internal CA cert is mounted + if [ -f /tmp/ca/ca.crt ]; then + cat /etc/pki/tls/certs/ca-bundle.crt /tmp/ca/ca.crt > /tmp/combined-ca.crt + export SSL_CERT_FILE=/tmp/combined-ca.crt + echo "Internal CA certificate added to trust store" + fi + + mkdir -p /tmp/tasks + envsubst < /opt/suites/mcp/default/eval.yaml > /tmp/eval.yaml + cp /opt/suites/mcp/default/tasks/*.yaml /tmp/tasks/ + envsubst < /opt/suites/mcp/default/mcp-config.yaml > /tmp/mcp-config.yaml + + cd /tmp + mcpchecker check eval.yaml \ + --mcp-config-file mcp-config.yaml \ + -o json || echo "mcpchecker exited with code $?" + workingDir: /tmp + env: + - name: HOME + value: /tmp + - name: MCP_URL + value: "${MCP_URL}" + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: ${AGENT_SECRET} + key: openai-api-key + - name: OPENAI_BASE_URL + valueFrom: + secretKeyRef: + name: ${AGENT_SECRET} + key: openai-base-url + - name: ANTHROPIC_API_KEY + valueFrom: + secretKeyRef: + name: ${AGENT_SECRET} + key: anthropic-api-key + - name: ANTHROPIC_BASE_URL + valueFrom: + secretKeyRef: + name: ${AGENT_SECRET} + key: anthropic-base-url + - name: AGENT_LLM_MODEL + valueFrom: + secretKeyRef: + name: ${AGENT_SECRET} + key: model-name + - name: JUDGE_LLM_MODEL + valueFrom: + secretKeyRef: + name: ${JUDGE_SECRET} + key: model-name + volumeMounts: + - name: internal-ca + mountPath: /tmp/ca + readOnly: true diff --git a/templates/mcp-conformance-eval-job.yaml b/templates/mcp-conformance-eval-job.yaml new file mode 100644 index 00000000..b89c25ab --- /dev/null +++ b/templates/mcp-conformance-eval-job.yaml @@ -0,0 +1,79 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: ${JOB_NAME} + namespace: ${NAMESPACE} +spec: + backoffLimit: 0 + activeDeadlineSeconds: ${JOB_TIMEOUT} + template: + spec: + restartPolicy: Never + containers: + - name: probe + image: ${IMAGE} + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + args: + - | + set -euo pipefail + + REPORTS_DIR="/tmp/reports" + mkdir -p "$REPORTS_DIR" + + echo "=== Conformance evaluation live probe against ${SERVER_URL} ===" + python -m scripts.mcp.probes.conformance_eval_probe \ + --server-url "${SERVER_URL}" \ + --reports-dir "$REPORTS_DIR" \ + --timeout "${TIMEOUT}" \ + --openapi-url "${OPENAPI_URL}" + + echo "=== Conformance evaluation Compass-consumed checks ===" + COMPASS_ARGS=() + compass_url="${COMPASS_URL:-}" + compass_entity_ref="${COMPASS_ENTITY_REF:-}" + if [ -n "$compass_url" ] && [ -n "$compass_entity_ref" ]; then + echo "Compass live mode: $compass_url (entity: $compass_entity_ref)" + COMPASS_ARGS=( + --compass-url "$compass_url" + --entity-ref "$compass_entity_ref" + ) + else + echo "No Compass source configured; consumed checks will report not_evaluated." + fi + python -m scripts.mcp.probes.conformance_eval_compass_probe \ + --reports-dir "$REPORTS_DIR" "${COMPASS_ARGS[@]}" + + echo "=== Probe reports ===" + for f in "$REPORTS_DIR"/*.json; do + [ -f "$f" ] || continue + printf '===REPORT_JSON:%s===\n' "$(basename "$f")" + # Keep each report on one line so the pipeline can extract the + # exact JSON record from logs without mixing it with adjacent + # reports or diagnostic output. + tr -d '\r\n' < "$f" + printf '\n' + done + echo "===REPORTS_END===" + workingDir: /tmp + env: + - name: HOME + value: /tmp + - name: COMPASS_API_KEY + valueFrom: + secretKeyRef: + name: ${COMPASS_SECRET} + key: api-key + optional: true + - name: COMPASS_URL + valueFrom: + secretKeyRef: + name: ${COMPASS_SECRET} + key: compass-url + optional: true + - name: COMPASS_ENTITY_REF + valueFrom: + secretKeyRef: + name: ${COMPASS_SECRET} + key: entity-ref + optional: true diff --git a/tests/test_mcp_phase2.py b/tests/test_mcp_conformance_eval.py similarity index 81% rename from tests/test_mcp_phase2.py rename to tests/test_mcp_conformance_eval.py index 0c321c99..4bd5cacb 100644 --- a/tests/test_mcp_phase2.py +++ b/tests/test_mcp_conformance_eval.py @@ -1,4 +1,4 @@ -"""Tests for MCP Phase 2 (deterministic contract/conformance). +"""Tests for MCP conformance evaluation (deterministic contract/conformance). Unit tests use synthetic JSON-RPC responses and a fake client, so they need no running server and are CI-safe. The integration test runs the real probe against @@ -14,14 +14,14 @@ import pytest from abevalflow.gates.base import GateMode -from abevalflow.mcp.checks import phase2_conformance_checks as phase2_probe -from abevalflow.mcp.gates import PHASE2_CHECKS, Phase2Gate, run_phase2 +from abevalflow.mcp.checks import conformance_eval_checks as conformance_probe +from abevalflow.mcp.gates import CONFORMANCE_EVAL_CHECKS, ConformanceEvalGate, run_conformance_eval from abevalflow.mcp.helpers import write_check_result from abevalflow.mcp.mcp_client import RpcResponse, _parse_sse_messages, _select_response from abevalflow.mcp.types import STATUS_FAIL, STATUS_NOT_EVALUATED, STATUS_PASS, CheckOutcome from abevalflow.schemas import GatePolicy -from scripts.mcp.probes import phase2_compass_probe as compass_fetch -from scripts.mcp.runners import phase2_gate as run_phase2_gates +from scripts.mcp.probes import conformance_eval_compass_probe as compass_fetch +from scripts.mcp.runners import conformance_eval_gate as run_conformance_eval_gates # --------------------------------------------------------------------------- # Helpers @@ -148,9 +148,9 @@ def _init_ok(): def test_http_support_pass_and_fail(): - assert phase2_probe.check_http_support(None, _init_ok()).status == STATUS_PASS + assert conformance_probe.check_http_support(None, _init_ok()).status == STATUS_PASS bad = _rpc(None, status=500, content_type="text/plain") - out = phase2_probe.check_http_support(None, bad) + out = conformance_probe.check_http_support(None, bad) assert out.status == STATUS_FAIL assert len(out.findings) == 2 # bad status + unparseable body @@ -162,7 +162,7 @@ def test_protocol_compliance_pass(): "tools/list": _rpc({"jsonrpc": "2.0", "id": 91, "error": {"code": -32600, "message": "bad"}}), } ) - out = phase2_probe.check_protocol_compliance(client, _init_ok()) + out = conformance_probe.check_protocol_compliance(client, _init_ok()) assert out.status == STATUS_PASS, out.findings @@ -174,7 +174,7 @@ def test_protocol_compliance_flags_missing_result_and_bad_id(): "tools/list": _rpc({"jsonrpc": "2.0", "id": 91, "error": {"code": -32600}}), } ) - out = phase2_probe.check_protocol_compliance(client, init) + out = conformance_probe.check_protocol_compliance(client, init) assert out.status == STATUS_FAIL rule_ids = {f["rule_id"] for f in out.findings} assert "id-echo" in rule_ids @@ -192,21 +192,21 @@ def post(self, *_a, **_kw): raise MCPConnectionError("connection reset") bad_init = _rpc({"jsonrpc": "2.0", "id": 999, "result": {}}) # wrong id + missing fields - out = phase2_probe.check_protocol_compliance(DroppingClient(), bad_init) + out = conformance_probe.check_protocol_compliance(DroppingClient(), bad_init) assert out.status == STATUS_FAIL assert out.findings # init findings preserved through the disconnect def test_schema_conformance_pass_and_fail(): good = _rpc({"result": {"tools": [{"name": "a", "inputSchema": {"type": "object", "properties": {}}}]}}) - assert phase2_probe.check_schema_conformance(None, good).status == STATUS_PASS + assert conformance_probe.check_schema_conformance(None, good).status == STATUS_PASS bad = _rpc({"result": {"tools": [{"name": "b"}]}}) # no inputSchema - out = phase2_probe.check_schema_conformance(None, bad) + out = conformance_probe.check_schema_conformance(None, bad) assert out.status == STATUS_FAIL empty = _rpc({"result": {"tools": []}}) - assert phase2_probe.check_schema_conformance(None, empty).status == STATUS_NOT_EVALUATED + assert conformance_probe.check_schema_conformance(None, empty).status == STATUS_NOT_EVALUATED def test_schema_conformance_fails_on_broken_tools_list(): @@ -214,12 +214,12 @@ def test_schema_conformance_fails_on_broken_tools_list(): # absence of tools: it must FAIL (not not_evaluated) so a broken endpoint cannot # slip through schema validation. errored = _rpc({"jsonrpc": "2.0", "id": 2, "error": {"code": -32603, "message": "boom"}}) - out = phase2_probe.check_schema_conformance(None, errored) + out = conformance_probe.check_schema_conformance(None, errored) assert out.status == STATUS_FAIL assert out.findings[0]["rule_id"] == "tools-list-malformed" malformed = _rpc({"jsonrpc": "2.0", "id": 2, "result": {"tools": "not-a-list"}}) - assert phase2_probe.check_schema_conformance(None, malformed).status == STATUS_FAIL + assert conformance_probe.check_schema_conformance(None, malformed).status == STATUS_FAIL # --------------------------------------------------------------------------- @@ -254,7 +254,7 @@ def test_openapi_pass_from_well_known_path(): "http://srv:8080/mcp", {"http://srv:8080/openapi.json": (200, "application/json", _VALID_OPENAPI)}, ) - out = phase2_probe.check_openapi_conformance(client, None) + out = conformance_probe.check_openapi_conformance(client, None) assert out.status == STATUS_PASS @@ -262,7 +262,7 @@ def test_openapi_pass_from_explicit_url(): client = _OpenApiClient( "http://srv:8080/mcp", {"http://host/custom.json": (200, "application/json", _VALID_OPENAPI)} ) - out = phase2_probe.check_openapi_conformance(client, None, openapi_url="http://host/custom.json") + out = conformance_probe.check_openapi_conformance(client, None, openapi_url="http://host/custom.json") assert out.status == STATUS_PASS @@ -271,7 +271,7 @@ def test_openapi_pass_from_mcp_native_contract(): # equivalent satisfies "OpenAPI (or equivalent)" per the ADR. client = _OpenApiClient("http://srv:8080/mcp", {}) # all well-known paths unreachable tools = _rpc({"result": {"tools": [{"name": "a", "inputSchema": {"type": "object"}}]}}) - out = phase2_probe.check_openapi_conformance(client, tools) + out = conformance_probe.check_openapi_conformance(client, tools) assert out.status == STATUS_PASS assert "MCP-native equivalent" in out.reason @@ -280,7 +280,7 @@ def test_openapi_fail_when_neither_rest_nor_mcp(): # No REST contract AND no usable MCP tools/list -> no contract at all. client = _OpenApiClient("http://srv:8080/mcp", {}) empty_tools = _rpc({"result": {"tools": []}}) - out = phase2_probe.check_openapi_conformance(client, empty_tools) + out = conformance_probe.check_openapi_conformance(client, empty_tools) assert out.status == STATUS_FAIL assert out.findings[0]["rule_id"] == "openapi-missing" @@ -288,7 +288,7 @@ def test_openapi_fail_when_neither_rest_nor_mcp(): def test_openapi_fail_when_absent_and_no_tools_resp(): # No REST contract AND tools_resp is None -> FAIL. client = _OpenApiClient("http://srv:8080/mcp", {}) - out = phase2_probe.check_openapi_conformance(client, None) + out = conformance_probe.check_openapi_conformance(client, None) assert out.status == STATUS_FAIL assert out.findings[0]["rule_id"] == "openapi-missing" @@ -299,7 +299,7 @@ def test_openapi_skips_unparseable_well_known_path(): client = _OpenApiClient( "http://srv:8080/mcp", {"http://srv:8080/openapi.json": (200, "application/json", "{ not json")} ) - out = phase2_probe.check_openapi_conformance(client, None) + out = conformance_probe.check_openapi_conformance(client, None) assert out.status == STATUS_FAIL assert out.findings[0]["rule_id"] == "openapi-missing" @@ -307,7 +307,7 @@ def test_openapi_skips_unparseable_well_known_path(): def test_openapi_explicit_url_fails_when_unparseable(): # An explicit --openapi-url that is unparseable is a real FAIL (the operator pointed at it). client = _OpenApiClient("http://srv:8080/mcp", {"http://host/api.json": (200, "application/json", "{ not json")}) - out = phase2_probe.check_openapi_conformance(client, None, openapi_url="http://host/api.json") + out = conformance_probe.check_openapi_conformance(client, None, openapi_url="http://host/api.json") assert out.status == STATUS_FAIL assert out.findings[0]["rule_id"] == "openapi-unparseable" @@ -316,7 +316,7 @@ def test_openapi_skips_non_openapi3_well_known_path(): # A Swagger 2.0 doc at a well-known path is skipped; falls through to missing. doc = json.dumps({"swagger": "2.0", "info": {}, "paths": {}}) client = _OpenApiClient("http://srv:8080/mcp", {"http://srv:8080/openapi.json": (200, "application/json", doc)}) - out = phase2_probe.check_openapi_conformance(client, None) + out = conformance_probe.check_openapi_conformance(client, None) assert out.status == STATUS_FAIL assert out.findings[0]["rule_id"] == "openapi-missing" @@ -324,7 +324,7 @@ def test_openapi_skips_non_openapi3_well_known_path(): def test_openapi_explicit_url_fails_when_not_openapi3(): doc = json.dumps({"swagger": "2.0", "info": {}, "paths": {}}) client = _OpenApiClient("http://srv:8080/mcp", {"http://host/api.json": (200, "application/json", doc)}) - out = phase2_probe.check_openapi_conformance(client, None, openapi_url="http://host/api.json") + out = conformance_probe.check_openapi_conformance(client, None, openapi_url="http://host/api.json") assert out.status == STATUS_FAIL assert out.findings[0]["rule_id"] == "openapi-invalid" @@ -334,34 +334,34 @@ def test_schema_conformance_output_schema_when_present(): both = _rpc( {"result": {"tools": [{"name": "a", "inputSchema": {"type": "object"}, "outputSchema": {"type": "object"}}]}} ) - assert phase2_probe.check_schema_conformance(None, both).status == STATUS_PASS + assert conformance_probe.check_schema_conformance(None, both).status == STATUS_PASS # Present-but-malformed output schema -> fail (optional, but must be well-formed when declared). bad_out = _rpc({"result": {"tools": [{"name": "b", "inputSchema": {"type": "object"}, "outputSchema": "nope"}]}}) - out = phase2_probe.check_schema_conformance(None, bad_out) + out = conformance_probe.check_schema_conformance(None, bad_out) assert out.status == STATUS_FAIL assert out.findings[0]["rule_id"] == "output-schema-shape" # No output schema at all -> not a violation (optional in the MCP spec). no_out = _rpc({"result": {"tools": [{"name": "c", "inputSchema": {"type": "object"}}]}}) - assert phase2_probe.check_schema_conformance(None, no_out).status == STATUS_PASS + assert conformance_probe.check_schema_conformance(None, no_out).status == STATUS_PASS def test_schema_is_object_rejects_conflicting_type_and_bad_properties(): # Declared type conflicts with object -> not an object schema. - assert phase2_probe._schema_is_object({"type": "array", "properties": {}}) is False + assert conformance_probe._schema_is_object({"type": "array", "properties": {}}) is False # type object but properties is not a mapping -> malformed. - assert phase2_probe._schema_is_object({"type": "object", "properties": []}) is False + assert conformance_probe._schema_is_object({"type": "object", "properties": []}) is False # No type but an object keyword present -> accepted. - assert phase2_probe._schema_is_object({"properties": {}}) is True + assert conformance_probe._schema_is_object({"properties": {}}) is True # No type and no object keyword -> not an object schema. - assert phase2_probe._schema_is_object({"description": "x"}) is False + assert conformance_probe._schema_is_object({"description": "x"}) is False # Not a dict at all. - assert phase2_probe._schema_is_object("nope") is False + assert conformance_probe._schema_is_object("nope") is False # JSON Schema allows a type array; one that permits "object" is accepted. - assert phase2_probe._schema_is_object({"type": ["object", "null"], "properties": {}}) is True + assert conformance_probe._schema_is_object({"type": ["object", "null"], "properties": {}}) is True # A type array that does NOT include "object" is rejected. - assert phase2_probe._schema_is_object({"type": ["array", "null"]}) is False + assert conformance_probe._schema_is_object({"type": ["array", "null"]}) is False def test_protocol_compliance_flags_non_object_error(): @@ -372,7 +372,7 @@ def test_protocol_compliance_flags_non_object_error(): "tools/list": _rpc({"jsonrpc": "2.0", "id": 91, "error": {"code": -32600}}), } ) - out = phase2_probe.check_protocol_compliance(client, _init_ok()) + out = conformance_probe.check_protocol_compliance(client, _init_ok()) assert out.status == STATUS_FAIL assert "error-not-object" in {f["rule_id"] for f in out.findings} @@ -380,17 +380,17 @@ def test_protocol_compliance_flags_non_object_error(): def test_tool_annotations_absent_is_not_evaluated(): # Annotations are optional in the MCP spec: none declared -> not_evaluated. resp = _rpc({"result": {"tools": [{"name": "a"}, {"name": "b"}]}}) - assert phase2_probe.check_tool_annotations(None, resp).status == STATUS_NOT_EVALUATED + assert conformance_probe.check_tool_annotations(None, resp).status == STATUS_NOT_EVALUATED def test_tool_annotations_wellformed_passes(): resp = _rpc({"result": {"tools": [{"name": "a", "annotations": {"readOnlyHint": True}}, {"name": "b"}]}}) - assert phase2_probe.check_tool_annotations(None, resp).status == STATUS_PASS + assert conformance_probe.check_tool_annotations(None, resp).status == STATUS_PASS def test_tool_annotations_malformed_fails(): resp = _rpc({"result": {"tools": [{"name": "a", "annotations": {"readOnlyHint": "yes"}}]}}) - out = phase2_probe.check_tool_annotations(None, resp) + out = conformance_probe.check_tool_annotations(None, resp) assert out.status == STATUS_FAIL assert out.findings[0]["rule_id"] == "annotation-malformed" @@ -400,7 +400,7 @@ class Burst: def list_tools(self, request_id=0): return _rpc({}, status=429 if request_id == 1005 else 200) - assert phase2_probe.check_rate_limiting(Burst(), burst=10).status == STATUS_PASS + assert conformance_probe.check_rate_limiting(Burst(), burst=10).status == STATUS_PASS def test_rate_limiting_not_evaluated_without_throttle(): @@ -408,13 +408,13 @@ class Burst: def list_tools(self, request_id=0): return _rpc({}, status=200) - assert phase2_probe.check_rate_limiting(Burst(), burst=5).status == STATUS_NOT_EVALUATED + assert conformance_probe.check_rate_limiting(Burst(), burst=5).status == STATUS_NOT_EVALUATED def test_deferred_checks_are_not_evaluated(): - # Behavioral checks deferred to Phase 3 (can't be verified black-box in Phase 2). - assert phase2_probe.check_response_size_limit(None).status == STATUS_NOT_EVALUATED - assert phase2_probe.check_mandatory_timeouts(None).status == STATUS_NOT_EVALUATED + # Behavioral checks deferred to behavioral evaluation (can't be verified black-box in conformance eval). + assert conformance_probe.check_response_size_limit(None).status == STATUS_NOT_EVALUATED + assert conformance_probe.check_mandatory_timeouts(None).status == STATUS_NOT_EVALUATED # --------------------------------------------------------------------------- @@ -474,7 +474,7 @@ def test_map_facts_oauth_explicit_false_fails_only_for_false(): def test_map_results_pass_and_fail(): - from abevalflow.mcp.checks.phase2_compass_checks import map_results + from abevalflow.mcp.checks.conformance_eval_compass_checks import map_results results = [ {"checkId": "mcp-tool-schema-check", "state": "passed", "details": {"notes": {"data": "All names valid"}}}, @@ -487,7 +487,7 @@ def test_map_results_pass_and_fail(): def test_map_results_missing_check_is_not_evaluated(): - from abevalflow.mcp.checks.phase2_compass_checks import map_results + from abevalflow.mcp.checks.conformance_eval_compass_checks import map_results # Empty results — entity not registered yet by_check = {o.check: o for o in map_results([])} @@ -496,7 +496,7 @@ def test_map_results_missing_check_is_not_evaluated(): def test_map_results_error_state_is_not_evaluated(): - from abevalflow.mcp.checks.phase2_compass_checks import map_results + from abevalflow.mcp.checks.conformance_eval_compass_checks import map_results results = [ {"checkId": "mcp-tool-schema-check", "state": "ERROR", "details": {}}, @@ -544,9 +544,9 @@ def test_gate_pass_fail_not_evaluated(tmp_path): # remaining checks have no file -> not_evaluated, must not fail policy = GatePolicy(default_mode=GateMode.BLOCK) - results = {r.get_policy_key(): r for r in run_phase2(tmp_path, policy)} + results = {r.get_policy_key(): r for r in run_conformance_eval(tmp_path, policy)} - assert len(results) == len(PHASE2_CHECKS) + assert len(results) == len(CONFORMANCE_EVAL_CHECKS) assert results["http-support"].passed is True assert results["protocol-compliance"].passed is False # fail in block mode assert results["openapi-conformance"].passed is True # not_evaluated never penalized @@ -555,7 +555,7 @@ def test_gate_pass_fail_not_evaluated(tmp_path): def test_corrupt_result_file_blocks_only_in_block_mode(tmp_path): (tmp_path / "http-support-check.json").write_text("{ not valid json ") - gate = Phase2Gate("http-support") + gate = ConformanceEvalGate("http-support") warn = gate.evaluate(tmp_path, GatePolicy(default_mode=GateMode.WARN)) assert warn.details["status"] == STATUS_FAIL assert warn.passed is True # warn never blocks, even on a corrupt file @@ -566,29 +566,51 @@ def test_corrupt_result_file_blocks_only_in_block_mode(tmp_path): def test_fail_without_findings_scores_zero(tmp_path): # A bare fail (no findings) must not report a perfect score. write_check_result(tmp_path, CheckOutcome("http-support", STATUS_FAIL, "boom", findings=[])) - res = Phase2Gate("http-support").evaluate(tmp_path, GatePolicy(default_mode=GateMode.BLOCK)) + res = ConformanceEvalGate("http-support").evaluate(tmp_path, GatePolicy(default_mode=GateMode.BLOCK)) assert res.passed is False assert res.score == 0.0 def test_not_evaluated_does_not_fail_runner(tmp_path, monkeypatch): _write(tmp_path, "http-support", STATUS_PASS) - for check in PHASE2_CHECKS: + for check in CONFORMANCE_EVAL_CHECKS: if check != "http-support": _write(tmp_path, check, STATUS_NOT_EVALUATED) - monkeypatch.setattr("sys.argv", ["run_phase2_gates", "--reports-dir", str(tmp_path), "--mode", "block"]) - assert run_phase2_gates.main() == 0 - summary = json.loads((tmp_path / "phase2-summary.json").read_text()) + monkeypatch.setattr( + "sys.argv", + [ + "run_conformance_eval_gates", + "--results-dir", + str(tmp_path), + "--reports-dir", + str(tmp_path), + "--mode", + "block", + ], + ) + assert run_conformance_eval_gates.main() == 0 + summary = json.loads((tmp_path / "conformance-eval-summary.json").read_text()) assert summary["passed"] is True - assert summary["counts"]["not_evaluated"] == len(PHASE2_CHECKS) - 1 + assert summary["counts"]["not_evaluated"] == len(CONFORMANCE_EVAL_CHECKS) - 1 def test_real_fail_exits_nonzero_in_block(tmp_path, monkeypatch): - for check in PHASE2_CHECKS: + for check in CONFORMANCE_EVAL_CHECKS: _write(tmp_path, check, STATUS_PASS) _write(tmp_path, "protocol-compliance", STATUS_FAIL, [{"severity": "high", "message": "m", "rule_id": "r"}]) - monkeypatch.setattr("sys.argv", ["run_phase2_gates", "--reports-dir", str(tmp_path), "--mode", "block"]) - assert run_phase2_gates.main() == 1 + monkeypatch.setattr( + "sys.argv", + [ + "run_conformance_eval_gates", + "--results-dir", + str(tmp_path), + "--reports-dir", + str(tmp_path), + "--mode", + "block", + ], + ) + assert run_conformance_eval_gates.main() == 1 # --------------------------------------------------------------------------- @@ -602,7 +624,7 @@ def test_real_fail_exits_nonzero_in_block(tmp_path, monkeypatch): def test_probe_against_live_server(tmp_path): # Smoke against a real server, e.g. generic-mock-mcp-server run with # --transport streamable-http on :8080 (endpoint http://localhost:8080/mcp). - outcomes = phase2_probe.probe_all(_SERVER_URL, timeout=10.0) + outcomes = conformance_probe.probe_all(_SERVER_URL, timeout=10.0) by_check = {o.check: o for o in outcomes} assert by_check["http-support"].status == STATUS_PASS assert by_check["protocol-compliance"].status in (STATUS_PASS, STATUS_FAIL) diff --git a/tests/test_mcp_phase1.py b/tests/test_mcp_static_eval.py similarity index 95% rename from tests/test_mcp_phase1.py rename to tests/test_mcp_static_eval.py index 16e97219..939acf24 100644 --- a/tests/test_mcp_phase1.py +++ b/tests/test_mcp_static_eval.py @@ -1,4 +1,4 @@ -"""Tests for MCP Phase 1 static / build-time checks. +"""Tests for MCP static evaluation / build-time checks. Two layers: @@ -23,12 +23,12 @@ LicenseGate, NoUserCodeGate, SecretsGate, - run_phase1, + run_static_eval, ) from abevalflow.schemas import GateMode, GatePolicy -from scripts.mcp.scanners import phase1_license_scan as license_scan -from scripts.mcp.scanners import phase1_no_user_code_scan as no_user_code_scan -from scripts.mcp.scanners import phase1_secrets_scan as secrets_scan +from scripts.mcp.scanners import static_eval_license_scan as license_scan +from scripts.mcp.scanners import static_eval_no_user_code_scan as no_user_code_scan +from scripts.mcp.scanners import static_eval_secrets_scan as secrets_scan BLOCK = GatePolicy(default_mode=GateMode.BLOCK) @@ -108,21 +108,21 @@ def test_secrets_gate_passes_when_clean(tmp_path): assert result.score == 1.0 -def test_run_phase1_all_clean_passes(tmp_path): +def test_run_static_eval_all_clean_passes(tmp_path): _write(tmp_path, "secrets-scan.json", []) _write(tmp_path, "no-user-code-scan.json", []) _write(tmp_path, "license-scan.json", []) - results = run_phase1(tmp_path, BLOCK) + results = run_static_eval(tmp_path, BLOCK) assert len(results) == 3 assert all(r.passed for r in results) assert {r.policy_key for r in results} == {"mcp-secrets", "mcp-no-user-code", "mcp-license"} -def test_run_phase1_one_dirty_fails_that_gate(tmp_path): +def test_run_static_eval_one_dirty_fails_that_gate(tmp_path): _write(tmp_path, "secrets-scan.json", []) _write(tmp_path, "no-user-code-scan.json", [{"severity": "high", "message": "eval", "rule_id": "x"}]) _write(tmp_path, "license-scan.json", []) - results = {r.policy_key: r for r in run_phase1(tmp_path, BLOCK)} + results = {r.policy_key: r for r in run_static_eval(tmp_path, BLOCK)} assert results["mcp-secrets"].passed is True assert results["mcp-no-user-code"].passed is False assert results["mcp-license"].passed is True