diff --git a/src/bcbench/agent/shared/mcp_gateway.py b/src/bcbench/agent/shared/mcp_gateway.py index 20d14a853..b5cb0aae2 100644 --- a/src/bcbench/agent/shared/mcp_gateway.py +++ b/src/bcbench/agent/shared/mcp_gateway.py @@ -19,7 +19,7 @@ import time from http.client import HTTPConnection from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from typing import cast +from typing import cast, override from urllib.parse import urlsplit from bcbench_core.container import ContainerConfig @@ -236,6 +236,7 @@ def _build_handler(gateway: BcMcpGateway) -> type[BaseHTTPRequestHandler]: class _ProxyHandler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" + @override def log_message(self, format: str, *args: object) -> None: # match stdlib signature; silence access log pass diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index 3c51e6dd6..93dc3d751 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -1,7 +1,7 @@ import logging import random from pathlib import Path -from typing import Annotated, cast +from typing import Annotated, cast, override import typer from bcbench_core.filesystem import prepare_run_dir @@ -346,12 +346,15 @@ class MockEvaluationPipeline(EvaluationPipeline[BaseDatasetEntry]): It randomly generates different scenarios to test result handling and serialization. """ + @override def setup_workspace(self, entry: BaseDatasetEntry, repo_path: Path) -> None: logger.info("Mock pipeline: Skipping workspace setup") + @override def setup(self, context: EvaluationContext[BaseDatasetEntry]) -> None: logger.info("Mock pipeline: Skipping setup") + @override def run_agent(self, context: EvaluationContext[BaseDatasetEntry], agent_runner: AgentRunner[BaseDatasetEntry]) -> None: """Generate random agent metrics and experiment configuration.""" logger.info("Mock pipeline: Generating random metrics and experiment configuration") @@ -382,6 +385,7 @@ def run_agent(self, context: EvaluationContext[BaseDatasetEntry], agent_runner: logger.info(f"Using agent metrics: {context.metrics}") logger.info(f"Using experiment configuration: {context.experiment}") + @override def evaluate(self, context: EvaluationContext[BaseDatasetEntry]) -> None: """Create random evaluation result to test different outcome scenarios.""" logger.info("Mock pipeline: Generating random evaluation result") diff --git a/src/bcbench/dataset/codereview.py b/src/bcbench/dataset/codereview.py index 55a6242f4..c08b77ebf 100644 --- a/src/bcbench/dataset/codereview.py +++ b/src/bcbench/dataset/codereview.py @@ -1,7 +1,7 @@ from __future__ import annotations from enum import StrEnum -from typing import Annotated, Self +from typing import Annotated, Self, override from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator @@ -81,6 +81,7 @@ def _coerce_severity(cls, value: object) -> Severity | None: def severity_label(self) -> str: return self.severity.value if self.severity is not None else "unspecified" + @override def __str__(self) -> str: loc = f"{self.file}:{self.line_start}" if self.line_end and self.line_end != self.line_start: @@ -129,9 +130,11 @@ def _validate_article_annotations(self) -> Self: ) return self + @override def get_task(self) -> str: return self.patch + @override def get_expected_output(self) -> str: return "\n".join(str(c) for c in self.expected_comments) diff --git a/src/bcbench/dataset/dataset_entry.py b/src/bcbench/dataset/dataset_entry.py index 0162f7c9f..1310d3d43 100644 --- a/src/bcbench/dataset/dataset_entry.py +++ b/src/bcbench/dataset/dataset_entry.py @@ -4,7 +4,7 @@ import re from abc import abstractmethod from pathlib import Path -from typing import Annotated, Literal, Self +from typing import Annotated, Literal, Self, override from bcbench_core.dataset import TestEntry from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -118,6 +118,7 @@ class RepoGroundedEntry(BaseDatasetEntry): patch: Annotated[str, Field(min_length=1, pattern=r"^[^\x00]*$")] @property + @override def customization_profile(self) -> str: return self.repo.replace("/", "-") @@ -125,6 +126,7 @@ def customization_profile(self) -> str: def problem_statement_dir(self) -> Path: return _config.paths.problem_statement_dir / self.instance_id + @override def get_task(self) -> str: readme_path = self.problem_statement_dir / _config.file_patterns.problem_statement_readme return readme_path.read_text(encoding="utf-8") @@ -158,6 +160,7 @@ def validate_baseapp_patches_are_w1_only(self) -> Self: class BugFixEntry(_BugFixTestGenBase): """Dataset entry for the bug-fix category.""" + @override def get_expected_output(self) -> str: return self.patch @@ -165,6 +168,7 @@ def get_expected_output(self) -> str: class TestGenEntry(_BugFixTestGenBase): """Dataset entry for the test-generation category.""" + @override def get_expected_output(self) -> str: return self.test_patch @@ -178,12 +182,15 @@ class NL2ALEntry(BaseDatasetEntry): audience: Literal["Business", "Technical", "Both"] @property + @override def customization_profile(self) -> str: return "nl2al" + @override def get_task(self) -> str: return self.nl_prompt + @override def get_expected_output(self) -> Checklist: return {"assertions": self.expected} @@ -204,11 +211,14 @@ class DataQueryEntry(BaseDatasetEntry): ordered: bool = False @property + @override def customization_profile(self) -> str: return "dataquery" + @override def get_task(self) -> str: return self.nl_prompt + @override def get_expected_output(self) -> str: return self.gold_query diff --git a/src/bcbench/dataset/extensibility_request.py b/src/bcbench/dataset/extensibility_request.py index aa5a6e58d..3811adee5 100644 --- a/src/bcbench/dataset/extensibility_request.py +++ b/src/bcbench/dataset/extensibility_request.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Annotated, Literal +from typing import Annotated, Literal, override from pydantic import Field @@ -20,12 +20,14 @@ class ExtRequestAdvisorEntry(RepoGroundedEntry): comments: str = "" expected: Annotated[list[ChecklistAssertion], Field(min_length=1)] + @override def get_task(self) -> str: sections = [f"# {self.title}", "", self.description.rstrip()] if self.comments.strip(): sections += ["", "## Additional requester context", "", self.comments.rstrip()] return "\n".join(sections) + @override def get_expected_output(self) -> Checklist: return {"assertions": self.expected} @@ -43,6 +45,7 @@ class ExtRequestImplementEntry(RepoGroundedEntry): # LLM-judge checklist: expected event/signature/placement and expected layer propagation. expected: Annotated[list[ChecklistAssertion], Field(min_length=1)] + @override def get_expected_output(self) -> Checklist: return {"assertions": self.expected} @@ -83,6 +86,7 @@ class ExtRequestTriageEntry(RepoGroundedEntry): # LLM-judge checklist: expected labels_to_set, issue_state and advisory-comment substance. expected: Annotated[list[ChecklistAssertion], Field(min_length=1)] + @override def get_task(self) -> str: sections = [f"# {self.title}", "", self.description.rstrip()] if self.current_labels: @@ -91,5 +95,6 @@ def get_task(self) -> str: sections += ["", "## Follow-up conversation", "", self.comments.rstrip()] return "\n".join(sections) + @override def get_expected_output(self) -> Checklist: return {"assertions": self.expected} diff --git a/src/bcbench/evaluate/bugfix.py b/src/bcbench/evaluate/bugfix.py index c9a403f85..4f513f40f 100644 --- a/src/bcbench/evaluate/bugfix.py +++ b/src/bcbench/evaluate/bugfix.py @@ -1,5 +1,6 @@ import logging from pathlib import Path +from typing import override from bcbench_core.bc import build_and_publish_projects from bcbench_core.exceptions import BuildError, TestExecutionError @@ -21,11 +22,13 @@ class BugFixPipeline(EvaluationPipeline[BugFixEntry]): """Pipeline for bug-fix evaluation category.""" + @override def setup_workspace(self, entry: BugFixEntry, repo_path: Path) -> None: setup_repo_prebuild(entry, repo_path) copy_problem_statement_folder(entry, repo_path) set_runtime_version(repo_path, entry.project_paths) + @override def setup(self, context: EvaluationContext[BugFixEntry]) -> None: setup_repo_prebuild(context.entry, context.repo_path) @@ -39,10 +42,12 @@ def setup(self, context: EvaluationContext[BugFixEntry]) -> None: copy_problem_statement_folder(context.entry, context.repo_path) set_runtime_version(context.repo_path, context.entry.project_paths) + @override def run_agent(self, context: EvaluationContext[BugFixEntry], agent_runner: AgentRunner[BugFixEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): context.metrics, context.experiment = agent_runner(context) + @override def evaluate(self, context: EvaluationContext[BugFixEntry]) -> None: container = context.get_container() test_projects, _app_projects = categorize_projects(context.entry.project_paths) diff --git a/src/bcbench/evaluate/codereview.py b/src/bcbench/evaluate/codereview.py index bbc72797f..8ad821415 100644 --- a/src/bcbench/evaluate/codereview.py +++ b/src/bcbench/evaluate/codereview.py @@ -1,6 +1,7 @@ import logging import subprocess from pathlib import Path +from typing import override from bcbench_core.git import apply_patch, fetch_commit_if_missing @@ -31,6 +32,7 @@ class CodeReviewPipeline(EvaluationPipeline[CodeReviewEntry]): as local git changes so the agent can review the branch diff directly. """ + @override def setup_workspace(self, entry: CodeReviewEntry, repo_path: Path) -> None: """Setup workspace for code review by applying the entry patch as local changes.""" # Code-review base commits are pre-squash PR commits, so they might be missing from local dev setups. @@ -48,13 +50,16 @@ def setup_workspace(self, entry: CodeReviewEntry, repo_path: Path) -> None: check=True, ) + @override def setup(self, context: EvaluationContext[CodeReviewEntry]) -> None: self.setup_workspace(context.entry, context.repo_path) + @override def run_agent(self, context: EvaluationContext[CodeReviewEntry], agent_runner: AgentRunner[CodeReviewEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): context.metrics, context.experiment = agent_runner(context) + @override def evaluate(self, context: EvaluationContext[CodeReviewEntry]) -> None: review_output_file: Path = context.repo_path / REVIEW_OUTPUT_FILE diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index e5ded1c93..61df1a724 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -3,6 +3,7 @@ from collections.abc import Callable, Mapping, Sequence from decimal import Decimal, InvalidOperation from pathlib import Path +from typing import override from bcbench_core.filesystem import clear_directory @@ -94,17 +95,21 @@ class DataQueryPipeline(EvaluationPipeline[DataQueryEntry]): genuinely querying the environment. """ + @override def setup_workspace(self, entry: DataQueryEntry, repo_path: Path) -> None: # The workspace is shared into the running container, so its contents are cleared in place. clear_directory(repo_path) + @override def setup(self, context: EvaluationContext[DataQueryEntry]) -> None: self.setup_workspace(context.entry, context.repo_path) + @override def run_agent(self, context: EvaluationContext[DataQueryEntry], agent_runner: Callable) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): context.metrics, context.experiment = agent_runner(context) + @override def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: query_file = context.repo_path / GENERATED_QUERY_FILE # query.al is only an inspection artifact now; scoring is on the data the agent retrieved. diff --git a/src/bcbench/evaluate/ext_request_advisor.py b/src/bcbench/evaluate/ext_request_advisor.py index 9fc798c5a..7c98a12d0 100644 --- a/src/bcbench/evaluate/ext_request_advisor.py +++ b/src/bcbench/evaluate/ext_request_advisor.py @@ -1,5 +1,6 @@ import logging from pathlib import Path +from typing import override from bcbench.dataset import ExtRequestAdvisorEntry from bcbench.evaluate.base import AgentRunner, EvaluationPipeline @@ -18,17 +19,21 @@ class ExtRequestAdvisorPipeline(EvaluationPipeline[ExtRequestAdvisorEntry]): """Offline single-shot proxy for the interactive extensibility advisor.""" + @override def setup_workspace(self, entry: ExtRequestAdvisorEntry, repo_path: Path) -> None: setup_repo_prebuild(entry, repo_path) (repo_path / ADVISOR_RESULT_FILE).unlink(missing_ok=True) + @override def setup(self, context: EvaluationContext[ExtRequestAdvisorEntry]) -> None: self.setup_workspace(context.entry, context.repo_path) + @override def run_agent(self, context: EvaluationContext[ExtRequestAdvisorEntry], agent_runner: AgentRunner[ExtRequestAdvisorEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): context.metrics, context.experiment = agent_runner(context) + @override def evaluate(self, context: EvaluationContext[ExtRequestAdvisorEntry]) -> None: result_path = context.repo_path / ADVISOR_RESULT_FILE raw = result_path.read_text(encoding="utf-8").strip() if result_path.exists() else "" diff --git a/src/bcbench/evaluate/ext_request_implement.py b/src/bcbench/evaluate/ext_request_implement.py index e95c91541..79d6ee20c 100644 --- a/src/bcbench/evaluate/ext_request_implement.py +++ b/src/bcbench/evaluate/ext_request_implement.py @@ -1,5 +1,6 @@ import logging from pathlib import Path +from typing import override from bcbench_core.exceptions import EmptyDiffError from bcbench_core.git import stage_and_get_diff @@ -24,18 +25,22 @@ class ExtRequestImplementPipeline(EvaluationPipeline[ExtRequestImplementEntry]): by an LLM judge against the entry checklist. """ + @override def setup_workspace(self, entry: ExtRequestImplementEntry, repo_path: Path) -> None: setup_repo_prebuild(entry, repo_path) copy_problem_statement_folder(entry, repo_path) set_runtime_version(repo_path, entry.project_paths) + @override def setup(self, context: EvaluationContext[ExtRequestImplementEntry]) -> None: self.setup_workspace(context.entry, context.repo_path) + @override def run_agent(self, context: EvaluationContext[ExtRequestImplementEntry], agent_runner: AgentRunner[ExtRequestImplementEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): context.metrics, context.experiment = agent_runner(context) + @override def evaluate(self, context: EvaluationContext[ExtRequestImplementEntry]) -> None: try: generated_patch = stage_and_get_diff(context.repo_path, exclude=("**/app.json", "*.docx", "*.md")) diff --git a/src/bcbench/evaluate/ext_request_triage.py b/src/bcbench/evaluate/ext_request_triage.py index d0dfdc562..1743916e8 100644 --- a/src/bcbench/evaluate/ext_request_triage.py +++ b/src/bcbench/evaluate/ext_request_triage.py @@ -10,6 +10,7 @@ import logging from pathlib import Path +from typing import override from bcbench.dataset import ExtRequestTriageEntry from bcbench.evaluate.base import AgentRunner, EvaluationPipeline @@ -28,17 +29,21 @@ class ExtRequestTriagePipeline(EvaluationPipeline[ExtRequestTriageEntry]): """Pipeline for the extensibility-request-triage category — no BC container, no build, no tests.""" + @override def setup_workspace(self, entry: ExtRequestTriageEntry, repo_path: Path) -> None: setup_repo_prebuild(entry, repo_path) (repo_path / TRIAGE_RESULT_FILE).unlink(missing_ok=True) + @override def setup(self, context: EvaluationContext[ExtRequestTriageEntry]) -> None: self.setup_workspace(context.entry, context.repo_path) + @override def run_agent(self, context: EvaluationContext[ExtRequestTriageEntry], agent_runner: AgentRunner[ExtRequestTriageEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): context.metrics, context.experiment = agent_runner(context) + @override def evaluate(self, context: EvaluationContext[ExtRequestTriageEntry]) -> None: result_path = context.repo_path / TRIAGE_RESULT_FILE raw = result_path.read_text(encoding="utf-8").strip() if result_path.exists() else "" diff --git a/src/bcbench/evaluate/nl2al.py b/src/bcbench/evaluate/nl2al.py index 2848f6fe3..e04e4c65b 100644 --- a/src/bcbench/evaluate/nl2al.py +++ b/src/bcbench/evaluate/nl2al.py @@ -2,6 +2,7 @@ import os import subprocess from pathlib import Path +from typing import override from bcbench_core.artifacts import copy_symbol_apps from bcbench_core.exceptions import EmptyDiffError @@ -56,20 +57,24 @@ def _git_init_and_commit(repo_path: Path) -> None: class NL2ALPipeline(EvaluationPipeline[NL2ALEntry]): """Pipeline for NL2AL evaluation category — generate AL code from natural language.""" + @override def setup_workspace(self, entry: NL2ALEntry, repo_path: Path) -> None: _reset_repo_path(repo_path) copy_symbol_apps(repo_path / entry.project_paths[0], entry.environment_setup_version) _git_init_and_commit(repo_path) + @override def setup(self, context: EvaluationContext[NL2ALEntry]) -> None: self.setup_workspace(context.entry, context.repo_path) + @override def run_agent(self, context: EvaluationContext[NL2ALEntry], agent_runner: AgentRunner[NL2ALEntry]) -> None: # Single attempt — retries are disabled. An empty diff (the agent asked for clarification # instead of editing) is scored as a failure in evaluate(), not re-run. with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): context.metrics, context.experiment = agent_runner(context) + @override def evaluate(self, context: EvaluationContext[NL2ALEntry]) -> None: try: generated_patch = stage_and_get_diff(context.repo_path, exclude=("**/app.json", "*.docx", "*.md")) diff --git a/src/bcbench/evaluate/testgeneration.py b/src/bcbench/evaluate/testgeneration.py index 3ccef67eb..2e8c3f45e 100644 --- a/src/bcbench/evaluate/testgeneration.py +++ b/src/bcbench/evaluate/testgeneration.py @@ -1,5 +1,6 @@ import logging from pathlib import Path +from typing import override import yaml from bcbench_core.bc import build_and_publish_projects, run_test_suite @@ -54,11 +55,13 @@ def _apply_input_postbuild(self, entry: TestGenEntry, repo_path: Path) -> None: case _: raise ValueError(f"Unhandled test generation input mode: {input_mode}") + @override def setup_workspace(self, entry: TestGenEntry, repo_path: Path) -> None: setup_repo_prebuild(entry, repo_path) self._apply_input_postbuild(entry, repo_path) set_runtime_version(repo_path, entry.project_paths) + @override def setup(self, context: EvaluationContext[TestGenEntry]) -> None: setup_repo_prebuild(context.entry, context.repo_path) @@ -72,10 +75,12 @@ def setup(self, context: EvaluationContext[TestGenEntry]) -> None: self._apply_input_postbuild(context.entry, context.repo_path) set_runtime_version(context.repo_path, context.entry.project_paths) + @override def run_agent(self, context: EvaluationContext[TestGenEntry], agent_runner: AgentRunner[TestGenEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): context.metrics, context.experiment = agent_runner(context) + @override def evaluate(self, context: EvaluationContext[TestGenEntry]) -> None: container = context.get_container() test_projects, app_projects = categorize_projects(context.entry.project_paths) diff --git a/src/bcbench/results/base.py b/src/bcbench/results/base.py index 4d113dcd3..165acb0fe 100644 --- a/src/bcbench/results/base.py +++ b/src/bcbench/results/base.py @@ -3,7 +3,7 @@ import json import logging from pathlib import Path -from typing import Any, Self, cast +from typing import Any, Self, cast, override from pydantic import BaseModel, model_validator @@ -126,12 +126,14 @@ def create_result(cls, context: "EvaluationContext", output: str, *, build: bool return cls(**cls._base_fields(context), output=output, build=build, resolved=resolved, error_message=error_message) @property + @override def status_label(self) -> str: if self.timeout: return "Timeout" return "Success" if self.resolved else "Failed" @property + @override def category_metrics(self) -> dict[str, int | float | bool]: return {"resolved": self.resolved, "build": self.build} @@ -158,10 +160,12 @@ def restore_missing_timeout_judge_model(cls, payload: object) -> object: return {**payload, "judge_model": judge_model} @classmethod + @override def _base_fields(cls, context: "EvaluationContext") -> dict[str, Any]: return {**super()._base_fields(context), "judge_model": context.category.judge_model} @property + @override def export_metadata(self) -> dict[str, str | int | float | bool | None]: return {"judge_model": self.judge_model} @@ -186,6 +190,7 @@ def create_empty_output(cls, context: "EvaluationContext") -> Self: return cls(**cls._base_fields(context), output="") @property + @override def status_label(self) -> str: if self.timeout: return "Timeout" diff --git a/src/bcbench/results/codereview.py b/src/bcbench/results/codereview.py index 89f8bb60a..80ae8f3e9 100644 --- a/src/bcbench/results/codereview.py +++ b/src/bcbench/results/codereview.py @@ -1,5 +1,5 @@ from collections.abc import Sequence -from typing import NamedTuple, Self +from typing import NamedTuple, Self, override import numpy as np from bcbench_core.scoring import f1_score, f_beta_score, precision_recall @@ -233,6 +233,7 @@ def create_invalid( ) @property + @override def category_metrics(self) -> dict[str, int | float | bool]: return { "generated_comment_count": len(self.generated_comments), @@ -251,6 +252,7 @@ def category_metrics(self) -> dict[str, int | float | bool]: } @property + @override def display_row(self) -> dict[str, str]: return { "Generated": str(len(self.generated_comments)), @@ -315,6 +317,7 @@ def metric(value: float | None, digits: int = 1) -> str: "\n" ) + @override def render_github_metrics_markdown(self) -> str: micro_p = self.precision * 100 micro_r = self.recall * 100 @@ -356,6 +359,7 @@ def render_github_metrics_markdown(self) -> str: f"{_METRIC_EXPLANATIONS}" ) + @override def render_console_metrics(self) -> RenderableType: metric_columns = ["Precision", "Recall", "F1", "Fβ (β=0.5)", "Fβ (β=2)"] @@ -420,6 +424,7 @@ def render_console_metrics(self) -> RenderableType: ) @classmethod + @override def from_results(cls, results: Sequence[BaseEvaluationResult], run_id: str) -> "CodeReviewResultSummary": summary = super().from_results(results, run_id) assert isinstance(summary, CodeReviewResultSummary) diff --git a/src/bcbench/results/leaderboard.py b/src/bcbench/results/leaderboard.py index 782c77e57..b327a59bc 100644 --- a/src/bcbench/results/leaderboard.py +++ b/src/bcbench/results/leaderboard.py @@ -4,7 +4,7 @@ from collections import defaultdict from collections.abc import Sequence from pathlib import Path -from typing import Any +from typing import Any, override from bcbench_core.scoring import pass_hat_k from bcbench_core.stats import bootstrap_ci @@ -91,6 +91,7 @@ class ExecutionBasedLeaderboardAggregate(LeaderboardAggregate): pass_hat_5: float | None = None @classmethod + @override def from_runs(cls, runs: Sequence[EvaluationResultSummary]) -> "ExecutionBasedLeaderboardAggregate": base = super().from_runs(runs) assert isinstance(base, ExecutionBasedLeaderboardAggregate) @@ -123,6 +124,7 @@ class JudgeBasedLeaderboardAggregate(LeaderboardAggregate): judge_model: str @classmethod + @override def _base_fields(cls, runs: Sequence[EvaluationResultSummary]) -> dict[str, Any]: from bcbench.results.summary import JudgeBasedEvaluationResultSummary @@ -157,6 +159,7 @@ class CodeReviewLeaderboardAggregate(JudgeBasedLeaderboardAggregate): average_ai_credits: float | None = None @classmethod + @override def from_runs(cls, runs: Sequence[EvaluationResultSummary]) -> "CodeReviewLeaderboardAggregate": from bcbench.results.codereview import CodeReviewResultSummary diff --git a/src/bcbench/results/summary.py b/src/bcbench/results/summary.py index eea4d3e6f..ecff8e365 100644 --- a/src/bcbench/results/summary.py +++ b/src/bcbench/results/summary.py @@ -7,7 +7,7 @@ from datetime import UTC, date, datetime from importlib.metadata import PackageNotFoundError, version from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, override from pydantic import BaseModel, Field @@ -164,10 +164,12 @@ class ExecutionBasedEvaluationResultSummary(EvaluationResultSummary): # Per-instance pass/fail for aggregate metrics (pass^k, CI) instance_results: dict[str, bool] = Field(default_factory=dict) + @override def render_github_metrics_markdown(self) -> str: return f"## Result Summary\n- Resolved: {self.resolved}\n- Failed: {self.failed}\n- Build: {self.build}\n- Pass Rate: {self.percentage}%\n" @classmethod + @override def from_results(cls, results: Sequence[BaseEvaluationResult], run_id: str) -> "ExecutionBasedEvaluationResultSummary": from bcbench.results.base import ExecutionBasedEvaluationResult @@ -196,6 +198,7 @@ class JudgeBasedEvaluationResultSummary(EvaluationResultSummary): judge_model: str @classmethod + @override def _base_fields(cls, results: Sequence[BaseEvaluationResult], run_id: str) -> dict[str, Any]: from bcbench.results.base import JudgeScoredEvaluationResult @@ -203,10 +206,12 @@ def _base_fields(cls, results: Sequence[BaseEvaluationResult], run_id: str) -> d assert isinstance(first_result, JudgeScoredEvaluationResult) return {**super()._base_fields(results, run_id), "judge_model": first_result.judge_model} + @override def combination_key(self) -> tuple[str | None, ...]: """Runs judged by different models are aggregated separately.""" return (*super().combination_key(), self.judge_model) + @override def render_github_metrics_markdown(self) -> str: """Judge scoring happens externally, so there are no in-run metrics to surface.""" return "" diff --git a/src/bcbench/results/testgeneration.py b/src/bcbench/results/testgeneration.py index a703752ed..24b5153df 100644 --- a/src/bcbench/results/testgeneration.py +++ b/src/bcbench/results/testgeneration.py @@ -1,4 +1,4 @@ -from typing import Self +from typing import Self, override from bcbench.results.base import ExecutionBasedEvaluationResult from bcbench.types import EvaluationContext @@ -11,10 +11,12 @@ class TestGenerationResult(ExecutionBasedEvaluationResult): post_patch_passed: bool = False @property + @override def category_metrics(self) -> dict[str, int | float | bool]: return {**super().category_metrics, "pre_patch_failed": self.pre_patch_failed, "post_patch_passed": self.post_patch_passed} @property + @override def display_row(self) -> dict[str, str]: return { "Pre-Patch Failed": "Yes" if self.pre_patch_failed else "No", @@ -22,6 +24,7 @@ def display_row(self) -> dict[str, str]: } @classmethod + @override def create_success(cls, context: "EvaluationContext", output: str) -> Self: return cls(**cls._base_fields(context), output=output, resolved=True, build=True, pre_patch_failed=True, post_patch_passed=True) diff --git a/tests/test_evaluate_pipeline.py b/tests/test_evaluate_pipeline.py index 131437b04..00cd01890 100644 --- a/tests/test_evaluate_pipeline.py +++ b/tests/test_evaluate_pipeline.py @@ -2,6 +2,7 @@ import json from pathlib import Path +from typing import override from unittest.mock import patch import pytest @@ -24,18 +25,22 @@ def __init__(self, *, raise_in_evaluate: Exception | None = None, raise_in_run_a self.run_agent_called = False self.evaluate_called = False + @override def setup_workspace(self, entry: E, repo_path: Path) -> None: pass + @override def setup(self, context: EvaluationContext[E]) -> None: self.setup_called = True + @override def run_agent(self, context: EvaluationContext[E], agent_runner: AgentRunner[E]) -> None: self.run_agent_called = True if self.raise_in_run_agent is not None: raise self.raise_in_run_agent context.metrics, context.experiment = agent_runner(context) + @override def evaluate(self, context: EvaluationContext[E]) -> None: self.evaluate_called = True if self.raise_in_evaluate is not None: diff --git a/tests/test_mcp_gateway.py b/tests/test_mcp_gateway.py index 1d00234b1..d2e718f25 100644 --- a/tests/test_mcp_gateway.py +++ b/tests/test_mcp_gateway.py @@ -4,6 +4,7 @@ import time from http.client import HTTPConnection from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import override from urllib.parse import urlsplit import pytest @@ -33,6 +34,7 @@ class _RecordingServer(ThreadingHTTPServer): class _UpstreamHandler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" + @override def log_message(self, format: str, *args: object) -> None: # match stdlib signature; silence access log pass @@ -163,6 +165,7 @@ def test_counts_forwarded_requests(self, gateway): class _McpHandler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" + @override def log_message(self, format: str, *args: object) -> None: # match stdlib signature; silence access log pass @@ -252,6 +255,7 @@ class _EmptyThenToolsHandler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" tools_list_calls = 0 + @override def log_message(self, format: str, *args: object) -> None: # match stdlib signature; silence access log pass @@ -307,6 +311,7 @@ class _HeldOpenSseHandler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" + @override def log_message(self, format: str, *args: object) -> None: # match stdlib signature; silence access log pass @@ -325,6 +330,7 @@ class _HeldOpenPostSseHandler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" + @override def log_message(self, format: str, *args: object) -> None: # match stdlib signature; silence access log pass @@ -378,6 +384,7 @@ class _InitializeExperimentalHandler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" + @override def log_message(self, format: str, *args: object) -> None: # match stdlib signature; silence access log pass