From ccfdc0a2a3586143d24cb82f39a874f694af8c1b Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Wed, 2 Sep 2026 12:36:44 -0700 Subject: [PATCH 01/16] feat(evaluations): add LD judge event support --- packages/client/pyproject.toml | 2 +- .../src/launchdarkly_ai_server/__init__.py | 4 + .../evaluations/__init__.py | 3 + .../evaluations/events.py | 73 ++++ .../evaluations/judges.py | 114 ++++++ .../evaluations/module.py | 25 +- .../evaluations/runner.py | 342 +++++++++++++++++- .../evaluations/types.py | 10 + packages/client/tests/test_evaluations_run.py | 228 +++++++++++- uv.lock | 2 + 10 files changed, 788 insertions(+), 15 deletions(-) create mode 100644 packages/client/src/launchdarkly_ai_server/evaluations/events.py create mode 100644 packages/client/src/launchdarkly_ai_server/evaluations/judges.py diff --git a/packages/client/pyproject.toml b/packages/client/pyproject.toml index 9ea3ce74..ee8f995c 100644 --- a/packages/client/pyproject.toml +++ b/packages/client/pyproject.toml @@ -2,7 +2,7 @@ name = "launchdarkly-ai-server" version = "0.1.3" requires-python = ">=3.12" -dependencies = ["opentelemetry-api>=1.25"] +dependencies = ["opentelemetry-api>=1.25", "pydantic>=2"] description = "LaunchDarkly AI SDK core client for Python" readme = "README.md" license = "Apache-2.0" diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 9a07b056..94267723 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -26,7 +26,9 @@ EvaluationsError, EvaluationsModule, GenerationConfig, + Judge, RunSummary, + Scorer, init_evaluations, ) from .graph import GraphInstance, graph, resolve_graph @@ -168,7 +170,9 @@ "EvaluationsError", "EvaluationsModule", "GenerationConfig", + "Judge", "RunSummary", + "Scorer", "init_evaluations", # utils "create_handler", diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py index 6516f4a0..f6623983 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py @@ -9,6 +9,7 @@ Transport, urllib_transport, ) +from .judges import Judge, Scorer from .module import EvaluationsModule, init_evaluations from .types import EvalRunResult, GenerationConfig, RunSummary, Usage @@ -19,9 +20,11 @@ "EvaluationsModule", "GenerationConfig", "HttpResponse", + "Judge", "LDApiClient", "LDApiError", "RunSummary", + "Scorer", "Transport", "Usage", "init_evaluations", diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/events.py b/packages/client/src/launchdarkly_ai_server/evaluations/events.py new file mode 100644 index 00000000..ba2d3e04 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/evaluations/events.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from enum import StrEnum +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class EvaluationStatus(StrEnum): + COMPLETE = "COMPLETE" + ERROR = "ERROR" + + +class EvaluationEventKind(StrEnum): + JUDGE = "judge" + SCORER = "scorer" + + +class TokenUsage(BaseModel): + """Token usage reported by an LD Judge provider call.""" + + model_config = ConfigDict(populate_by_name=True, extra="forbid") + + input_tokens: int = Field(alias="inputTokens") + output_tokens: int = Field(alias="outputTokens") + + +class EvaluationEventPayload(BaseModel): + """Common fields emitted for every SDK-run evaluation criterion result.""" + + model_config = ConfigDict( + populate_by_name=True, extra="forbid", use_enum_values=True + ) + + project_key: str = Field(alias="projectKey") + evaluation_id: str = Field(alias="evaluationId") + evaluation_run_id: str = Field(alias="evaluationRunId") + run_id: str = Field(alias="runId") + dataset_id: str = Field(alias="datasetId") + row_index: int = Field(alias="rowIndex") + criterion_type: str = Field(alias="criterionType") + kind: EvaluationEventKind + event_id: str = Field(alias="eventId") + emitted_at: str = Field(alias="emittedAt") + evaluation_key: str = Field(alias="evaluationKey") + evaluation_version: int | None = Field(default=None, alias="evaluationVersion") + dataset_key: str = Field(alias="datasetKey") + status: EvaluationStatus + started_at: str = Field(alias="startedAt") + evaluated_at: str = Field(alias="evaluatedAt") + latency_ms: int = Field(alias="latencyMs") + score: float | int | None = None + reason: str | None = None + error: dict[str, Any] | None = None + + def to_track_payload(self) -> dict[str, Any]: + return self.model_dump(by_alias=True, exclude_none=True) + + +class LDJudgeEvaluationEventPayload(EvaluationEventPayload): + """Payload for one LaunchDarkly AI Judge result on one dataset row.""" + + kind: EvaluationEventKind = EvaluationEventKind.JUDGE + judge_key: str = Field(alias="judgeKey") + variation_key: str = Field(alias="variationKey") + version: int | None = None + usage: TokenUsage | None = None + + +class DeterministicScorerEvaluationEventPayload(EvaluationEventPayload): + """Payload for one local deterministic scorer result on one dataset row.""" + + kind: EvaluationEventKind = EvaluationEventKind.SCORER diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/judges.py b/packages/client/src/launchdarkly_ai_server/evaluations/judges.py new file mode 100644 index 00000000..44230df3 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/evaluations/judges.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from typing import Any + +type ScorerFn = Callable[ + [Mapping[str, Any], Any], float | bool | Awaitable[float | bool] +] + + +@dataclass(frozen=True) +class Judge: + """Reference to a LaunchDarkly AI Judge config to run for each eval row. + + The SDK does not create or provide built-in judges. Pass the key of a judge + that exists in LaunchDarkly. Resolution uses LaunchDarkly flag delivery for + the currently served variation. + """ + + key: str + threshold: float | None = None + pass_rate_threshold: float | None = None + ground_truth_context: str | None = None + + def __post_init__(self) -> None: + if not isinstance(self.key, str) or not self.key.strip(): + raise ValueError("judge key must not be blank") + _validate_thresholds( + threshold=self.threshold, + pass_rate_threshold=self.pass_rate_threshold, + ) + + @property + def criterion_type(self) -> str: + return self.key + + def to_criteria_wire(self) -> dict[str, Any]: + options = _criteria_options( + threshold=self.threshold, + pass_rate_threshold=self.pass_rate_threshold, + ground_truth_context=self.ground_truth_context, + ) + return {"criterionType": self.criterion_type, "options": options} + + +@dataclass(frozen=True) +class Scorer: + """Local deterministic scorer run for each generated evaluation row. + + ``fn`` may be sync or async and receives ``(row, output)``. It must return a + boolean or a numeric score from 0 to 1. Boolean results are converted to 1.0 + or 0.0 before being emitted as evaluation events. + """ + + name: str + fn: ScorerFn + threshold: float | None = 1.0 + pass_rate_threshold: float | None = None + + def __post_init__(self) -> None: + if not isinstance(self.name, str) or not self.name.strip(): + raise ValueError("scorer name must not be blank") + if not callable(self.fn): + raise ValueError("scorer fn must be callable") + _validate_thresholds( + threshold=self.threshold, + pass_rate_threshold=self.pass_rate_threshold, + ) + + @property + def criterion_type(self) -> str: + return self.name + + def to_criteria_wire(self) -> dict[str, Any]: + return { + "criterionType": self.criterion_type, + "options": _criteria_options( + threshold=self.threshold, + pass_rate_threshold=self.pass_rate_threshold, + ), + } + + +type JudgeReference = Judge | Scorer + + +def _validate_thresholds( + *, + threshold: float | None, + pass_rate_threshold: float | None, +) -> None: + for name, value in ( + ("threshold", threshold), + ("pass_rate_threshold", pass_rate_threshold), + ): + if value is not None and (value < 0 or value > 1): + raise ValueError(f"{name} must be between 0 and 1") + + +def _criteria_options( + *, + threshold: float | None, + pass_rate_threshold: float | None, + ground_truth_context: str | None = None, +) -> dict[str, Any]: + options: dict[str, Any] = {} + if threshold is not None: + options["threshold"] = threshold + if pass_rate_threshold is not None: + options["passRateThreshold"] = pass_rate_threshold + if ground_truth_context is not None: + options["groundTruthContext"] = ground_truth_context + return options diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 4a3ffad3..7f8067da 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -17,6 +17,7 @@ Transport, urllib_transport, ) +from .judges import Judge, JudgeReference from .runner import EvalHandler, EvaluationsRunner, ToolImplementation, _segment from .types import EvalRunResult, GenerationConfig, RunSummary @@ -93,6 +94,7 @@ async def run( handler: EvalHandler, generation: GenerationConfig, tools: Mapping[str, ToolImplementation] | None = None, + judges: list[JudgeReference] | None = None, concurrency: int = 10, poll_interval_seconds: float | None = None, poll_timeout_seconds: float | None = None, @@ -121,14 +123,17 @@ async def run( poll_timeout_seconds=poll_timeout_seconds, ) run_tools = dict(tools or {}) + run_judges = list(judges or []) + ld_judges = [judge for judge in run_judges if isinstance(judge, Judge)] client = await self._resolve_client() # The management API client is synchronous; running it in a worker thread # keeps the caller's event loop free. - # Tool verification is deliberately first: a typo must not create records. + # Tool/judge verification is deliberately first: a typo must not create records. resolved_tools = await asyncio.to_thread( self._runner._resolve_tools, project_key, run_tools ) + resolved_judges = await self._runner._resolve_judges(ld_judges) dataset_ref = await asyncio.to_thread( self._runner._fetch_dataset, project_key, dataset ) @@ -141,12 +146,12 @@ async def run( key, generation, resolved_tools, + run_judges, ) evaluation_run = await asyncio.to_thread( self._runner._create_evaluation_run, project_key, evaluation.id, - len(rows), dataset_ref.id, ) config = self._runner._build_handler_config(generation, resolved_tools) @@ -165,6 +170,22 @@ async def run( dataset=dataset_ref, results=results, ) + if run_judges: + judge_results = await self._runner._run_judges_for_results( + results, + handler, + run_tools, + run_judges, + resolved_judges, + ) + self._runner._emit_evaluation_events( + client, + project_key=project_key, + evaluation=evaluation, + evaluation_run=evaluation_run, + dataset=dataset_ref, + results=judge_results, + ) flush_result = client.flush() if inspect.isawaitable(flush_result): await flush_result diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 951213b5..132d8361 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -2,6 +2,7 @@ import asyncio import hashlib +import inspect import json import time import urllib.parse @@ -9,21 +10,44 @@ from datetime import UTC, datetime from typing import Any +from ..lifecycle import extract_variation from ..types import NativeTool -from ..utils import parse_template, parse_usage, to_ld_context +from ..utils import ( + parse_json_with_possible_fences, + parse_template, + parse_usage, + to_ld_context, +) from .api import EvaluationsError, LDApiClient, LDApiError +from .events import ( + DeterministicScorerEvaluationEventPayload, + EvaluationEventPayload, + LDJudgeEvaluationEventPayload, + TokenUsage, +) +from .judges import Judge, JudgeReference, Scorer from .types import ( DatasetRef, DatasetRow, EvaluationRef, EvaluationRunRef, GenerationConfig, + ResolvedJudge, ResolvedTool, RunSummary, ) DATASET_PAGE_SIZE = 200 GENERATION_EVENT_NAME = "$ld:ai:offline-evals:generation" +EVALUATION_EVENT_NAME = "$ld:ai:offline-evals:evaluation" +JUDGE_FORMATTING_INSTRUCTIONS = "\n".join( + [ + "Your response MUST be in valid JSON format with the following structure:", + '{ "score": , "reasoning": }', + "The output must be valid, parseable JSON. Do not include additional tags, comments, formatting, or newlines.", + "Do not include ```json tags.", + ] +) EvalHandler = Callable[..., Awaitable[dict[str, Any]]] ToolImplementation = Callable[..., Any] | NativeTool @@ -124,6 +148,40 @@ def _resolve_tools( ) return resolved + async def _resolve_judges( + self, + judges: list[Judge], + ) -> dict[str, ResolvedJudge]: + """Resolve LD Judge configs before any evaluation records are created.""" + resolved: dict[str, ResolvedJudge] = {} + context: dict[str, Any] = {} + for judge in judges: + try: + variation = await extract_variation(judge.key, context) + except Exception as error: + raise EvaluationsError( + f"LaunchDarkly judge {judge.key!r} was not found or could not be resolved " + "for this project. Create the judge in the LaunchDarkly UI and try again." + ) from error + config = variation.get("config") + meta_value = variation.get("meta") + meta: Mapping[str, Any] = ( + meta_value if isinstance(meta_value, Mapping) else {} + ) + if not isinstance(config, Mapping): + raise EvaluationsError( + f"LaunchDarkly judge {judge.key!r} returned an invalid AI config variation" + ) + resolved[judge.key] = ResolvedJudge( + key=judge.key, + config=dict(config), + variation_key=str(meta.get("variationKey") or ""), + version=int(meta["version"]) + if isinstance(meta.get("version"), int) + else None, + ) + return resolved + def _fetch_dataset(self, project_key: str, dataset_key: str) -> DatasetRef: path = f"projects/{_segment(project_key)}/datasets/{_segment(dataset_key)}" try: @@ -231,6 +289,7 @@ def _create_evaluation( key: str, generation: GenerationConfig, tools: Mapping[str, ResolvedTool], + judges: list[JudgeReference] | None = None, ) -> EvaluationRef: body: dict[str, Any] = { "name": key, @@ -253,6 +312,8 @@ def _create_evaluation( body["tools"] = [ {"key": tool.key, "version": tool.version} for tool in tools.values() ] + if judges: + body["criteria"] = [judge.to_criteria_wire() for judge in judges] path = f"projects/{_segment(project_key)}/evaluations" raw = _mapping(self._api.post(path, body=body), description="evaluation") @@ -269,22 +330,18 @@ def _create_evaluation_run( self, project_key: str, evaluation_id: str, - row_count: int, dataset_id: str, ) -> EvaluationRunRef: path = ( f"projects/{_segment(project_key)}/evaluations/" f"{_segment(evaluation_id)}/runs" ) + body: dict[str, Any] = { + "source": "api", + "datasetId": dataset_id, + } raw = _mapping( - self._api.post( - path, - body={ - "source": "api", - "rowCount": row_count, - "datasetId": dataset_id, - }, - ), + self._api.post(path, body=body), description="evaluation run", ) return self._run_ref(raw) @@ -480,6 +537,271 @@ def _emit_generation_events( flush=True, ) + def _judge_variables( + self, + row_result: Mapping[str, Any], + judge: Judge, + ) -> dict[str, Any]: + variables = dict(row_result.get("variables") or {}) + output = row_result.get("output") + expected = row_result.get("expected_output") + ground_truth = judge.ground_truth_context + if ground_truth is not None: + ground_truth = parse_template(ground_truth, variables) + elif expected is not None: + ground_truth = str(expected) + variables.update( + { + "input": row_result.get("input"), + "response_to_evaluate": output, + "message_history": "\n\n".join( + str(value) + for value in (row_result.get("input"), output) + if value is not None + ), + "expected_output": expected, + "ground_truth_context": ground_truth, + } + ) + return variables + + def _render_config_value(self, value: Any, variables: Mapping[str, Any]) -> Any: + if isinstance(value, str): + return parse_template(value, dict(variables)) + if isinstance(value, list): + return [self._render_config_value(item, variables) for item in value] + if isinstance(value, Mapping): + return { + str(key): self._render_config_value(item, variables) + for key, item in value.items() + } + return value + + def _criterion_error_result( + self, + base: Mapping[str, Any], + started_clock: float, + code: str, + message: str, + ) -> dict[str, Any]: + completed = datetime.now(UTC) + return { + **base, + "status": "ERROR", + "error": {"code": code, "message": message}, + "evaluated_at": completed.isoformat().replace("+00:00", "Z"), + "latency_ms": round((time.perf_counter() - started_clock) * 1000), + } + + async def _run_scorer_for_result( + self, + row: Mapping[str, Any], + scorer: Scorer, + ) -> dict[str, Any]: + started = datetime.now(UTC) + started_clock = time.perf_counter() + base: dict[str, Any] = { + "row_index": row["row_index"], + "criterion_type": scorer.criterion_type, + "kind": "scorer", + "started_at": started.isoformat().replace("+00:00", "Z"), + } + if row.get("status") != "COMPLETE": + return self._criterion_error_result( + base, started_clock, "scorer_error", "generation did not complete" + ) + try: + score_value = scorer.fn(row, row.get("output")) + if inspect.isawaitable(score_value): + score_value = await score_value + if isinstance(score_value, bool): + score: float = 1.0 if score_value else 0.0 + elif isinstance(score_value, int | float): + score = float(score_value) + else: + raise TypeError("scorer fn must return a bool or numeric score") + if score < 0 or score > 1: + raise ValueError("scorer fn score must be between 0 and 1") + completed = datetime.now(UTC) + return { + **base, + "status": "COMPLETE", + "score": score, + "reason": None, + "evaluated_at": completed.isoformat().replace("+00:00", "Z"), + "latency_ms": round((time.perf_counter() - started_clock) * 1000), + } + except Exception as error: + return self._criterion_error_result( + base, started_clock, "scorer_error", str(error) + ) + + async def _run_ld_judge_for_result( + self, + row: Mapping[str, Any], + handler: EvalHandler, + tool_handlers: dict[str, ToolImplementation], + judge: Judge, + resolved: ResolvedJudge, + ) -> dict[str, Any]: + started = datetime.now(UTC) + started_clock = time.perf_counter() + base: dict[str, Any] = { + "row_index": row["row_index"], + "criterion_type": judge.criterion_type, + "kind": "judge", + "judge_key": judge.key, + "started_at": started.isoformat().replace("+00:00", "Z"), + "variation_key": resolved.variation_key, + "version": resolved.version, + } + if row.get("status") != "COMPLETE": + return self._criterion_error_result( + base, started_clock, "judge_error", "generation did not complete" + ) + try: + variables = self._judge_variables(row, judge) + rendered_config = self._render_config_value(resolved.config, variables) + result = await handler( + rendered_config, + row.get("output"), + tool_handlers, + { + **variables, + "formatting_instructions": JUDGE_FORMATTING_INSTRUCTIONS, + }, + ) + if not isinstance(result, Mapping): + raise TypeError("judge handler result must be a mapping") + raw = result.get("output", result.get("response")) + parsed = ( + parse_json_with_possible_fences(raw) + if isinstance(raw, str) + else raw + if isinstance(raw, Mapping) + else None + ) + if not isinstance(parsed, Mapping): + raise ValueError("Invalid JSON from judge") + completed = datetime.now(UTC) + event = { + **base, + "status": "COMPLETE", + "score": parsed.get("score"), + "reason": str(parsed.get("reasoning", parsed.get("reason", ""))), + "evaluated_at": completed.isoformat().replace("+00:00", "Z"), + "latency_ms": round((time.perf_counter() - started_clock) * 1000), + } + usage = result.get("usage") + if isinstance(usage, Mapping): + event["usage"] = dict(usage) + return event + except Exception as error: + return self._criterion_error_result( + base, started_clock, "judge_error", str(error) + ) + + async def _run_judges_for_results( + self, + rows: list[dict[str, Any]], + handler: EvalHandler, + tool_handlers: dict[str, ToolImplementation], + judge_refs: list[JudgeReference], + resolved_judges: Mapping[str, ResolvedJudge], + ) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for row in rows: + for judge in judge_refs: + if isinstance(judge, Scorer): + results.append(await self._run_scorer_for_result(row, judge)) + else: + results.append( + await self._run_ld_judge_for_result( + row, + handler, + tool_handlers, + judge, + resolved_judges[judge.key], + ) + ) + return results + + def _emit_evaluation_events( + self, + client: Any, + *, + project_key: str, + evaluation: EvaluationRef, + evaluation_run: EvaluationRunRef, + dataset: DatasetRef, + results: list[dict[str, Any]], + ) -> None: + context = to_ld_context( + client, + { + "kind": "evaluation", + "key": evaluation_run.id, + "projectKey": project_key, + "evaluationId": evaluation.id, + }, + ) + for result in results: + identity = { + "projectKey": project_key, + "evaluationId": evaluation.id, + "evaluationRunId": evaluation_run.id, + "runId": evaluation_run.id, + "datasetId": dataset.id, + "rowIndex": result["row_index"], + "criterionType": result["criterion_type"], + } + event_id = hashlib.sha256( + json.dumps(identity, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + emitted_at = datetime.now(UTC).isoformat().replace("+00:00", "Z") + usage: TokenUsage | None = None + if result["kind"] == "judge" and isinstance(result.get("usage"), Mapping): + normalized_usage = parse_usage(dict(result["usage"])) + usage = TokenUsage( + inputTokens=normalized_usage["input"], + outputTokens=normalized_usage["output"], + ) + common_payload = { + **identity, + "eventId": event_id, + "emittedAt": emitted_at, + "evaluationKey": evaluation.key, + "evaluationVersion": evaluation.version, + "datasetKey": dataset.key, + "status": result["status"], + "startedAt": result["started_at"], + "evaluatedAt": result["evaluated_at"], + "latencyMs": result["latency_ms"], + "score": result.get("score"), + "reason": result.get("reason"), + "error": result.get("error"), + } + payload_model: EvaluationEventPayload + if result["kind"] == "judge": + payload_model = LDJudgeEvaluationEventPayload( + **common_payload, + judgeKey=result["judge_key"], + variationKey=result["variation_key"], + version=result.get("version"), + usage=usage, + ) + else: + payload_model = DeterministicScorerEvaluationEventPayload( + **common_payload + ) + client.track( + EVALUATION_EVENT_NAME, context, payload_model.to_track_payload(), 1 + ) + print( + f"{EVALUATION_EVENT_NAME} emittedAt={emitted_at} eventId={event_id}", + flush=True, + ) + def _get_summary( self, project_key: str, evaluation_id: str, run_id: str ) -> RunSummary: diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index 5f4f4d5c..b5d0e647 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -67,6 +67,16 @@ class ResolvedTool: schema: dict[str, Any] = field(default_factory=dict) +@dataclass +class ResolvedJudge: + """A LaunchDarkly AI Judge config variation resolved for an evaluation run.""" + + key: str + config: dict[str, Any] + variation_key: str = "" + version: int | None = None + + @dataclass class EvaluationRef: """Identifiers returned after creating an evaluation.""" diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 5784559e..62a3b614 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -1,7 +1,7 @@ from __future__ import annotations import json -from collections.abc import Callable +from collections.abc import Callable, Mapping from datetime import datetime from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -11,6 +11,8 @@ from launchdarkly_ai_server.evaluations import ( EvaluationsError, HttpResponse, + Judge, + Scorer, init_evaluations, ) @@ -278,7 +280,6 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( ) assert transport.requests[5]["body"] == { "source": "api", - "rowCount": 2, "datasetId": "33333333-3333-3333-3333-333333333333", } @@ -1033,3 +1034,226 @@ async def fake_init_client(options: dict[str, Any]) -> MagicMock: assert "inputTokens" not in error_event assert "outputTokens" not in error_event assert {"input", "expected_output", "metadata", "variables"}.isdisjoint(error_event) + + +@pytest.mark.asyncio +async def test_run_with_ld_judge_emits_per_criterion_evaluation_event( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, +) -> None: + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [ + { + "rowIndex": 42, + "input": "Question {{id}}", + "expectedOutput": "Answer {{id}}", + "variables": {"id": "A"}, + } + ], + total=1, + ), + ), + response(201, {"id": "evaluation-id", "name": "support-qa", "version": 3}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + {"statusCounts": {"total": 1, "passed": 1, "error": 0, "pending": 0}}, + ), + ] + ) + + async def fake_extract_variation( + key: str, context: dict[str, Any] + ) -> dict[str, Any]: + assert key == "$ld:ai:judge:accuracy" + assert context == {} + return { + "config": { + "provider": {"name": "OpenAI"}, + "model": {"name": "gpt-4o"}, + "instructions": "Judge {{response_to_evaluate}} against {{expected_output}}", + }, + "meta": {"variationKey": "default", "version": 12}, + } + + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.runner.extract_variation", + fake_extract_variation, + ) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + async def handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + if "Judge" in config.get("instructions", ""): + assert user_input == "generated" + assert variables["response_to_evaluate"] == "generated" + assert variables["expected_output"] == "Answer A" + assert "generated" in config["instructions"] + return { + "output": '{"score": 0.86, "reasoning": "matches policy"}', + "usage": {"input_tokens": 640, "output_tokens": 48}, + } + return { + "output": "generated", + "usage": {"input_tokens": 10, "output_tokens": 4}, + } + + result = await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + judges=[Judge(key="$ld:ai:judge:accuracy")], + ) + + assert result.passed is True + assert transport.requests[2]["body"]["criteria"] == [ + { + "criterionType": "$ld:ai:judge:accuracy", + "options": {}, + } + ] + assert transport.requests[3]["body"] == {"source": "api", "datasetId": "dataset-id"} + events = [call.args[2] for call in stub_sdk_client.track.call_args_list] + judge_event = next(event for event in events if event.get("kind") == "judge") + assert judge_event["criterionType"] == "$ld:ai:judge:accuracy" + assert judge_event["judgeKey"] == "$ld:ai:judge:accuracy" + assert judge_event["status"] == "COMPLETE" + assert judge_event["score"] == 0.86 + assert judge_event["reason"] == "matches policy" + assert judge_event["usage"] == {"inputTokens": 640, "outputTokens": 48} + assert judge_event["variationKey"] == "default" + assert judge_event["version"] == 12 + assert len(judge_event["eventId"]) == 64 + + +@pytest.mark.asyncio +async def test_missing_ld_judge_aborts_before_mutating_request( + monkeypatch: pytest.MonkeyPatch, +) -> None: + transport = SequencedTransport([]) + + async def fake_extract_variation( + key: str, context: dict[str, Any] + ) -> dict[str, Any]: + raise RuntimeError("not found") + + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.runner.extract_variation", + fake_extract_variation, + ) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + with pytest.raises( + EvaluationsError, match="Create the judge in the LaunchDarkly UI" + ): + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + judges=[Judge(key="security-judge")], + ) + + assert transport.requests == [] + + +@pytest.mark.asyncio +async def test_run_with_deterministic_scorer_emits_scorer_evaluation_event( + stub_sdk_client: MagicMock, +) -> None: + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "support-golden-v3"}), + response( + 200, + dataset_page( + [ + { + "rowIndex": 42, + "input": "Ticket {{id}}", + "expectedOutput": "refund row", + "variables": {"id": "A"}, + } + ], + total=1, + ), + ), + response(201, {"id": "evaluation-id", "name": "support-qa", "version": 3}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + {"statusCounts": {"total": 1, "passed": 1, "error": 0, "pending": 0}}, + ), + ] + ) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + async def handler(*args: object) -> dict[str, Any]: + return { + "output": "refund exists", + "usage": {"input_tokens": 10, "output_tokens": 4}, + } + + def check_refund(row: Mapping[str, Any], output: Any) -> bool: + assert row["row_index"] == 42 + assert row["input"] == "Ticket A" + assert output == "refund exists" + return "refund" in str(output) + + result = await evals.run( + project_key="proj", + key="support-qa", + dataset="support-golden-v3", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + judges=[Scorer(name="refund-exists", fn=check_refund)], + ) + + assert result.passed is True + assert transport.requests[2]["body"]["criteria"] == [ + {"criterionType": "refund-exists", "options": {"threshold": 1.0}} + ] + assert transport.requests[3]["body"] == {"source": "api", "datasetId": "dataset-id"} + events = [call.args[2] for call in stub_sdk_client.track.call_args_list] + scorer_event = next(event for event in events if event.get("kind") == "scorer") + assert scorer_event["projectKey"] == "proj" + assert scorer_event["evaluationId"] == "evaluation-id" + assert scorer_event["evaluationRunId"] == "run-id" + assert scorer_event["runId"] == "run-id" + assert scorer_event["datasetId"] == "dataset-id" + assert scorer_event["rowIndex"] == 42 + assert scorer_event["criterionType"] == "refund-exists" + assert scorer_event["evaluationKey"] == "support-qa" + assert scorer_event["evaluationVersion"] == 3 + assert scorer_event["datasetKey"] == "support-golden-v3" + assert scorer_event["status"] == "COMPLETE" + assert scorer_event["score"] == 1 + assert "reason" not in scorer_event + assert "usage" not in scorer_event + assert scorer_event["latencyMs"] >= 0 + assert scorer_event["startedAt"].endswith("Z") + assert scorer_event["evaluatedAt"].endswith("Z") + assert "judgeKey" not in scorer_event + assert "variationKey" not in scorer_event + assert "version" not in scorer_event diff --git a/uv.lock b/uv.lock index 7d93a3cd..fa21dbca 100644 --- a/uv.lock +++ b/uv.lock @@ -922,6 +922,7 @@ version = "0.1.3" source = { editable = "packages/client" } dependencies = [ { name = "opentelemetry-api" }, + { name = "pydantic" }, ] [package.optional-dependencies] @@ -935,6 +936,7 @@ requires-dist = [ { name = "opentelemetry-api", specifier = ">=1.25" }, { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otel'", specifier = ">=1.25" }, { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.25" }, + { name = "pydantic", specifier = ">=2" }, ] provides-extras = ["otel"] From f63cb3cafd2c77775e1d455f040b9f31f2f80073 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Wed, 2 Sep 2026 16:45:45 -0700 Subject: [PATCH 02/16] refactor(evaluations): criteria naming + shared judge scoring contract Rename the public run() parameter judges= to criteria= (with JudgeReference -> Criterion and evaluations/judges.py -> criteria.py): the wire format already calls these criteria, and a Scorer is not a judge. Scorer callbacks now receive the public DatasetRow instead of the internal result dict, so internal key renames cannot break customer scorers; Criterion and DatasetRow are exported. Extract the judge response contract (formatting instructions, JSON score parsing, finite-number guard) into judge_scoring.py shared by the online judge path and the offline evaluations runner. The two copies had already drifted textually. Co-Authored-By: Claude Fable 5 --- .../src/launchdarkly_ai_server/__init__.py | 4 ++ .../evaluations/__init__.py | 6 +- .../evaluations/{judges.py => criteria.py} | 22 +++--- .../evaluations/module.py | 20 +++--- .../evaluations/runner.py | 61 ++++++++--------- .../launchdarkly_ai_server/judge_scoring.py | 61 +++++++++++++++++ .../src/launchdarkly_ai_server/judges.py | 67 ++++++------------- packages/client/tests/test_evaluations_run.py | 15 +++-- packages/client/tests/test_judges.py | 16 ++--- 9 files changed, 157 insertions(+), 115 deletions(-) rename packages/client/src/launchdarkly_ai_server/evaluations/{judges.py => criteria.py} (80%) create mode 100644 packages/client/src/launchdarkly_ai_server/judge_scoring.py diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 94267723..9b2fce80 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -22,6 +22,8 @@ set_conversation_id_if_absent, ) from .evaluations import ( + Criterion, + DatasetRow, EvalRunResult, EvaluationsError, EvaluationsModule, @@ -167,6 +169,8 @@ "VariationMeta", # evaluations "EvalRunResult", + "Criterion", + "DatasetRow", "EvaluationsError", "EvaluationsModule", "GenerationConfig", diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py index f6623983..b110a550 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py @@ -9,12 +9,14 @@ Transport, urllib_transport, ) -from .judges import Judge, Scorer +from .criteria import Criterion, Judge, Scorer from .module import EvaluationsModule, init_evaluations -from .types import EvalRunResult, GenerationConfig, RunSummary, Usage +from .types import DatasetRow, EvalRunResult, GenerationConfig, RunSummary, Usage __all__ = [ "DEFAULT_BASE_URI", + "Criterion", + "DatasetRow", "EvalRunResult", "EvaluationsError", "EvaluationsModule", diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/judges.py b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py similarity index 80% rename from packages/client/src/launchdarkly_ai_server/evaluations/judges.py rename to packages/client/src/launchdarkly_ai_server/evaluations/criteria.py index 44230df3..f77d6415 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/judges.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py @@ -1,12 +1,12 @@ from __future__ import annotations -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable from dataclasses import dataclass from typing import Any -type ScorerFn = Callable[ - [Mapping[str, Any], Any], float | bool | Awaitable[float | bool] -] +from .types import DatasetRow + +type ScorerFn = Callable[[DatasetRow, Any], float | bool | Awaitable[float | bool]] @dataclass(frozen=True) @@ -48,9 +48,15 @@ def to_criteria_wire(self) -> dict[str, Any]: class Scorer: """Local deterministic scorer run for each generated evaluation row. - ``fn`` may be sync or async and receives ``(row, output)``. It must return a - boolean or a numeric score from 0 to 1. Boolean results are converted to 1.0 - or 0.0 before being emitted as evaluation events. + ``fn`` may be sync or async and receives ``(row, output)``, where ``row`` + is the :class:`~launchdarkly_ai_server.evaluations.types.DatasetRow` the + output was generated from and ``output`` is the generated output. It must + return a boolean or a numeric score from 0 to 1. Boolean results are + converted to 1.0 or 0.0 before being emitted as evaluation events. + + ``threshold`` defaults to 1.0: a row passes only on a perfect score, which + matches the common case of boolean scorers. Pass a lower threshold for + graded numeric scorers. """ name: str @@ -82,7 +88,7 @@ def to_criteria_wire(self) -> dict[str, Any]: } -type JudgeReference = Judge | Scorer +type Criterion = Judge | Scorer def _validate_thresholds( diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 7f8067da..9bb3399c 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -17,7 +17,7 @@ Transport, urllib_transport, ) -from .judges import Judge, JudgeReference +from .criteria import Criterion, Judge from .runner import EvalHandler, EvaluationsRunner, ToolImplementation, _segment from .types import EvalRunResult, GenerationConfig, RunSummary @@ -94,7 +94,7 @@ async def run( handler: EvalHandler, generation: GenerationConfig, tools: Mapping[str, ToolImplementation] | None = None, - judges: list[JudgeReference] | None = None, + criteria: list[Criterion] | None = None, concurrency: int = 10, poll_interval_seconds: float | None = None, poll_timeout_seconds: float | None = None, @@ -123,8 +123,10 @@ async def run( poll_timeout_seconds=poll_timeout_seconds, ) run_tools = dict(tools or {}) - run_judges = list(judges or []) - ld_judges = [judge for judge in run_judges if isinstance(judge, Judge)] + run_criteria = list(criteria or []) + ld_judges = [ + criterion for criterion in run_criteria if isinstance(criterion, Judge) + ] client = await self._resolve_client() # The management API client is synchronous; running it in a worker thread @@ -146,7 +148,7 @@ async def run( key, generation, resolved_tools, - run_judges, + run_criteria, ) evaluation_run = await asyncio.to_thread( self._runner._create_evaluation_run, @@ -170,12 +172,12 @@ async def run( dataset=dataset_ref, results=results, ) - if run_judges: - judge_results = await self._runner._run_judges_for_results( + if run_criteria: + criterion_results = await self._runner._run_criteria_for_results( results, handler, run_tools, - run_judges, + run_criteria, resolved_judges, ) self._runner._emit_evaluation_events( @@ -184,7 +186,7 @@ async def run( evaluation=evaluation, evaluation_run=evaluation_run, dataset=dataset_ref, - results=judge_results, + results=criterion_results, ) flush_result = client.flush() if inspect.isawaitable(flush_result): diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 132d8361..99c4e673 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -10,22 +10,25 @@ from datetime import UTC, datetime from typing import Any +from ..judge_scoring import ( + FORMATTING_INSTRUCTIONS, + parse_judge_response, +) from ..lifecycle import extract_variation from ..types import NativeTool from ..utils import ( - parse_json_with_possible_fences, parse_template, parse_usage, to_ld_context, ) from .api import EvaluationsError, LDApiClient, LDApiError +from .criteria import Criterion, Judge, Scorer from .events import ( DeterministicScorerEvaluationEventPayload, EvaluationEventPayload, LDJudgeEvaluationEventPayload, TokenUsage, ) -from .judges import Judge, JudgeReference, Scorer from .types import ( DatasetRef, DatasetRow, @@ -40,14 +43,6 @@ DATASET_PAGE_SIZE = 200 GENERATION_EVENT_NAME = "$ld:ai:offline-evals:generation" EVALUATION_EVENT_NAME = "$ld:ai:offline-evals:evaluation" -JUDGE_FORMATTING_INSTRUCTIONS = "\n".join( - [ - "Your response MUST be in valid JSON format with the following structure:", - '{ "score": , "reasoning": }', - "The output must be valid, parseable JSON. Do not include additional tags, comments, formatting, or newlines.", - "Do not include ```json tags.", - ] -) EvalHandler = Callable[..., Awaitable[dict[str, Any]]] ToolImplementation = Callable[..., Any] | NativeTool @@ -289,7 +284,7 @@ def _create_evaluation( key: str, generation: GenerationConfig, tools: Mapping[str, ResolvedTool], - judges: list[JudgeReference] | None = None, + criteria: list[Criterion] | None = None, ) -> EvaluationRef: body: dict[str, Any] = { "name": key, @@ -312,8 +307,8 @@ def _create_evaluation( body["tools"] = [ {"key": tool.key, "version": tool.version} for tool in tools.values() ] - if judges: - body["criteria"] = [judge.to_criteria_wire() for judge in judges] + if criteria: + body["criteria"] = [criterion.to_criteria_wire() for criterion in criteria] path = f"projects/{_segment(project_key)}/evaluations" raw = _mapping(self._api.post(path, body=body), description="evaluation") @@ -611,7 +606,14 @@ async def _run_scorer_for_result( base, started_clock, "scorer_error", "generation did not complete" ) try: - score_value = scorer.fn(row, row.get("output")) + dataset_row = DatasetRow( + row_index=row["row_index"], + input=row.get("input"), + expected_output=row.get("expected_output"), + variables=dict(row.get("variables") or {}), + metadata=row.get("metadata"), + ) + score_value = scorer.fn(dataset_row, row.get("output")) if inspect.isawaitable(score_value): score_value = await score_value if isinstance(score_value, bool): @@ -668,27 +670,20 @@ async def _run_ld_judge_for_result( tool_handlers, { **variables, - "formatting_instructions": JUDGE_FORMATTING_INSTRUCTIONS, + "formatting_instructions": FORMATTING_INSTRUCTIONS, }, ) if not isinstance(result, Mapping): raise TypeError("judge handler result must be a mapping") - raw = result.get("output", result.get("response")) - parsed = ( - parse_json_with_possible_fences(raw) - if isinstance(raw, str) - else raw - if isinstance(raw, Mapping) - else None + score, reason = parse_judge_response( + result.get("output", result.get("response")) ) - if not isinstance(parsed, Mapping): - raise ValueError("Invalid JSON from judge") completed = datetime.now(UTC) event = { **base, "status": "COMPLETE", - "score": parsed.get("score"), - "reason": str(parsed.get("reasoning", parsed.get("reason", ""))), + "score": score, + "reason": reason, "evaluated_at": completed.isoformat().replace("+00:00", "Z"), "latency_ms": round((time.perf_counter() - started_clock) * 1000), } @@ -701,27 +696,27 @@ async def _run_ld_judge_for_result( base, started_clock, "judge_error", str(error) ) - async def _run_judges_for_results( + async def _run_criteria_for_results( self, rows: list[dict[str, Any]], handler: EvalHandler, tool_handlers: dict[str, ToolImplementation], - judge_refs: list[JudgeReference], + criteria: list[Criterion], resolved_judges: Mapping[str, ResolvedJudge], ) -> list[dict[str, Any]]: results: list[dict[str, Any]] = [] for row in rows: - for judge in judge_refs: - if isinstance(judge, Scorer): - results.append(await self._run_scorer_for_result(row, judge)) + for criterion in criteria: + if isinstance(criterion, Scorer): + results.append(await self._run_scorer_for_result(row, criterion)) else: results.append( await self._run_ld_judge_for_result( row, handler, tool_handlers, - judge, - resolved_judges[judge.key], + criterion, + resolved_judges[criterion.key], ) ) return results diff --git a/packages/client/src/launchdarkly_ai_server/judge_scoring.py b/packages/client/src/launchdarkly_ai_server/judge_scoring.py new file mode 100644 index 00000000..8375b413 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/judge_scoring.py @@ -0,0 +1,61 @@ +"""Shared scoring contract for LaunchDarkly AI Judge invocations. + +Both judge execution paths — the online path (``judges.run_judges``, sampled +per invocation) and the offline evaluations path (``evaluations.runner``) — +prompt a judge model for the same ``{"score": <0-1>, "reasoning": }`` +JSON shape and must parse it the same way. This module owns that contract so +the two paths cannot drift. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from math import isfinite +from typing import Any + +from .utils import parse_json_with_possible_fences + +FORMATTING_INSTRUCTIONS = "\n".join( + [ + "Your response MUST be in valid JSON format with the following structure:", + '{ "score": , "reasoning": }', + "The output must be valid, parseable JSON. Do not include additional tags, comments, " + "formatting, or newlines.", + "It should be returned in a format that is immediately parseable by a JSON parsing " + "function. Do not include ```json tags.", + ] +) + + +def numeric_score(score: Any) -> float | None: + """Return ``score`` as a float only when it already is a finite number. + + Never raises. A judge that returns ``"0.9 (high)"`` or ``None`` must not take down the + evaluation metric track that follows, and must not put a string where semconv defines a double. + """ + if isinstance(score, bool) or not isinstance(score, (int, float)): + return None + value = float(score) + return value if isfinite(value) else None + + +def parse_judge_response(raw: Any) -> tuple[Any, str]: + """Parse a judge model response into ``(score, reasoning)``. + + Accepts a JSON string (possibly wrapped in markdown fences) or an + already-decoded mapping. The score is returned untouched — callers apply + their own policy to non-numeric values via :func:`numeric_score`. + + Raises ``ValueError`` when the response is not a non-empty JSON object. + """ + parsed: Any + if isinstance(raw, Mapping): + parsed = raw + elif isinstance(raw, str): + parsed = parse_json_with_possible_fences(raw) + else: + parsed = None + if not isinstance(parsed, Mapping) or not parsed: + raise ValueError("Invalid JSON from judge") + reasoning = parsed.get("reasoning") or parsed.get("reason") or "" + return parsed.get("score"), str(reasoning) diff --git a/packages/client/src/launchdarkly_ai_server/judges.py b/packages/client/src/launchdarkly_ai_server/judges.py index ecff6b13..934c2716 100644 --- a/packages/client/src/launchdarkly_ai_server/judges.py +++ b/packages/client/src/launchdarkly_ai_server/judges.py @@ -3,10 +3,14 @@ import logging import random from collections.abc import Callable -from math import isfinite from typing import Any from .conversation import with_judge_evaluation +from .judge_scoring import ( + FORMATTING_INSTRUCTIONS, + numeric_score, + parse_judge_response, +) from .types import ( AiConfigRep, JudgeResult, @@ -22,7 +26,6 @@ ) from .utils import ( normalize_mode, - parse_json_with_possible_fences, to_ld_context, to_usage_dict, ) @@ -38,29 +41,6 @@ def _provider_matches(handler: ProviderHandler, provider: str | None) -> bool: logger = logging.getLogger(__name__) -_FORMATTING_INSTRUCTIONS = "\n".join( - [ - "Your response MUST be in valid JSON format with the following structure:", - '{ "score": , "reasoning": }', - "The output must be valid, parseable JSON. Do not include additional tags, comments, " - "formatting, or newlines.", - "It should be returned in a format that is immediately parseable by a JSON parsing " - "function. Do not include ```json tags.", - ] -) - - -def _numeric_score(score: Any) -> float | None: - """Return ``score`` as a float only when it already is a finite number. - - Never raises. A judge that returns ``"0.9 (high)"`` or ``None`` must not take down the - evaluation metric track that follows, and must not put a string where semconv defines a double. - """ - if isinstance(score, bool) or not isinstance(score, (int, float)): - return None - value = float(score) - return value if isfinite(value) else None - async def run_judges( *, @@ -166,7 +146,7 @@ async def run_judges( ) message_history = "\n\n".join( - filter(None, [user_input, llm_response, _FORMATTING_INSTRUCTIONS]) + filter(None, [user_input, llm_response, FORMATTING_INSTRUCTIONS]) ) async with with_judge_evaluation(judge_key) as record_evaluation: @@ -185,24 +165,16 @@ async def run_judges( }, ) - raw = result["response"] - judge_response = raw if isinstance(raw, str) else str(raw) - - parsed = parse_json_with_possible_fences(judge_response) - if not parsed: - raise ValueError("Invalid JSON from judge") - - score = parsed.get("score") - reasoning = parsed.get("reasoning", "") + score, reasoning = parse_judge_response(result["response"]) judge_results[judge_key] = JudgeResult( usage=to_usage_dict(result["usage"]), response=reasoning, score=score, ) - numeric_score = _numeric_score(score) - if numeric_score is not None: + metric_score = numeric_score(score) + if metric_score is not None: record_evaluation( - numeric_score, + metric_score, reasoning if judge_handler.capture_content else None, ) @@ -403,7 +375,7 @@ def _matches(h: ProviderHandler) -> bool: ) message_history = "\n\n".join( - filter(None, [task.actual_output, _FORMATTING_INSTRUCTIONS]) + filter(None, [task.actual_output, FORMATTING_INSTRUCTIONS]) ) async with with_judge_evaluation(task.config_key) as record_evaluation: @@ -422,18 +394,17 @@ def _matches(h: ProviderHandler) -> bool: }, ) - raw = result["response"] - judge_response = raw if isinstance(raw, str) else str(raw) - parsed = parse_json_with_possible_fences(judge_response) - if not parsed: + try: + score, reasoning = parse_judge_response(result["response"]) + except ValueError: return None - score = parsed.get("score", 0.0) - reasoning = parsed.get("reasoning", "") - numeric_score = _numeric_score(score) - if numeric_score is not None: + if score is None: + score = 0.0 + metric_score = numeric_score(score) + if metric_score is not None: record_evaluation( - numeric_score, + metric_score, reasoning if judge_handler.capture_content else None, ) raw_usage = result["usage"] diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 62a3b614..dbbabab0 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -1,7 +1,7 @@ from __future__ import annotations import json -from collections.abc import Callable, Mapping +from collections.abc import Callable from datetime import datetime from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -9,6 +9,7 @@ import pytest from launchdarkly_ai_server.evaluations import ( + DatasetRow, EvaluationsError, HttpResponse, Judge, @@ -1116,7 +1117,7 @@ async def handler( dataset="golden", handler=handler, generation={"provider": "OpenAI", "model": "gpt-4o"}, - judges=[Judge(key="$ld:ai:judge:accuracy")], + criteria=[Judge(key="$ld:ai:judge:accuracy")], ) assert result.passed is True @@ -1169,7 +1170,7 @@ async def handler(*args: object) -> dict[str, Any]: dataset="golden", handler=handler, generation={"provider": "OpenAI", "model": "gpt-4o"}, - judges=[Judge(key="security-judge")], + criteria=[Judge(key="security-judge")], ) assert transport.requests == [] @@ -1215,9 +1216,9 @@ async def handler(*args: object) -> dict[str, Any]: "usage": {"input_tokens": 10, "output_tokens": 4}, } - def check_refund(row: Mapping[str, Any], output: Any) -> bool: - assert row["row_index"] == 42 - assert row["input"] == "Ticket A" + def check_refund(row: DatasetRow, output: Any) -> bool: + assert row.row_index == 42 + assert row.input == "Ticket A" assert output == "refund exists" return "refund" in str(output) @@ -1227,7 +1228,7 @@ def check_refund(row: Mapping[str, Any], output: Any) -> bool: dataset="support-golden-v3", handler=handler, generation={"provider": "OpenAI", "model": "gpt-4o"}, - judges=[Scorer(name="refund-exists", fn=check_refund)], + criteria=[Scorer(name="refund-exists", fn=check_refund)], ) assert result.passed is True diff --git a/packages/client/tests/test_judges.py b/packages/client/tests/test_judges.py index 582db513..01998bf5 100644 --- a/packages/client/tests/test_judges.py +++ b/packages/client/tests/test_judges.py @@ -391,18 +391,18 @@ class TestScoreGuard: """`float(score)` used to sit ahead of the evaluation-metric track, so a junk score killed it.""" def test_rejects_non_numeric_scores_without_raising(self) -> None: - from launchdarkly_ai_server.judges import _numeric_score + from launchdarkly_ai_server.judge_scoring import numeric_score for junk in ("0.9 (high)", "85%", None, {"v": 1}, [], True, False): - assert _numeric_score(junk) is None + assert numeric_score(junk) is None def test_accepts_finite_numbers(self) -> None: from math import inf, nan - from launchdarkly_ai_server.judges import _numeric_score + from launchdarkly_ai_server.judge_scoring import numeric_score - assert _numeric_score(0.9) == 0.9 - assert _numeric_score(1) == 1.0 - assert _numeric_score(0) == 0.0 - assert _numeric_score(inf) is None - assert _numeric_score(nan) is None + assert numeric_score(0.9) == 0.9 + assert numeric_score(1) == 1.0 + assert numeric_score(0) == 0.0 + assert numeric_score(inf) is None + assert numeric_score(nan) is None From 1eb197615739c45dd9a7c261a4573ae92a9f6479 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Wed, 2 Sep 2026 16:50:22 -0700 Subject: [PATCH 03/16] fix(evaluations): correct judge resolution, score validation, and rendering - Resolve LD judges with a valid {kind: evaluation, key: project} context. The previous empty context is invalid to the real LD SDK, so every resolution returned the None default and failed as 'not found'. The resolution error now also carries the underlying cause instead of always claiming the judge does not exist. - Validate judge scores when the judge responds: non-JSON output, non-numeric, non-finite, and out-of-range scores become per-criterion ERROR events with cause codes (invalid_judge_output, invalid_score, handler_raised, generation_incomplete, scorer_raised) instead of crashing the run at event-build time after all LLM spend. - Pass judge configs to the handler unrendered. The handler owns the single template pass, so {{...}} sequences inside generated output or dataset values can no longer be expanded into the judge prompt. Absent judge variables render as empty strings rather than leaving literal mustache in the prompt. - Reject duplicate criterion identities (judge keys / scorer names) before any records are created; they would share event identity. - Flush queued generation events in a finally so they reach LaunchDarkly even when the criteria phase fails. Co-Authored-By: Claude Fable 5 --- .../evaluations/module.py | 70 +++-- .../evaluations/runner.py | 176 +++++++----- packages/client/tests/test_evaluations_run.py | 269 +++++++++++++++++- 3 files changed, 423 insertions(+), 92 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 9bb3399c..4b4dbe78 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -124,6 +124,7 @@ async def run( ) run_tools = dict(tools or {}) run_criteria = list(criteria or []) + self._validate_criteria(run_criteria) ld_judges = [ criterion for criterion in run_criteria if isinstance(criterion, Judge) ] @@ -135,7 +136,7 @@ async def run( resolved_tools = await asyncio.to_thread( self._runner._resolve_tools, project_key, run_tools ) - resolved_judges = await self._runner._resolve_judges(ld_judges) + resolved_judges = await self._runner._resolve_judges(project_key, ld_judges) dataset_ref = await asyncio.to_thread( self._runner._fetch_dataset, project_key, dataset ) @@ -164,33 +165,37 @@ async def run( run_tools, concurrency, ) - self._runner._emit_generation_events( - client, - project_key=project_key, - evaluation=evaluation, - evaluation_run=evaluation_run, - dataset=dataset_ref, - results=results, - ) - if run_criteria: - criterion_results = await self._runner._run_criteria_for_results( - results, - handler, - run_tools, - run_criteria, - resolved_judges, - ) - self._runner._emit_evaluation_events( + try: + self._runner._emit_generation_events( client, project_key=project_key, evaluation=evaluation, evaluation_run=evaluation_run, dataset=dataset_ref, - results=criterion_results, + results=results, ) - flush_result = client.flush() - if inspect.isawaitable(flush_result): - await flush_result + if run_criteria: + criterion_results = await self._runner._run_criteria_for_results( + results, + handler, + run_tools, + run_criteria, + resolved_judges, + ) + self._runner._emit_evaluation_events( + client, + project_key=project_key, + evaluation=evaluation, + evaluation_run=evaluation_run, + dataset=dataset_ref, + results=criterion_results, + ) + finally: + # Generation results already queued on the SDK event buffer must + # reach LaunchDarkly even when the criteria phase fails. + flush_result = client.flush() + if inspect.isawaitable(flush_result): + await flush_result summary = await self._poll_summary_until_terminal( project_key, evaluation.id, @@ -265,6 +270,27 @@ async def _resolve_client(self) -> Any: ) return await init_client({"sdkKey": self._sdk_key}) + @staticmethod + def _validate_criteria(criteria: list[Criterion]) -> None: + """Reject duplicate criterion identities before any records are created. + + A judge key and a scorer name that collide would share a criterionType, + and with it the deterministic event identity of their results. + """ + seen: set[str] = set() + duplicates: list[str] = [] + for criterion in criteria: + criterion_type = criterion.criterion_type + if criterion_type in seen and criterion_type not in duplicates: + duplicates.append(criterion_type) + seen.add(criterion_type) + if duplicates: + raise EvaluationsError( + "Duplicate evaluation criteria: " + + ", ".join(repr(name) for name in duplicates) + + ". Judge keys and scorer names must be unique within a run." + ) + @staticmethod def _validate_run_args( *, diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 99c4e673..3fbd49cb 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -4,6 +4,7 @@ import hashlib import inspect import json +import logging import time import urllib.parse from collections.abc import Awaitable, Callable, Mapping @@ -12,6 +13,7 @@ from ..judge_scoring import ( FORMATTING_INSTRUCTIONS, + numeric_score, parse_judge_response, ) from ..lifecycle import extract_variation @@ -40,6 +42,8 @@ RunSummary, ) +logger = logging.getLogger(__name__) + DATASET_PAGE_SIZE = 200 GENERATION_EVENT_NAME = "$ld:ai:offline-evals:generation" EVALUATION_EVENT_NAME = "$ld:ai:offline-evals:evaluation" @@ -145,18 +149,22 @@ def _resolve_tools( async def _resolve_judges( self, + project_key: str, judges: list[Judge], ) -> dict[str, ResolvedJudge]: """Resolve LD Judge configs before any evaluation records are created.""" resolved: dict[str, ResolvedJudge] = {} - context: dict[str, Any] = {} + # variation() rejects a context without kind and key; use the same + # context shape the emitted evaluation events are attributed to. + context: dict[str, Any] = {"kind": "evaluation", "key": project_key} for judge in judges: try: variation = await extract_variation(judge.key, context) except Exception as error: raise EvaluationsError( - f"LaunchDarkly judge {judge.key!r} was not found or could not be resolved " - "for this project. Create the judge in the LaunchDarkly UI and try again." + f"Failed to resolve LaunchDarkly judge {judge.key!r}: {error} " + f"If the judge does not exist in project {project_key!r}, " + "create it in the LaunchDarkly UI and try again." ) from error config = variation.get("config") meta_value = variation.get("meta") @@ -537,6 +545,12 @@ def _judge_variables( row_result: Mapping[str, Any], judge: Judge, ) -> dict[str, Any]: + """Variables available to the judge config's ``{{...}}`` placeholders. + + Absent values become empty strings: ``parse_template`` leaves a + placeholder with a ``None`` value as-is, and literal mustache text must + not reach the judge model. + """ variables = dict(row_result.get("variables") or {}) output = row_result.get("output") expected = row_result.get("expected_output") @@ -547,31 +561,21 @@ def _judge_variables( ground_truth = str(expected) variables.update( { - "input": row_result.get("input"), - "response_to_evaluate": output, + "input": row_result.get("input") or "", + "response_to_evaluate": output if output is not None else "", "message_history": "\n\n".join( str(value) for value in (row_result.get("input"), output) if value is not None ), - "expected_output": expected, - "ground_truth_context": ground_truth, + "expected_output": expected if expected is not None else "", + "ground_truth_context": ( + ground_truth if ground_truth is not None else "" + ), } ) return variables - def _render_config_value(self, value: Any, variables: Mapping[str, Any]) -> Any: - if isinstance(value, str): - return parse_template(value, dict(variables)) - if isinstance(value, list): - return [self._render_config_value(item, variables) for item in value] - if isinstance(value, Mapping): - return { - str(key): self._render_config_value(item, variables) - for key, item in value.items() - } - return value - def _criterion_error_result( self, base: Mapping[str, Any], @@ -603,40 +607,55 @@ async def _run_scorer_for_result( } if row.get("status") != "COMPLETE": return self._criterion_error_result( - base, started_clock, "scorer_error", "generation did not complete" + base, + started_clock, + "generation_incomplete", + "generation did not complete", ) + dataset_row = DatasetRow( + row_index=row["row_index"], + input=row.get("input"), + expected_output=row.get("expected_output"), + variables=dict(row.get("variables") or {}), + metadata=row.get("metadata"), + ) try: - dataset_row = DatasetRow( - row_index=row["row_index"], - input=row.get("input"), - expected_output=row.get("expected_output"), - variables=dict(row.get("variables") or {}), - metadata=row.get("metadata"), - ) score_value = scorer.fn(dataset_row, row.get("output")) if inspect.isawaitable(score_value): score_value = await score_value - if isinstance(score_value, bool): - score: float = 1.0 if score_value else 0.0 - elif isinstance(score_value, int | float): - score = float(score_value) - else: - raise TypeError("scorer fn must return a bool or numeric score") - if score < 0 or score > 1: - raise ValueError("scorer fn score must be between 0 and 1") - completed = datetime.now(UTC) - return { - **base, - "status": "COMPLETE", - "score": score, - "reason": None, - "evaluated_at": completed.isoformat().replace("+00:00", "Z"), - "latency_ms": round((time.perf_counter() - started_clock) * 1000), - } except Exception as error: return self._criterion_error_result( - base, started_clock, "scorer_error", str(error) + base, started_clock, "scorer_raised", f"scorer fn raised: {error}" ) + if isinstance(score_value, bool): + score: float = 1.0 if score_value else 0.0 + else: + maybe_score = numeric_score(score_value) + if maybe_score is None: + return self._criterion_error_result( + base, + started_clock, + "invalid_score", + "scorer fn must return a bool or a finite number, " + f"got {score_value!r}", + ) + score = maybe_score + if score < 0 or score > 1: + return self._criterion_error_result( + base, + started_clock, + "invalid_score", + f"scorer fn score must be between 0 and 1, got {score_value!r}", + ) + completed = datetime.now(UTC) + return { + **base, + "status": "COMPLETE", + "score": score, + "reason": None, + "evaluated_at": completed.isoformat().replace("+00:00", "Z"), + "latency_ms": round((time.perf_counter() - started_clock) * 1000), + } async def _run_ld_judge_for_result( self, @@ -659,13 +678,18 @@ async def _run_ld_judge_for_result( } if row.get("status") != "COMPLETE": return self._criterion_error_result( - base, started_clock, "judge_error", "generation did not complete" + base, + started_clock, + "generation_incomplete", + "generation did not complete", ) + # The config is passed unrendered: the handler owns the single + # parse_template pass, so ``{{...}}`` sequences inside generated output + # or dataset values are never re-expanded into the judge prompt. + variables = self._judge_variables(row, judge) try: - variables = self._judge_variables(row, judge) - rendered_config = self._render_config_value(resolved.config, variables) result = await handler( - rendered_config, + dict(resolved.config), row.get("output"), tool_handlers, { @@ -673,28 +697,46 @@ async def _run_ld_judge_for_result( "formatting_instructions": FORMATTING_INSTRUCTIONS, }, ) - if not isinstance(result, Mapping): - raise TypeError("judge handler result must be a mapping") - score, reason = parse_judge_response( + except Exception as error: + return self._criterion_error_result( + base, started_clock, "handler_raised", f"judge handler raised: {error}" + ) + if not isinstance(result, Mapping): + return self._criterion_error_result( + base, + started_clock, + "invalid_judge_output", + "judge handler result must be a mapping", + ) + try: + raw_score, reason = parse_judge_response( result.get("output", result.get("response")) ) - completed = datetime.now(UTC) - event = { - **base, - "status": "COMPLETE", - "score": score, - "reason": reason, - "evaluated_at": completed.isoformat().replace("+00:00", "Z"), - "latency_ms": round((time.perf_counter() - started_clock) * 1000), - } - usage = result.get("usage") - if isinstance(usage, Mapping): - event["usage"] = dict(usage) - return event - except Exception as error: + except ValueError as error: return self._criterion_error_result( - base, started_clock, "judge_error", str(error) + base, started_clock, "invalid_judge_output", str(error) ) + score = numeric_score(raw_score) + if score is None or score < 0 or score > 1: + return self._criterion_error_result( + base, + started_clock, + "invalid_score", + f"judge score must be a number between 0 and 1, got {raw_score!r}", + ) + completed = datetime.now(UTC) + event = { + **base, + "status": "COMPLETE", + "score": score, + "reason": reason, + "evaluated_at": completed.isoformat().replace("+00:00", "Z"), + "latency_ms": round((time.perf_counter() - started_clock) * 1000), + } + usage = result.get("usage") + if isinstance(usage, Mapping): + event["usage"] = dict(usage) + return event async def _run_criteria_for_results( self, diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index dbbabab0..4dcbf743 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -1075,7 +1075,9 @@ async def fake_extract_variation( key: str, context: dict[str, Any] ) -> dict[str, Any]: assert key == "$ld:ai:judge:accuracy" - assert context == {} + # An empty or kindless context is invalid to the real LD SDK and would + # make every judge resolution fail. + assert context == {"kind": "evaluation", "key": "proj"} return { "config": { "provider": {"name": "OpenAI"}, @@ -1101,7 +1103,14 @@ async def handler( assert user_input == "generated" assert variables["response_to_evaluate"] == "generated" assert variables["expected_output"] == "Answer A" - assert "generated" in config["instructions"] + # The SDK hands the judge config over unrendered; the handler owns + # the single template pass. + assert config["instructions"] == ( + "Judge {{response_to_evaluate}} against {{expected_output}}" + ) + assert variables["formatting_instructions"].startswith( + "Your response MUST be in valid JSON" + ) return { "output": '{"score": 0.86, "reasoning": "matches policy"}', "usage": {"input_tokens": 640, "output_tokens": 48}, @@ -1162,7 +1171,8 @@ async def handler(*args: object) -> dict[str, Any]: return {"output": "generated"} with pytest.raises( - EvaluationsError, match="Create the judge in the LaunchDarkly UI" + EvaluationsError, + match=r"Failed to resolve LaunchDarkly judge 'security-judge': not found", ): await evals.run( project_key="proj", @@ -1258,3 +1268,256 @@ def check_refund(row: DatasetRow, output: Any) -> bool: assert "judgeKey" not in scorer_event assert "variationKey" not in scorer_event assert "version" not in scorer_event + + +def judge_run_transport(*, summary: dict[str, Any] | None = None) -> SequencedTransport: + """Transport for a one-row run that resolves a dataset, evaluation, and run.""" + return SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [ + { + "rowIndex": 7, + "input": "Question {{id}}", + "expectedOutput": "Answer {{id}}", + "variables": {"id": "A"}, + } + ], + total=1, + ), + ), + response(201, {"id": "evaluation-id", "name": "support-qa", "version": 3}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + summary + or { + "statusCounts": {"total": 1, "passed": 1, "error": 0, "pending": 0} + }, + ), + ] + ) + + +def accuracy_judge_variation(monkeypatch: pytest.MonkeyPatch) -> None: + async def fake_extract_variation( + key: str, context: dict[str, Any] + ) -> dict[str, Any]: + return { + "config": { + "provider": {"name": "OpenAI"}, + "model": {"name": "gpt-4o"}, + "instructions": "Judge {{response_to_evaluate}} against {{expected_output}}", + }, + "meta": {"variationKey": "default", "version": 12}, + } + + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.runner.extract_variation", + fake_extract_variation, + ) + + +@pytest.mark.parametrize( + ("judge_output", "expected_code"), + [ + ('{"score": "high (0.9)", "reasoning": "confident"}', "invalid_score"), + ('{"score": 3, "reasoning": "confident"}', "invalid_score"), + ('{"score": NaN, "reasoning": "confident"}', "invalid_score"), + ("the answer looks right to me", "invalid_judge_output"), + ], +) +@pytest.mark.asyncio +async def test_bad_judge_output_emits_error_event_instead_of_crashing( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, + judge_output: str, + expected_code: str, +) -> None: + transport = judge_run_transport() + accuracy_judge_variation(monkeypatch) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + async def handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + if "Judge" in config.get("instructions", ""): + return {"output": judge_output} + return {"output": "generated"} + + result = await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + ) + + assert result.passed is True + events = [call.args[2] for call in stub_sdk_client.track.call_args_list] + judge_event = next(event for event in events if event.get("kind") == "judge") + assert judge_event["status"] == "ERROR" + assert judge_event["error"]["code"] == expected_code + assert "score" not in judge_event + stub_sdk_client.flush.assert_awaited() + + +@pytest.mark.asyncio +async def test_generated_placeholders_are_not_expanded_into_judge_prompt( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, +) -> None: + from launchdarkly_ai_server import parse_template + + transport = judge_run_transport() + accuracy_judge_variation(monkeypatch) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + async def handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + if "Judge" in config.get("instructions", ""): + rendered = parse_template(config["instructions"], variables) + # The placeholder smuggled in via the generated output must stay + # literal text after the handler's single render pass. + assert rendered == "Judge {{expected_output}} leaked? against Answer A" + return {"output": '{"score": 1, "reasoning": "ok"}'} + return {"output": "{{expected_output}} leaked?"} + + result = await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + ) + + assert result.passed is True + events = [call.args[2] for call in stub_sdk_client.track.call_args_list] + judge_event = next(event for event in events if event.get("kind") == "judge") + assert judge_event["status"] == "COMPLETE" + assert judge_event["score"] == 1.0 + + +@pytest.mark.asyncio +async def test_missing_expected_output_renders_empty_judge_variables( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, +) -> None: + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page([{"rowIndex": 7, "input": "Question"}], total=1), + ), + response(201, {"id": "evaluation-id", "name": "support-qa", "version": 3}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + {"statusCounts": {"total": 1, "passed": 1, "error": 0, "pending": 0}}, + ), + ] + ) + accuracy_judge_variation(monkeypatch) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + async def handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + if "Judge" in config.get("instructions", ""): + assert variables["expected_output"] == "" + assert variables["ground_truth_context"] == "" + return {"output": '{"score": 1, "reasoning": "ok"}'} + return {"output": "generated"} + + result = await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + ) + assert result.passed is True + + +@pytest.mark.asyncio +async def test_duplicate_criteria_rejected_before_any_request() -> None: + transport = SequencedTransport([]) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + with pytest.raises(EvaluationsError, match="Duplicate evaluation criteria"): + await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[ + Judge(key="accuracy"), + Scorer(name="accuracy", fn=lambda row, output: True), + ], + ) + + assert transport.requests == [] + + +@pytest.mark.asyncio +async def test_errored_generation_row_emits_generation_incomplete_criterion_event( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, +) -> None: + transport = judge_run_transport( + summary={"statusCounts": {"total": 1, "passed": 0, "error": 1, "pending": 0}} + ) + accuracy_judge_variation(monkeypatch) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + async def handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + if "Judge" in config.get("instructions", ""): + raise AssertionError("judges must not run for errored generations") + raise RuntimeError("provider unavailable") + + result = await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + ) + + assert result.passed is False + events = [call.args[2] for call in stub_sdk_client.track.call_args_list] + judge_event = next(event for event in events if event.get("kind") == "judge") + assert judge_event["status"] == "ERROR" + assert judge_event["error"]["code"] == "generation_incomplete" From 46bfa001b24f1266105b2bbf6b2c1ff27afe98be Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Wed, 2 Sep 2026 16:52:44 -0700 Subject: [PATCH 04/16] refactor(evaluations): drop pydantic, run criteria concurrently Rewrite the evaluation event payloads as frozen dataclasses with an explicit to_track_payload(), matching the Usage/RunSummary wire pattern used everywhere else in the SDK, and remove the pydantic>=2 dependency. The models validated the SDK's own dicts, and a validation failure surfaced as a run-aborting crash at emission time; payload construction is now also wrapped per result so one bad criterion result is logged and skipped instead of dropping the whole batch. ERROR events carry a top-level errorMessage for parity with generation events. Run (row x criterion) pairs through the same ConcurrencyController and concurrency parameter the generation phase uses, instead of one criterion at a time: a 200-row dataset with 3 judges was 600 serial LLM calls. Co-Authored-By: Claude Fable 5 --- packages/client/pyproject.toml | 2 +- .../evaluations/events.py | 101 +++++++---- .../evaluations/module.py | 1 + .../evaluations/runner.py | 165 +++++++++++------- packages/client/tests/test_evaluations_run.py | 105 +++++++++++ uv.lock | 2 - 6 files changed, 277 insertions(+), 99 deletions(-) diff --git a/packages/client/pyproject.toml b/packages/client/pyproject.toml index ee8f995c..9ea3ce74 100644 --- a/packages/client/pyproject.toml +++ b/packages/client/pyproject.toml @@ -2,7 +2,7 @@ name = "launchdarkly-ai-server" version = "0.1.3" requires-python = ">=3.12" -dependencies = ["opentelemetry-api>=1.25", "pydantic>=2"] +dependencies = ["opentelemetry-api>=1.25"] description = "LaunchDarkly AI SDK core client for Python" readme = "README.md" license = "Apache-2.0" diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/events.py b/packages/client/src/launchdarkly_ai_server/evaluations/events.py index ba2d3e04..97de0938 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/events.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/events.py @@ -1,10 +1,9 @@ from __future__ import annotations +from dataclasses import dataclass from enum import StrEnum from typing import Any -from pydantic import BaseModel, ConfigDict, Field - class EvaluationStatus(StrEnum): COMPLETE = "COMPLETE" @@ -16,57 +15,95 @@ class EvaluationEventKind(StrEnum): SCORER = "scorer" -class TokenUsage(BaseModel): +@dataclass(frozen=True) +class TokenUsage: """Token usage reported by an LD Judge provider call.""" - model_config = ConfigDict(populate_by_name=True, extra="forbid") + input_tokens: int + output_tokens: int - input_tokens: int = Field(alias="inputTokens") - output_tokens: int = Field(alias="outputTokens") + def to_wire(self) -> dict[str, int]: + return { + "inputTokens": self.input_tokens, + "outputTokens": self.output_tokens, + } -class EvaluationEventPayload(BaseModel): +@dataclass(frozen=True, kw_only=True) +class EvaluationEventPayload: """Common fields emitted for every SDK-run evaluation criterion result.""" - model_config = ConfigDict( - populate_by_name=True, extra="forbid", use_enum_values=True - ) - - project_key: str = Field(alias="projectKey") - evaluation_id: str = Field(alias="evaluationId") - evaluation_run_id: str = Field(alias="evaluationRunId") - run_id: str = Field(alias="runId") - dataset_id: str = Field(alias="datasetId") - row_index: int = Field(alias="rowIndex") - criterion_type: str = Field(alias="criterionType") + project_key: str + evaluation_id: str + evaluation_run_id: str + run_id: str + dataset_id: str + row_index: int + criterion_type: str kind: EvaluationEventKind - event_id: str = Field(alias="eventId") - emitted_at: str = Field(alias="emittedAt") - evaluation_key: str = Field(alias="evaluationKey") - evaluation_version: int | None = Field(default=None, alias="evaluationVersion") - dataset_key: str = Field(alias="datasetKey") + event_id: str + emitted_at: str + evaluation_key: str + dataset_key: str status: EvaluationStatus - started_at: str = Field(alias="startedAt") - evaluated_at: str = Field(alias="evaluatedAt") - latency_ms: int = Field(alias="latencyMs") - score: float | int | None = None + started_at: str + evaluated_at: str + latency_ms: int + evaluation_version: int | None = None + score: float | None = None reason: str | None = None error: dict[str, Any] | None = None + error_message: str | None = None def to_track_payload(self) -> dict[str, Any]: - return self.model_dump(by_alias=True, exclude_none=True) - - + payload: dict[str, Any] = { + "projectKey": self.project_key, + "evaluationId": self.evaluation_id, + "evaluationRunId": self.evaluation_run_id, + "runId": self.run_id, + "datasetId": self.dataset_id, + "rowIndex": self.row_index, + "criterionType": self.criterion_type, + "kind": self.kind.value, + "eventId": self.event_id, + "emittedAt": self.emitted_at, + "evaluationKey": self.evaluation_key, + "evaluationVersion": self.evaluation_version, + "datasetKey": self.dataset_key, + "status": self.status.value, + "startedAt": self.started_at, + "evaluatedAt": self.evaluated_at, + "latencyMs": self.latency_ms, + "score": self.score, + "reason": self.reason, + "error": self.error, + "errorMessage": self.error_message, + } + return {key: value for key, value in payload.items() if value is not None} + + +@dataclass(frozen=True, kw_only=True) class LDJudgeEvaluationEventPayload(EvaluationEventPayload): """Payload for one LaunchDarkly AI Judge result on one dataset row.""" kind: EvaluationEventKind = EvaluationEventKind.JUDGE - judge_key: str = Field(alias="judgeKey") - variation_key: str = Field(alias="variationKey") + judge_key: str + variation_key: str version: int | None = None usage: TokenUsage | None = None + def to_track_payload(self) -> dict[str, Any]: + payload = super().to_track_payload() + payload["judgeKey"] = self.judge_key + payload["variationKey"] = self.variation_key + if self.version is not None: + payload["version"] = self.version + if self.usage is not None: + payload["usage"] = self.usage.to_wire() + return payload + +@dataclass(frozen=True, kw_only=True) class DeterministicScorerEvaluationEventPayload(EvaluationEventPayload): """Payload for one local deterministic scorer result on one dataset row.""" diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 4b4dbe78..3ee59d09 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -181,6 +181,7 @@ async def run( run_tools, run_criteria, resolved_judges, + concurrency, ) self._runner._emit_evaluation_events( client, diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 3fbd49cb..0aeaee18 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -28,6 +28,7 @@ from .events import ( DeterministicScorerEvaluationEventPayload, EvaluationEventPayload, + EvaluationStatus, LDJudgeEvaluationEventPayload, TokenUsage, ) @@ -745,23 +746,33 @@ async def _run_criteria_for_results( tool_handlers: dict[str, ToolImplementation], criteria: list[Criterion], resolved_judges: Mapping[str, ResolvedJudge], + concurrency: int, ) -> list[dict[str, Any]]: - results: list[dict[str, Any]] = [] - for row in rows: - for criterion in criteria: + """Run every (row, criterion) pair, bounded by the run's concurrency.""" + controller = ConcurrencyController(concurrency) + + async def run_one( + row: Mapping[str, Any], criterion: Criterion + ) -> dict[str, Any]: + await controller.acquire() + try: if isinstance(criterion, Scorer): - results.append(await self._run_scorer_for_result(row, criterion)) - else: - results.append( - await self._run_ld_judge_for_result( - row, - handler, - tool_handlers, - criterion, - resolved_judges[criterion.key], - ) - ) - return results + return await self._run_scorer_for_result(row, criterion) + return await self._run_ld_judge_for_result( + row, + handler, + tool_handlers, + criterion, + resolved_judges[criterion.key], + ) + finally: + controller.release() + + return list( + await asyncio.gather( + *(run_one(row, criterion) for row in rows for criterion in criteria) + ) + ) def _emit_evaluation_events( self, @@ -783,57 +794,83 @@ def _emit_evaluation_events( }, ) for result in results: - identity = { - "projectKey": project_key, - "evaluationId": evaluation.id, - "evaluationRunId": evaluation_run.id, - "runId": evaluation_run.id, - "datasetId": dataset.id, - "rowIndex": result["row_index"], - "criterionType": result["criterion_type"], - } - event_id = hashlib.sha256( - json.dumps(identity, sort_keys=True, separators=(",", ":")).encode() - ).hexdigest() - emitted_at = datetime.now(UTC).isoformat().replace("+00:00", "Z") - usage: TokenUsage | None = None - if result["kind"] == "judge" and isinstance(result.get("usage"), Mapping): - normalized_usage = parse_usage(dict(result["usage"])) - usage = TokenUsage( - inputTokens=normalized_usage["input"], - outputTokens=normalized_usage["output"], - ) - common_payload = { - **identity, - "eventId": event_id, - "emittedAt": emitted_at, - "evaluationKey": evaluation.key, - "evaluationVersion": evaluation.version, - "datasetKey": dataset.key, - "status": result["status"], - "startedAt": result["started_at"], - "evaluatedAt": result["evaluated_at"], - "latencyMs": result["latency_ms"], - "score": result.get("score"), - "reason": result.get("reason"), - "error": result.get("error"), - } - payload_model: EvaluationEventPayload - if result["kind"] == "judge": - payload_model = LDJudgeEvaluationEventPayload( - **common_payload, - judgeKey=result["judge_key"], - variationKey=result["variation_key"], - version=result.get("version"), - usage=usage, + # One bad criterion result must not abort the run or drop the + # events queued for the results that preceded it. + try: + identity = { + "projectKey": project_key, + "evaluationId": evaluation.id, + "evaluationRunId": evaluation_run.id, + "runId": evaluation_run.id, + "datasetId": dataset.id, + "rowIndex": result["row_index"], + "criterionType": result["criterion_type"], + } + event_id = hashlib.sha256( + json.dumps(identity, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + emitted_at = datetime.now(UTC).isoformat().replace("+00:00", "Z") + usage: TokenUsage | None = None + if result["kind"] == "judge" and isinstance( + result.get("usage"), Mapping + ): + normalized_usage = parse_usage(dict(result["usage"])) + usage = TokenUsage( + input_tokens=normalized_usage["input"], + output_tokens=normalized_usage["output"], + ) + error = result.get("error") + error_message: str | None = None + if result["status"] == "ERROR": + if isinstance(error, Mapping) and error.get("message"): + error_message = str(error["message"]) + else: + error_message = str(error) if error else "Unknown error" + common_payload: dict[str, Any] = { + "project_key": project_key, + "evaluation_id": evaluation.id, + "evaluation_run_id": evaluation_run.id, + "run_id": evaluation_run.id, + "dataset_id": dataset.id, + "row_index": result["row_index"], + "criterion_type": result["criterion_type"], + "event_id": event_id, + "emitted_at": emitted_at, + "evaluation_key": evaluation.key, + "evaluation_version": evaluation.version, + "dataset_key": dataset.key, + "status": EvaluationStatus(result["status"]), + "started_at": result["started_at"], + "evaluated_at": result["evaluated_at"], + "latency_ms": result["latency_ms"], + "score": result.get("score"), + "reason": result.get("reason"), + "error": error, + "error_message": error_message, + } + payload_model: EvaluationEventPayload + if result["kind"] == "judge": + payload_model = LDJudgeEvaluationEventPayload( + **common_payload, + judge_key=result["judge_key"], + variation_key=result["variation_key"], + version=result.get("version"), + usage=usage, + ) + else: + payload_model = DeterministicScorerEvaluationEventPayload( + **common_payload + ) + client.track( + EVALUATION_EVENT_NAME, context, payload_model.to_track_payload(), 1 ) - else: - payload_model = DeterministicScorerEvaluationEventPayload( - **common_payload + except Exception: + logger.exception( + "Skipping evaluation event for row %s criterion %s", + result.get("row_index"), + result.get("criterion_type"), ) - client.track( - EVALUATION_EVENT_NAME, context, payload_model.to_track_payload(), 1 - ) + continue print( f"{EVALUATION_EVENT_NAME} emittedAt={emitted_at} eventId={event_id}", flush=True, diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 4dcbf743..ac73ed59 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -1368,6 +1368,7 @@ async def handler( judge_event = next(event for event in events if event.get("kind") == "judge") assert judge_event["status"] == "ERROR" assert judge_event["error"]["code"] == expected_code + assert judge_event["errorMessage"] == judge_event["error"]["message"] assert "score" not in judge_event stub_sdk_client.flush.assert_awaited() @@ -1521,3 +1522,107 @@ async def handler( judge_event = next(event for event in events if event.get("kind") == "judge") assert judge_event["status"] == "ERROR" assert judge_event["error"]["code"] == "generation_incomplete" + + +@pytest.mark.asyncio +async def test_failed_evaluation_event_tracking_skips_event_but_completes_run( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, +) -> None: + transport = judge_run_transport() + accuracy_judge_variation(monkeypatch) + + def track(event_name: str, *args: Any) -> None: + if event_name == "$ld:ai:offline-evals:evaluation": + raise RuntimeError("event pipeline unavailable") + + stub_sdk_client.track = MagicMock(side_effect=track) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + async def handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + if "Judge" in config.get("instructions", ""): + return {"output": '{"score": 1, "reasoning": "ok"}'} + return {"output": "generated"} + + result = await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + ) + + assert result.passed is True + stub_sdk_client.flush.assert_awaited() + + +@pytest.mark.asyncio +async def test_criteria_run_concurrently_within_the_concurrency_bound( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, +) -> None: + import asyncio + + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [ + {"rowIndex": index, "input": f"Question {index}"} + for index in range(3) + ], + total=3, + ), + ), + response(201, {"id": "evaluation-id", "name": "support-qa", "version": 3}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + {"statusCounts": {"total": 3, "passed": 3, "error": 0, "pending": 0}}, + ), + ] + ) + accuracy_judge_variation(monkeypatch) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + in_flight = 0 + max_in_flight = 0 + + async def handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + nonlocal in_flight, max_in_flight + if "Judge" in config.get("instructions", ""): + in_flight += 1 + max_in_flight = max(max_in_flight, in_flight) + await asyncio.sleep(0.01) + in_flight -= 1 + return {"output": '{"score": 1, "reasoning": "ok"}'} + return {"output": "generated"} + + result = await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + concurrency=2, + ) + + assert result.passed is True + assert max_in_flight == 2 diff --git a/uv.lock b/uv.lock index fa21dbca..7d93a3cd 100644 --- a/uv.lock +++ b/uv.lock @@ -922,7 +922,6 @@ version = "0.1.3" source = { editable = "packages/client" } dependencies = [ { name = "opentelemetry-api" }, - { name = "pydantic" }, ] [package.optional-dependencies] @@ -936,7 +935,6 @@ requires-dist = [ { name = "opentelemetry-api", specifier = ">=1.25" }, { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otel'", specifier = ">=1.25" }, { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.25" }, - { name = "pydantic", specifier = ">=2" }, ] provides-extras = ["otel"] From da69d336dca0e6ae936bb262aeec477feee3b14a Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Wed, 2 Sep 2026 16:53:54 -0700 Subject: [PATCH 05/16] polish(evaluations): event logging, docstrings, judge scoring tests Emit per-event telemetry lines through the module logger instead of printing to the host application's stdout, refresh the run() docstring (no longer generation-only), and cover the shared judge response parser with unit tests. Co-Authored-By: Claude Fable 5 --- .../evaluations/module.py | 8 ++- .../evaluations/runner.py | 16 +++--- packages/client/tests/test_evaluations_run.py | 13 +++-- packages/client/tests/test_judge_scoring.py | 54 +++++++++++++++++++ 4 files changed, 80 insertions(+), 11 deletions(-) create mode 100644 packages/client/tests/test_judge_scoring.py diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 3ee59d09..57a77523 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -100,7 +100,13 @@ async def run( poll_timeout_seconds: float | None = None, ) -> EvalRunResult: """ - Create and run a generation-only evaluation in the caller's process. + Create and run an evaluation in the caller's process. + + Each dataset row is generated with ``handler``; every entry in + ``criteria`` — LaunchDarkly :class:`Judge` references and local + deterministic :class:`Scorer` functions — is then run against each + generated row, and one evaluation event is emitted per + ``(row, criterion)`` result. The returned pass/fail result is derived from LaunchDarkly's run summary. A CI script can exit with ``0 if result.passed else 1`` after awaiting diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 0aeaee18..5a2e4ee1 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -536,9 +536,11 @@ def _emit_generation_events( if "usage" in generated: payload["usage"] = generated["usage"] client.track(GENERATION_EVENT_NAME, context, payload, 1) - print( - f"{GENERATION_EVENT_NAME} emittedAt={emitted_at} eventId={event_id}", - flush=True, + logger.info( + "%s emittedAt=%s eventId=%s", + GENERATION_EVENT_NAME, + emitted_at, + event_id, ) def _judge_variables( @@ -871,9 +873,11 @@ def _emit_evaluation_events( result.get("criterion_type"), ) continue - print( - f"{EVALUATION_EVENT_NAME} emittedAt={emitted_at} eventId={event_id}", - flush=True, + logger.info( + "%s emittedAt=%s eventId=%s", + EVALUATION_EVENT_NAME, + emitted_at, + event_id, ) def _get_summary( diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index ac73ed59..38104dc5 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -120,8 +120,9 @@ def lookup_order(order_id: str) -> str: @pytest.mark.asyncio async def test_complete_run_with_zero_failed_and_error_rows_passes( monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], + caplog: pytest.LogCaptureFixture, ) -> None: + caplog.set_level("INFO", logger="launchdarkly_ai_server.evaluations.runner") monkeypatch.delenv("LD_SDK_KEY", raising=False) init_client = AsyncMock() monkeypatch.setattr( @@ -308,9 +309,13 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( assert event["emittedAt"].endswith("Z") assert datetime.fromisoformat(event["emittedAt"]).tzinfo is not None assert {"input", "expected_output", "metadata", "variables"}.isdisjoint(event) - output_lines = capsys.readouterr().out.splitlines() - assert len(output_lines) == 2 - assert output_lines[0] == ( + emit_logs = [ + record.getMessage() + for record in caplog.records + if record.name == "launchdarkly_ai_server.evaluations.runner" + ] + assert len(emit_logs) == 2 + assert emit_logs[0] == ( "$ld:ai:offline-evals:generation " f"emittedAt={event['emittedAt']} eventId={event['eventId']}" ) diff --git a/packages/client/tests/test_judge_scoring.py b/packages/client/tests/test_judge_scoring.py new file mode 100644 index 00000000..4613b679 --- /dev/null +++ b/packages/client/tests/test_judge_scoring.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from launchdarkly_ai_server.judge_scoring import parse_judge_response + + +class TestParseJudgeResponse: + def test_parses_plain_json(self) -> None: + assert parse_judge_response('{"score": 0.9, "reasoning": "solid"}') == ( + 0.9, + "solid", + ) + + def test_parses_fenced_json(self) -> None: + raw = '```json\n{"score": 1, "reasoning": "ok"}\n```' + assert parse_judge_response(raw) == (1, "ok") + + def test_accepts_already_decoded_mapping(self) -> None: + assert parse_judge_response({"score": 0.5, "reasoning": "meh"}) == ( + 0.5, + "meh", + ) + + def test_falls_back_to_reason_key(self) -> None: + assert parse_judge_response({"score": 0.5, "reason": "alt key"}) == ( + 0.5, + "alt key", + ) + + def test_null_reasoning_becomes_empty_string_not_none_literal(self) -> None: + assert parse_judge_response({"score": 0.5, "reasoning": None}) == (0.5, "") + + def test_score_returned_untouched_for_caller_policy(self) -> None: + score, _ = parse_judge_response({"score": "high", "reasoning": "?"}) + assert score == "high" + + @pytest.mark.parametrize( + "raw", + [ + "the answer looks correct", + "{}", + {}, + None, + 42, + ["not", "a", "mapping"], + '["not", "a", "mapping"]', + ], + ) + def test_rejects_non_object_responses(self, raw: Any) -> None: + with pytest.raises(ValueError, match="Invalid JSON from judge"): + parse_judge_response(raw) From 6509971612c600c77e7018bf74e8c026cc89df45 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Tue, 8 Sep 2026 17:23:53 -0700 Subject: [PATCH 06/16] Capture and forward judge success direction (isInverted) The SDK has no way to learn whether a judge is lower-is-better or upper-is-better today. Gonfalon is adding isInverted as a top-level key in the flag-variation payload (same delivery path already used for variationKey/version); this wires the client side up to consume it. - ResolvedJudge gains is_inverted, read from the resolved variation's raw config dict (already passed through untouched by extract_variation/parse_ai_config, so no change needed there). - _resolve_judges populates it via config.get("isInverted"), tolerant of the key being absent (older Gonfalon) or non-bool. - LDJudgeEvaluationEventPayload gains success_direction, emitted as successDirection only when non-None, matching how usage/version are conditionally included. - _emit_evaluation_events maps is_inverted -> "lower_is_better" / "upper_is_better" / omitted (unresolved), never trusting client-side direction for verdict computation - only for what gets reported. Direction never affects the score itself; it's forwarded for ClickHouse ingestion in ai-evaluator, which is deferred/out of scope here. Co-Authored-By: Claude Sonnet 5 --- .../evaluations/events.py | 7 ++ .../evaluations/runner.py | 13 +++ .../evaluations/types.py | 1 + packages/client/tests/test_evaluations_run.py | 97 +++++++++++++++++++ 4 files changed, 118 insertions(+) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/events.py b/packages/client/src/launchdarkly_ai_server/evaluations/events.py index 97de0938..3ac1ce9b 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/events.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/events.py @@ -91,6 +91,11 @@ class LDJudgeEvaluationEventPayload(EvaluationEventPayload): variation_key: str version: int | None = None usage: TokenUsage | None = None + # "lower_is_better" or "upper_is_better"; None when direction is unresolved (e.g. + # Gonfalon hasn't deployed the flag-payload change yet, or a custom judge with no + # direction set). Direction is authoritative from LaunchDarkly, never trusted from + # the client for verdict computation. + success_direction: str | None = None def to_track_payload(self) -> dict[str, Any]: payload = super().to_track_payload() @@ -100,6 +105,8 @@ def to_track_payload(self) -> dict[str, Any]: payload["version"] = self.version if self.usage is not None: payload["usage"] = self.usage.to_wire() + if self.success_direction is not None: + payload["successDirection"] = self.success_direction return payload diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 5a2e4ee1..44c4dd2f 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -183,6 +183,11 @@ async def _resolve_judges( version=int(meta["version"]) if isinstance(meta.get("version"), int) else None, + # Absent (e.g. Gonfalon hasn't deployed the flag-payload change yet, or a + # custom judge with no direction set) resolves to None, not a raised error. + is_inverted=config.get("isInverted") + if isinstance(config.get("isInverted"), bool) + else None, ) return resolved @@ -678,6 +683,7 @@ async def _run_ld_judge_for_result( "started_at": started.isoformat().replace("+00:00", "Z"), "variation_key": resolved.variation_key, "version": resolved.version, + "is_inverted": resolved.is_inverted, } if row.get("status") != "COMPLETE": return self._criterion_error_result( @@ -852,12 +858,19 @@ def _emit_evaluation_events( } payload_model: EvaluationEventPayload if result["kind"] == "judge": + is_inverted = result.get("is_inverted") + success_direction: str | None = None + if is_inverted is True: + success_direction = "lower_is_better" + elif is_inverted is False: + success_direction = "upper_is_better" payload_model = LDJudgeEvaluationEventPayload( **common_payload, judge_key=result["judge_key"], variation_key=result["variation_key"], version=result.get("version"), usage=usage, + success_direction=success_direction, ) else: payload_model = DeterministicScorerEvaluationEventPayload( diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index b5d0e647..057a37e9 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -75,6 +75,7 @@ class ResolvedJudge: config: dict[str, Any] variation_key: str = "" version: int | None = None + is_inverted: bool | None = None @dataclass diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 38104dc5..4d5f909a 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -1153,6 +1153,103 @@ async def handler( assert judge_event["variationKey"] == "default" assert judge_event["version"] == 12 assert len(judge_event["eventId"]) == 64 + # The fake resolved config never set isInverted (e.g. Gonfalon hasn't deployed the + # flag-payload change yet); direction is unresolved, so the key must be omitted + # rather than sent as successDirection=None. + assert "successDirection" not in judge_event + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("is_inverted", "expected_direction"), + [ + (True, "lower_is_better"), + (False, "upper_is_better"), + ], +) +async def test_run_with_ld_judge_emits_success_direction_from_is_inverted( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, + is_inverted: bool, + expected_direction: str, +) -> None: + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [ + { + "rowIndex": 42, + "input": "Question {{id}}", + "expectedOutput": "Answer {{id}}", + "variables": {"id": "A"}, + } + ], + total=1, + ), + ), + response(201, {"id": "evaluation-id", "name": "support-qa", "version": 3}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + {"statusCounts": {"total": 1, "passed": 1, "error": 0, "pending": 0}}, + ), + ] + ) + + async def fake_extract_variation( + key: str, context: dict[str, Any] + ) -> dict[str, Any]: + return { + "config": { + "provider": {"name": "OpenAI"}, + "model": {"name": "gpt-4o"}, + "instructions": "Judge {{response_to_evaluate}} against {{expected_output}}", + "isInverted": is_inverted, + }, + "meta": {"variationKey": "default", "version": 12}, + } + + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.runner.extract_variation", + fake_extract_variation, + ) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + async def handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + if "Judge" in config.get("instructions", ""): + return { + "output": '{"score": 0.86, "reasoning": "matches policy"}', + "usage": {"input_tokens": 640, "output_tokens": 48}, + } + return { + "output": "generated", + "usage": {"input_tokens": 10, "output_tokens": 4}, + } + + result = await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + ) + + assert result.passed is True + events = [call.args[2] for call in stub_sdk_client.track.call_args_list] + judge_event = next(event for event in events if event.get("kind") == "judge") + assert judge_event["successDirection"] == expected_direction @pytest.mark.asyncio From b26a3074aae06f47c3bb49c7a73932cf7f4b5f54 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Wed, 9 Sep 2026 13:48:17 -0700 Subject: [PATCH 07/16] Compute judge verdict client-side instead of emitting successDirection Course change: rather than reporting a raw score/threshold/direction for ai-evaluator to compute a verdict from server-side, the SDK now computes pass/fail itself using the judge's resolved isInverted, and sends that verdict directly. A missing threshold still means "no verdict" (score-only reporting), and an unresolved direction defaults to upper-is-better, matching Gonfalon's own default for a judge with no stored isInverted. verdict lives on the shared EvaluationEventPayload base class rather than just the judge payload, since deterministic scorers will need the same field once they compute their own verdicts. Co-Authored-By: Claude Sonnet 5 --- .../evaluations/events.py | 21 ++++++---- .../evaluations/runner.py | 22 ++++++---- packages/client/tests/test_evaluations_run.py | 42 +++++++++++-------- 3 files changed, 52 insertions(+), 33 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/events.py b/packages/client/src/launchdarkly_ai_server/evaluations/events.py index 3ac1ce9b..cf01ec37 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/events.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/events.py @@ -15,6 +15,18 @@ class EvaluationEventKind(StrEnum): SCORER = "scorer" +class EvaluationVerdict(StrEnum): + """Pass/fail outcome for one criterion result on one row. + + Computed by the SDK, not the server: for a judge, the SDK compares score + against threshold using the judge's own direction (isInverted) before this + verdict is ever put on the wire. + """ + + PASS = "pass" + FAIL = "fail" + + @dataclass(frozen=True) class TokenUsage: """Token usage reported by an LD Judge provider call.""" @@ -51,6 +63,7 @@ class EvaluationEventPayload: latency_ms: int evaluation_version: int | None = None score: float | None = None + verdict: EvaluationVerdict | None = None reason: str | None = None error: dict[str, Any] | None = None error_message: str | None = None @@ -75,6 +88,7 @@ def to_track_payload(self) -> dict[str, Any]: "evaluatedAt": self.evaluated_at, "latencyMs": self.latency_ms, "score": self.score, + "verdict": self.verdict.value if self.verdict is not None else None, "reason": self.reason, "error": self.error, "errorMessage": self.error_message, @@ -91,11 +105,6 @@ class LDJudgeEvaluationEventPayload(EvaluationEventPayload): variation_key: str version: int | None = None usage: TokenUsage | None = None - # "lower_is_better" or "upper_is_better"; None when direction is unresolved (e.g. - # Gonfalon hasn't deployed the flag-payload change yet, or a custom judge with no - # direction set). Direction is authoritative from LaunchDarkly, never trusted from - # the client for verdict computation. - success_direction: str | None = None def to_track_payload(self) -> dict[str, Any]: payload = super().to_track_payload() @@ -105,8 +114,6 @@ def to_track_payload(self) -> dict[str, Any]: payload["version"] = self.version if self.usage is not None: payload["usage"] = self.usage.to_wire() - if self.success_direction is not None: - payload["successDirection"] = self.success_direction return payload diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 44c4dd2f..a93e61db 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -29,6 +29,7 @@ DeterministicScorerEvaluationEventPayload, EvaluationEventPayload, EvaluationStatus, + EvaluationVerdict, LDJudgeEvaluationEventPayload, TokenUsage, ) @@ -683,7 +684,6 @@ async def _run_ld_judge_for_result( "started_at": started.isoformat().replace("+00:00", "Z"), "variation_key": resolved.variation_key, "version": resolved.version, - "is_inverted": resolved.is_inverted, } if row.get("status") != "COMPLETE": return self._criterion_error_result( @@ -733,11 +733,23 @@ async def _run_ld_judge_for_result( "invalid_score", f"judge score must be a number between 0 and 1, got {raw_score!r}", ) + verdict: EvaluationVerdict | None = None + if judge.threshold is not None: + # An unresolved direction (e.g. Gonfalon hasn't deployed the flag-payload + # change yet, or a custom judge with none set) defaults to upper-is-better, + # matching Gonfalon's own default for a judge with no stored isInverted. + passed = ( + score <= judge.threshold + if resolved.is_inverted + else score >= judge.threshold + ) + verdict = EvaluationVerdict.PASS if passed else EvaluationVerdict.FAIL completed = datetime.now(UTC) event = { **base, "status": "COMPLETE", "score": score, + "verdict": verdict, "reason": reason, "evaluated_at": completed.isoformat().replace("+00:00", "Z"), "latency_ms": round((time.perf_counter() - started_clock) * 1000), @@ -852,25 +864,19 @@ def _emit_evaluation_events( "evaluated_at": result["evaluated_at"], "latency_ms": result["latency_ms"], "score": result.get("score"), + "verdict": result.get("verdict"), "reason": result.get("reason"), "error": error, "error_message": error_message, } payload_model: EvaluationEventPayload if result["kind"] == "judge": - is_inverted = result.get("is_inverted") - success_direction: str | None = None - if is_inverted is True: - success_direction = "lower_is_better" - elif is_inverted is False: - success_direction = "upper_is_better" payload_model = LDJudgeEvaluationEventPayload( **common_payload, judge_key=result["judge_key"], variation_key=result["variation_key"], version=result.get("version"), usage=usage, - success_direction=success_direction, ) else: payload_model = DeterministicScorerEvaluationEventPayload( diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 4d5f909a..f1181b69 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -1153,25 +1153,29 @@ async def handler( assert judge_event["variationKey"] == "default" assert judge_event["version"] == 12 assert len(judge_event["eventId"]) == 64 - # The fake resolved config never set isInverted (e.g. Gonfalon hasn't deployed the - # flag-payload change yet); direction is unresolved, so the key must be omitted - # rather than sent as successDirection=None. - assert "successDirection" not in judge_event + # No threshold was set on the Judge, so there's nothing to compare the score + # against -- verdict must be omitted rather than sent as verdict=None. + assert "verdict" not in judge_event @pytest.mark.asyncio @pytest.mark.parametrize( - ("is_inverted", "expected_direction"), + ("is_inverted", "threshold", "expected_verdict"), [ - (True, "lower_is_better"), - (False, "upper_is_better"), + # score is fixed at 0.86 in the handler below. + (False, 0.8, "pass"), # upper-is-better, 0.86 >= 0.8 + (False, 0.9, "fail"), # upper-is-better, 0.86 < 0.9 + (True, 0.9, "pass"), # lower-is-better, 0.86 <= 0.9 + (True, 0.5, "fail"), # lower-is-better, 0.86 > 0.5 + (None, 0.8, "pass"), # unresolved direction defaults to upper-is-better ], ) -async def test_run_with_ld_judge_emits_success_direction_from_is_inverted( +async def test_run_with_ld_judge_computes_verdict_from_score_threshold_and_direction( monkeypatch: pytest.MonkeyPatch, stub_sdk_client: MagicMock, - is_inverted: bool, - expected_direction: str, + is_inverted: bool | None, + threshold: float, + expected_verdict: str, ) -> None: transport = SequencedTransport( [ @@ -1205,13 +1209,15 @@ async def test_run_with_ld_judge_emits_success_direction_from_is_inverted( async def fake_extract_variation( key: str, context: dict[str, Any] ) -> dict[str, Any]: + config: dict[str, Any] = { + "provider": {"name": "OpenAI"}, + "model": {"name": "gpt-4o"}, + "instructions": "Judge {{response_to_evaluate}} against {{expected_output}}", + } + if is_inverted is not None: + config["isInverted"] = is_inverted return { - "config": { - "provider": {"name": "OpenAI"}, - "model": {"name": "gpt-4o"}, - "instructions": "Judge {{response_to_evaluate}} against {{expected_output}}", - "isInverted": is_inverted, - }, + "config": config, "meta": {"variationKey": "default", "version": 12}, } @@ -1243,13 +1249,13 @@ async def handler( dataset="golden", handler=handler, generation={"provider": "OpenAI", "model": "gpt-4o"}, - criteria=[Judge(key="$ld:ai:judge:accuracy")], + criteria=[Judge(key="$ld:ai:judge:accuracy", threshold=threshold)], ) assert result.passed is True events = [call.args[2] for call in stub_sdk_client.track.call_args_list] judge_event = next(event for event in events if event.get("kind") == "judge") - assert judge_event["successDirection"] == expected_direction + assert judge_event["verdict"] == expected_verdict @pytest.mark.asyncio From c49de2a2577cee9399a47d307630e4e32727bdc8 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Wed, 9 Sep 2026 20:57:08 -0700 Subject: [PATCH 08/16] Report criterion scores and let LaunchDarkly rule on them Reverses the client-side verdict from #78 and finishes the wire contract around it. Why the course change back: verdict policy will keep moving -- run-level pass rates are already specified and unbuilt, and warn bands and per-criterion policy are the obvious next asks. Anything the SDK computes is frozen at each customer's installed version and cannot be re-derived, because only the answer was stored. Server-side it is one implementation that applies to every SDK version and to runs already recorded. The reason #78 gave for moving it into the SDK -- that ai-evaluator has no way to learn a judge's direction -- is answered by hydrating successDirection onto the criterion through the Gonfalon proxy instead, the same way tool versions are already pinned there. - runner/events/types: judge results carry score and reason, never a verdict; ResolvedJudge drops is_inverted and _resolve_judges no longer reads isInverted off the served config. - Event key is now $ld:ai:offline-evals:criterion, matching the worker that polls for it; payload classes renamed to match. The previous ":evaluation" name was never read by anything, so nothing was ingested. - criteria: the create body carries kind and judgeKey, without which ai-evaluator defaults the criterion to deepeval and either rejects a judge key outright or silently registers "toxicity" as a server-judged metric and dispatches it to the judging worker. - Scorer declares success_direction (default higher_is_better). A scorer has no AI Config for the proxy to read a direction from, so the caller is the only source, and ai-evaluator needs one to rule at all. - Judge.threshold defaults to 0.5. A criterion with no threshold leaves nothing to compare a score against, so it would be stored and never ruled on. - The gate now counts failed_rows. It was omissible while runs were generation-only -- a row either generated or errored, and nothing produced a fail -- and with criteria it is the normal way a run fails, so a run where every row failed its judge was exiting 0. This reverses test_generation_failed_rows_do_not_fail_the_result, which asserted the old behavior without stating a rationale. - New test pins that judges resolve once per run, not per row: extract_variation reads flag delivery, so per-row resolution would let a mid-run edit change rubric, model, and provider between rows. Co-Authored-By: Claude Opus 5 --- .../evaluations/__init__.py | 3 +- .../evaluations/criteria.py | 34 +++- .../evaluations/events.py | 32 +-- .../evaluations/module.py | 11 +- .../evaluations/runner.py | 47 ++--- .../evaluations/types.py | 1 - packages/client/tests/test_evaluations_run.py | 183 ++++++++++++++++-- 7 files changed, 235 insertions(+), 76 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py index b110a550..bd3eb0a4 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py @@ -9,7 +9,7 @@ Transport, urllib_transport, ) -from .criteria import Criterion, Judge, Scorer +from .criteria import Criterion, Judge, Scorer, SuccessDirection from .module import EvaluationsModule, init_evaluations from .types import DatasetRow, EvalRunResult, GenerationConfig, RunSummary, Usage @@ -27,6 +27,7 @@ "LDApiError", "RunSummary", "Scorer", + "SuccessDirection", "Transport", "Usage", "init_evaluations", diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py index f77d6415..94acd8ee 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py @@ -2,12 +2,20 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass -from typing import Any +from typing import Any, Literal from .types import DatasetRow type ScorerFn = Callable[[DatasetRow, Any], float | bool | Awaitable[float | bool]] +type SuccessDirection = Literal["higher_is_better", "lower_is_better"] + +#: The threshold a judge is held to when its reference sets none. A criterion +#: with no threshold gives LaunchDarkly nothing to compare a score against, so +#: it would be recorded and never ruled on; defaulting is what keeps a judge +#: usable as a gate without every caller restating the obvious. +DEFAULT_JUDGE_THRESHOLD = 0.5 + @dataclass(frozen=True) class Judge: @@ -16,10 +24,15 @@ class Judge: The SDK does not create or provide built-in judges. Pass the key of a judge that exists in LaunchDarkly. Resolution uses LaunchDarkly flag delivery for the currently served variation. + + A judge's success direction is not set here. It lives on the judge's AI + Config as ``isInverted`` and is injected onto the criterion by LaunchDarkly + when the evaluation is created, so the one input a verdict is ruled on stays + server-attested even though the score beside it is client-reported. """ key: str - threshold: float | None = None + threshold: float | None = DEFAULT_JUDGE_THRESHOLD pass_rate_threshold: float | None = None ground_truth_context: str | None = None @@ -41,7 +54,12 @@ def to_criteria_wire(self) -> dict[str, Any]: pass_rate_threshold=self.pass_rate_threshold, ground_truth_context=self.ground_truth_context, ) - return {"criterionType": self.criterion_type, "options": options} + return { + "criterionType": self.criterion_type, + "kind": "judge", + "judgeKey": self.key, + "options": options, + } @dataclass(frozen=True) @@ -57,12 +75,20 @@ class Scorer: ``threshold`` defaults to 1.0: a row passes only on a perfect score, which matches the common case of boolean scorers. Pass a lower threshold for graded numeric scorers. + + ``success_direction`` says which way the score points, and defaults to + higher-is-better. Unlike a judge, a scorer has no LaunchDarkly-side config + to read a direction from, so the declaration here is the only source -- + LaunchDarkly derives each row's verdict by comparing score to threshold in + this direction. Set ``"lower_is_better"`` for a scorer that counts + something unwanted, e.g. a regex hit count or an edit distance. """ name: str fn: ScorerFn threshold: float | None = 1.0 pass_rate_threshold: float | None = None + success_direction: SuccessDirection = "higher_is_better" def __post_init__(self) -> None: if not isinstance(self.name, str) or not self.name.strip(): @@ -81,6 +107,8 @@ def criterion_type(self) -> str: def to_criteria_wire(self) -> dict[str, Any]: return { "criterionType": self.criterion_type, + "kind": "scorer", + "successDirection": self.success_direction, "options": _criteria_options( threshold=self.threshold, pass_rate_threshold=self.pass_rate_threshold, diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/events.py b/packages/client/src/launchdarkly_ai_server/evaluations/events.py index cf01ec37..4098e61b 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/events.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/events.py @@ -5,28 +5,16 @@ from typing import Any -class EvaluationStatus(StrEnum): +class CriterionStatus(StrEnum): COMPLETE = "COMPLETE" ERROR = "ERROR" -class EvaluationEventKind(StrEnum): +class CriterionEventKind(StrEnum): JUDGE = "judge" SCORER = "scorer" -class EvaluationVerdict(StrEnum): - """Pass/fail outcome for one criterion result on one row. - - Computed by the SDK, not the server: for a judge, the SDK compares score - against threshold using the judge's own direction (isInverted) before this - verdict is ever put on the wire. - """ - - PASS = "pass" - FAIL = "fail" - - @dataclass(frozen=True) class TokenUsage: """Token usage reported by an LD Judge provider call.""" @@ -42,7 +30,7 @@ def to_wire(self) -> dict[str, int]: @dataclass(frozen=True, kw_only=True) -class EvaluationEventPayload: +class CriterionEventPayload: """Common fields emitted for every SDK-run evaluation criterion result.""" project_key: str @@ -52,18 +40,17 @@ class EvaluationEventPayload: dataset_id: str row_index: int criterion_type: str - kind: EvaluationEventKind + kind: CriterionEventKind event_id: str emitted_at: str evaluation_key: str dataset_key: str - status: EvaluationStatus + status: CriterionStatus started_at: str evaluated_at: str latency_ms: int evaluation_version: int | None = None score: float | None = None - verdict: EvaluationVerdict | None = None reason: str | None = None error: dict[str, Any] | None = None error_message: str | None = None @@ -88,7 +75,6 @@ def to_track_payload(self) -> dict[str, Any]: "evaluatedAt": self.evaluated_at, "latencyMs": self.latency_ms, "score": self.score, - "verdict": self.verdict.value if self.verdict is not None else None, "reason": self.reason, "error": self.error, "errorMessage": self.error_message, @@ -97,10 +83,10 @@ def to_track_payload(self) -> dict[str, Any]: @dataclass(frozen=True, kw_only=True) -class LDJudgeEvaluationEventPayload(EvaluationEventPayload): +class LDJudgeCriterionEventPayload(CriterionEventPayload): """Payload for one LaunchDarkly AI Judge result on one dataset row.""" - kind: EvaluationEventKind = EvaluationEventKind.JUDGE + kind: CriterionEventKind = CriterionEventKind.JUDGE judge_key: str variation_key: str version: int | None = None @@ -118,7 +104,7 @@ def to_track_payload(self) -> dict[str, Any]: @dataclass(frozen=True, kw_only=True) -class DeterministicScorerEvaluationEventPayload(EvaluationEventPayload): +class DeterministicScorerCriterionEventPayload(CriterionEventPayload): """Payload for one local deterministic scorer result on one dataset row.""" - kind: EvaluationEventKind = EvaluationEventKind.SCORER + kind: CriterionEventKind = CriterionEventKind.SCORER diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 57a77523..143036c1 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -215,7 +215,16 @@ async def run( f"{_segment(evaluation.id)}/runs/{_segment(evaluation_run.id)}" ) return EvalRunResult( - passed=(summary.error_rows == 0 and summary.pending_rows == 0), + # failed_rows counts rows whose criteria were scored and did not + # meet their threshold, so a gate that ignores it exits 0 on a run + # where every row failed its judge. It was omissible while runs were + # generation-only -- a row either generated or errored, and nothing + # produced a fail -- and stops being so the moment criteria exist. + passed=( + summary.error_rows == 0 + and summary.failed_rows == 0 + and summary.pending_rows == 0 + ), url=url, run_id=evaluation_run.id, summary=summary, diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index a93e61db..73c18a37 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -26,11 +26,10 @@ from .api import EvaluationsError, LDApiClient, LDApiError from .criteria import Criterion, Judge, Scorer from .events import ( - DeterministicScorerEvaluationEventPayload, - EvaluationEventPayload, - EvaluationStatus, - EvaluationVerdict, - LDJudgeEvaluationEventPayload, + CriterionEventPayload, + CriterionStatus, + DeterministicScorerCriterionEventPayload, + LDJudgeCriterionEventPayload, TokenUsage, ) from .types import ( @@ -48,7 +47,7 @@ DATASET_PAGE_SIZE = 200 GENERATION_EVENT_NAME = "$ld:ai:offline-evals:generation" -EVALUATION_EVENT_NAME = "$ld:ai:offline-evals:evaluation" +CRITERION_EVENT_NAME = "$ld:ai:offline-evals:criterion" EvalHandler = Callable[..., Awaitable[dict[str, Any]]] ToolImplementation = Callable[..., Any] | NativeTool @@ -184,11 +183,6 @@ async def _resolve_judges( version=int(meta["version"]) if isinstance(meta.get("version"), int) else None, - # Absent (e.g. Gonfalon hasn't deployed the flag-payload change yet, or a - # custom judge with no direction set) resolves to None, not a raised error. - is_inverted=config.get("isInverted") - if isinstance(config.get("isInverted"), bool) - else None, ) return resolved @@ -733,23 +727,17 @@ async def _run_ld_judge_for_result( "invalid_score", f"judge score must be a number between 0 and 1, got {raw_score!r}", ) - verdict: EvaluationVerdict | None = None - if judge.threshold is not None: - # An unresolved direction (e.g. Gonfalon hasn't deployed the flag-payload - # change yet, or a custom judge with none set) defaults to upper-is-better, - # matching Gonfalon's own default for a judge with no stored isInverted. - passed = ( - score <= judge.threshold - if resolved.is_inverted - else score >= judge.threshold - ) - verdict = EvaluationVerdict.PASS if passed else EvaluationVerdict.FAIL completed = datetime.now(UTC) + # No verdict: the SDK reports the score and LaunchDarkly rules on it. The + # criterion carries the threshold and the judge's success direction, and + # ai-evaluator compares them at ingest -- so pass/fail policy is one + # server-side implementation that applies to every SDK version and to runs + # already recorded, rather than one frozen into each release of each + # language's SDK. event = { **base, "status": "COMPLETE", "score": score, - "verdict": verdict, "reason": reason, "evaluated_at": completed.isoformat().replace("+00:00", "Z"), "latency_ms": round((time.perf_counter() - started_clock) * 1000), @@ -859,19 +847,18 @@ def _emit_evaluation_events( "evaluation_key": evaluation.key, "evaluation_version": evaluation.version, "dataset_key": dataset.key, - "status": EvaluationStatus(result["status"]), + "status": CriterionStatus(result["status"]), "started_at": result["started_at"], "evaluated_at": result["evaluated_at"], "latency_ms": result["latency_ms"], "score": result.get("score"), - "verdict": result.get("verdict"), "reason": result.get("reason"), "error": error, "error_message": error_message, } - payload_model: EvaluationEventPayload + payload_model: CriterionEventPayload if result["kind"] == "judge": - payload_model = LDJudgeEvaluationEventPayload( + payload_model = LDJudgeCriterionEventPayload( **common_payload, judge_key=result["judge_key"], variation_key=result["variation_key"], @@ -879,11 +866,11 @@ def _emit_evaluation_events( usage=usage, ) else: - payload_model = DeterministicScorerEvaluationEventPayload( + payload_model = DeterministicScorerCriterionEventPayload( **common_payload ) client.track( - EVALUATION_EVENT_NAME, context, payload_model.to_track_payload(), 1 + CRITERION_EVENT_NAME, context, payload_model.to_track_payload(), 1 ) except Exception: logger.exception( @@ -894,7 +881,7 @@ def _emit_evaluation_events( continue logger.info( "%s emittedAt=%s eventId=%s", - EVALUATION_EVENT_NAME, + CRITERION_EVENT_NAME, emitted_at, event_id, ) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index 057a37e9..b5d0e647 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -75,7 +75,6 @@ class ResolvedJudge: config: dict[str, Any] variation_key: str = "" version: int | None = None - is_inverted: bool | None = None @dataclass diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index f1181b69..05ee31a2 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -812,7 +812,15 @@ async def handler(*args: object) -> dict[str, Any]: @pytest.mark.asyncio -async def test_generation_failed_rows_do_not_fail_the_result() -> None: +async def test_failed_rows_fail_the_result() -> None: + """A row the server scored and marked failed must fail the gate. + + This reverses the previous assertion, which was written when runs were + generation-only -- a row then either generated or errored, and nothing + produced a "failed", so excluding failed_rows was unobservable. With + criteria it is the normal way a run fails, and a gate that ignores it exits + 0 on a run where every row failed its judge. + """ transport = SequencedTransport( [ response(200, {"id": "dataset-id", "name": "golden"}), @@ -856,7 +864,7 @@ async def handler(*args: object) -> dict[str, Any]: ) assert result.summary.failed_rows == 1 - assert result.passed is True + assert result.passed is False @pytest.mark.asyncio @@ -1135,10 +1143,16 @@ async def handler( ) assert result.passed is True + # kind and judgeKey are what let ai-evaluator store this as a judge rather + # than default it to a deepeval metric and reject the key. successDirection + # is deliberately absent: LaunchDarkly injects it from the judge's AI Config + # on the way through, so the SDK must not assert a direction of its own. assert transport.requests[2]["body"]["criteria"] == [ { "criterionType": "$ld:ai:judge:accuracy", - "options": {}, + "kind": "judge", + "judgeKey": "$ld:ai:judge:accuracy", + "options": {"threshold": 0.5}, } ] assert transport.requests[3]["body"] == {"source": "api", "datasetId": "dataset-id"} @@ -1153,30 +1167,37 @@ async def handler( assert judge_event["variationKey"] == "default" assert judge_event["version"] == 12 assert len(judge_event["eventId"]) == 64 - # No threshold was set on the Judge, so there's nothing to compare the score - # against -- verdict must be omitted rather than sent as verdict=None. + # The SDK reports the score and never rules on it: ai-evaluator derives the + # verdict at ingest from the criterion's stored threshold and direction. assert "verdict" not in judge_event @pytest.mark.asyncio @pytest.mark.parametrize( - ("is_inverted", "threshold", "expected_verdict"), + ("is_inverted", "threshold"), [ - # score is fixed at 0.86 in the handler below. - (False, 0.8, "pass"), # upper-is-better, 0.86 >= 0.8 - (False, 0.9, "fail"), # upper-is-better, 0.86 < 0.9 - (True, 0.9, "pass"), # lower-is-better, 0.86 <= 0.9 - (True, 0.5, "fail"), # lower-is-better, 0.86 > 0.5 - (None, 0.8, "pass"), # unresolved direction defaults to upper-is-better + # The score is fixed at 0.86 below. Under every direction and on both + # sides of the threshold, the SDK reports the same thing: a score. + (False, 0.8), + (False, 0.9), + (True, 0.9), + (True, 0.5), + (None, 0.8), ], ) -async def test_run_with_ld_judge_computes_verdict_from_score_threshold_and_direction( +async def test_run_with_ld_judge_never_sends_a_verdict( monkeypatch: pytest.MonkeyPatch, stub_sdk_client: MagicMock, is_inverted: bool | None, threshold: float, - expected_verdict: str, ) -> None: + """Pass/fail is ai-evaluator's ruling, not the SDK's. + + Parametrized over isInverted -- including the served-payload value -- to + pin that the SDK does not compare even when it could: verdict policy has to + be able to change server-side and apply to runs already recorded, which it + cannot if each SDK release freezes its own comparison. + """ transport = SequencedTransport( [ response(200, {"id": "dataset-id", "name": "golden"}), @@ -1255,7 +1276,128 @@ async def handler( assert result.passed is True events = [call.args[2] for call in stub_sdk_client.track.call_args_list] judge_event = next(event for event in events if event.get("kind") == "judge") - assert judge_event["verdict"] == expected_verdict + assert judge_event["score"] == 0.86 + assert "verdict" not in judge_event + assert "successDirection" not in judge_event + + +@pytest.mark.asyncio +async def test_judges_resolve_once_per_run_not_once_per_row( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, +) -> None: + """One resolution for the whole run, however many rows it has. + + extract_variation reads flag delivery, an in-memory store that updates + within seconds of a UI edit, so resolving per row would let an edit + mid-run change the rubric text, judge model, and provider between one row + and the next -- rows in a single run scored against different judges. The + online path does resolve per invocation (judges.build_judge_tasks), so + routing the offline runner through it for convenience is a live way to + reintroduce this. + """ + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [ + {"rowIndex": 0, "input": "one", "variables": {}}, + {"rowIndex": 1, "input": "two", "variables": {}}, + {"rowIndex": 2, "input": "three", "variables": {}}, + ], + total=3, + ), + ), + response(201, {"id": "evaluation-id", "name": "support-qa"}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + {"statusCounts": {"total": 3, "passed": 3, "error": 0, "pending": 0}}, + ), + ] + ) + + resolutions: list[str] = [] + + async def fake_extract_variation( + key: str, context: dict[str, Any] + ) -> dict[str, Any]: + resolutions.append(key) + return { + "config": { + "provider": {"name": "OpenAI"}, + "model": {"name": "gpt-4o"}, + "instructions": "Judge {{response_to_evaluate}}", + }, + "meta": {"variationKey": "default", "version": 12}, + } + + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.runner.extract_variation", + fake_extract_variation, + ) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + async def handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + if "Judge" in config.get("instructions", ""): + return {"output": '{"score": 0.9, "reasoning": "fine"}'} + return {"output": "generated"} + + await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + ) + + assert resolutions == ["$ld:ai:judge:accuracy"] + + +def test_scorer_lower_is_better_reaches_the_criteria_wire() -> None: + """A scorer counting something unwanted -- regex hits, edit distance -- + inverts, and only the SDK knows: there is no AI Config for the proxy to + read a scorer's direction off, so what the caller declares is the sole + source ai-evaluator derives its verdict from.""" + + def count_violations(row: DatasetRow, output: Any) -> float: + return 0.0 + + scorer = Scorer( + name="policy-violations", + fn=count_violations, + threshold=0.0, + success_direction="lower_is_better", + ) + + assert scorer.to_criteria_wire() == { + "criterionType": "policy-violations", + "kind": "scorer", + "successDirection": "lower_is_better", + "options": {"threshold": 0.0}, + } + + +def test_judge_threshold_defaults_so_a_criterion_is_always_rulable() -> None: + """A judge with no threshold gives LaunchDarkly nothing to compare against, + so the criterion would be stored and never ruled on.""" + assert Judge(key="$ld:ai:judge:accuracy").to_criteria_wire() == { + "criterionType": "$ld:ai:judge:accuracy", + "kind": "judge", + "judgeKey": "$ld:ai:judge:accuracy", + "options": {"threshold": 0.5}, + } @pytest.mark.asyncio @@ -1350,8 +1492,15 @@ def check_refund(row: DatasetRow, output: Any) -> bool: ) assert result.passed is True + # A scorer has no LaunchDarkly-side config, so unlike a judge it declares + # its own direction and the proxy leaves it alone. assert transport.requests[2]["body"]["criteria"] == [ - {"criterionType": "refund-exists", "options": {"threshold": 1.0}} + { + "criterionType": "refund-exists", + "kind": "scorer", + "successDirection": "higher_is_better", + "options": {"threshold": 1.0}, + } ] assert transport.requests[3]["body"] == {"source": "api", "datasetId": "dataset-id"} events = [call.args[2] for call in stub_sdk_client.track.call_args_list] @@ -1641,7 +1790,7 @@ async def test_failed_evaluation_event_tracking_skips_event_but_completes_run( accuracy_judge_variation(monkeypatch) def track(event_name: str, *args: Any) -> None: - if event_name == "$ld:ai:offline-evals:evaluation": + if event_name == "$ld:ai:offline-evals:criterion": raise RuntimeError("event pipeline unavailable") stub_sdk_client.track = MagicMock(side_effect=track) From 4126d7b8bc33adf848663ea1789ff7a36abff37b Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Thu, 10 Sep 2026 16:49:55 -0700 Subject: [PATCH 09/16] fix(evaluations): dedup criteria case-insensitively, matching the API The API's own duplicate-criterion check lowercases criterionType before comparing (the worker's retry gate does the same), so two criteria differing only by case still collide server-side. The SDK's local fail-fast check compared case-sensitively, so a case-variant duplicate would pass the SDK check, pay for judge resolution and a dataset fetch, and only then be rejected by the API. Co-Authored-By: Claude Sonnet 5 --- .../evaluations/module.py | 13 ++++++--- packages/client/tests/test_evaluations_run.py | 28 +++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 143036c1..698ff7bd 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -291,20 +291,25 @@ def _validate_criteria(criteria: list[Criterion]) -> None: """Reject duplicate criterion identities before any records are created. A judge key and a scorer name that collide would share a criterionType, - and with it the deterministic event identity of their results. + and with it the deterministic event identity of their results. Case- + insensitive, matching the API's own dedup: the worker's retry gate + lowercases criterion types, so two criteria differing only by case + would still collide there even though they look distinct here. """ seen: set[str] = set() duplicates: list[str] = [] for criterion in criteria: criterion_type = criterion.criterion_type - if criterion_type in seen and criterion_type not in duplicates: + normalized = criterion_type.lower() + if normalized in seen and criterion_type not in duplicates: duplicates.append(criterion_type) - seen.add(criterion_type) + seen.add(normalized) if duplicates: raise EvaluationsError( "Duplicate evaluation criteria: " + ", ".join(repr(name) for name in duplicates) - + ". Judge keys and scorer names must be unique within a run." + + ". Judge keys and scorer names must be unique within a run " + "(case-insensitive)." ) @staticmethod diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 05ee31a2..9efba1f6 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -1744,6 +1744,34 @@ async def handler(*args: object) -> dict[str, Any]: assert transport.requests == [] +@pytest.mark.asyncio +async def test_duplicate_criteria_rejected_case_insensitively() -> None: + """Matches the API's own dedup, which lowercases criterionType before + comparing: the worker's retry gate does the same, so criteria differing + only by case would still collide there even though they'd look distinct + to a case-sensitive check.""" + transport = SequencedTransport([]) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + with pytest.raises(EvaluationsError, match="Duplicate evaluation criteria"): + await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[ + Judge(key="Accuracy"), + Scorer(name="accuracy", fn=lambda row, output: True), + ], + ) + + assert transport.requests == [] + + @pytest.mark.asyncio async def test_errored_generation_row_emits_generation_incomplete_criterion_event( monkeypatch: pytest.MonkeyPatch, From 16b5d22feee9be1b3c7faeb02a5187809ff494ad Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Thu, 10 Sep 2026 20:55:57 -0700 Subject: [PATCH 10/16] TEMP(evaluations): hardcode judge successDirection for proxy testing Dylan has a gonfalon-judge-proxy PR adding the successDirection injection middleware but its enabling flag is off, so ai-evaluator's create-time validation 400s on a judge criterion with no successDirection. Hardcoding higher_is_better on the wire unblocks testing that proxy PR end to end. DO NOT MERGE THIS COMMIT: Judge.to_criteria_wire deliberately omits successDirection so the proxy can inject a server-attested value from the judge's AI Config; revert this alongside its two updated test assertions once the proxy lands. Co-Authored-By: Claude Sonnet 5 --- .../src/launchdarkly_ai_server/evaluations/criteria.py | 5 +++++ packages/client/tests/test_evaluations_run.py | 10 ++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py index 94acd8ee..8e3c2359 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py @@ -58,6 +58,11 @@ def to_criteria_wire(self) -> dict[str, Any]: "criterionType": self.criterion_type, "kind": "judge", "judgeKey": self.key, + # TEMPORARY: hardcoded to unblock testing the gonfalon-judge-proxy + # successDirection-injection middleware while its enabling flag is + # off. The proxy is expected to overwrite this; remove once that + # lands and ai-evaluator no longer needs it on the raw request. + "successDirection": "higher_is_better", "options": options, } diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 9efba1f6..eb0ca47a 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -1145,13 +1145,16 @@ async def handler( assert result.passed is True # kind and judgeKey are what let ai-evaluator store this as a judge rather # than default it to a deepeval metric and reject the key. successDirection - # is deliberately absent: LaunchDarkly injects it from the judge's AI Config - # on the way through, so the SDK must not assert a direction of its own. + # is TEMPORARILY hardcoded on the wire (see Judge.to_criteria_wire) to + # unblock testing the gonfalon-judge-proxy injection middleware while its + # enabling flag is off; remove this key here alongside that hardcode once + # the proxy is live and LaunchDarkly injects the direction itself. assert transport.requests[2]["body"]["criteria"] == [ { "criterionType": "$ld:ai:judge:accuracy", "kind": "judge", "judgeKey": "$ld:ai:judge:accuracy", + "successDirection": "higher_is_better", "options": {"threshold": 0.5}, } ] @@ -1392,10 +1395,13 @@ def count_violations(row: DatasetRow, output: Any) -> float: def test_judge_threshold_defaults_so_a_criterion_is_always_rulable() -> None: """A judge with no threshold gives LaunchDarkly nothing to compare against, so the criterion would be stored and never ruled on.""" + # successDirection is TEMPORARILY hardcoded on the wire; see the matching + # comment on Judge.to_criteria_wire. assert Judge(key="$ld:ai:judge:accuracy").to_criteria_wire() == { "criterionType": "$ld:ai:judge:accuracy", "kind": "judge", "judgeKey": "$ld:ai:judge:accuracy", + "successDirection": "higher_is_better", "options": {"threshold": 0.5}, } From 0b81f4e3c6eda8ed84888d376f8c86d9eb522d93 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Thu, 10 Sep 2026 21:18:29 -0700 Subject: [PATCH 11/16] fix(evaluations): offline judge message_history must carry FORMATTING_INSTRUCTIONS _judge_variables built message_history from just input+output, then exposed FORMATTING_INSTRUCTIONS as a separate formatting_instructions variable. But no judge -- including every one of the AI Library's default templates (accuracy, relevance, toxicity) and anything cloned from them -- references {{formatting_instructions}}; they all reference {{message_history}}, because judges.run_judges (the online path) already bakes FORMATTING_INSTRUCTIONS into that value: message_history = "\n\n".join(filter(None, [user_input, llm_response, FORMATTING_INSTRUCTIONS])) Offline diverged from that and never asked the model for the {score, reasoning} JSON shape, so every judge response came back as free-text prose and failed to parse (invalid_judge_output) -- for any judge built the standard way, not just a misconfigured one. Match the online construction so an unmodified default-template judge scores correctly through both paths, per judge_scoring.py's own "the two paths cannot drift" contract. Co-Authored-By: Claude Sonnet 5 --- .../launchdarkly_ai_server/evaluations/runner.py | 15 +++++++++++++-- packages/client/tests/test_evaluations_run.py | 9 +++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 73c18a37..6cae4a79 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -562,14 +562,25 @@ def _judge_variables( ground_truth = parse_template(ground_truth, variables) elif expected is not None: ground_truth = str(expected) + # message_history carries FORMATTING_INSTRUCTIONS the same way the + # online path builds it (judges.run_judges), because that -- not the + # standalone formatting_instructions variable below -- is what every + # judge built from the AI Library's default templates (accuracy, + # relevance, toxicity, and any judge cloned from them) actually + # references. A judge authored before this variable existed must keep + # getting scored without edits. variables.update( { "input": row_result.get("input") or "", "response_to_evaluate": output if output is not None else "", "message_history": "\n\n".join( str(value) - for value in (row_result.get("input"), output) - if value is not None + for value in ( + row_result.get("input"), + output, + FORMATTING_INSTRUCTIONS, + ) + if value ), "expected_output": expected if expected is not None else "", "ground_truth_context": ( diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index eb0ca47a..3974231b 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -1124,6 +1124,15 @@ async def handler( assert variables["formatting_instructions"].startswith( "Your response MUST be in valid JSON" ) + # message_history must carry the formatting instructions the same + # way judges.run_judges (the online path) builds it: every judge + # built from the AI Library's default templates references + # {{message_history}}, not the standalone formatting_instructions + # variable above, to ask for the {score, reasoning} JSON shape. + assert ( + "Your response MUST be in valid JSON format" + in (variables["message_history"]) + ) return { "output": '{"score": 0.86, "reasoning": "matches policy"}', "usage": {"input_tokens": 640, "output_tokens": 48}, From 1a5e0999c3b772d377110c4fa93c2494b6a9041f Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Tue, 15 Sep 2026 10:43:41 -0700 Subject: [PATCH 12/16] fix(evaluations): route judge configs to a compatible handler Address PR #63 review feedback. A judge is an independent AI Config and may be served by a different provider or mode than the evaluation's generation config, but the offline path invoked the generation handler for every judge. An OpenAI handler handed an Anthropic judge config called OpenAI with an Anthropic model name, so affected rows recorded handler_raised instead of a score -- after the generation spend had already happened. ResolvedJudge now carries the judge's own provider and normalized mode, and each judge is paired with a handler at resolution time, before any records are created: a judge handler matching its provider and mode, an agent-mode handler for a messages-mode judge (with messages collapsed, as the online path does), or the generation handler when it covers the judge's provider. A judge no handler covers fails the run up front. Handlers for other providers are passed as run(judge_handlers=[...]); a plain callable with no provides_for still routes itself, unchanged. Also from review: - Criterion event emission failures are collected and raised together after every result is attempted. Dropping them left the backend short of a result per (row, criterion), so run() polled to its 180s timeout and hid the tracking exception. - NaN thresholds are rejected. Both range comparisons are false for NaN, so it reached the wire as a bare NaN literal the management API rejects. - run_judge no longer coerces a null judge score to 0.0. It recorded a gen_ai.evaluation of 0 -- indistinguishable from a hard fail -- where every other non-numeric judge output skips the metric. Co-Authored-By: Claude Opus 5 --- .../evaluations/criteria.py | 11 +- .../evaluations/module.py | 42 ++- .../evaluations/runner.py | 172 ++++++++++-- .../evaluations/types.py | 12 +- .../src/launchdarkly_ai_server/judges.py | 6 +- packages/client/tests/test_evaluations_run.py | 254 +++++++++++++++++- packages/client/tests/test_judges.py | 72 +++++ 7 files changed, 541 insertions(+), 28 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py index 8e3c2359..e0a1dd11 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py @@ -2,6 +2,7 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass +from math import isnan from typing import Any, Literal from .types import DatasetRow @@ -133,8 +134,14 @@ def _validate_thresholds( ("threshold", threshold), ("pass_rate_threshold", pass_rate_threshold), ): - if value is not None and (value < 0 or value > 1): - raise ValueError(f"{name} must be between 0 and 1") + if value is None: + continue + # NaN passes both range comparisons, so without an explicit check it + # reaches to_criteria_wire and is serialized as a bare ``NaN`` literal + # the management API rejects -- an evaluation that fails to be created + # rather than a threshold that fails to validate. + if isnan(value) or value < 0 or value > 1: + raise ValueError(f"{name} must be a number between 0 and 1") def _criteria_options( diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 698ff7bd..80d22ece 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -18,7 +18,13 @@ urllib_transport, ) from .criteria import Criterion, Judge -from .runner import EvalHandler, EvaluationsRunner, ToolImplementation, _segment +from .runner import ( + EvalHandler, + EvaluationsRunner, + ToolImplementation, + _provides_for, + _segment, +) from .types import EvalRunResult, GenerationConfig, RunSummary logger = logging.getLogger(__name__) @@ -95,6 +101,7 @@ async def run( generation: GenerationConfig, tools: Mapping[str, ToolImplementation] | None = None, criteria: list[Criterion] | None = None, + judge_handlers: list[EvalHandler] | None = None, concurrency: int = 10, poll_interval_seconds: float | None = None, poll_timeout_seconds: float | None = None, @@ -108,6 +115,12 @@ async def run( generated row, and one evaluation event is emitted per ``(row, criterion)`` result. + A :class:`Judge` is an independent AI Config and may be served by a + different provider or mode than ``generation``. ``handler`` runs a judge + only when it provides for that judge's provider; pass handlers for any + other providers your judges use in ``judge_handlers``. A judge no + handler covers fails the run before any records are created. + The returned pass/fail result is derived from LaunchDarkly's run summary. A CI script can exit with ``0 if result.passed else 1`` after awaiting this method. Large datasets may need a longer ``poll_timeout_seconds`` @@ -130,7 +143,9 @@ async def run( ) run_tools = dict(tools or {}) run_criteria = list(criteria or []) + run_judge_handlers = list(judge_handlers or []) self._validate_criteria(run_criteria) + self._validate_judge_handlers(run_judge_handlers) ld_judges = [ criterion for criterion in run_criteria if isinstance(criterion, Judge) ] @@ -142,7 +157,9 @@ async def run( resolved_tools = await asyncio.to_thread( self._runner._resolve_tools, project_key, run_tools ) - resolved_judges = await self._runner._resolve_judges(project_key, ld_judges) + resolved_judges = await self._runner._resolve_judges( + project_key, ld_judges, handler, run_judge_handlers + ) dataset_ref = await asyncio.to_thread( self._runner._fetch_dataset, project_key, dataset ) @@ -183,7 +200,6 @@ async def run( if run_criteria: criterion_results = await self._runner._run_criteria_for_results( results, - handler, run_tools, run_criteria, resolved_judges, @@ -312,6 +328,26 @@ def _validate_criteria(criteria: list[Criterion]) -> None: "(case-insensitive)." ) + @staticmethod + def _validate_judge_handlers(judge_handlers: list[EvalHandler]) -> None: + """Reject judge handlers that cannot be routed by provider and mode. + + A judge handler is only ever chosen by matching its ``provides_for`` + against the judge's resolved provider and mode. One without that + metadata could never be selected, so it would silently fall through to + the generation handler instead of running the judge it was passed for. + """ + for index, candidate in enumerate(judge_handlers): + if not callable(candidate): + raise EvaluationsError(f"judge_handlers[{index}] must be callable") + if _provides_for(candidate) is None: + raise EvaluationsError( + f"judge_handlers[{index}] does not declare provides_for. " + "Build judge handlers with create_handler() (or a provider " + "package's create_*_handler()) so they can be matched to a " + "judge's provider and mode." + ) + @staticmethod def _validate_run_args( *, diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 6cae4a79..75456f13 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -8,8 +8,9 @@ import time import urllib.parse from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass from datetime import UTC, datetime -from typing import Any +from typing import Any, Literal from ..judge_scoring import ( FORMATTING_INSTRUCTIONS, @@ -19,6 +20,8 @@ from ..lifecycle import extract_variation from ..types import NativeTool from ..utils import ( + collapse_messages_to_instructions, + normalize_mode, parse_template, parse_usage, to_ld_context, @@ -53,6 +56,89 @@ ToolImplementation = Callable[..., Any] | NativeTool +@dataclass(frozen=True) +class JudgeExecution: + """A resolved judge paired with the handler selected to run its config.""" + + resolved: ResolvedJudge + handler: EvalHandler + collapse_messages: bool = False + + +def _provides_for( + handler: EvalHandler, +) -> tuple[str, Literal["agent", "messages"]] | None: + provides_for = getattr(handler, "provides_for", None) + if ( + isinstance(provides_for, tuple | list) + and len(provides_for) == 2 + and isinstance(provides_for[0], str) + ): + return (provides_for[0], normalize_mode(provides_for[1])) + return None + + +def _covers_provider( + provides_for: tuple[str, Literal["agent", "messages"]], + provider: str | None, +) -> bool: + return provides_for[0] == provider or provides_for[0] == "*" + + +def _select_judge_handler( + resolved: ResolvedJudge, + handler: EvalHandler, + judge_handlers: list[EvalHandler], +) -> JudgeExecution | None: + """Pick the handler that can run this judge's config, or ``None``. + + A judge is an independent AI Config: it may resolve to a different provider + and mode than the evaluation's generation config, and a handler built for + one provider cannot execute another's config. The priority mirrors the + online path (``judges.run_judges``): + + 1. a judge handler matching the judge's provider (or a wildcard) and mode; + 2. an agent-mode judge handler for a messages-mode judge, whose messages + are collapsed into a single instructions block; + 3. the generation handler, when it covers the judge's provider. + + A handler that declares no ``provides_for`` is a plain callable doing its + own routing -- the same contract it already honours for the generation + config -- so it is treated as covering every judge. + """ + for candidate in judge_handlers: + provides_for = _provides_for(candidate) + if ( + provides_for + and _covers_provider(provides_for, resolved.provider) + and provides_for[1] == resolved.mode + ): + return JudgeExecution(resolved=resolved, handler=candidate) + if resolved.mode == "messages": + for candidate in judge_handlers: + provides_for = _provides_for(candidate) + if ( + provides_for + and _covers_provider(provides_for, resolved.provider) + and provides_for[1] == "agent" + ): + return JudgeExecution( + resolved=resolved, handler=candidate, collapse_messages=True + ) + generation_provides_for = _provides_for(handler) + if generation_provides_for is None: + return JudgeExecution(resolved=resolved, handler=handler) + if _covers_provider(generation_provides_for, resolved.provider): + return JudgeExecution( + resolved=resolved, + handler=handler, + collapse_messages=( + generation_provides_for[1] == "agent" and resolved.mode == "messages" + ), + ) + return None + + def _segment(value: str) -> str: return urllib.parse.quote(value, safe="") @@ -152,9 +238,17 @@ async def _resolve_judges( self, project_key: str, judges: list[Judge], - ) -> dict[str, ResolvedJudge]: - """Resolve LD Judge configs before any evaluation records are created.""" - resolved: dict[str, ResolvedJudge] = {} + handler: EvalHandler, + judge_handlers: list[EvalHandler] | None = None, + ) -> dict[str, JudgeExecution]: + """Resolve LD Judge configs before any evaluation records are created. + + Each judge is paired with the handler that can execute its config here, + rather than at scoring time, so a judge no handler covers fails the run + before any records exist or any generation spend happens. + """ + available_judge_handlers = list(judge_handlers or []) + resolved: dict[str, JudgeExecution] = {} # variation() rejects a context without kind and key; use the same # context shape the emitted evaluation events are attributed to. context: dict[str, Any] = {"kind": "evaluation", "key": project_key} @@ -176,14 +270,36 @@ async def _resolve_judges( raise EvaluationsError( f"LaunchDarkly judge {judge.key!r} returned an invalid AI config variation" ) - resolved[judge.key] = ResolvedJudge( + provider_value = config.get("provider") + provider = ( + provider_value.get("name") + if isinstance(provider_value, Mapping) + else None + ) + resolved_judge = ResolvedJudge( key=judge.key, config=dict(config), variation_key=str(meta.get("variationKey") or ""), version=int(meta["version"]) if isinstance(meta.get("version"), int) else None, + provider=str(provider) if isinstance(provider, str) else None, + mode=normalize_mode( + meta.get("mode") if isinstance(meta.get("mode"), str) else None + ), ) + execution = _select_judge_handler( + resolved_judge, handler, available_judge_handlers + ) + if execution is None: + raise EvaluationsError( + f"No handler can run LaunchDarkly judge {judge.key!r}: its " + f"config is served by provider {resolved_judge.provider!r} in " + f"{resolved_judge.mode!r} mode, which neither the generation " + "handler nor any judge_handlers entry provides for. Pass a " + "handler for that provider to run(judge_handlers=[...])." + ) + resolved[judge.key] = execution return resolved def _fetch_dataset(self, project_key: str, dataset_key: str) -> DatasetRef: @@ -674,11 +790,11 @@ async def _run_scorer_for_result( async def _run_ld_judge_for_result( self, row: Mapping[str, Any], - handler: EvalHandler, tool_handlers: dict[str, ToolImplementation], judge: Judge, - resolved: ResolvedJudge, + execution: JudgeExecution, ) -> dict[str, Any]: + resolved = execution.resolved started = datetime.now(UTC) started_clock = time.perf_counter() base: dict[str, Any] = { @@ -701,9 +817,17 @@ async def _run_ld_judge_for_result( # parse_template pass, so ``{{...}}`` sequences inside generated output # or dataset values are never re-expanded into the judge prompt. variables = self._judge_variables(row, judge) + # An agent-mode handler standing in for a messages-mode judge needs the + # messages folded into one instructions block, exactly as the online + # path does before handing a judge config to an agent handler. + judge_config = ( + collapse_messages_to_instructions(resolved.config) + if execution.collapse_messages + else resolved.config + ) try: - result = await handler( - dict(resolved.config), + result = await execution.handler( + dict(judge_config), row.get("output"), tool_handlers, { @@ -761,10 +885,9 @@ async def _run_ld_judge_for_result( async def _run_criteria_for_results( self, rows: list[dict[str, Any]], - handler: EvalHandler, tool_handlers: dict[str, ToolImplementation], criteria: list[Criterion], - resolved_judges: Mapping[str, ResolvedJudge], + resolved_judges: Mapping[str, JudgeExecution], concurrency: int, ) -> list[dict[str, Any]]: """Run every (row, criterion) pair, bounded by the run's concurrency.""" @@ -779,7 +902,6 @@ async def run_one( return await self._run_scorer_for_result(row, criterion) return await self._run_ld_judge_for_result( row, - handler, tool_handlers, criterion, resolved_judges[criterion.key], @@ -812,9 +934,12 @@ def _emit_evaluation_events( "evaluationId": evaluation.id, }, ) + failures: list[str] = [] for result in results: - # One bad criterion result must not abort the run or drop the - # events queued for the results that preceded it. + # One bad criterion result must not stop the remaining results from + # being emitted, or drop the events already queued for the ones + # before it -- so every result is attempted and the failures are + # raised together once the loop is done. try: identity = { "projectKey": project_key, @@ -883,12 +1008,16 @@ def _emit_evaluation_events( client.track( CRITERION_EVENT_NAME, context, payload_model.to_track_payload(), 1 ) - except Exception: + except Exception as error: logger.exception( - "Skipping evaluation event for row %s criterion %s", + "Failed to emit evaluation event for row %s criterion %s", result.get("row_index"), result.get("criterion_type"), ) + failures.append( + f"row {result.get('row_index')} criterion " + f"{result.get('criterion_type')!r}: {error}" + ) continue logger.info( "%s emittedAt=%s eventId=%s", @@ -896,6 +1025,17 @@ def _emit_evaluation_events( emitted_at, event_id, ) + if failures: + # The evaluation was created with a fixed criterion list, so the + # backend needs one result per (row, criterion) before it can finish + # row accounting. A dropped event is never converted into an error + # result by anything downstream -- swallowing it here would leave + # run() polling to its timeout instead of reporting what failed. + raise EvaluationsError( + f"Failed to emit {len(failures)} of {len(results)} evaluation " + "criterion events, so the run cannot be fully accounted for: " + + "; ".join(failures) + ) def _get_summary( self, project_key: str, evaluation_id: str, run_id: str diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index b5d0e647..f397859b 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -2,7 +2,7 @@ from collections.abc import Mapping from dataclasses import dataclass, field -from typing import Any, TypedDict +from typing import Any, Literal, TypedDict @dataclass @@ -69,12 +69,20 @@ class ResolvedTool: @dataclass class ResolvedJudge: - """A LaunchDarkly AI Judge config variation resolved for an evaluation run.""" + """A LaunchDarkly AI Judge config variation resolved for an evaluation run. + + ``provider`` and ``mode`` come from the judge's own variation, not the + evaluation's generation config: a judge is an independent AI Config and may + be served by a different provider in a different mode. They are kept here + because they are what selects the handler that can actually run this config. + """ key: str config: dict[str, Any] variation_key: str = "" version: int | None = None + provider: str | None = None + mode: Literal["agent", "messages"] = "messages" @dataclass diff --git a/packages/client/src/launchdarkly_ai_server/judges.py b/packages/client/src/launchdarkly_ai_server/judges.py index 934c2716..5878d6b1 100644 --- a/packages/client/src/launchdarkly_ai_server/judges.py +++ b/packages/client/src/launchdarkly_ai_server/judges.py @@ -399,8 +399,10 @@ def _matches(h: ProviderHandler) -> bool: except ValueError: return None - if score is None: - score = 0.0 + # The score is reported as the judge gave it. A missing or null score + # is not a zero: coercing it would record a gen_ai.evaluation of 0 -- + # indistinguishable from a judge that scored the output a hard fail -- + # where every other non-numeric judge output skips the metric instead. metric_score = numeric_score(score) if metric_score is not None: record_evaluation( diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 3974231b..a02955a4 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -8,6 +8,7 @@ import pytest +from launchdarkly_ai_server import create_handler from launchdarkly_ai_server.evaluations import ( DatasetRow, EvaluationsError, @@ -1825,15 +1826,25 @@ async def handler( @pytest.mark.asyncio -async def test_failed_evaluation_event_tracking_skips_event_but_completes_run( +async def test_failed_evaluation_event_tracking_raises_after_attempting_every_result( monkeypatch: pytest.MonkeyPatch, stub_sdk_client: MagicMock, ) -> None: + """A dropped criterion event is a delivery failure, not a silent skip. + + The evaluation is created with a fixed criterion list, so the backend needs + one result per (row, criterion) before row accounting can finish. Swallowing + the failure leaves run() polling to its timeout and hides the cause, so the + run attempts every result, flushes what it queued, and then raises. + """ transport = judge_run_transport() accuracy_judge_variation(monkeypatch) + attempted: list[str] = [] + def track(event_name: str, *args: Any) -> None: if event_name == "$ld:ai:offline-evals:criterion": + attempted.append(args[1]["criterionType"]) raise RuntimeError("event pipeline unavailable") stub_sdk_client.track = MagicMock(side_effect=track) @@ -1849,17 +1860,254 @@ async def handler( return {"output": '{"score": 1, "reasoning": "ok"}'} return {"output": "generated"} + with pytest.raises(EvaluationsError) as error: + await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[ + Judge(key="$ld:ai:judge:accuracy"), + Scorer(name="nonempty", fn=lambda row, output: bool(output)), + ], + ) + + # Every criterion is attempted before the failures are reported together, + # so one bad result never drops the ones behind it. + assert attempted == ["$ld:ai:judge:accuracy", "nonempty"] + assert "Failed to emit 2 of 2 evaluation criterion events" in str(error.value) + assert "event pipeline unavailable" in str(error.value) + stub_sdk_client.flush.assert_awaited() + + +@pytest.mark.parametrize("value", [float("nan"), -0.1, 1.1]) +@pytest.mark.parametrize("field", ["threshold", "pass_rate_threshold"]) +def test_criteria_reject_thresholds_outside_zero_to_one( + field: str, value: float +) -> None: + """NaN passes both range comparisons, so it needs its own rejection. + + Left in, it is serialized into the criteria wire payload as a bare ``NaN`` + literal and the management API rejects the whole evaluation. + """ + with pytest.raises(ValueError, match=f"{field} must be a number between 0 and 1"): + Judge(key="$ld:ai:judge:accuracy", **{field: value}) + with pytest.raises(ValueError, match=f"{field} must be a number between 0 and 1"): + Scorer(name="nonempty", fn=lambda row, output: True, **{field: value}) + + +def judge_variation( + monkeypatch: pytest.MonkeyPatch, + *, + provider: str, + mode: str | None = None, + config: dict[str, Any] | None = None, +) -> None: + """Serve one judge variation for the given provider and mode.""" + + async def fake_extract_variation( + key: str, context: dict[str, Any] + ) -> dict[str, Any]: + meta: dict[str, Any] = {"variationKey": "default", "version": 12} + if mode is not None: + meta["mode"] = mode + return { + "config": { + "provider": {"name": provider}, + "model": {"name": "judge-model"}, + **(config or {"instructions": "Judge {{response_to_evaluate}}"}), + }, + "meta": meta, + } + + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.runner.extract_variation", + fake_extract_variation, + ) + + +async def _generation_only( + config: dict[str, Any], + user_input: str | None = None, + tool_handlers: dict[str, Callable[..., Any]] | None = None, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + return {"output": "generated"} + + +@pytest.mark.asyncio +async def test_judge_on_another_provider_fails_before_any_records_are_created( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A provider handler cannot execute another provider's judge config. + + Passing it anyway spent the generation budget and then recorded every row + as handler_raised, so the mismatch is caught while it is still only a + configuration error: before the dataset is read or any record is created. + """ + transport = judge_run_transport() + judge_variation(monkeypatch, provider="Anthropic") + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + with pytest.raises(EvaluationsError) as error: + await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=create_handler(("OpenAI", "messages"), _generation_only), + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + ) + + assert "No handler can run LaunchDarkly judge" in str(error.value) + assert "'Anthropic'" in str(error.value) + assert transport.requests == [] + + +@pytest.mark.asyncio +async def test_judge_handlers_route_a_judge_to_its_own_provider( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, +) -> None: + transport = judge_run_transport() + judge_variation(monkeypatch, provider="Anthropic") + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + judged: list[dict[str, Any]] = [] + + async def anthropic_judge( + config: dict[str, Any], + user_input: str | None = None, + tool_handlers: dict[str, Callable[..., Any]] | None = None, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> dict[str, Any]: + judged.append(config) + return {"output": '{"score": 0.75, "reasoning": "ok"}'} + result = await evals.run( project_key="proj", key="support-qa", dataset="golden", - handler=handler, + handler=create_handler(("OpenAI", "messages"), _generation_only), generation={"provider": "OpenAI", "model": "gpt-4o"}, criteria=[Judge(key="$ld:ai:judge:accuracy")], + judge_handlers=[create_handler(("Anthropic", "messages"), anthropic_judge)], ) assert result.passed is True - stub_sdk_client.flush.assert_awaited() + assert [config["provider"]["name"] for config in judged] == ["Anthropic"] + events = [call.args[2] for call in stub_sdk_client.track.call_args_list] + judge_event = next(event for event in events if event.get("kind") == "judge") + assert judge_event["status"] == "COMPLETE" + assert judge_event["score"] == 0.75 + + +@pytest.mark.asyncio +async def test_agent_handler_runs_a_messages_mode_judge_with_collapsed_messages( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, +) -> None: + """Mirrors the online path's agent-mode fallback for a messages-mode judge.""" + transport = judge_run_transport() + judge_variation( + monkeypatch, + provider="Anthropic", + mode="messages", + config={ + "messages": [ + {"role": "system", "content": "Grade strictly."}, + {"role": "user", "content": "Judge {{response_to_evaluate}}"}, + ] + }, + ) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + judged: list[dict[str, Any]] = [] + + async def anthropic_agent_judge( + config: dict[str, Any], + user_input: str | None = None, + tool_handlers: dict[str, Callable[..., Any]] | None = None, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> dict[str, Any]: + judged.append(config) + return {"output": '{"score": 1, "reasoning": "ok"}'} + + result = await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=create_handler(("OpenAI", "messages"), _generation_only), + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + judge_handlers=[create_handler(("Anthropic", "agent"), anthropic_agent_judge)], + ) + + assert result.passed is True + assert judged[0]["instructions"] == ( + "Grade strictly.\n\nJudge {{response_to_evaluate}}" + ) + assert judged[0]["messages"] == [] + + +@pytest.mark.asyncio +async def test_generation_handler_runs_a_judge_on_the_same_provider( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, +) -> None: + transport = judge_run_transport() + judge_variation(monkeypatch, provider="OpenAI") + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + calls: list[str | None] = [] + + async def openai_handler( + config: dict[str, Any], + user_input: str | None = None, + tool_handlers: dict[str, Callable[..., Any]] | None = None, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> dict[str, Any]: + calls.append(config.get("instructions")) + if "Judge" in (config.get("instructions") or ""): + return {"output": '{"score": 0.9, "reasoning": "ok"}'} + return {"output": "generated"} + + result = await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=create_handler(("OpenAI", "messages"), openai_handler), + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + ) + + assert result.passed is True + assert any("Judge" in (instructions or "") for instructions in calls) + + +@pytest.mark.asyncio +async def test_judge_handlers_must_declare_the_provider_they_serve( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An unrouted judge handler would silently never be selected.""" + transport = judge_run_transport() + accuracy_judge_variation(monkeypatch) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + with pytest.raises(EvaluationsError, match="does not declare provides_for"): + await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=_generation_only, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + judge_handlers=[_generation_only], + ) + + assert transport.requests == [] @pytest.mark.asyncio diff --git a/packages/client/tests/test_judges.py b/packages/client/tests/test_judges.py index 01998bf5..38b289c8 100644 --- a/packages/client/tests/test_judges.py +++ b/packages/client/tests/test_judges.py @@ -406,3 +406,75 @@ def test_accepts_finite_numbers(self) -> None: assert numeric_score(0) == 0.0 assert numeric_score(inf) is None assert numeric_score(nan) is None + + +class TestRunJudgeScoreReporting: + """A judge that returns no score has not scored the output a zero.""" + + @pytest.mark.asyncio + async def test_null_score_skips_the_metric_instead_of_recording_zero( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from contextlib import asynccontextmanager + + import launchdarkly_ai_server.judges as judges_module + import launchdarkly_ai_server.tracking as tracking_module + from launchdarkly_ai_server import JudgeTask, run_judge + + recorded: list[tuple[float, str | None]] = [] + + @asynccontextmanager + async def fake_with_judge_evaluation(name: str) -> Any: + def record(score: float, explanation: str | None = None) -> None: + recorded.append((score, explanation)) + + yield record + + monkeypatch.setattr( + judges_module, "with_judge_evaluation", fake_with_judge_evaluation + ) + + async def fake_execute_and_track(**kwargs: Any) -> dict[str, Any]: + return { + "response": '{"score": null, "reasoning": "cannot tell"}', + "usage": {"input_tokens": 1, "output_tokens": 1}, + "track_data": {"runId": "run-1"}, + } + + monkeypatch.setattr( + tracking_module, "execute_and_track", fake_execute_and_track + ) + + async def judge_fn( + config, user_input, tool_handlers, variables, history=None + ) -> dict: # type: ignore[override] + raise AssertionError("execute_and_track is stubbed") + + handler = ProviderHandler( + fn=judge_fn, provides_for=("TestProvider", "messages") + ) # type: ignore[arg-type] + + task = JudgeTask( + config_key="judge-key", + judge_config={ + "model": {"name": "gpt-4"}, + "provider": {"name": "TestProvider"}, + "instructions": "judge", + }, + judge_meta={"enabled": True, "variationKey": "j1", "version": 1}, + actual_output="response", + user_context=CONTEXT, + judge_provider="TestProvider", + judge_mode="messages", + collapse_messages=False, + parent_track_data={"runId": "run-1"}, + ) + + result = await run_judge(task, [handler]) + + assert result is not None + # A gen_ai.evaluation of 0 is indistinguishable from a judge that + # scored the output a hard fail, so no metric is recorded at all. + assert recorded == [] + assert result.score is None + assert result.response == "cannot tell" From 462b712422401bf8bf7763efd03b295bb3b389b5 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Tue, 15 Sep 2026 13:30:52 -0700 Subject: [PATCH 13/16] Revert "TEMP(evaluations): hardcode judge successDirection for proxy testing" This reverts commit 16b5d22feee9be1b3c7faeb02a5187809ff494ad. The hardcode existed only to get past ai-evaluator's create-time validation while the gonfalon-judge-proxy injection middleware was behind an off flag. Judge.to_criteria_wire omits successDirection again so the proxy hydrates it from the judge's AI Config: the one input a verdict is ruled on stays server-attested, rather than being asserted by whichever SDK version happened to create the evaluation. A Scorer still declares its own direction, since it has no LaunchDarkly-side config to read one from. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/evaluations/criteria.py | 5 ----- packages/client/tests/test_evaluations_run.py | 10 ++-------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py index e0a1dd11..ee0962fd 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py @@ -59,11 +59,6 @@ def to_criteria_wire(self) -> dict[str, Any]: "criterionType": self.criterion_type, "kind": "judge", "judgeKey": self.key, - # TEMPORARY: hardcoded to unblock testing the gonfalon-judge-proxy - # successDirection-injection middleware while its enabling flag is - # off. The proxy is expected to overwrite this; remove once that - # lands and ai-evaluator no longer needs it on the raw request. - "successDirection": "higher_is_better", "options": options, } diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index a02955a4..a3458d35 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -1155,16 +1155,13 @@ async def handler( assert result.passed is True # kind and judgeKey are what let ai-evaluator store this as a judge rather # than default it to a deepeval metric and reject the key. successDirection - # is TEMPORARILY hardcoded on the wire (see Judge.to_criteria_wire) to - # unblock testing the gonfalon-judge-proxy injection middleware while its - # enabling flag is off; remove this key here alongside that hardcode once - # the proxy is live and LaunchDarkly injects the direction itself. + # is deliberately absent: LaunchDarkly injects it from the judge's AI Config + # on the way through, so the SDK must not assert a direction of its own. assert transport.requests[2]["body"]["criteria"] == [ { "criterionType": "$ld:ai:judge:accuracy", "kind": "judge", "judgeKey": "$ld:ai:judge:accuracy", - "successDirection": "higher_is_better", "options": {"threshold": 0.5}, } ] @@ -1405,13 +1402,10 @@ def count_violations(row: DatasetRow, output: Any) -> float: def test_judge_threshold_defaults_so_a_criterion_is_always_rulable() -> None: """A judge with no threshold gives LaunchDarkly nothing to compare against, so the criterion would be stored and never ruled on.""" - # successDirection is TEMPORARILY hardcoded on the wire; see the matching - # comment on Judge.to_criteria_wire. assert Judge(key="$ld:ai:judge:accuracy").to_criteria_wire() == { "criterionType": "$ld:ai:judge:accuracy", "kind": "judge", "judgeKey": "$ld:ai:judge:accuracy", - "successDirection": "higher_is_better", "options": {"threshold": 0.5}, } From c61b3a545b48265c7942f1344307b883fdd5ffef Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Tue, 15 Sep 2026 13:36:39 -0700 Subject: [PATCH 14/16] fix(evaluations): prefer an exact-provider judge handler over a wildcard Address PR #63 review feedback. _select_judge_handler took the first judge handler whose provider matched or was a wildcard, so with judge_handlers=[LangChain("*"), OpenAI("OpenAI")] an OpenAI judge ran through the LangChain adapter and reversing the list changed which handler scored it. A wildcard is a fallback for multi-provider adapters: the handler contract gives an exact provider name priority, and config() already routes generation configs that way. Handler lookup is now two passes -- exact provider, then wildcard -- applied to both the judge's own mode and the messages-to-agent fallback, so the order a caller lists its handlers in no longer decides. Co-Authored-By: Claude Opus 5 --- .../evaluations/runner.py | 56 ++++++++----- packages/client/tests/test_evaluations_run.py | 81 +++++++++++++++++++ 2 files changed, 118 insertions(+), 19 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 75456f13..164ea57d 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -85,6 +85,33 @@ def _covers_provider( return provides_for[0] == provider or provides_for[0] == "*" +def _find_judge_handler( + judge_handlers: list[EvalHandler], + provider: str | None, + mode: Literal["agent", "messages"], +) -> EvalHandler | None: + """Find a handler for ``provider`` in ``mode``, exact match before wildcard. + + A wildcard handler is a fallback for multi-provider adapters, so it is only + chosen when no handler names the provider outright -- the priority + ``config()`` already applies to a generation config. Searching in one pass + would instead let the order the caller happened to list its handlers in + decide, sending an OpenAI judge through a LangChain adapter that was merely + listed first. + """ + for exact in (True, False): + for candidate in judge_handlers: + provides_for = _provides_for(candidate) + if provides_for is None or provides_for[1] != mode: + continue + if exact: + if provides_for[0] == provider: + return candidate + elif provides_for[0] == "*": + return candidate + return None + + def _select_judge_handler( resolved: ResolvedJudge, handler: EvalHandler, @@ -97,7 +124,8 @@ def _select_judge_handler( one provider cannot execute another's config. The priority mirrors the online path (``judges.run_judges``): - 1. a judge handler matching the judge's provider (or a wildcard) and mode; + 1. a judge handler in the judge's mode, naming its provider outright + before any wildcard adapter; 2. an agent-mode judge handler for a messages-mode judge, whose messages are collapsed into a single instructions block; 3. the generation handler, when it covers the judge's provider. @@ -106,25 +134,15 @@ def _select_judge_handler( own routing -- the same contract it already honours for the generation config -- so it is treated as covering every judge. """ - for candidate in judge_handlers: - provides_for = _provides_for(candidate) - if ( - provides_for - and _covers_provider(provides_for, resolved.provider) - and provides_for[1] == resolved.mode - ): - return JudgeExecution(resolved=resolved, handler=candidate) + match = _find_judge_handler(judge_handlers, resolved.provider, resolved.mode) + if match is not None: + return JudgeExecution(resolved=resolved, handler=match) if resolved.mode == "messages": - for candidate in judge_handlers: - provides_for = _provides_for(candidate) - if ( - provides_for - and _covers_provider(provides_for, resolved.provider) - and provides_for[1] == "agent" - ): - return JudgeExecution( - resolved=resolved, handler=candidate, collapse_messages=True - ) + agent_fallback = _find_judge_handler(judge_handlers, resolved.provider, "agent") + if agent_fallback is not None: + return JudgeExecution( + resolved=resolved, handler=agent_fallback, collapse_messages=True + ) generation_provides_for = _provides_for(handler) if generation_provides_for is None: return JudgeExecution(resolved=resolved, handler=handler) diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index a3458d35..c7ca278b 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -1998,6 +1998,87 @@ async def anthropic_judge( assert judge_event["score"] == 0.75 +@pytest.mark.asyncio +@pytest.mark.parametrize("wildcard_first", [True, False]) +async def test_exact_provider_judge_handler_beats_a_wildcard_adapter( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, + wildcard_first: bool, +) -> None: + """A wildcard is a fallback, so the order handlers are listed in cannot decide. + + Taking the first provider-or-wildcard match would send an Anthropic judge + through a multi-provider adapter that merely happened to be listed first. + """ + transport = judge_run_transport() + judge_variation(monkeypatch, provider="Anthropic") + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + chosen: list[str] = [] + + def judge_handler(name: str) -> Any: + async def run( + config: dict[str, Any], + user_input: str | None = None, + tool_handlers: dict[str, Callable[..., Any]] | None = None, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> dict[str, Any]: + chosen.append(name) + return {"output": '{"score": 1, "reasoning": "ok"}'} + + return run + + wildcard = create_handler(("*", "messages"), judge_handler("wildcard")) + exact = create_handler(("Anthropic", "messages"), judge_handler("exact")) + + result = await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=create_handler(("OpenAI", "messages"), _generation_only), + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + judge_handlers=[wildcard, exact] if wildcard_first else [exact, wildcard], + ) + + assert result.passed is True + assert chosen == ["exact"] + + +@pytest.mark.asyncio +async def test_wildcard_judge_handler_runs_a_judge_no_handler_names( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, +) -> None: + transport = judge_run_transport() + judge_variation(monkeypatch, provider="Anthropic") + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + judged: list[dict[str, Any]] = [] + + async def wildcard_judge( + config: dict[str, Any], + user_input: str | None = None, + tool_handlers: dict[str, Callable[..., Any]] | None = None, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> dict[str, Any]: + judged.append(config) + return {"output": '{"score": 1, "reasoning": "ok"}'} + + result = await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + handler=create_handler(("OpenAI", "messages"), _generation_only), + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + judge_handlers=[create_handler(("*", "messages"), wildcard_judge)], + ) + + assert result.passed is True + assert [config["provider"]["name"] for config in judged] == ["Anthropic"] + + @pytest.mark.asyncio async def test_agent_handler_runs_a_messages_mode_judge_with_collapsed_messages( monkeypatch: pytest.MonkeyPatch, From 5f5a626a2406774010485a070546ab27cc5d86fb Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Tue, 15 Sep 2026 14:31:06 -0700 Subject: [PATCH 15/16] docs(evaluations): document criteria, judge handlers, and scoring policy The evaluations section still described a generation-only harness, so the criteria this PR adds were undocumented and two claims had gone stale: events are logged on launchdarkly_ai_server.evaluations.runner rather than printed to stdout, and EvalRunResult.passed now treats failed_rows as a gate failure. Adds a "Score rows with judges and scorers" section covering the Judge and Scorer contracts and their threshold defaults, the criterion event, why the SDK reports a score and LaunchDarkly rules on it, per-judge handler routing via judge_handlers, and which failures fail the run up front versus becoming a cause-coded per-criterion ERROR event. The barrel package README picks up the criterion types it re-exports and the second event name. Both snippets were executed against the package: the imports resolve, and the judge criterion wire payload carries no successDirection. Co-Authored-By: Claude Opus 5 --- packages/ai/README.md | 10 ++++++--- packages/client/README.md | 44 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/packages/ai/README.md b/packages/ai/README.md index 7bbf073f..151b29df 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -54,10 +54,10 @@ Never raises. Returns `{"enabled": bool, "config": dict | None, "meta": dict | N ## Evaluations from code -`init_evaluations` and the evaluations result types are also re-exported: +`init_evaluations`, the criterion types, and the evaluations result types are all re-exported: ```python -from launchdarkly_ai_python import init_evaluations +from launchdarkly_ai_python import Judge, Scorer, init_evaluations evals = init_evaluations() result = await evals.run( @@ -66,10 +66,14 @@ result = await evals.run( dataset="golden-dataset", handler=my_handler, generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[ + Judge(key="$ld:ai:judge:accuracy"), + Scorer(name="mentions-policy", fn=lambda row, output: "policy" in (output or "")), + ], ) ``` -`LD_API_TOKEN` is required. Configure `LD_SDK_KEY` — or initialize your own client with `init_client(client=...)` — to emit one `$ld:ai:offline-evals:generation` event per generated row through the standard SDK event transport. Use `LD_API_BASE_URI` for staging or local management API traffic; it is separate from the SDK delivery setting `LD_BASE_URI`. Evaluation-run links use the explicit `ui_base_uri` option or `LD_UI_BASE_URI` (for example, `https://ld-stg.launchdarkly.com` in staging), defaulting to `https://app.launchdarkly.com`. See the [core evaluations guide](../client/README.md#run-an-evaluation-from-code). +`LD_API_TOKEN` is required. Configure `LD_SDK_KEY` — or initialize your own client with `init_client(client=...)` — to emit one `$ld:ai:offline-evals:generation` event per generated row, plus one `$ld:ai:offline-evals:criterion` event per `(row, criterion)` when `criteria` are supplied, through the standard SDK event transport. The SDK reports scores; LaunchDarkly rules on them at ingest. A judge served by a different provider than `generation` needs a handler for it in `judge_handlers`. Use `LD_API_BASE_URI` for staging or local management API traffic; it is separate from the SDK delivery setting `LD_BASE_URI`. Evaluation-run links use the explicit `ui_base_uri` option or `LD_UI_BASE_URI` (for example, `https://ld-stg.launchdarkly.com` in staging), defaulting to `https://app.launchdarkly.com`. See the [core evaluations guide](../client/README.md#run-an-evaluation-from-code). --- diff --git a/packages/client/README.md b/packages/client/README.md index 852f5eff..f0259c42 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -46,9 +46,9 @@ No code changes are required — `init_client()` detects the packages at runtime ### Run an evaluation from code -The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. Each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error (`errorMessage` is included for `ERROR` rows), nested `usage.inputTokens`/`usage.outputTokens`, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event prints a line to stdout with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. +The evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. Rows can then be scored by LaunchDarkly judges and local scorer functions; see [Score rows with judges and scorers](#score-rows-with-judges-and-scorers). Each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error (`errorMessage` is included for `ERROR` rows), nested `usage.inputTokens`/`usage.outputTokens`, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event is logged at `INFO` on the `launchdarkly_ai_server.evaluations.runner` logger with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time once that logger is enabled. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. -Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. After flushing generation events, the harness polls the run summary endpoint until passed + failed + error rows fully account for a nonzero total with no pending rows, polling every `poll_interval_seconds` (default 2s) up to `poll_timeout_seconds` (default 180s); pass either to `run()` to widen both for large datasets. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. A generation result passes only when the completed summary has no error or pending rows. Evaluation keys must be unique because every call creates a new evaluation with `POST`. +Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. After flushing generation events, the harness polls the run summary endpoint until passed + failed + error rows fully account for a nonzero total with no pending rows, polling every `poll_interval_seconds` (default 2s) up to `poll_timeout_seconds` (default 180s); pass either to `run()` to widen both for large datasets. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. A run passes only when the completed summary has no failed, error, or pending rows. `failed_rows` counts rows whose criteria were scored and did not meet their threshold, so a gate that ignored it would exit 0 on a run where every row failed its judge. Evaluation keys must be unique because every call creates a new evaluation with `POST`. ```python import asyncio @@ -80,7 +80,45 @@ sys.exit(asyncio.run(main())) `project_key` is supplied per run rather than during initialization. `generation.instructions` is shorthand for one system message; use `generation.messages` instead for a full message list, but do not supply both. The harness never retries a handler invocation because doing so could repeat tool side effects. Its retries apply only to LaunchDarkly management API requests. -Generation events are the only path by which row results reach LaunchDarkly, so `init_evaluations()` raises rather than creating a run that can never complete unless it can resolve an event transport: either an SDK key (`sdk_key` or `LD_SDK_KEY`) or a client already initialized through `init_client(client=...)`. Bringing your own client lets a process emit evaluation events without an SDK key in scope. Every generated row is emitted and flushed unconditionally; no feature flag gates event publishing. The harness then polls the summary endpoint until row accounting shows processing is complete. +Generation and criterion events are the only path by which row results reach LaunchDarkly, so `init_evaluations()` raises rather than creating a run that can never complete unless it can resolve an event transport: either an SDK key (`sdk_key` or `LD_SDK_KEY`) or a client already initialized through `init_client(client=...)`. Bringing your own client lets a process emit evaluation events without an SDK key in scope. Every generated row is emitted and flushed unconditionally; no feature flag gates event publishing. The harness then polls the summary endpoint until row accounting shows processing is complete. + +### Score rows with judges and scorers + +Pass `criteria` to `run()` to score every generated row. A `Judge` references an AI Judge config that already exists in LaunchDarkly — the SDK creates no judges and ships none of its own — and a `Scorer` wraps a local function, so a run can mix model-graded and deterministic checks. Each criterion runs once per generated row, bounded by the same `concurrency` as generation, and emits one `$ld:ai:offline-evals:criterion` event per `(row, criterion)` carrying the criterion identity, the judge's variation key and version, the validated score, its reason, usage, and timings. + +```python +from launchdarkly_ai_claude_messages import create_claude_messages_handler +from launchdarkly_ai_openai_messages import create_openai_messages_handler +from launchdarkly_ai_server import DatasetRow, Judge, Scorer, init_evaluations + + +def mentions_policy(row: DatasetRow, output: str | None) -> bool: + return "refund policy" in (output or "").lower() + + +result = await init_evaluations().run( + project_key="my-project", + key="support-qa-2026-08-20", + dataset="support-golden", + handler=create_openai_messages_handler(), + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[ + Judge(key="$ld:ai:judge:accuracy", threshold=0.8), + Scorer(name="mentions-policy", fn=mentions_policy), + ], + # Needed only because this judge is served by a different provider than + # the generation config above. + judge_handlers=[create_claude_messages_handler()], +) +``` + +`Scorer.fn` receives the `DatasetRow` the output was generated from plus the generated output, may be sync or async, and must return a bool or a number from 0 to 1; booleans become 1.0 or 0.0. `Judge.threshold` defaults to 0.5 and `Scorer.threshold` to 1.0 — a perfect score, which is what a boolean scorer wants — and both accept an optional `pass_rate_threshold`. Judge keys and scorer names share one `criterionType` namespace and must be unique within a run, case-insensitively, because that name is part of each result's deterministic event identity. `Judge.ground_truth_context` overrides what the judge is graded against when the dataset row's expected output is not it. + +**The SDK reports scores and never rules on them.** LaunchDarkly derives each row's verdict at ingest by comparing the score against the criterion's stored threshold and success direction, so pass/fail policy is one server-side implementation that applies to every SDK version and to runs already recorded. A judge's direction lives on its AI Config and is injected server-side, keeping the one input a verdict turns on server-attested; a `Scorer` has no LaunchDarkly-side config to read, so it declares its own `success_direction` (default `"higher_is_better"` — set `"lower_is_better"` for a scorer that counts something unwanted, like a regex hit count). + +**Judges are independent AI Configs, so handlers are routed per judge.** A judge may resolve to a different provider or mode than `generation`, and a handler built for one provider cannot execute another's config. `handler` runs a judge when it provides for that judge's provider; pass handlers for any other providers in `judge_handlers`. Selection prefers a handler naming the judge's provider outright over a wildcard multi-provider adapter, and an agent-mode handler can serve a messages-mode judge with its messages collapsed into one instructions block. A plain callable that declares no `provides_for` routes itself, exactly as it already does for the generation config. + +Judges are resolved through flag delivery, and handlers are matched to them, **before** any evaluation records are created — a missing judge or one no handler covers fails the run up front rather than after the generation spend. After that point a criterion failure never aborts the run: an unparseable judge response, an out-of-range score, a raising handler or scorer, and a row whose generation errored each become a per-criterion `ERROR` event with a cause code (`invalid_judge_output`, `invalid_score`, `handler_raised`, `scorer_raised`, `generation_incomplete`) and a top-level `errorMessage`. Event *delivery* is different: the backend needs one result per `(row, criterion)` to finish row accounting, so if tracking a criterion event fails, every remaining result is still attempted and flushed and then `run()` raises — rather than polling to its timeout with the cause hidden. The client uses **lazy initialization**: importing the package does not connect to LaunchDarkly. The singleton is created automatically on the first API call that needs it (`config().invoke()`, `graph().invoke()`, `resolve_graph()`, etc.), as long as `LD_SDK_KEY` is set in the environment. From ceca049617b1cac362789aafbe7465759aed0762 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Tue, 15 Sep 2026 16:03:10 -0700 Subject: [PATCH 16/16] docs(evaluations): use a customer-style judge key in examples The examples passed "$ld:ai:judge:accuracy", a key from LaunchDarkly's own internal namespace, which reads as though callers must reach for that prefix. A judge key is just the key of a judge in the reader's project, so the examples now use "accuracy-judge". Co-Authored-By: Claude Opus 5 --- packages/ai/README.md | 2 +- packages/client/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ai/README.md b/packages/ai/README.md index 151b29df..b1b47a70 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -67,7 +67,7 @@ result = await evals.run( handler=my_handler, generation={"provider": "OpenAI", "model": "gpt-4o"}, criteria=[ - Judge(key="$ld:ai:judge:accuracy"), + Judge(key="accuracy-judge"), Scorer(name="mentions-policy", fn=lambda row, output: "policy" in (output or "")), ], ) diff --git a/packages/client/README.md b/packages/client/README.md index f0259c42..0c5c4b26 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -103,7 +103,7 @@ result = await init_evaluations().run( handler=create_openai_messages_handler(), generation={"provider": "OpenAI", "model": "gpt-4o"}, criteria=[ - Judge(key="$ld:ai:judge:accuracy", threshold=0.8), + Judge(key="accuracy-judge", threshold=0.8), Scorer(name="mentions-policy", fn=mentions_policy), ], # Needed only because this judge is served by a different provider than