From 27ef12f85faa2790ed6ecac8c23d78bd7a2304ad Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 25 Aug 2026 16:04:25 -0400 Subject: [PATCH 01/22] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20retrieval=20through=20an=20injectable=20store?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second of five slices. Adds the layer that turns a reference into content: an injectable store seam, integrity verification of everything it serves, and a body-free telemetry seam for the failures. - `get_skill(key, *, version=None)` returns one verified skill, or None. - `get_skills(refs)` is the batch form, accepting references and bare keys. - `all_skills()` returns every verified skill the store holds, one per key. - `SkillStore` is the structural interface content arrives through — `get_object(kind, key, version=None)`, `all_objects(kind)`, and an optional `add_listener(kind, fn)` — configured with `init_client(options={"skillStore": store})`. `InMemorySkillStore` ships for local development and testing. A delivery transport drops in behind the same seam with no public API change. Store data is untrusted. Key and version are revalidated, size is bounded, and the sha256 of the verbatim bytes must match the delivered `contentHash`; anything that does not verify is withheld and treated as missing, so no unverified content is ever returned. The wire object delivers content as a JSON string; the UTF-8 encode happens exactly once, inside verification, and the `Skill` handed to user code carries the verified verbatim bytes (`Skill.content: bytes`) — the exact byte sequence that was hashed, never a re-derived value. Content carrying an unpaired surrogate has no UTF-8 encoding at all and is withheld too — `str.encode` is called strictly, never with an error handler that would fabricate bytes a hash comparison could then accept. `verified_bytes` also accepts already-bytes content, hashing it directly, for the pre-write re-verification pass a later slice adds. Integrity failures are reported through a private telemetry seam carrying hashes and byte counts only, never the skill body. The two properties copied off the wire, `skill_key` and `expected_hash`, are shape-checked and replaced when malformed, so a hostile store cannot use either one to publish the body through a signal that is otherwise body-free. The default emitter is a no-op: nothing leaves the process in this release, and the three signal names are an allowlist maintained in one section of one module. Version is part of the lookup identity rather than a filter applied to the answer. A delivery payload carries the newest version of every skill plus every version any variation currently pins, so two versions of one key coexist routinely; a seam keyed by key alone would answer a pinned reference with the newest object and then reject it, turning the primary use case into a missing skill. `InMemorySkillStore` holds several versions of a key, `get_object` takes the wanted version, and `version=None` means "the newest you hold". The equality check afterwards is kept as a defense — the store is untrusted, so an answer that is not the version asked for is withheld. `all_objects` returns one entry per key-and-version under keys that are opaque to this SDK; identity is read off each object's own fields. `newest_by_key` is the single place that collapses the result to one object per key. A run that withheld anything now logs a count at WARN. Every individual withholding already records a signal and an error line, but a caller reading logs at WARN saw neither, and a payload where nothing verifies otherwise returns an empty result indistinguishable from "this project has no skills". `SKILL_OBJECT_KIND` is deliberately **not** exported from the package root. It is the string this SDK hands a store, and an adapter maps whatever the transport underneath calls a skill onto it; publishing it would advertise an SDK-side seam value as the wire contract. An adapter that needs to agree with it reaches it through `skills_core`. `MAX_SKILL_CONTENT_BYTES` stays internal for the adjacent reason. Note one behaviour change: `shutdown()` clears the configured skill store along with the client. `init_client` applies `skillStore` on every successful call, even the idempotent ones, which is what lets a lazily auto-initialized client be given a store afterwards. Testing: `uv run pytest` → 1145 passed. `ruff check`, `ruff format --check`, and `mypy packages/*/src` all clean. Co-Authored-By: Claude Fable 5 --- packages/client/README.md | 61 +- packages/client/agents.md | 112 +- .../src/launchdarkly_ai_server/__init__.py | 10 + .../src/launchdarkly_ai_server/lifecycle.py | 42 +- .../src/launchdarkly_ai_server/skills.py | 242 +++- .../src/launchdarkly_ai_server/skills_core.py | 628 ++++++++++ packages/client/tests/conftest.py | 112 ++ packages/client/tests/test_skills.py | 1019 ++++++++++++++++- 8 files changed, 2203 insertions(+), 23 deletions(-) create mode 100644 packages/client/src/launchdarkly_ai_server/skills_core.py diff --git a/packages/client/README.md b/packages/client/README.md index fbd40045..f16a1eb2 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -223,22 +223,46 @@ asyncio.run(main()) ### Agent Skills Skills are versioned `SKILL.md` documents managed in LaunchDarkly and attached to AI Config -variations by reference. This release adds the first layer: discovering which skills a -resolved config references. Retrieving their content and materializing them onto disk follow. +variations by reference. The SDK surfaces which skills a config references and retrieves +their content. Materializing them onto disk, where agent runtimes discover them, follows. ```python import asyncio +import hashlib -from launchdarkly_ai_server import init_client, inspect_config, skill_refs +from launchdarkly_ai_server import ( + init_client, inspect_config, skill_refs, get_skill, get_skills, + InMemorySkillStore, +) + +SKILL_MD = "---\nname: PDF Extraction\n---\nExtract text from PDFs.\n" async def main(): - await init_client() + # A store supplies skill content. InMemorySkillStore is the dict-backed + # store for local development, testing, and bring-your-own-content use. + store = InMemorySkillStore() + store.put({ + "key": "pdf-extraction", + "version": 2, + "content": SKILL_MD, + # sha256, lowercase hex, over the verbatim utf-8 bytes. Content whose hash + # does not match is withheld, so this is not optional. + "contentHash": hashlib.sha256(SKILL_MD.encode("utf-8")).hexdigest(), + }) + await init_client(options={"skillStore": store}) + # 1. Which skills does this config reference? Pure projection — no I/O. info = await inspect_config("doc-agent", {"kind": "user", "key": "user-123"}) - refs = skill_refs(info["config"]) # [SkillReference(key='pdf-extraction', version=2)] + refs = skill_refs(info["config"]) # [SkillReference(key='pdf-extraction', version=2)] - for ref in refs: - print(ref.key, ref.version) + # 2. Fetch content. Returns None rather than raising when a skill is unavailable. + skill = await get_skill("pdf-extraction") + if skill is not None: + print(skill.content) + + # 3. Or resolve the config's references in one call. + for s in await get_skills(refs): + print(s.key, s.version) asyncio.run(main()) ``` @@ -249,9 +273,32 @@ integer ≥ 1): the whole variation is rejected, `inspect_config` returns `confi `extract_variation` raises. A variation that previously carried its own custom `skills` field of a different shape must rename it before upgrading. +**Integrity is not optional.** Content is only returned after its sha256 (lowercase hex, +over the verbatim UTF-8 bytes) matches the delivered `contentHash`, its key and version +revalidate, and its size is within 64 KiB. Anything that fails is withheld and treated as +missing — no unverified content ever reaches your code. A retrieval that withheld anything +logs a count at WARN, so a run that resolved nothing is not silent. + +**Versions are selected, not filtered.** A store may hold several versions of one key at +once, because a delivery payload does: the newest version of every skill, plus every +version a variation currently pins. `get_skill("k", version=1)` asks the store for version +1 and gets it even when a newer one is also held. + | Export | Description | |---|---| | `skill_refs(config)` | Project a config's `skills` array into `list[SkillReference]`. Pure — no client, store, or network needed. Returns `[]` when absent. | +| `get_skill(key, *, version=None)` | One verified skill, or `None`. `version=None` means newest available; a specific `version` matches exactly. Raises only when no store is configured. | +| `get_skills(refs)` | Batch form. Accepts `SkillReference` values and bare key strings (string = latest). Results follow input order; missing or unverifiable entries are omitted. | +| `all_skills()` | Every verified skill the store holds, one per key at its newest version. | +| `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)`. | +| `InMemorySkillStore(objects=None)` | A dict-backed store with `put(raw)`, for local development and testing. Holds several versions of a key. | + +Configure the store with `init_client(options={"skillStore": store})`. With none configured, +the accessors raise `RuntimeError` explaining what to do. `shutdown()` clears it. + +`all_objects` returns one entry per `(key, version)` under keys that are **opaque** to the +SDK — identity is read from each object's own `key` and `version` fields, so a store is free +to key its own map however the transport underneath does. > `Skill.content` is `bytes` — the verified verbatim bytes LaunchDarkly delivered, exactly > what was hashed. The SDK never parses or interprets them; if you want the frontmatter, diff --git a/packages/client/agents.md b/packages/client/agents.md index 8b55f173..f284c425 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -28,7 +28,8 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/graph.py` | `graph()`, `resolve_graph()`, `GraphInstance` | | `src/launchdarkly_ai_server/types.py` | All shared Python types — `AiConfigRep`, `ProviderHandler`, `LDContext`, `NativeTool`, etc. | | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | -| `src/launchdarkly_ai_server/skills.py` | Agent Skills — `skill_refs`, the projection of a config's `skills` array into typed references | +| `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | +| `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | | `src/launchdarkly_ai_server/registry.py` | `Registry`, `global_registry`, `compose`, `resolve_handlers`, `resolve_tools` | | `src/launchdarkly_ai_server/judges.py` | `run_judges`, `build_judge_tasks`, `run_judge` | @@ -73,9 +74,22 @@ from launchdarkly_ai_server import execute_and_track, execute_and_stream, wrap_t from launchdarkly_ai_server import config, graph, resolve_graph # Agent Skills -from launchdarkly_ai_server import skill_refs +from launchdarkly_ai_server import ( + skill_refs, get_skill, get_skills, all_skills, + SkillStore, InMemorySkillStore, +) ``` +`MAX_SKILL_CONTENT_BYTES` is deliberately *not* among them: it is a local enforcement +bound on content the platform produces, not a value this SDK defines, so exporting it +would semver-lock a number this side does not own. Keep it internal to `skills_core`. + +`SKILL_OBJECT_KIND` is not exported either, for a different reason: it is the string this +SDK hands a store, and a store adapter maps whatever the transport underneath calls a skill +onto it. Publishing it would advertise an SDK-side seam value as the wire contract — a claim +this side cannot make, and hard to walk back once a caller depends on it. An adapter that +needs to agree with it reaches it through `launchdarkly_ai_server.skills_core`. + When adding a new export, add it to `__init__.py`'s imports and `__all__`. Handler packages must never import from sub-paths (e.g. `launchdarkly_ai_server.client`). --- @@ -170,20 +184,103 @@ Three layers, in increasing order of blast radius. Only the first is implemented typed `SkillReference` values. Pure: no network, no client, no store, no telemetry. Validation of the array itself lives in `parse_ai_config` and is **fail closed** — one malformed reference fails the whole config parse. -2. **Content accessors** — reading skill content through a store seam. +2. **Content accessors** — `get_skill`, `get_skills`, `all_skills` read through the + `SkillStore` seam. Configure a store with + `init_client(options={"skillStore": store})`; with none configured the accessors raise + an actionable `RuntimeError`. A delivery transport can be added behind the seam + without touching the public API. 3. **Materialization** — writing skills onto disk under a manifest. +### The store seam, and why version is part of the lookup + +`SkillStore` is `get_object(kind, key, version=None)`, `all_objects(kind)`, and an optional +`add_listener(kind, fn)`. Version is part of the **lookup identity**, not a filter applied +to the answer, and that is load-bearing: a delivery payload carries the newest version of +every skill *plus* every version any variation currently pins, so two versions of one key +coexist routinely. A seam keyed by key alone would answer a pinned reference with the newest +object, and the caller would then have to reject it — turning the primary use case, a +version-pinned attachment, into a missing skill. `version=None` asks for the newest held. + +The equality check in `resolve_from_store` stays, now as a **defense** rather than as the +selection mechanism: the store is untrusted, so an answer that is not the version asked for +is withheld. + +`all_objects` returns one entry per `(key, version)` under keys that are **opaque** to this +SDK. Do not parse them and do not assume one per skill key; identity is read off each +object's own `key` and `version`, which are revalidated anyway. `newest_by_key` is the +one place that collapses the result to one object per key, because both whole-store +consumers need it — `all_skills`, since a list holding two versions of one key is not a set +of skills, and the `"*"` reconcile, since `//SKILL.md` is a single path. + ### Security posture — do not relax any of this +Store data is **untrusted input**; the transport is not part of the trust boundary. + - **Skill content is an opaque byte buffer.** `Skill.content` is `bytes` — the verified - verbatim bytes LaunchDarkly delivered, exactly what was hashed. This SDK never parses, - decodes, or interprets them anywhere: not in the integrity path, not in an accessor, not - during materialization. Consumers who want frontmatter parse it themselves. + verbatim bytes, exactly what was hashed. The wire object delivers content as a JSON + string; the UTF-8 encode happens once, during verification, and from then on the SDK + never parses, decodes, or interprets the bytes anywhere: not in the integrity path, not + in an accessor, not during materialization. Consumers who want frontmatter parse it + themselves. +- **Integrity is mandatory and doubled, through one implementation.** Every raw object is + verified at the accessor boundary (key pattern and length, integer version >= 1, content + at most 64 KiB, sha256 lowercase hex over the verbatim bytes against `contentHash`) + and the hash is re-verified immediately before a write, both through + `skills_core.verified_bytes`, so the integrity signal's property set cannot depend on + which layer caught the defect. A `Skill` is only ever constructed from content that + passed. Nothing unverified reaches user code. +- **`contentHash` is required.** An object without one is withheld, not accepted on trust. + A payload built before the field is populated therefore yields nothing, which is why a + withholding run logs a run-level count at WARN — an empty result would otherwise be + indistinguishable from "this project has no skills". +- **No unencodable string ever reaches an encode.** `json.loads` turns a `\ud800` escape + into an unpaired surrogate with no UTF-8 representation; every `.encode("utf-8")` site + treats that as a verification failure. Never reach for `errors="surrogatepass"` — + fabricating bytes could satisfy the hash comparison. +- **Attacker-controlled strings are never echoed into telemetry.** `contentHash` and `key` + come off the wire, so a store could put the skill body in either; both are shape-checked + and redacted before they reach a signal or a log line. - **A key is untrusted input everywhere it appears.** `skill_key_rejection_reason` is the single canonical explanation, so the config parser and the reference projection reject a key for the same stated reason — and so does every layer added later. A silently shortened projection is not acceptable: every dropped entry is logged. +### Telemetry seam + +Skills telemetry goes through a private emitter with one method, +`record(signal, properties)`, whose default implementation is a **no-op** — nothing leaves +the process in this release. `client.track()` is deliberately *not* used: it needs an LD +context, spends the customer's event volume, lands in their data export, and is silenced by +offline mode. No LD context is involved anywhere in this feature. + +Exactly three signals exist, and the list is an **allowlist, not a floor**: + +| Signal | When | Properties | +|---|---|---| +| `AgentControl Skill Integrity Failure` | any hash/size/shape verification failure | `skill_key`, `version?`, `expected_hash?`, `observed_hash?`, `language` | +| `AgentControl Skill Materialized` | each `written` / `updated` / `skipped_current` | `skill_key`, `content_bytes`, `content_hash`, `reconcile_action`, `language` | +| `AgentControl Skill Revoked Received` | prune removes a formerly managed skill | `skill_key`, `version`, `removed_from_disk`, `language` | + +The last two belong to the materialization layer and have no caller yet; they live here +with the first so the allowlist is one section of one file rather than three sites to audit. + +`AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` +were considered and **deliberately excluded from SDK emission** — both are observable +server-side. Do not add them. The skill body never appears in a signal, a log line, or +an error message, and no signal carries a filesystem path. An emitter that raises is caught +and logged; it never fails the operation. + +Module state lives in `skills_core.py`, so there is exactly one store and one emitter +however the feature is entered. All three signals are emitted from the `record_*` functions +next to the seam there — nothing outside that module calls `emit`, so the allowlist is +enforced in one place. + +The injection path is deliberately narrower than the state's location: `skills.py` owns +`_set_store`, `_set_emitter_for_testing` and `_clear_state`, which delegate to +`skills_core`. `init_client` and `shutdown` use those, tests inject through those +(`skills._set_store(store)` is the same setter `init_client` uses), and neither should +reach into `skills_core` directly. + --- ## OTel Setup @@ -332,3 +429,6 @@ on their side of the boundary. - Do not weaken the `parse_ai_config` validation — handler packages rely on `config` being valid when they receive it. - `parse_usage` must continue to accept `input_tokens/output_tokens`, `inputTokens/outputTokens`, and `input/output` as all existing handlers return one of these variants. - `Skill.content` is opaque `bytes`. Do not add anything that parses or interprets it — no YAML library in this package's dependencies at any tier, and no accessor that decodes content. +- Do not route skills telemetry through `client.track()`, and do not introduce an LD context anywhere in the skills path. Signals go through the `skills_core.py` emitter seam, whose default is a no-op, and only via its `record_*` functions. +- Do not add a signal name outside the three in the Agent Skills table above — the list is an allowlist. `AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` were considered and deliberately excluded from SDK emission. +- Do not make `SkillStore` lookups key-only. Version is part of the lookup identity because a payload holds several versions of one key; a key-only seam cannot express a version-pinned reference. diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 3faff304..02c858f2 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -38,8 +38,13 @@ resolve_tools, ) from .skills import ( + InMemorySkillStore, + all_skills, + get_skill, + get_skills, skill_refs, ) +from .skills_core import SkillStore from .tracking import execute_and_stream, execute_and_track, wrap_tool_handlers from .types import ( NATIVE_TOOL_KEY, @@ -208,4 +213,9 @@ "GraphInstance", # skills "skill_refs", + "get_skill", + "get_skills", + "all_skills", + "SkillStore", + "InMemorySkillStore", ] diff --git a/packages/client/src/launchdarkly_ai_server/lifecycle.py b/packages/client/src/launchdarkly_ai_server/lifecycle.py index 8969f2ea..28b7e81e 100644 --- a/packages/client/src/launchdarkly_ai_server/lifecycle.py +++ b/packages/client/src/launchdarkly_ai_server/lifecycle.py @@ -6,6 +6,7 @@ import os from typing import Any +from . import skills from .types import InitClientOptions logger = logging.getLogger(__name__) @@ -130,13 +131,43 @@ async def init_client( - Pass *client* directly (BYOC) to skip the LaunchDarkly Python SDK path. - Otherwise, reads ``LD_SDK_KEY`` from env or ``options['sdkKey']``. + - ``options['skillStore']`` configures the store the Agent Skills accessors + read from. Absent by default, in which case they raise an actionable error. + + This function is idempotent for the client singleton: a second call returns + the existing client without re-initializing, and every option is ignored — + **except** ``skillStore``, which is applied on every successful call. That + asymmetry is deliberate, and it is what lets a client that was lazily + auto-initialized, or initialized without a store, be given one afterwards. + A ``skillStore`` of ``None`` (or absent) never clears an already-configured + store; use ``shutdown()`` for that. The store is installed only once + initialization has succeeded: a call that raises leaves no global state + behind, so a failed init cannot leave the skill accessors working against a + store the application believes was never installed. Returns the initialized ``LDClientInterface`` instance. """ - global _client - opts = options or {} + ld_client = await _resolve_client(opts, client) + + # The single success point: every path that raises returns before here, so + # "installed only on success" is one statement rather than a copy per exit. + skill_store = opts.get("skillStore") + if skill_store is not None: + skills._set_store(skill_store) + return ld_client + + +async def _resolve_client(opts: InitClientOptions, client: Any) -> Any: + """ + Returns the singleton client, initializing it on first call. + + Split from ``init_client`` so that function has exactly one success point to + hang the ``skillStore`` carve-out on. + """ + global _client + # Idempotent — if already initialized, return the existing client if _client is not None: return _client @@ -198,12 +229,18 @@ async def shutdown() -> None: """ Shuts down the singleton client. Idempotent — safe to call multiple times even if the client was never initialized or already shut down. + + Also clears the configured skill store (and telemetry emitter): after a + shutdown, re-pass ``skillStore`` to the next ``init_client`` if the skill + accessors should keep working. """ global _client, _tracer_provider local_client = _client local_provider = _tracer_provider + skills._clear_state() + # Null the singleton before any awaits so a second call is a no-op _client = None _tracer_provider = None @@ -240,6 +277,7 @@ def _reset_for_testing() -> None: global _client, _tracer_provider _client = None _tracer_provider = None + skills._clear_state() async def inspect_config( diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py index a3294bd2..eb3946c2 100644 --- a/packages/client/src/launchdarkly_ai_server/skills.py +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -1,16 +1,43 @@ """ -Agent Skills — reference discovery. +Agent Skills — reference discovery and content accessors. -Projects the skill references a resolved AI Config carries into typed values. A -pure projection: no network, no client, no store, no telemetry. Retrieving the -content those references point at is a separate layer. +The public retrieval surface: projecting the skill references a resolved AI +Config carries, and retrieving skill content through an injectable store seam. + +The three layers of the feature sit in three modules, and the dependencies run +one way only: + +- ``skills_core.py`` — the store and telemetry seams, module state, integrity + verification, and store resolution. Shared, and imports neither of the others. +- ``skills.py`` (this file) — ``skill_refs``, the accessors, and + ``InMemorySkillStore``. +- ``skills_fs.py`` — the highest-blast-radius layer, the one that writes to a + customer's disk. It owns the manifest format and the on-disk filenames; + nothing here knows about the filesystem. + +``_set_store``, ``_set_emitter_for_testing`` and ``_clear_state`` live here +because this module is the documented injection path; the +state they mutate lives in ``skills_core``. """ from __future__ import annotations import logging +from collections.abc import Callable, Sequence +from typing import Any -from .types import AiConfigRep, SkillReference +from . import skills_core +from .skills_core import ( + SKILL_OBJECT_KIND, + list_raw_objects, + log_withholding_summary, + newest_by_key, + reference_target, + require_store, + resolve_from_store, + verify_raw_skill, +) +from .types import AiConfigRep, Skill, SkillReference from .types_validation import ( is_valid_skill_key, is_valid_skill_version, @@ -19,16 +46,140 @@ logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Injection points +# --------------------------------------------------------------------------- +# +# These three names are the documented seam: ``init_client`` and ``shutdown`` +# call them, and tests inject through them. They delegate to ``skills_core``, +# which owns the state, so that there is exactly one store and one emitter no +# matter which layer reaches for them. + + +# Bound directly to the implementations rather than wrapped: a one-line +# delegation per name would give every state mutation two definitions and two +# docstrings to keep in agreement, which is the drift these names exist to +# avoid. ``_set_emitter_for_testing`` keeps its distinct name because it has no +# production caller. +_set_store = skills_core.set_store +_set_emitter_for_testing = skills_core.set_emitter +_clear_state = skills_core.clear_state + + +class InMemorySkillStore: + """ + A skill store backed by plain dicts. + + Ships for local development, tests, and bring-your-own-content injection. + Holds raw wire objects verbatim and performs no validation of its own — + verification belongs at the accessor boundary, where it applies to every + store equally. + + Several versions of one key coexist here, because they coexist in a real + delivery payload: the newest version of every skill, plus every version a + variation currently pins. ``get_object`` therefore selects on + ``(key, version)``, and ``version=None`` means "the newest held". + + An object whose ``version`` is not an integer >= 1 is still accepted and + still served, under its key alone. Withholding it is verification's job, not + the store's: a store that quietly refused it would make a malformed object + indistinguishable from an absent one, and no integrity signal would be + recorded. + """ + + def __init__(self, objects: dict[str, dict[str, Any]] | None = None) -> None: + self._versions: dict[str, dict[int, dict[str, Any]]] = {} + self._loose: dict[str, dict[str, Any]] = {} + self._listeners: dict[str, list[Callable[[dict[str, Any]], Any]]] = {} + for object_key, raw in (objects or {}).items(): + self._place(object_key, raw) + + def _place(self, fallback_key: str, raw: dict[str, Any]) -> None: + """Files one raw object under its own identity, verbatim.""" + key = raw.get("key") if isinstance(raw, dict) else None + if not isinstance(key, str): + key = fallback_key + version = raw.get("version") if isinstance(raw, dict) else None + if is_valid_skill_version(version): + self._versions.setdefault(key, {})[version] = raw + else: + self._loose[key] = raw + + def put(self, raw: dict[str, Any]) -> None: + """ + Adds or replaces a raw skill object, keyed by its own ``key`` and + ``version`` fields. + + Putting a second version of a key keeps both; putting the same + ``(key, version)`` twice replaces it. + + Notifies every skill-kind listener with the raw object as a single + positional argument. No validation happens here — verification belongs at + the accessor boundary, where it applies to every store equally — so a + listener sees exactly what was put, unverified. + """ + key = raw.get("key") + if not isinstance(key, str): + raise ValueError("a raw skill object must carry a string 'key'") + self._place(key, raw) + for listener in self._listeners.get(SKILL_OBJECT_KIND, []): + listener(raw) + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + if kind != SKILL_OBJECT_KIND: + return None + held = self._versions.get(key, {}) + if version is not None: + # Fall through to the version-less entry when the pin does not match + # anything well-formed, so a malformed object reaches verification and + # is withheld with a signal rather than reading as simply absent. + return held.get(version) or self._loose.get(key) + if held: + return held[max(held)] + return self._loose.get(key) + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + """ + Every object held, one entry per ``(key, version)``. + + The dict keys are opaque store-internal identifiers, as ``SkillStore`` + documents. Do not parse them and do not assume one entry per skill key. + """ + if kind != SKILL_OBJECT_KIND: + return {} + out: dict[str, dict[str, Any]] = { + f"{key}:{version}": raw + for key, versions in self._versions.items() + for version, raw in versions.items() + } + out.update(self._loose) + return out + + def add_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Registers *fn* to be called with each raw object ``put`` under *kind*. + + Only ``kind == SKILL_OBJECT_KIND`` is ever notified, because ``put`` only + accepts skill objects; a listener registered under any other kind is + recorded and never fires. + """ + self._listeners.setdefault(kind, []).append(fn) + # --------------------------------------------------------------------------- # Reference discovery # --------------------------------------------------------------------------- + + def skill_refs(config: AiConfigRep | None) -> list[SkillReference]: """ Projects a resolved AI Config's ``skills`` array into typed references. A pure projection — no network, no client, no store, no telemetry. Returns - ``[]`` when the config carries no skills. + ``[]`` when the config carries no skills. Compose it with the accessors for + per-context resolution: ``await get_skills(skill_refs(config))``. A config that came through ``parse_ai_config`` never contains an invalid entry — parsing fails closed on one. A hand-built dict can, and a silently @@ -70,3 +221,82 @@ def skill_refs(config: AiConfigRep | None) -> list[SkillReference]: else: refs.append(SkillReference(key=key, version=version)) return refs + + +# --------------------------------------------------------------------------- +# Content accessors +# --------------------------------------------------------------------------- + + +async def get_skill(key: str, *, version: int | None = None) -> Skill | None: + """ + Retrieves one verified skill by key. + + ``version=None`` means the newest version the store holds; a specific + ``version`` asks the store for that version and returns it only when the + store answers with it. A payload holding several versions of one key + resolves a pin to the pinned version, not to the newest. + Returns ``None`` — never raises — when the skill is missing, the requested + version is not the one held, or verification fails. Raises ``RuntimeError`` + only when no skill store is configured. + + There is no context parameter: skills have no targeting, so the SDK + credentials fully determine availability. Compose per-context resolution + explicitly with ``get_skills(skill_refs(config))``. + """ + return resolve_from_store(require_store(), key, version).skill + + +async def get_skills(refs: Sequence[SkillReference | str]) -> list[Skill]: + """ + Retrieves a batch of verified skills. + + Accepts a mixed sequence of ``SkillReference`` values and bare key strings, + where a string means "the latest version". Results follow input order for + the skills that were found; entries that are missing, are the wrong version, + or fail verification are omitted rather than returned as placeholders — and + a run that omitted anything logs a count at WARN, so a batch that resolved + nothing is not silent. + """ + if isinstance(refs, str): + # str satisfies Sequence[str], so this type-checks; iterating it would + # silently look up one skill per character. + raise TypeError( + "get_skills takes a sequence of references; pass [key] rather than a " + f"bare string. Got {refs!r}." + ) + + store = require_store() + + requests = list(refs) + skills: list[Skill] = [] + for ref in requests: + key, wanted = reference_target(ref) + skill = resolve_from_store(store, key, wanted).skill + if skill is not None: + skills.append(skill) + log_withholding_summary("requested skills", len(requests), len(skills)) + return skills + + +async def all_skills() -> list[Skill]: + """ + Retrieves every verified skill the store currently holds. + + Skills that fail verification are omitted. Raises ``RuntimeError`` only when + no skill store is configured. + """ + objects, error = list_raw_objects(require_store()) + if error is not None: + return [] + + # One entry per key at its newest version: ``all_objects`` may hold several + # versions of one key, and a list carrying two of them is not a set of skills. + candidates = newest_by_key(objects) + skills: list[Skill] = [] + for _object_key, raw in candidates: + skill = verify_raw_skill(raw) + if skill is not None: + skills.append(skill) + log_withholding_summary("skills held by the store", len(candidates), len(skills)) + return skills diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py new file mode 100644 index 00000000..c478be93 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -0,0 +1,628 @@ +""" +Agent Skills — the internals ``skills`` and ``skills_fs`` both need. + +Extracted so the two layers above it share one implementation through an +explicit surface instead of reaching into each other's privates. Everything +here is package-internal — nothing in this module is exported from +``launchdarkly_ai_server`` except the two constants that are public API — and +the dependency runs one way: this module imports neither ``skills`` nor +``skills_fs``. + +What lives here, and why it has to be one copy: + +- **The store seam and the configured store.** One place holds the store, so + the accessors and the materialization path cannot disagree about whether one + is configured. +- **The telemetry seam.** Every signal the feature can emit is constructed by a + ``record_*`` function in this file and nowhere else, which is what makes the + three-signal allowlist enforceable by reading one section. ``emit`` is never + called from outside this module. +- **Integrity verification.** ``verified_bytes`` runs twice per skill by design + — once at the accessor boundary, and again immediately before a write, since a + ``Skill`` can also be constructed directly by a caller. Sharing the + implementation is what keeps the two passes from drifting — the signal's + property keys must match whichever layer caught the defect. +- **Store resolution.** ``resolve_from_store`` is the fetch-and-verify sequence + the accessors and the reconcile share, so its call sites cannot drift apart — + in particular on how a raising store is handled. + +Everything the store hands back is untrusted input; the transport is not part of +the trust boundary. Key, version, size, and content hash are revalidated here on +every pass. + +The store and emitter are injected through ``skills._set_store`` and +``skills._set_emitter_for_testing`` — those names are the documented seam, +and they delegate here. +""" + +from __future__ import annotations + +import hashlib +import logging +import re +from dataclasses import dataclass +from typing import Any, Protocol + +from .types import Skill, SkillReference +from .types_validation import is_valid_skill_key, is_valid_skill_version + +logger = logging.getLogger(__name__) + +SKILL_OBJECT_KIND = "skill" +""" +The kind this SDK asks a store for. + +An **internal seam value**, deliberately not exported from the package root. It +is the string ``skills.py`` and ``skills_fs.py`` pass to ``SkillStore.get_object`` +and ``SkillStore.all_objects``, and a store adapter is free to map it onto +whatever the transport underneath actually uses — a delivery payload may well +carry skills under a broader kind with a narrower category, in which case +translating that pair to this one value is the adapter's job. + +Exporting it would publish an SDK-side seam string as though it were the wire +contract, which is a claim this side cannot make and would be hard to walk back +once a caller depends on it. A store that needs to agree on a kind agrees with +whatever the SDK hands it, which is this constant reached through +``launchdarkly_ai_server.skills_core``. +""" + +MAX_SKILL_CONTENT_BYTES = 64 * 1024 +""" +Hard cap on skill content. Legitimately delivered skills are well under this +bound, so anything larger is withheld regardless of whether its hash checks out. + +Deliberately **not** exported from the package root, unlike the on-disk and +on-the-wire constants beside it. Those are values this SDK defines and a caller +may need to agree with; this one is a local enforcement bound on content the +platform produces, so publishing it would semver-lock a number this side does +not own — and a caller pre-flighting "will my skill fit?" against it would be +reading the client's guess rather than the real limit. The reason string from +``verified_bytes`` already reports the bound when it is what withheld content. +""" + +_LANGUAGE = "python" + +_SHA256_HEX = re.compile(r"\A[0-9a-f]{64}\Z") +"""What a legitimate content hash looks like. Anything else is redacted before +it reaches telemetry — ``contentHash`` is attacker-controlled, and a store that +put the skill body there would otherwise leak it into a signal.""" + +_SIGNAL_INTEGRITY_FAILURE = "AgentControl Skill Integrity Failure" +_SIGNAL_MATERIALIZED = "AgentControl Skill Materialized" +_SIGNAL_REVOKED = "AgentControl Skill Revoked Received" + +NO_STORE_MESSAGE = ( + "No skill store is configured, so skill content cannot be retrieved. Configure " + 'one with init_client(options={"skillStore": store}) — InMemorySkillStore is ' + "available for local development and testing." +) + + +# --------------------------------------------------------------------------- +# The store seam +# --------------------------------------------------------------------------- + + +class SkillStore(Protocol): + """ + Structural interface every source of skill content satisfies. + + Duck-typed on purpose, mirroring how the LaunchDarkly client interface works + in this package: pass any object carrying these methods. + + ``add_listener(kind, fn)`` is part of the seam but + **optional**, which is why it is deliberately not declared here: a Protocol + member is required for structural compatibility, so declaring it would reject + every store that does not implement it. Nothing in this module calls it — it + exists for the delivery transport to push updates through. + + The raw objects a store serves are wire-shaped, with camelCase field names + identical across language implementations:: + + {"key": "pdf-extraction", "version": 2, "content": "---\\n...", + "contentHash": "9f3a...", "name": "PDF Extraction", "description": "..."} + + **Version is part of the lookup identity, not a filter applied afterwards.** + A delivery payload holds the newest version of every skill *and* every + version any variation currently pins, so two versions of one key coexist + routinely. A seam keyed by key alone cannot express "the one this variation + pinned": it would answer with the newest and the caller would then have to + reject it, which turns a pinned reference into a missing skill. So + ``get_object`` takes the wanted version, and ``version=None`` means "the + newest you hold". + + ``all_objects`` returns one entry per *(key, version)* the store holds. Its + dict keys are **opaque store-internal identifiers** — do not parse them, and + do not assume one entry per skill key. Identity is read off each object's own + ``key`` and ``version`` fields, which are revalidated here anyway because + everything a store serves is untrusted. + """ + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: ... + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: ... + + +# --------------------------------------------------------------------------- +# Telemetry seam +# --------------------------------------------------------------------------- + + +class _TelemetryEmitter(Protocol): + def record(self, signal: str, properties: dict[str, Any]) -> None: ... + + +class _NoOpEmitter: + """ + The default emitter. + + No skills telemetry leaves the process in this release: ``client.track()`` is + the wrong channel (it needs an LD context, spends the customer's event + volume, and lands in their data export), and the diagnostic-event channel + that would be right has no wrapper-SDK extension point yet. Signals are + recorded through this seam so the eventual transport drops in behind it + without touching a single call site. + """ + + def record(self, signal: str, properties: dict[str, Any]) -> None: + return None + + +_NOOP_EMITTER: _TelemetryEmitter = _NoOpEmitter() + +# --------------------------------------------------------------------------- +# Module state +# --------------------------------------------------------------------------- + +_store: SkillStore | None = None +_emitter: _TelemetryEmitter = _NOOP_EMITTER +"""Never ``None``: "no emitter installed" is spelled as the no-op, so ``emit`` +has one code path instead of re-deciding on every signal.""" + + +def set_store(store: Any) -> None: + """ + Replaces the configured store. + + Reached through ``skills._set_store``, which is the documented seam; see that + function for who calls it and why it has no test-only twin. + """ + global _store + _store = store + + +def set_emitter(emitter: Any) -> None: + """Replaces the telemetry emitter. Reached through + ``skills._set_emitter_for_testing``.""" + global _emitter + _emitter = emitter + + +def clear_state() -> None: + """Drops both the store and the emitter. Reached through ``skills._clear_state``.""" + global _store, _emitter + _store = None + _emitter = _NOOP_EMITTER + + +def get_store() -> SkillStore | None: + """The configured store, or ``None``. The only reader of the global.""" + return _store + + +def require_store() -> SkillStore: + store = get_store() + if store is None: + raise RuntimeError(NO_STORE_MESSAGE) + return store + + +def emit(signal: str, properties: dict[str, Any]) -> None: + """ + Records one signal. Never raises into the calling operation — a broken + emitter must not be able to fail a retrieval or a reconcile. + """ + try: + _emitter.record(signal, properties) + except Exception: + logger.warning("Skills telemetry emitter raised; ignoring", exc_info=True) + + +def record_integrity_failure( + skill_key: str, + reason: str, + *, + version: Any = None, + expected_hash: Any = None, + observed_hash: str | None = None, +) -> None: + """ + Records an integrity failure. Carries hashes and byte counts only — the skill + body never appears in a signal, a log line, or an error message. + """ + # Both of these come off the wire, so neither may be echoed verbatim: a store + # that set contentHash (or key) to the skill body would otherwise publish the + # body itself. Shape-check, then redact. + safe_key = skill_key if is_valid_skill_key(skill_key) else "" + properties: dict[str, Any] = {"skill_key": safe_key, "language": _LANGUAGE} + if is_valid_skill_version(version): + properties["version"] = version + if isinstance(expected_hash, str): + properties["expected_hash"] = ( + expected_hash + if _SHA256_HEX.match(expected_hash) + else "" + ) + if observed_hash is not None: + properties["observed_hash"] = observed_hash + + logger.error("Skill '%s' failed integrity verification: %s", safe_key, reason) + emit(_SIGNAL_INTEGRITY_FAILURE, properties) + + +def record_materialized( + skill_key: str, content_bytes: int, content_hash: str, reconcile_action: str +) -> None: + """ + Records a materialization. Deliberately carries no ``target_path`` and no + filesystem path of any kind — the same reasoning that keeps the skill body + out of telemetry keeps the customer's directory layout out. Paths live in the + returned ``ReconcileReport``, which is user-facing API rather than telemetry. + """ + emit( + _SIGNAL_MATERIALIZED, + { + "skill_key": skill_key, + "content_bytes": content_bytes, + "content_hash": content_hash, + "reconcile_action": reconcile_action, + "language": _LANGUAGE, + }, + ) + + +def record_revoked(skill_key: str, version: Any) -> None: + """ + Records a revocation — a prune that removed a formerly managed skill. + + Lives here with the other two recorders rather than at the prune site so the + signal allowlist is maintained in one place: every signal this SDK can emit + is visible in this section of this module, and nothing outside it touches + ``emit``. + """ + # Both fields come off the manifest, which is untrusted — same rule as + # ``record_integrity_failure``: shape-check, then redact, so a hand-edited + # manifest cannot plant an arbitrary string in a signal. + safe_key = skill_key if is_valid_skill_key(skill_key) else "" + properties: dict[str, Any] = { + "skill_key": safe_key, + "removed_from_disk": True, + "language": _LANGUAGE, + } + if is_valid_skill_version(version): + properties["version"] = version + emit(_SIGNAL_REVOKED, properties) + + +# --------------------------------------------------------------------------- +# Integrity verification +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class VerifiedContent: + """Content that passed integrity verification.""" + + encoded: bytes + """The verbatim bytes, exactly as hashed.""" + content_hash: str + """The locally computed sha256 — never the caller's expected value.""" + + +@dataclass(frozen=True) +class VerificationFailure: + """Why content did not pass. The reason is safe to show a caller.""" + + reason: str + + +def verified_bytes( + key: str, content: str | bytes, expected_hash: str, version: int +) -> VerifiedContent | VerificationFailure: + """ + The whole content half of integrity verification: encode, size, hash. + + Accepts either shape content legitimately arrives in. Wire-shaped ``str`` + input — a raw store object's JSON string — is UTF-8 encoded here, once, and + this is the only place that encode happens. ``bytes`` input is an already + verified ``Skill.content`` being re-verified, and is hashed directly: those + bytes are the verbatim value, so re-encoding does not apply. + + Returns the verbatim bytes and their locally computed sha256, or a + human-readable reason — having already recorded the integrity signal, so the + signal's property set cannot depend on which caller noticed. The hash handed + back is the one computed here, never the caller's expected value: the two are + equal on this path by construction, and returning the locally derived one + keeps an attacker-supplied string out of ``Skill``. + + The two outcomes are distinct types rather than a ``tuple | str`` union so a + call site reads as "verification failed" instead of "the result is a string", + and so a future success payload carrying a ``str`` cannot silently invert the + discrimination. + + This runs twice per skill by design: once at the accessor boundary, and again + immediately before a write, because a ``Skill`` can also be constructed + directly by a caller. Sharing the implementation is what keeps those two + passes from drifting — the property keys must match. + + The second pass re-hashes bytes the first pass already hashed. That + redundancy is deliberate: it is negligible next to the write it guards, and + carrying the first pass's verdict forward would put a "trust the value + computed upstream" branch inside the one function whose entire job is not to. + """ + if isinstance(content, bytes): + encoded = content + else: + try: + encoded = content.encode("utf-8") + except UnicodeEncodeError: + # json.loads turns a "\ud800" escape into an unpaired surrogate, which + # has no UTF-8 encoding. There are no bytes the server could have + # hashed, so this is not authentic content. Never use + # errors="surrogatepass" here: that would fabricate bytes and could + # satisfy the hash comparison. + reason = "content is not encodable as UTF-8" + record_integrity_failure( + key, reason, version=version, expected_hash=expected_hash + ) + return VerificationFailure(reason) + + if len(encoded) > MAX_SKILL_CONTENT_BYTES: + reason = ( + f"content is {len(encoded)} bytes, over the " + f"{MAX_SKILL_CONTENT_BYTES} byte cap" + ) + record_integrity_failure( + key, reason, version=version, expected_hash=expected_hash + ) + return VerificationFailure(reason) + + # sha256, lowercase hex, over the verbatim bytes — no canonicalization and + # no content parsing of any kind anywhere in the integrity path. + observed_hash = hashlib.sha256(encoded).hexdigest() + if observed_hash != expected_hash: + record_integrity_failure( + key, + "content hash mismatch", + version=version, + expected_hash=expected_hash, + observed_hash=observed_hash, + ) + return VerificationFailure("content hash mismatch") + + return VerifiedContent(encoded=encoded, content_hash=observed_hash) + + +def verify_raw_skill(raw: Any) -> Skill | None: + """ + Turns one untrusted raw store object into a ``Skill``, or withholds it. + + On any failure the skill is treated as missing, the integrity signal is + recorded, and an error is logged. No unverified content is ever returned to + user code. + """ + if not isinstance(raw, dict): + record_integrity_failure("", "raw skill object is not an object") + return None + + key = raw.get("key") + if not is_valid_skill_key(key): + record_integrity_failure( + key if isinstance(key, str) else "", + "key is not a valid skill key", + ) + return None + + version = raw.get("version") + if not is_valid_skill_version(version): + record_integrity_failure(key, "version is not an integer >= 1") + return None + + content = raw.get("content") + if not isinstance(content, str): + record_integrity_failure( + key, "content is missing or not a string", version=version + ) + return None + + expected_hash = raw.get("contentHash") + if not isinstance(expected_hash, str): + record_integrity_failure( + key, "contentHash is missing or not a string", version=version + ) + return None + + verified = verified_bytes(key, content, expected_hash, version) + if isinstance(verified, VerificationFailure): + return None + + name = raw.get("name") + description = raw.get("description") + return Skill( + key=key, + version=version, + content=verified.encoded, + content_hash=verified.content_hash, + name=name if isinstance(name, str) else None, + description=description if isinstance(description, str) else None, + ) + + +def log_withholding_summary(subject: str, requested: int, resolved: int) -> None: + """ + One WARN per run when content was withheld, naming the counts. + + Every individual withholding already records an integrity signal and an error + log line, but a caller reading logs at WARN sees neither. That matters most in + the case where *nothing* verified — a payload built before ``contentHash`` is + populated, say — because the feature then returns an empty result that is + indistinguishable from "this project has no skills". A run-level summary is + the difference between a silent no-op and a visible one. + + Called once per batch retrieval, not once per skill, so a large withholding + run does not itself become the noise. + """ + withheld = requested - resolved + if withheld <= 0: + return + if resolved == 0: + logger.warning( + "All %d %s were withheld and no skill content is available. Every " + "object failed verification — check that the delivered objects carry " + "a contentHash matching the sha256 of their content.", + requested, + subject, + ) + return + logger.warning( + "%d of %d %s were withheld and are unavailable; see the preceding errors " + "for the per-skill reason.", + withheld, + requested, + subject, + ) + + +def store_raised(exc: Exception) -> str: + """The one wording for "the store could not answer", used by every path.""" + return f"the skill store raised {type(exc).__name__}: {exc}" + + +def list_raw_objects( + store: SkillStore, +) -> tuple[dict[str, dict[str, Any]], str | None]: + """ + Every raw object the store holds, or the reason it could not answer. + + One entry per *(key, version)*, under keys that are opaque to this SDK — see + ``SkillStore``. Callers that need one skill per key have to collapse the + result themselves; ``newest_by_key`` does it. + + Returns the reason rather than raising, because both callers need the + distinction between "no skills" and "the store is broken" — and they need it + worded identically. Letting the exception out instead would make each of + them re-derive the log line and the message, which is the drift this module + exists to prevent. + """ + try: + objects = store.all_objects(SKILL_OBJECT_KIND) + except Exception as exc: + logger.error("Skill store raised while listing skills", exc_info=True) + return {}, store_raised(exc) + return (objects if isinstance(objects, dict) else {}), None + + +def newest_by_key(objects: dict[str, dict[str, Any]]) -> list[tuple[str, Any]]: + """ + One raw object per skill key — the highest version of each, paired with the + store key it was served under. + + ``all_objects`` may hold several versions of one key, and both callers that + consume the whole store want one skill per key: ``all_skills`` because a list + holding two versions of one key is not a set of skills, and the ``"*"`` + reconcile because ``//SKILL.md`` is a single path and writing it + twice in one run is a bug rather than a policy. + + The store key is carried through rather than discarded because the reconcile + attributes a failure to it when the object's own key is unusable. + + Objects too malformed to carry a usable key and version are **kept**, not + dropped, so verification is what withholds them: a silently dropped object + falls out of the requested set, and prune would then delete the last + known-good copy already on disk. + """ + best: dict[str, tuple[str, Any]] = {} + unusable: list[tuple[str, Any]] = [] + for object_key, raw in objects.items(): + skill_key = raw.get("key") if isinstance(raw, dict) else None + version = raw.get("version") if isinstance(raw, dict) else None + if not is_valid_skill_key(skill_key) or not is_valid_skill_version(version): + unusable.append((object_key, raw)) + continue + held = best.get(skill_key) + if held is None or version > held[1]["version"]: + best[skill_key] = (object_key, raw) + return list(best.values()) + unusable + + +# --------------------------------------------------------------------------- +# Resolution internals — shared with the materialization path +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Resolution: + """One key resolved against a store: the skill, or why there is none.""" + + skill: Skill | None = None + error: str | None = None + unavailable: bool = False + """ + ``True`` when the *store* could not answer — it raised — rather than when it + answered "no". Only the former suppresses pruning: deleting managed files + because a lookup failed would turn an outage into data loss. + """ + + +def resolve_from_store( + store: SkillStore, key: str, wanted_version: int | None +) -> Resolution: + """ + Fetches one key and verifies it — the sequence the accessors and the + materialization path share. + + Written once on purpose, so the call sites cannot drift apart — in + particular on the policy for a raising store. + + ``wanted_version`` goes *into* the lookup, because a store may hold several + versions of one key and only it can pick between them; ``None`` asks for the + newest. The equality check afterwards is kept as a **defense**, not as the + selection mechanism: the store is untrusted, so an answer that is not the + version that was asked for is withheld rather than returned. + """ + try: + raw = store.get_object(SKILL_OBJECT_KIND, key, wanted_version) + except Exception as exc: + logger.error("Skill store raised while retrieving '%s'", key, exc_info=True) + return Resolution(error=store_raised(exc), unavailable=True) + + if not isinstance(raw, dict): + return Resolution( + error=f"skill '{key}' is not available from the configured skill store" + ) + + skill = verify_raw_skill(raw) + if skill is None: + return Resolution( + error=f"skill '{key}' failed integrity verification and was withheld" + ) + if wanted_version is not None and skill.version != wanted_version: + return Resolution( + error=( + f"skill '{key}' version {wanted_version} is not available " + f"(the store holds version {skill.version})" + ) + ) + return Resolution(skill=skill) + + +def reference_target(item: SkillReference | str) -> tuple[str, int | None]: + """Normalises a reference-or-key into ``(key, wanted version)``. + + A bare string means "the latest version the store holds". + """ + if isinstance(item, str): + return item, None + return item.key, item.version diff --git a/packages/client/tests/conftest.py b/packages/client/tests/conftest.py index 6c140200..17fbb68b 100644 --- a/packages/client/tests/conftest.py +++ b/packages/client/tests/conftest.py @@ -1,7 +1,14 @@ +import hashlib +from collections.abc import Iterator +from typing import Any from unittest.mock import AsyncMock, MagicMock import pytest +import launchdarkly_ai_server.lifecycle as lifecycle_module +import launchdarkly_ai_server.skills as skills_module +from launchdarkly_ai_server import InMemorySkillStore + @pytest.fixture def mock_ld_client() -> MagicMock: @@ -36,3 +43,108 @@ def mock_tracer(mock_span: MagicMock) -> MagicMock: tracer.start_as_current_span.return_value.__exit__ = MagicMock(return_value=False) tracer.start_span.return_value = mock_span return tracer + + +# --------------------------------------------------------------------------- +# Agent Skills helpers +# +# Exposed as fixtures rather than importable module-level helpers: pytest runs +# with --import-mode=importlib and the tests directory is not a package, so +# sibling imports from conftest are not reliable. +# --------------------------------------------------------------------------- + +SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" + + +class _RecordingEmitter: + """Telemetry seam double — records (signal, properties) pairs.""" + + def __init__(self) -> None: + self.records: list[tuple[str, dict[str, Any]]] = [] + + def record(self, signal: str, properties: dict[str, Any]) -> None: + self.records.append((signal, properties)) + + def signals(self, name: str) -> list[dict[str, Any]]: + return [props for sig, props in self.records if sig == name] + + +class _ThrowingEmitter: + """Telemetry seam double whose record() always raises.""" + + def record(self, signal: str, properties: dict[str, Any]) -> None: + raise RuntimeError("emitter exploded") + + +@pytest.fixture +def make_raw_skill() -> Any: + """Factory for wire-shaped raw store objects with a correct contentHash.""" + + def _make( + key: str = "test-skill", + version: int = 1, + content: str = SKILL_BODY, + **overrides: Any, + ) -> dict[str, Any]: + obj: dict[str, Any] = { + "key": key, + "version": version, + "content": content, + "contentHash": hashlib.sha256(content.encode("utf-8")).hexdigest(), + "name": "Test Skill", + "description": "A skill used in tests.", + } + obj.update(overrides) + return obj + + return _make + + +@pytest.fixture +def store() -> InMemorySkillStore: + """An in-memory store, wired in as the configured store for the test.""" + s = InMemorySkillStore() + skills_module._set_store(s) + return s + + +class _ExplodingStore: + """Store double whose every read raises — the "transport is down" case.""" + + def get_object(self, kind: str, key: str) -> dict[str, Any] | None: + raise RuntimeError("transport failure") + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + raise RuntimeError("transport failure") + + +@pytest.fixture +def exploding_store() -> _ExplodingStore: + """A raising store, wired in as the configured store for the test.""" + s = _ExplodingStore() + skills_module._set_store(s) + return s + + +@pytest.fixture +def reset_skill_state() -> Iterator[None]: + """Clears client, store, and emitter module state around one test. + + Opted into per module with ``pytestmark = pytest.mark.usefixtures(...)`` + rather than being autouse here: autouse would newly reset lifecycle state + for every test in every module in this directory, which is a behaviour change + well outside the skills tests. + """ + lifecycle_module._reset_for_testing() + yield + lifecycle_module._reset_for_testing() + + +@pytest.fixture +def recording_emitter() -> _RecordingEmitter: + return _RecordingEmitter() + + +@pytest.fixture +def throwing_emitter() -> _ThrowingEmitter: + return _ThrowingEmitter() diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 38ca34f2..e99a8957 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -1,24 +1,57 @@ """ -Agent Skills — reference discovery: the types and the projection from a -resolved AI Config. +Tests for Agent Skills types, reference discovery, content accessors, +integrity verification, and the telemetry seam. """ from __future__ import annotations import dataclasses import hashlib +import json from typing import Any +from unittest.mock import MagicMock import pytest +import launchdarkly_ai_server.lifecycle as lifecycle_module +import launchdarkly_ai_server.skills as skills_module from launchdarkly_ai_server import ( + InMemorySkillStore, Skill, SkillReference, + all_skills, + get_client, + get_skill, + get_skills, + init_client, + shutdown, skill_refs, ) SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" +INTEGRITY_SIGNAL = "AgentControl Skill Integrity Failure" +MATERIALIZED_SIGNAL = "AgentControl Skill Materialized" +REVOKED_SIGNAL = "AgentControl Skill Revoked Received" + +# The three signal names are an allowlist, not a floor: any +# other name reaching the emitter is a regression. +APPROVED_SIGNALS = frozenset({INTEGRITY_SIGNAL, MATERIALIZED_SIGNAL, REVOKED_SIGNAL}) + +# These two were considered and deliberately excluded from SDK emission — +# named explicitly rather than relying on the subset check to be read as +# covering them. +REMOVED_SIGNALS = frozenset( + { + "AgentControl Skill SDK Reference Returned", + "AgentControl Skill Content Retrieved", + } +) + + +pytestmark = pytest.mark.usefixtures("reset_skill_state") +"""Every test in this module runs against freshly cleared module state.""" + def _hash(content: str) -> str: """sha256, lowercase hex, over verbatim utf-8 bytes.""" @@ -40,6 +73,61 @@ def _skill( ) +# --------------------------------------------------------------------------- + +UNENCODABLE_BODIES = ( + json.loads(r'"hi \ud800 there"'), # lone high surrogate + # A lone *low* surrogate, which is the one range errors="surrogateescape" + # smuggles through (as a raw 0x80 byte) while raising on everything else. + json.loads(r'"hi \udc80 there"'), +) + +NON_STRICT_HANDLERS = ( + "surrogatepass", + "surrogateescape", + "replace", + "ignore", + "backslashreplace", + "xmlcharrefreplace", + "namereplace", +) +"""Every ``str.encode`` error handler that is not ``strict``. + +``verified_bytes`` must use none of them: each one *fabricates* bytes for input +that has no encoding, and fabricated bytes can satisfy the hash comparison. +""" + + +def _fabricated_hash_cases() -> list[Any]: + """One case per (body, handler) pair the handler can actually encode. + + Each carries the sha256 of the bytes *that* handler would have produced, so + the case is not vacuous: an implementation that reached for the handler + would encode successfully, match the pinned hash, and return content + LaunchDarkly never delivered. Handlers that raise on a given body are + skipped — for that input they are as strict as ``strict``, so there is + nothing to detect. + """ + cases: list[Any] = [] + for index, body in enumerate(UNENCODABLE_BODIES): + for handler in NON_STRICT_HANDLERS: + try: + fabricated = body.encode("utf-8", errors=handler) + except UnicodeEncodeError: + continue + cases.append( + pytest.param( + body, + hashlib.sha256(fabricated).hexdigest(), + id=f"body{index}-{handler}", + ) + ) + return cases + + +FABRICATED_HASH_CASES = _fabricated_hash_cases() + + class TestSkillTypes: """Immutability and optional metadata.""" @@ -106,6 +194,11 @@ def test_returns_typed_references_in_order(self) -> None: ] assert all(isinstance(r, SkillReference) for r in refs) + def test_emits_no_telemetry(self, recording_emitter: Any) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + skill_refs(self._config(skills=[{"key": "a", "version": 1}])) + assert recording_emitter.records == [] + def test_dropped_entries_are_logged(self, caplog: pytest.LogCaptureFixture) -> None: """A shortened projection is never silent. @@ -137,3 +230,925 @@ def test_requires_no_client_or_store(self, mock_ld_client: Any) -> None: refs = skill_refs(self._config(skills=[{"key": "a", "version": 2}])) assert refs == [SkillReference(key="a", version=2)] mock_ld_client.track.assert_not_called() + + +class TestPackageExports: + """ + What is and is not part of the public surface. + + The literal values are spelled out on purpose: this is the one place the + constants themselves are asserted, so importing them to build the + expectation would make the assertion circular. + """ + + def test_content_cap_is_not_public_api(self) -> None: + """The content cap stays internal to ``skills_core`` — see the + ``MAX_SKILL_CONTENT_BYTES`` docstring there for why it is not exported.""" + import launchdarkly_ai_server as package + from launchdarkly_ai_server import skills_core + + assert skills_core.MAX_SKILL_CONTENT_BYTES == 65536 + assert "MAX_SKILL_CONTENT_BYTES" not in package.__all__ + assert not hasattr(package, "MAX_SKILL_CONTENT_BYTES") + + def test_object_kind_is_not_public_api(self) -> None: + """The kind is an SDK-side seam value, not the wire contract. + + A store adapter maps whatever the transport calls a skill onto the value + this SDK passes it, so publishing the string would advertise a contract + this side does not own — and one that would be hard to walk back once a + caller depended on it. It stays reachable through ``skills_core`` for the + adapter that needs to agree with it. + """ + import launchdarkly_ai_server as package + from launchdarkly_ai_server import skills_core + + assert skills_core.SKILL_OBJECT_KIND == "skill" + assert "SKILL_OBJECT_KIND" not in package.__all__ + assert not hasattr(package, "SKILL_OBJECT_KIND") + + def test_retrieval_surface_is_exported_from_the_package_root(self) -> None: + """A name absent from ``__all__`` is not part of the public surface.""" + import launchdarkly_ai_server as package + + expected = { + "skill_refs", + "get_skill", + "get_skills", + "all_skills", + "SkillStore", + "InMemorySkillStore", + "Skill", + "SkillReference", + } + assert expected <= set(package.__all__) + + +class TestInMemorySkillStore: + """The public in-memory store implementation.""" + + def test_get_object_round_trips(self, make_raw_skill: Any) -> None: + raw = make_raw_skill(key="pdf-extraction", version=2) + s = InMemorySkillStore({"pdf-extraction": raw}) + assert s.get_object("skill", "pdf-extraction") == raw + + def test_get_object_unknown_key_returns_none(self) -> None: + assert InMemorySkillStore().get_object("skill", "nope") is None + + def test_put_then_get(self, make_raw_skill: Any) -> None: + s = InMemorySkillStore() + raw = make_raw_skill(key="a") + s.put(raw) + assert s.get_object("skill", "a") == raw + + def test_all_objects_returns_everything(self, make_raw_skill: Any) -> None: + """Asserted on the object bodies, not the dict keys. + + ``all_objects`` keys are opaque store-internal identifiers — the seam + documents them as such — so a test that pinned their spelling would be + asserting an implementation detail the contract disclaims. + """ + s = InMemorySkillStore() + first = make_raw_skill(key="a") + second = make_raw_skill(key="b") + s.put(first) + s.put(second) + held = s.all_objects("skill").values() + assert len(held) == 2 + assert first in held + assert second in held + + def test_all_objects_holds_every_version_of_one_key( + self, make_raw_skill: Any + ) -> None: + s = InMemorySkillStore() + v1 = make_raw_skill(key="a", version=1, content="one\n") + v2 = make_raw_skill(key="a", version=2, content="two\n") + s.put(v1) + s.put(v2) + held = list(s.all_objects("skill").values()) + assert len(held) == 2 + assert v1 in held + assert v2 in held + + def test_put_replaces_only_the_same_key_and_version( + self, make_raw_skill: Any + ) -> None: + s = InMemorySkillStore() + s.put(make_raw_skill(key="a", version=1, content="first\n")) + replacement = make_raw_skill(key="a", version=1, content="second\n") + s.put(replacement) + assert list(s.all_objects("skill").values()) == [replacement] + + def test_get_object_with_a_version_selects_that_version( + self, make_raw_skill: Any + ) -> None: + s = InMemorySkillStore() + v1 = make_raw_skill(key="a", version=1, content="one\n") + v2 = make_raw_skill(key="a", version=3, content="three\n") + s.put(v1) + s.put(v2) + assert s.get_object("skill", "a", 1) == v1 + assert s.get_object("skill", "a", 3) == v2 + + def test_get_object_without_a_version_selects_the_newest( + self, make_raw_skill: Any + ) -> None: + s = InMemorySkillStore() + s.put(make_raw_skill(key="a", version=1, content="one\n")) + newest = make_raw_skill(key="a", version=7, content="seven\n") + s.put(newest) + s.put(make_raw_skill(key="a", version=4, content="four\n")) + assert s.get_object("skill", "a") == newest + + def test_get_object_unknown_version_falls_back_to_a_malformed_object( + self, make_raw_skill: Any + ) -> None: + """A malformed object must reach verification, not read as absent. + + An object whose ``version`` is unusable is filed under its key alone. A + pinned lookup that finds nothing well-formed serves it anyway, so + verification withholds it and records an integrity signal — a store that + returned ``None`` here would make tampering indistinguishable from a + skill that was never delivered. + """ + s = InMemorySkillStore() + malformed = make_raw_skill(key="a", version="two") + s.put(malformed) + assert s.get_object("skill", "a", 2) == malformed + assert s.get_object("skill", "a") == malformed + + def test_all_objects_unknown_kind_is_empty(self, make_raw_skill: Any) -> None: + s = InMemorySkillStore() + s.put(make_raw_skill(key="a")) + assert s.all_objects("flag") == {} + + def test_put_notifies_skill_kind_listeners(self, make_raw_skill: Any) -> None: + """``add_listener`` is part of the seam, so its one + implementation carries a smoke test for the callback contract: the raw + object, verbatim and unverified, as a single positional argument.""" + s = InMemorySkillStore() + seen: list[dict[str, Any]] = [] + s.add_listener("skill", seen.append) + raw = make_raw_skill(key="a") + + s.put(raw) + + assert seen == [raw] + + def test_put_does_not_notify_other_kind_listeners( + self, make_raw_skill: Any + ) -> None: + s = InMemorySkillStore() + seen: list[dict[str, Any]] = [] + s.add_listener("flag", seen.append) + + s.put(make_raw_skill(key="a")) + + assert seen == [] + + +class TestStoreConfiguration: + """Store wiring on the lifecycle layer.""" + + async def test_configured_via_init_client_option( + self, make_raw_skill: Any, mock_ld_client: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + await init_client(options={"skillStore": store}, client=mock_ld_client) + skill = await get_skill("a") + assert skill is not None + assert skill.key == "a" + + async def test_get_skill_raises_actionably_when_no_store(self) -> None: + with pytest.raises(RuntimeError, match="skill store"): + await get_skill("a") + + async def test_get_skills_raises_actionably_when_no_store(self) -> None: + with pytest.raises(RuntimeError, match="skill store"): + await get_skills([SkillReference(key="a", version=1)]) + + async def test_all_skills_raises_actionably_when_no_store(self) -> None: + with pytest.raises(RuntimeError, match="skill store"): + await all_skills() + + async def test_shutdown_clears_the_store( + self, make_raw_skill: Any, mock_ld_client: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + await init_client(options={"skillStore": store}, client=mock_ld_client) + assert await get_skill("a") is not None + + await shutdown() + + with pytest.raises(RuntimeError, match="skill store"): + await get_skill("a") + + async def test_skill_store_is_applied_on_every_init_client_call( + self, make_raw_skill: Any + ) -> None: + """``skillStore`` is the one option a second call applies. + + ``init_client`` is idempotent for the client singleton, and on a second + call every other option is ignored. ``skillStore`` is applied anyway, + on purpose: it is what lets a client that was lazily auto-initialized, + or initialized without a store, be given one afterwards. Both halves are + asserted on the same pair of calls, because each is meaningless without + the other. + """ + first_store = InMemorySkillStore() + first_store.put(make_raw_skill(key="first")) + second_store = InMemorySkillStore() + second_store.put(make_raw_skill(key="second")) + + first_client = MagicMock() + second_client = MagicMock() + + await init_client(options={"skillStore": first_store}, client=first_client) + await init_client(options={"skillStore": second_store}, client=second_client) + + # Half one: the client singleton is unchanged — the second call is a + # no-op for it, so the second client was discarded. + assert get_client() is first_client + + # Half two: the store was nevertheless swapped. + assert await get_skill("second") is not None + assert await get_skill("first") is None + + async def test_init_client_without_a_store_leaves_the_configured_one( + self, make_raw_skill: Any + ) -> None: + """Only a non-None ``skillStore`` replaces the configured store. + + Otherwise a bare ``init_client()`` from an unrelated code path — the + lazy auto-init, say — would silently unconfigure skills. + """ + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + await init_client(options={"skillStore": store}, client=MagicMock()) + + await init_client(client=MagicMock()) + + assert await get_skill("a") is not None + + async def test_failed_init_client_leaves_no_store_configured( + self, monkeypatch: pytest.MonkeyPatch, make_raw_skill: Any + ) -> None: + """A raising ``init_client`` must not leave global state behind. + + Installing the store before the SDK-key check would leave the accessors + working against a store the application believes was never installed, + masking a failed initialization. + """ + monkeypatch.delenv("LD_SDK_KEY", raising=False) + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + + with pytest.raises(RuntimeError, match="No LaunchDarkly SDK key"): + await init_client(options={"skillStore": store}) + + with pytest.raises(RuntimeError, match="skill store"): + await get_skill("a") + + async def test_reset_for_testing_clears_the_store( + self, make_raw_skill: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + skills_module._set_store(store) + assert await get_skill("a") is not None + + lifecycle_module._reset_for_testing() + + with pytest.raises(RuntimeError, match="skill store"): + await get_skill("a") + + +class TestAccessorArgumentErrors: + """A bare string is a type error, not a reference. + + ``str`` satisfies ``Sequence[str]``, so the annotation on ``get_skills`` + admits a bare string and only this runtime guard catches it; iterating one + would look up a skill per character. Deliberately a *different* class from + ``write_skills``'s bare-string rejection, which is a ``ValueError`` because + a string is an accepted argument type there. + """ + + async def test_bare_string_raises_type_error( + self, store: InMemorySkillStore + ) -> None: + with pytest.raises(TypeError) as excinfo: + await get_skills("pdf-extraction") # type: ignore[arg-type] + + # The message has to name the fix, not merely reject the input. + assert "[key]" in str(excinfo.value) + + async def test_bare_string_is_rejected_before_the_store_is_consulted( + self, make_raw_skill: Any + ) -> None: + """The guard is an argument check, so it precedes store resolution. + + Asserting the raise alone would also pass if the string were iterated + into single-character lookups that all missed, so pin that no lookup + happened at all. + """ + looked_up: list[str] = [] + + class _RecordingStore: + def get_object(self, kind: str, key: str) -> dict[str, Any] | None: + looked_up.append(key) + return None + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + return {} + + skills_module._set_store(_RecordingStore()) + + with pytest.raises(TypeError): + await get_skills("abc") # type: ignore[arg-type] + + assert looked_up == [] + + +class TestGetSkill: + """Single-skill accessor.""" + + async def test_returns_verified_skill( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="pdf-extraction", version=2)) + skill = await get_skill("pdf-extraction") + assert skill is not None + assert skill.key == "pdf-extraction" + assert skill.version == 2 + assert skill.content == SKILL_BODY.encode("utf-8") + assert skill.content_hash == _hash(SKILL_BODY) + assert skill.name == "Test Skill" + assert skill.description == "A skill used in tests." + + async def test_version_omitted_returns_newest_available( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=7)) + skill = await get_skill("a") + assert skill is not None + assert skill.version == 7 + + async def test_exact_version_match_returns_skill( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=3)) + skill = await get_skill("a", version=3) + assert skill is not None + assert skill.version == 3 + + async def test_other_version_returns_none( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=3)) + assert await get_skill("a", version=2) is None + assert await get_skill("a", version=4) is None + + async def test_missing_key_returns_none(self, store: InMemorySkillStore) -> None: + assert await get_skill("nope") is None + + async def test_multibyte_content_verifies( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + content = "---\nname: emoji\n---\n🚀 unicode ✅ body\n" + store.put(make_raw_skill(key="a", content=content)) + skill = await get_skill("a") + assert skill is not None + assert skill.content == content.encode("utf-8") + + +class TestGetSkills: + """Batch accessor.""" + + async def test_mixed_refs_and_strings( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=1)) + store.put(make_raw_skill(key="b", version=5)) + result = await get_skills([SkillReference(key="a", version=1), "b"]) + assert [s.key for s in result] == ["a", "b"] + + async def test_preserves_input_order( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + for k in ("a", "b", "c"): + store.put(make_raw_skill(key=k)) + result = await get_skills(["c", "a", "b"]) + assert [s.key for s in result] == ["c", "a", "b"] + + async def test_missing_entries_are_omitted( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a")) + result = await get_skills(["a", "missing"]) + assert [s.key for s in result] == ["a"] + + async def test_version_mismatch_is_omitted( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=2)) + result = await get_skills([SkillReference(key="a", version=1)]) + assert result == [] + + async def test_empty_input_returns_empty_list( + self, store: InMemorySkillStore + ) -> None: + assert await get_skills([]) == [] + + async def test_integrity_failure_omitted_and_signalled( + self, + store: InMemorySkillStore, + make_raw_skill: Any, + recording_emitter: Any, + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="good-a")) + store.put(make_raw_skill(key="bad", contentHash="0" * 64)) + store.put(make_raw_skill(key="good-b")) + + result = await get_skills(["good-a", "bad", "good-b"]) + + assert [s.key for s in result] == ["good-a", "good-b"] + failures = recording_emitter.signals(INTEGRITY_SIGNAL) + assert len(failures) == 1 + assert failures[0]["skill_key"] == "bad" + + +class TestAllSkills: + """All_skills accessor.""" + + async def test_returns_every_verified_skill( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + for k in ("a", "b", "c"): + store.put(make_raw_skill(key=k)) + result = await all_skills() + assert {s.key for s in result} == {"a", "b", "c"} + + async def test_empty_store_returns_empty_list( + self, store: InMemorySkillStore + ) -> None: + assert await all_skills() == [] + + async def test_omits_skills_that_fail_verification( + self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="good")) + store.put(make_raw_skill(key="bad", contentHash="deadbeef")) + result = await all_skills() + assert {s.key for s in result} == {"good"} + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + +class TestVersionPinning: + """ + A payload holds several versions of one key, and a pin has to resolve to the + version it names. + + Delivery serves the newest version of every skill *plus* every version any + variation currently pins, so this is the ordinary case rather than an edge + one. A seam keyed by key alone cannot express it: it answers with the newest + and the pin then reads as a missing skill. + """ + + async def _two_versions(self, store: Any, make_raw_skill: Any) -> None: + store.put(make_raw_skill(key="a", version=1, content="version one\n")) + store.put(make_raw_skill(key="a", version=2, content="version two\n")) + + async def test_pinned_old_version_resolves( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + skill = await get_skill("a", version=1) + assert skill is not None + assert skill.version == 1 + assert skill.content == b"version one\n" + + async def test_latest_resolves_alongside_the_pinned_old_version( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + skill = await get_skill("a") + assert skill is not None + assert skill.version == 2 + + async def test_both_lookups_succeed_against_one_store( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + pinned = await get_skill("a", version=1) + latest = await get_skill("a", version=2) + assert pinned is not None and pinned.version == 1 + assert latest is not None and latest.version == 2 + + async def test_get_skills_resolves_a_mix_of_pins( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + store.put(make_raw_skill(key="b", version=5, content="bee\n")) + skills = await get_skills( + [ + SkillReference(key="a", version=1), + SkillReference(key="b", version=5), + "a", + ] + ) + assert [(s.key, s.version) for s in skills] == [("a", 1), ("b", 5), ("a", 2)] + + async def test_pin_to_a_version_the_store_does_not_hold_returns_none( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + assert await get_skill("a", version=9) is None + + async def test_all_skills_returns_one_entry_per_key_at_the_newest_version( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + store.put(make_raw_skill(key="b", version=5, content="bee\n")) + skills = await all_skills() + assert sorted((s.key, s.version) for s in skills) == [("a", 2), ("b", 5)] + + async def test_a_store_answering_with_the_wrong_version_is_withheld( + self, make_raw_skill: Any + ) -> None: + """The post-fetch check is a defense, not the selection mechanism. + + The store is untrusted, so an answer that is not the version that was + asked for is withheld rather than returned. + """ + + class _WrongVersionStore: + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return make_raw_skill(key=key, version=99) + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + skills_module._set_store(_WrongVersionStore()) + assert await get_skill("a", version=1) is None + + +class TestWithholdingSummary: + """ + A run that withheld content says so at WARN, once. + + Every individual withholding already records an integrity signal and an + error line, but a caller reading logs at WARN sees neither — and the case + that matters most is a payload where *nothing* verifies, because the feature + then returns an empty result indistinguishable from "this project has no + skills". + """ + + def _tampered(self, make_raw_skill: Any, key: str = "a") -> dict[str, Any]: + raw = make_raw_skill(key=key) + raw["contentHash"] = "0" * 64 + return raw + + async def test_total_withholding_warns_and_names_the_hash( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + store.put(self._tampered(make_raw_skill)) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + assert await all_skills() == [] + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + assert "contentHash" in warnings[0].getMessage() + + async def test_partial_withholding_warns_with_the_counts( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + store.put(make_raw_skill(key="good")) + store.put(self._tampered(make_raw_skill, key="bad")) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + skills = await all_skills() + assert [s.key for s in skills] == ["good"] + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + assert "1 of 2" in warnings[0].getMessage() + + async def test_get_skills_warns_once_per_batch( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + store.put(self._tampered(make_raw_skill, key="a")) + store.put(self._tampered(make_raw_skill, key="b")) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + assert await get_skills(["a", "b"]) == [] + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + + async def test_a_fully_resolved_run_is_silent( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + store.put(make_raw_skill(key="a")) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + assert len(await all_skills()) == 1 + assert len(await get_skills(["a"])) == 1 + assert [r for r in caplog.records if r.levelname == "WARNING"] == [] + + +class TestIntegrityVerification: + """Mandatory verification at the accessor boundary.""" + + async def test_hash_mismatch_withholds_skill( + self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="a", contentHash="a" * 64)) + assert await get_skill("a") is None + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_tampered_content_withholds_skill( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + raw = make_raw_skill(key="a") + raw["content"] = raw["content"] + "x" # hash now stale by one byte + store.put(raw) + assert await get_skill("a") is None + + async def test_oversize_content_rejected( + self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + oversize = "x" * (64 * 1024 + 1) + store.put(make_raw_skill(key="a", content=oversize)) + assert await get_skill("a") is None + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_content_at_size_cap_is_accepted( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + at_cap = "x" * (64 * 1024) + store.put(make_raw_skill(key="a", content=at_cap)) + skill = await get_skill("a") + assert skill is not None + assert len(skill.content) == 64 * 1024 + + async def test_key_at_length_bound_from_store_accepted( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """The accepting side of the <= 256 bound. + + ``write_skills`` cannot reach this bound (a key is one directory name and + NAME_MAX is 255), so config validation and this accessor-side + revalidation are the only two layers where 256 is observable at all. The + rejecting side is ``test_invalid_key_from_store_rejected["x" * 257]``. + """ + key = "a" * 256 + store.put(make_raw_skill(key=key)) + skill = await get_skill(key) + assert skill is not None + assert skill.key == key + + @pytest.mark.parametrize( + "bad_key", + [ + "Evil", + "-leading-dash", + ".hidden", + "has space", + "a/b", + "../escape", + "", + "x" * 257, + ], + ) + async def test_invalid_key_from_store_rejected( + self, make_raw_skill: Any, bad_key: str + ) -> None: + """A hostile store may serve any key — the accessor revalidates.""" + raw = make_raw_skill(key="placeholder") + raw["key"] = bad_key + skills_module._set_store(InMemorySkillStore({bad_key: raw})) + assert await get_skill(bad_key) is None + + @pytest.mark.parametrize("bad_version", [0, -1, 2.5, "2", None, True]) + async def test_invalid_version_from_store_rejected( + self, make_raw_skill: Any, bad_version: Any + ) -> None: + raw = make_raw_skill(key="a", version=bad_version) + skills_module._set_store(InMemorySkillStore({"a": raw})) + assert await get_skill("a") is None + + async def test_missing_content_rejected(self, make_raw_skill: Any) -> None: + raw = make_raw_skill(key="a") + del raw["content"] + skills_module._set_store(InMemorySkillStore({"a": raw})) + assert await get_skill("a") is None + + async def test_missing_content_hash_rejected(self, make_raw_skill: Any) -> None: + raw = make_raw_skill(key="a") + del raw["contentHash"] + skills_module._set_store(InMemorySkillStore({"a": raw})) + assert await get_skill("a") is None + + async def test_uppercase_hash_rejected( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """Hashes are lowercase hex; a non-canonical hash is not authentic.""" + store.put(make_raw_skill(key="a", contentHash=_hash(SKILL_BODY).upper())) + assert await get_skill("a") is None + + @pytest.mark.parametrize(("body", "fabricated_hash"), FABRICATED_HASH_CASES) + async def test_unencodable_content_withheld( + self, recording_emitter: Any, body: str, fabricated_hash: str + ) -> None: + """Content with no UTF-8 encoding is withheld, and the signal recorded. + + ``str.encode`` raises on a lone surrogate, so ``verified_bytes`` has an + exception to catch and never sees bytes for this content at all. + + The parametrization is what makes that observable. The guard must never + pass ``errors="surrogatepass"``, or any other non-strict handler: each + of them fabricates bytes for input that has no encoding, and fabricated + bytes can satisfy the hash comparison. Every case here supplies the + sha256 of the bytes one such handler would have produced, so an + implementation that reached for one would verify this object + successfully and hand back content LaunchDarkly never sent. An + arbitrary wrong hash would not catch that — the mismatch check would + reject the input before the encoder guard was reached. + """ + with pytest.raises(UnicodeEncodeError): + body.encode("utf-8") # the premise: there is no encoding to hash + + skills_module._set_emitter_for_testing(recording_emitter) + skills_module._set_store( + InMemorySkillStore( + { + "a": { + "key": "a", + "version": 1, + "content": body, + "contentHash": fabricated_hash, + } + } + ) + ) + + assert await get_skill("a") is None + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_store_error_does_not_leak_content( + self, exploding_store: Any + ) -> None: + assert await get_skill("a") is None + assert await get_skills(["a"]) == [] + assert await all_skills() == [] + + +class TestTelemetrySeam: + """Internal emitter seam, no client.track, no context.""" + + async def test_default_emitter_is_noop( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", contentHash="0" * 64)) + assert await get_skill("a") is None # no emitter injected, no raise + + async def test_integrity_signal_properties( + self, + store: InMemorySkillStore, + make_raw_skill: Any, + recording_emitter: Any, + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="a", version=4, contentHash="b" * 64)) + + await get_skill("a") + + props = recording_emitter.signals(INTEGRITY_SIGNAL)[0] + assert props["skill_key"] == "a" + assert props["version"] == 4 + assert props["expected_hash"] == "b" * 64 + assert props["observed_hash"] == _hash(SKILL_BODY) + assert props["language"] == "python" + + async def test_skill_body_never_appears_in_signals( + self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="a", contentHash="c" * 64)) + + await get_skill("a") + + for _signal, props in recording_emitter.records: + for value in props.values(): + assert "Do the thing." not in str(value) + + # ``skill_key`` and ``expected_hash`` are copied off the wire, so a hostile + # store can smuggle the body through either one and publish it in a signal + # that is otherwise body-free. The sweep above cannot see that: it serves a + # well-formed 64-character digest under a valid key, so neither replacement + # branch ever runs, and it passes even against an implementation that copies + # both fields verbatim. These two cases are what make the rule observable. + # Both assert the body's *absence* rather than the placeholder's exact + # spelling, which is not part of the contract. + + async def test_body_smuggled_through_content_hash_is_redacted( + self, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + body = "UNIQUE-SECRET-BODY-VIA-HASH" + skills_module._set_store( + InMemorySkillStore( + {"a": {"key": "a", "version": 1, "content": body, "contentHash": body}} + ) + ) + + assert await get_skill("a") is None + + signals = recording_emitter.signals(INTEGRITY_SIGNAL) + assert len(signals) == 1 + for value in signals[0].values(): + assert body not in str(value) + + async def test_body_smuggled_through_the_key_is_redacted( + self, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + # Uppercase and a path separator, so this is not a valid skill key and + # the invalid-key branch is the one that has to redact it. + body = "UNIQUE-SECRET-BODY-VIA-KEY/../x" + skills_module._set_store( + InMemorySkillStore( + {body: {"key": body, "version": 1, "content": "x", "contentHash": "y"}} + ) + ) + + assert await get_skill(body) is None + + signals = recording_emitter.signals(INTEGRITY_SIGNAL) + assert len(signals) == 1 + for value in signals[0].values(): + assert body not in str(value) + + async def test_no_ld_track_calls_from_accessors( + self, make_raw_skill: Any, mock_ld_client: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + store.put(make_raw_skill(key="bad", contentHash="0" * 64)) + await init_client(options={"skillStore": store}, client=mock_ld_client) + + await get_skill("a") + await get_skill("bad") + await get_skills(["a"]) + await all_skills() + + mock_ld_client.track.assert_not_called() + + async def test_throwing_emitter_never_breaks_the_operation( + self, store: InMemorySkillStore, make_raw_skill: Any, throwing_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(throwing_emitter) + store.put(make_raw_skill(key="bad", contentHash="0" * 64)) + store.put(make_raw_skill(key="good")) + + assert await get_skill("bad") is None + good = await get_skill("good") + assert good is not None + assert good.key == "good" + + async def test_accessors_record_no_signal_outside_the_approved_set( + self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any + ) -> None: + """The three names are an allowlist, not a floor. + + Asserted over the recorded strings, so nothing here mandates a + particular module-level constant. The write-side half of this sweep is + ``test_write_skills_records_no_signal_outside_the_approved_set`` in + test_skills_fs.py, where all four reconcile actions can be staged. + + Guards the most likely regression: an implementation that also emits + ``AgentControl Skill Content Retrieved`` from ``get_skill``, or + ``AgentControl Skill SDK Reference Returned`` from ``skill_refs``, + passes every other test in this class. + """ + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="good")) + store.put(make_raw_skill(key="tampered", contentHash="0" * 64)) + + assert await get_skill("good") is not None + assert await get_skill("tampered") is None + await get_skills(["good", "tampered"]) + await all_skills() + skill_refs({"skills": [{"key": "good", "version": 1}]}) + + recorded = {signal for signal, _props in recording_emitter.records} + assert recorded <= APPROVED_SIGNALS, ( + f"unapproved signal(s): {sorted(recorded - APPROVED_SIGNALS)}" + ) + assert not recorded & REMOVED_SIGNALS + # Positive control: a subset assertion is satisfied vacuously by an + # implementation that records nothing at all. + assert INTEGRITY_SIGNAL in recorded From 1afb690e0048e824849166cbe7f4ebe793fa2ddc Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 28 Aug 2026 16:11:51 -0400 Subject: [PATCH 02/22] feat(client): a documented log record for skill integrity failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An integrity failure now writes a structured, machine-parseable ERROR record on the SDK's own logger, designed to be ingested by a SIEM and alerted on. This is the detection path that works when telemetry is off, and the only one that exists at all in an instance with no telemetry destination — so it is a documented contract rather than a debugging aid. The LD-side counter is left exactly as designed: opt-out respecting, no-op by default, property set unchanged. `reason_code` lives in the log record only. - `ld.skills.integrity_failure` is the stable event name, and it appears in the message text rather than only in `extra`. Severity cannot discriminate — a raising store also logs ERROR from this module — and the stdlib's default formatter drops `extra`, so an `extra`-only record is invisible under a plain `logging.basicConfig()`. - The message is the event name plus compact key-sorted JSON, so the line is greppable, `jq`-able, and byte-identical across LaunchDarkly's AI SDKs for the same input. The same mapping is attached as `extra["ld_skills"]`. - `reason_code` is a closed vocabulary of eight tokens, one per `record_integrity_failure` call site, typed as a `Literal` so a typo at a call site is a type error. - The record spreads the signal's properties rather than rebuilding them, so the two cannot drift on which fields are redacted or omitted. Optional fields are omitted, never nulled. No new untrusted value, and no path. Documented for customers in the README and for contributors in agents.md, including the full vocabulary, so a ninth reason cannot land in one language only. --- packages/client/README.md | 51 ++++ packages/client/agents.md | 54 ++++ .../src/launchdarkly_ai_server/skills_core.py | 128 ++++++++- packages/client/tests/test_skills.py | 269 ++++++++++++++++++ 4 files changed, 491 insertions(+), 11 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index f16a1eb2..ae572b3b 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -279,6 +279,57 @@ revalidate, and its size is within 64 KiB. Anything that fails is withheld and t missing — no unverified content ever reaches your code. A retrieval that withheld anything logs a count at WARN, so a run that resolved nothing is not silent. +#### Detecting integrity failures + +Every withheld skill emits one structured **ERROR** log record on the SDK's own logger +(`launchdarkly_ai_server.skills_core`), designed to be ingested by a SIEM and alerted on. +It is emitted **regardless of how telemetry is configured** — it is not conditional on any +opt-in, and it is the detection path that works when nothing leaves your process. + +The message text is the stable event name followed by compact JSON, so it is greppable and +`jq`-able under any handler configuration, and the same mapping is attached as +`extra["ld_skills"]` for a structured handler: + +``` +ERROR ld.skills.integrity_failure {"action":"withheld","event":"ld.skills.integrity_failure","expected_hash":"0000…0000","language":"python","observed_hash":"5fc8…6ec0","reason":"content hash mismatch","reason_code":"hash_mismatch","skill_key":"pdf-extraction","version":2} +``` + +**`ld.skills.integrity_failure` is a stability commitment.** It is the string to match on, +it will not be renamed, and the JSON keys are sorted so the line is byte-identical across +LaunchDarkly's AI SDKs for the same input. + +| Field | Description | +|---|---| +| `event` | Always `ld.skills.integrity_failure`. | +| `action` | Always `withheld` — the content was not returned to your code. | +| `skill_key` | The skill key, or `` when the delivered key was itself malformed. | +| `version` | The delivered version. Omitted when it was not a valid version. | +| `expected_hash` | The delivered `contentHash`, or `` when it was not one. Omitted when none was delivered. | +| `observed_hash` | The sha256 the SDK computed. Omitted when the failure happened before anything was hashed. | +| `reason_code` | A stable token naming the failure mode — see below. | +| `reason` | Human-readable detail, including byte counts where relevant. | +| `language` | Always `python`. | + +Absent optional fields are **omitted entirely** rather than emitted as `null`, so a field +existence check is meaningful. The skill body, and any attacker-controllable string that +could carry it, never appears in the record; neither does any filesystem path. + +| `reason_code` | Meaning | +|---|---| +| `not_an_object` | The delivered object was not a JSON object. | +| `invalid_key` | The key did not match `^[a-z0-9][a-z0-9-]*$` or exceeded 256 characters. | +| `invalid_version` | The version was not an integer ≥ 1. | +| `missing_content` | `content` was absent or not a string. | +| `missing_content_hash` | `contentHash` was absent or not a string. | +| `not_utf8` | The content string had no UTF-8 encoding, so there are no bytes that could have been hashed. | +| `over_size_cap` | The content exceeded the SDK's local size cap. | +| `hash_mismatch` | The computed sha256 did not match the delivered `contentHash`. | + +**`hash_mismatch` is the one worth paging on.** The other seven describe a malformed or +truncated payload; a mismatch means content was delivered whose bytes are not the bytes +LaunchDarkly hashed, which is a possible **active-tampering** signal. Alert on it, and +treat `expected_hash` / `observed_hash` as the evidence pair. + **Versions are selected, not filtered.** A store may hold several versions of one key at once, because a delivery payload does: the newest version of every skill, plus every version a variation currently pins. `get_skill("k", version=1)` asks the store for version diff --git a/packages/client/agents.md b/packages/client/agents.md index f284c425..b5733567 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -264,6 +264,59 @@ Exactly three signals exist, and the list is an **allowlist, not a floor**: The last two belong to the materialization layer and have no caller yet; they live here with the first so the allowlist is one section of one file rather than three sites to audit. +### The integrity-failure log record + +The signal above is product telemetry; the **log record** beside it is the customer-owned +detection path, and the more load-bearing of the two. It is the only integrity surface that +works when telemetry is off, and the only one that exists at all in an instance with no +telemetry destination, so it is a documented contract in the README rather than a debugging +aid. `record_integrity_failure` writes both, and is the only place either is constructed. + +One ERROR record per withheld skill, message text = `INTEGRITY_FAILURE_EVENT` + a space + +`json.dumps(record, sort_keys=True, separators=(",", ":"))`, plus the same mapping under +`extra={"ld_skills": record}`. Fields: `event`, `action` (always `withheld`), `skill_key`, +`version?`, `expected_hash?`, `observed_hash?`, `reason_code`, `reason`, `language`. + +Each of those choices is load-bearing; do not undo one as a simplification. + +- **The event name is in the message text**, not only in `extra`. Severity cannot + discriminate — `resolve_from_store` and `list_raw_objects` in the same module also log + ERROR for a raising store — and the stdlib's default formatter drops `extra` entirely, so + an `extra`-only record is invisible under a plain `logging.basicConfig()`. +- **`ld.skills.integrity_failure` is documented for customers to match on**, which makes it + a compatibility surface. It must never be renamed. +- **`sort_keys=True` is not cosmetic.** The other language implementations build the object + in alphabetical key order, so sorting makes the serialized line byte-identical across + SDKs for the same input, modulo `language`. +- **Optional fields are omitted, never nulled**, so a SIEM field-existence check means + something. +- **The record spreads the signal's properties** rather than rebuilding them, so the two + cannot drift on the fields they share — in particular on which are redacted. Anything + added later that comes off the wire needs the same shape-check-then-redact treatment. +- **`reason_code` is in the record only.** The signal's property set is the allowlist above + and does not grow; the local record is where the detection vocabulary lives. + +`reason_code` is a **closed vocabulary of exactly eight tokens** — `IntegrityReasonCode`, a +`Literal`, so a typo at a call site is a type error — one per `record_integrity_failure` +call site, and the same eight in every language implementation: + +| `reason_code` | Call site | +|---|---| +| `not_an_object` | `verify_raw_skill` — raw object is not a dict | +| `invalid_key` | `verify_raw_skill` — fails `is_valid_skill_key` | +| `invalid_version` | `verify_raw_skill` — fails `is_valid_skill_version` | +| `missing_content` | `verify_raw_skill` — `content` absent or not a string | +| `missing_content_hash` | `verify_raw_skill` — `contentHash` absent or not a string | +| `not_utf8` | `verified_bytes` — `UnicodeEncodeError` on encode (wire-`str` path only; a `Skill` already holds bytes) | +| `over_size_cap` | `verified_bytes` — over `MAX_SKILL_CONTENT_BYTES` | +| `hash_mismatch` | `verified_bytes` — observed sha256 != `contentHash` | + +Adding a ninth failure mode means widening `IntegrityReasonCode`, adding a case to +`REASON_CODE_CASES` in `test_skills.py` (whose exhaustiveness assertion fails otherwise), +documenting it in the README table, **and** doing the same in the other language SDKs. A +token added on one side only is a drift bug: a customer's detection rule stops matching +where they cannot see it. + `AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` were considered and **deliberately excluded from SDK emission** — both are observable server-side. Do not add them. The skill body never appears in a signal, a log line, or @@ -431,4 +484,5 @@ on their side of the boundary. - `Skill.content` is opaque `bytes`. Do not add anything that parses or interprets it — no YAML library in this package's dependencies at any tier, and no accessor that decodes content. - Do not route skills telemetry through `client.track()`, and do not introduce an LD context anywhere in the skills path. Signals go through the `skills_core.py` emitter seam, whose default is a no-op, and only via its `record_*` functions. - Do not add a signal name outside the three in the Agent Skills table above — the list is an allowlist. `AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` were considered and deliberately excluded from SDK emission. +- Do not rename `ld.skills.integrity_failure`, and do not add a ninth `reason_code` in one language only — both are documented compatibility surfaces. See "The integrity-failure log record" above. - Do not make `SkillStore` lookups key-only. Version is part of the lookup identity because a payload holds several versions of one key; a key-only seam cannot express a version-pinned reference. diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index c478be93..1e517e4e 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -38,10 +38,11 @@ from __future__ import annotations import hashlib +import json import logging import re from dataclasses import dataclass -from typing import Any, Protocol +from typing import Any, Literal, Protocol, get_args from .types import Skill, SkillReference from .types_validation import is_valid_skill_key, is_valid_skill_version @@ -91,6 +92,46 @@ _SIGNAL_MATERIALIZED = "AgentControl Skill Materialized" _SIGNAL_REVOKED = "AgentControl Skill Revoked Received" +INTEGRITY_FAILURE_EVENT = "ld.skills.integrity_failure" +""" +Stable event identity for the local integrity-failure log record. + +A **compatibility surface**, not an implementation detail: the README documents +it as the string a customer's SIEM matches on, so it must never be renamed. + +It appears verbatim **in the message text**, not only in ``extra``. Severity +alone cannot discriminate — ``list_raw_objects`` and ``resolve_from_store`` in +this module also log ERROR when a store raises — and the stdlib's default +formatter drops ``extra`` entirely, so under a plain ``logging.basicConfig()`` +an ``extra``-only record would be invisible. +""" + +_ACTION_WITHHELD = "withheld" +"""The only action an integrity failure results in: content is never returned.""" + +IntegrityReasonCode = Literal[ + "not_an_object", + "invalid_key", + "invalid_version", + "missing_content", + "missing_content_hash", + "not_utf8", + "over_size_cap", + "hash_mismatch", +] +""" +The closed ``reason_code`` vocabulary — one token per ``record_integrity_failure`` +call site, and the same eight tokens in every language implementation of this +feature, so a detection rule written against one SDK reads the others. + +A ``Literal`` rather than a bare ``str`` so a typo at a call site is a type +error, and so widening the vocabulary is a deliberate edit here rather than a +new string invented at the site that needed it. +""" + +INTEGRITY_REASON_CODES: frozenset[str] = frozenset(get_args(IntegrityReasonCode)) +"""``IntegrityReasonCode`` as a runtime set, derived rather than restated.""" + NO_STORE_MESSAGE = ( "No skill store is configured, so skill content cannot be retrieved. Configure " 'one with init_client(options={"skillStore": store}) — InMemorySkillStore is ' @@ -234,17 +275,39 @@ def record_integrity_failure( skill_key: str, reason: str, *, + reason_code: IntegrityReasonCode, version: Any = None, expected_hash: Any = None, observed_hash: str | None = None, ) -> None: """ - Records an integrity failure. Carries hashes and byte counts only — the skill - body never appears in a signal, a log line, or an error message. + Records an integrity failure on both surfaces: one local log record, one + product signal. + + Carries hashes and byte counts only — the skill body never appears in a + signal, a log line, or an error message. + + The two surfaces are deliberately different sizes, and the log record is the + more important of the two. The signal is product telemetry: opt-out + respecting, no-op by default, and its property set is a documented allowlist + (``agents.md``) that does not grow. The **log record** is the customer-owned + detection path — the only one that works when telemetry is off, and the only + one that exists at all in an instance with no telemetry destination — so it + is designed to be ingested and alerted on, and additionally carries the + stable event name, the action taken, the human-readable reason, and the + machine-parseable ``reason_code``. + + Emitted twice over, because neither form alone is sufficient: the message + text carries ``INTEGRITY_FAILURE_EVENT`` followed by compact JSON, so the + record survives ``logging.basicConfig()`` and is greppable and ``jq``-able + under any handler configuration; ``extra["ld_skills"]`` carries the same + mapping unflattened for a structured handler that would rather not reparse. """ # Both of these come off the wire, so neither may be echoed verbatim: a store # that set contentHash (or key) to the skill body would otherwise publish the - # body itself. Shape-check, then redact. + # body itself. Shape-check, then redact. Every field the log record adds on + # top is either a literal or SDK-authored, so the record introduces no new + # untrusted value — anything added later needs this same treatment. safe_key = skill_key if is_valid_skill_key(skill_key) else "" properties: dict[str, Any] = {"skill_key": safe_key, "language": _LANGUAGE} if is_valid_skill_version(version): @@ -258,7 +321,28 @@ def record_integrity_failure( if observed_hash is not None: properties["observed_hash"] = observed_hash - logger.error("Skill '%s' failed integrity verification: %s", safe_key, reason) + # Spread the signal's properties rather than rebuilding them, so the record + # cannot drift from the signal on the fields they share — in particular on + # which of them are redacted and which are omitted. Absent optional fields + # stay absent; the record never carries a null. + record: dict[str, Any] = { + "event": INTEGRITY_FAILURE_EVENT, + "action": _ACTION_WITHHELD, + "reason_code": reason_code, + "reason": reason, + **properties, + } + # ``sort_keys`` is load-bearing rather than cosmetic: the other language + # implementations build this object in alphabetical key order, so sorting + # here makes the serialized line byte-identical across SDKs for the same + # input, modulo ``language``. Do not drop it, and do not reorder the keys + # above expecting the output to follow. + logger.error( + "%s %s", + INTEGRITY_FAILURE_EVENT, + json.dumps(record, sort_keys=True, separators=(",", ":")), + extra={"ld_skills": record}, + ) emit(_SIGNAL_INTEGRITY_FAILURE, properties) @@ -375,7 +459,11 @@ def verified_bytes( # satisfy the hash comparison. reason = "content is not encodable as UTF-8" record_integrity_failure( - key, reason, version=version, expected_hash=expected_hash + key, + reason, + reason_code="not_utf8", + version=version, + expected_hash=expected_hash, ) return VerificationFailure(reason) @@ -385,7 +473,11 @@ def verified_bytes( f"{MAX_SKILL_CONTENT_BYTES} byte cap" ) record_integrity_failure( - key, reason, version=version, expected_hash=expected_hash + key, + reason, + reason_code="over_size_cap", + version=version, + expected_hash=expected_hash, ) return VerificationFailure(reason) @@ -396,6 +488,7 @@ def verified_bytes( record_integrity_failure( key, "content hash mismatch", + reason_code="hash_mismatch", version=version, expected_hash=expected_hash, observed_hash=observed_hash, @@ -414,7 +507,11 @@ def verify_raw_skill(raw: Any) -> Skill | None: user code. """ if not isinstance(raw, dict): - record_integrity_failure("", "raw skill object is not an object") + record_integrity_failure( + "", + "raw skill object is not an object", + reason_code="not_an_object", + ) return None key = raw.get("key") @@ -422,25 +519,34 @@ def verify_raw_skill(raw: Any) -> Skill | None: record_integrity_failure( key if isinstance(key, str) else "", "key is not a valid skill key", + reason_code="invalid_key", ) return None version = raw.get("version") if not is_valid_skill_version(version): - record_integrity_failure(key, "version is not an integer >= 1") + record_integrity_failure( + key, "version is not an integer >= 1", reason_code="invalid_version" + ) return None content = raw.get("content") if not isinstance(content, str): record_integrity_failure( - key, "content is missing or not a string", version=version + key, + "content is missing or not a string", + reason_code="missing_content", + version=version, ) return None expected_hash = raw.get("contentHash") if not isinstance(expected_hash, str): record_integrity_failure( - key, "contentHash is missing or not a string", version=version + key, + "contentHash is missing or not a string", + reason_code="missing_content_hash", + version=version, ) return None diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index e99a8957..203eeb4f 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -1006,6 +1006,275 @@ async def test_store_error_does_not_leak_content( assert await all_skills() == [] +# --------------------------------------------------------------------------- +# Gap 1 — the local integrity-failure log record +# --------------------------------------------------------------------------- + +INTEGRITY_EVENT = "ld.skills.integrity_failure" +"""The stable event name, spelled out rather than imported. + +The name is a compatibility surface documented for customers to match on in a +SIEM, so the test has to fail when it is renamed. Importing the constant would +rename in lockstep and assert nothing. +""" + +LOGGED_BODY = "UNIQUE-SECRET-BODY-THAT-MUST-NOT-BE-LOGGED" + + +def _raw_object(**overrides: Any) -> dict[str, Any]: + """A wire-shaped raw object with a correct ``contentHash``. + + The ``make_raw_skill`` fixture is the same factory, but a module-level + ``parametrize`` table cannot reach a fixture. + """ + raw: dict[str, Any] = { + "key": "a", + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + raw.update(overrides) + return raw + + +def _raw_without(field: str) -> dict[str, Any]: + raw = _raw_object() + del raw[field] + return raw + + +_OVERSIZE = "x" * (64 * 1024 + 1) + +REASON_CODE_CASES = [ + # Not a dict at all. Reachable through ``all_skills`` and not through + # ``get_skill``, which rejects a non-dict answer before verification. + pytest.param("not-an-object-at-all", "not_an_object", id="not_an_object"), + pytest.param(_raw_object(key="Evil/../x"), "invalid_key", id="invalid_key"), + pytest.param(_raw_object(version=0), "invalid_version", id="invalid_version"), + pytest.param(_raw_without("content"), "missing_content", id="missing_content"), + pytest.param( + _raw_without("contentHash"), "missing_content_hash", id="missing_content_hash" + ), + # A lone surrogate has no UTF-8 encoding. Only reachable on the wire-``str`` + # path: a ``Skill`` already holds bytes and skips the encode. + pytest.param( + _raw_object(content=json.loads(r'"hi \ud800 there"')), "not_utf8", id="not_utf8" + ), + # Correct hash for the oversize body, so the cap is what withheld it. + pytest.param( + _raw_object(content=_OVERSIZE, contentHash=_hash(_OVERSIZE)), + "over_size_cap", + id="over_size_cap", + ), + pytest.param( + _raw_object(contentHash="0" * 64), "hash_mismatch", id="hash_mismatch" + ), +] +"""One case per ``reason_code`` token, driven end to end through ``all_skills``. + +``all_skills`` rather than ``get_skill`` for every case so the table is uniform: +it verifies every object the store holds, including the ones too malformed to +carry a usable key, which is the only accessor path a non-dict reaches. +""" + + +def _integrity_records(caplog: pytest.LogCaptureFixture) -> list[dict[str, Any]]: + """Every integrity-failure record in *caplog*, parsed out of the message text. + + Read off the message rather than off ``record.ld_skills`` deliberately: the + message is what a customer sees under a plain ``logging.basicConfig()``, and + it is the surface the documented contract is about. The structured mirror is + asserted separately, against this. + """ + parsed: list[dict[str, Any]] = [] + for entry in caplog.records: + message = entry.getMessage() + if not message.startswith(f"{INTEGRITY_EVENT} "): + continue + parsed.append(json.loads(message[len(INTEGRITY_EVENT) + 1 :])) + return parsed + + +class TestIntegrityFailureLogRecord: + """ + The local log record is a documented detection surface, not a debugging aid. + + It is the only integrity signal that survives telemetry being switched off, + and the only one that exists at all in an instance with no telemetry + destination, so its shape is a contract: a stable event name in the message + text, a closed ``reason_code`` vocabulary, and no field a hostile store can + dictate. + """ + + @pytest.fixture(autouse=True) + def _capture_errors(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level("ERROR", logger="launchdarkly_ai_server.skills_core") + + async def _withhold(self, objects: dict[str, Any]) -> None: + skills_module._set_store(InMemorySkillStore(objects)) + assert await all_skills() == [] + + @pytest.mark.parametrize(("raw", "expected_code"), REASON_CODE_CASES) + async def test_one_record_per_reason_code( + self, caplog: pytest.LogCaptureFixture, raw: Any, expected_code: str + ) -> None: + await self._withhold({"a": raw}) + + records = _integrity_records(caplog) + assert len(records) == 1 + record = records[0] + assert record["reason_code"] == expected_code + assert record["event"] == INTEGRITY_EVENT + assert record["action"] == "withheld" + assert record["language"] == "python" + assert record["reason"] # the human-readable half, carrying byte counts + # Absent optional fields are omitted, never nulled: a SIEM field + # existence check has to mean something. + assert None not in record.values() + # The body never reaches a log line. Swept over every failure mode here; + # the two cases below are the ones that make the rule observable, since + # a well-formed key and digest never enter a redaction branch. + assert "Do the thing." not in json.dumps(record) + + def test_the_case_table_exhausts_the_vocabulary(self) -> None: + """The vocabulary is closed, and every token in it is reachable. + + Both directions matter. A ninth token added to the source without a call + site fails here, and so does a ninth call site that invented a token the + table does not cover — which is what keeps the Python and TypeScript + vocabularies from drifting apart one edit at a time. + """ + from launchdarkly_ai_server import skills_core + + covered = {case.values[1] for case in REASON_CODE_CASES} + assert covered == skills_core.INTEGRITY_REASON_CODES + assert len(covered) == 8 + + async def test_the_event_name_is_in_the_message_text( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Severity alone cannot discriminate, so the name has to be in the line. + + ``resolve_from_store`` and ``list_raw_objects`` also log ERROR from this + logger when a store raises, and the stdlib's default formatter drops + ``extra`` entirely — an ``extra``-only record would be invisible to a + customer running ``logging.basicConfig()``. + """ + await self._withhold({"a": _raw_object(contentHash="0" * 64)}) + + errors = [r for r in caplog.records if r.levelname == "ERROR"] + assert len(errors) == 1 + assert errors[0].getMessage().startswith(f"{INTEGRITY_EVENT} ") + + async def test_structured_handlers_get_the_same_record( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """``extra`` carries the record unflattened, and says the same thing.""" + await self._withhold({"a": _raw_object(contentHash="0" * 64)}) + + errors = [r for r in caplog.records if r.levelname == "ERROR"] + # extra lands in the record's __dict__, which is where a + # structured handler reads it from. + assert errors[0].__dict__["ld_skills"] == _integrity_records(caplog)[0] + + async def test_redaction_survives_into_the_record( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Both wire-sourced fields are still redacted in the JSON payload. + + ``key`` and ``contentHash`` are attacker-controlled, so a store can put + the skill body in either. The record must not have reopened a leak the + signal already closed — same treatment, same placeholders. + """ + await self._withhold( + { + "a": { + "key": f"{LOGGED_BODY}/../x", + "version": 1, + "content": LOGGED_BODY, + "contentHash": LOGGED_BODY, + } + } + ) + + record = _integrity_records(caplog)[0] + assert record["skill_key"] == "" + assert LOGGED_BODY not in json.dumps(record) + + async def test_a_non_sha256_expected_hash_is_redacted( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """The key is valid here, so ``expected_hash`` is the field under test.""" + await self._withhold({"a": _raw_object(contentHash=LOGGED_BODY)}) + + record = _integrity_records(caplog)[0] + assert record["expected_hash"] == "" + assert LOGGED_BODY not in json.dumps(record) + + async def test_observed_hash_is_absent_before_hashing( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Nothing was hashed, so there is no observed value to report.""" + await self._withhold({"a": _raw_without("contentHash")}) + + assert "observed_hash" not in _integrity_records(caplog)[0] + + async def test_observed_hash_is_present_on_a_mismatch( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """The one case that is a possible active-tampering signal. + + Positive control for the test above: an implementation that never + populated ``observed_hash`` would satisfy it vacuously. + """ + await self._withhold({"a": _raw_object(contentHash="0" * 64)}) + + record = _integrity_records(caplog)[0] + assert record["observed_hash"] == _hash(SKILL_BODY) + assert record["expected_hash"] == "0" * 64 + + async def test_the_serialized_payload_is_compact_and_key_sorted( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Sorted keys are what make the line byte-identical across SDKs. + + The other language implementations build this object in alphabetical + order, so a Python line following insertion order would differ byte for + byte on identical input. Compact separators are asserted alongside + because the two together are the serialization contract. + """ + await self._withhold({"a": _raw_object(contentHash="0" * 64)}) + + errors = [r for r in caplog.records if r.levelname == "ERROR"] + payload = errors[0].getMessage()[len(INTEGRITY_EVENT) + 1 :] + # json.loads preserves the document's order, so this is what was written. + keys = list(json.loads(payload)) + assert keys == sorted(keys) + assert ", " not in payload and ": " not in payload + + async def test_reason_code_stays_out_of_the_telemetry_signal( + self, recording_emitter: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """The signal's property set is an allowlist and does not grow. + + The record is the customer-owned detection path and carries the new + vocabulary; the LD-side counter is product telemetry and was left + exactly as designed. + """ + skills_module._set_emitter_for_testing(recording_emitter) + await self._withhold({"a": _raw_object(contentHash="0" * 64)}) + + props = recording_emitter.signals(INTEGRITY_SIGNAL)[0] + assert set(props) == { + "skill_key", + "version", + "expected_hash", + "observed_hash", + "language", + } + assert _integrity_records(caplog)[0]["reason_code"] == "hash_mismatch" + + class TestTelemetrySeam: """Internal emitter seam, no client.track, no context.""" From 8ad49f21b614022306436687d3ca65bdac0efa80 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 25 Aug 2026 16:06:16 -0400 Subject: [PATCH 03/22] feat(client): descriptor-pinned filesystem primitives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third of five slices. Adds `safe_fs.py`, the "write a file under a directory something else may be racing you for" problem solved once. Nothing here knows what a skill is; the materialization layer is its only caller, and it lands next. A path check is only as good as the last path resolution after it. Every `lstat` and containment check validates an inode, but a following `os.replace(tmp, dir / name)` re-resolves `dir` from its name — so anything holding write permission there can move the validated directory aside, leave a symlink in its place, and redirect the write or the unlink somewhere else. Narrowing that window is not a fix; the race is winnable at any width. So the checks hand off to a descriptor and nothing re-resolves a path afterwards. - `open_directory_nofollow` opens with `O_RDONLY | O_DIRECTORY | O_NOFOLLOW` and confirms `S_ISDIR` on the `fstat`, since not every platform defines `O_DIRECTORY`. `open_or_create_directory` adds `os.mkdir` plus an `lstat` on the `FileExistsError` path, because `Path.mkdir(exist_ok=True)` accepts a symlink-to-directory as "already there" and would reopen the hole the caller's check just closed. `pinned_directory` holds either for a block, so a caller states the platform split once and cannot forget the close. - `atomic_write` creates its temp file with `O_CREAT | O_EXCL | O_NOFOLLOW` at that descriptor, `fchmod`s the descriptor rather than a path, writes, fsyncs, renames, and fsyncs the directory so the rename survives a crash. Mode is set explicitly at 0644, never inherited from the umask and never executable. `os.replace` is the single rename call site and `os.rename` must not be substituted for it. - `unlink_file` probes and unlinks descriptor-relative. `unlink` never follows a trailing symlink but it does resolve the directory above it, so the same swap turns a removal into a delete of an arbitrary file. A symlink found where this SDK expects its own file raises `SymlinkRefused` rather than being tidied away — the state on disk is not what the caller believes, and that is the caller's to report. `SUPPORTS_DIR_FD` gates all of it, and the probe deliberately names `os.rename`/`os.stat` rather than the `os.replace`/`os.lstat` this module calls: `os.supports_dir_fd` is populated per underlying syscall, and CPython registers `renameat` under `rename` only and `fstatat` under `stat` only. Probing the names actually called reports "unsupported" on every POSIX platform and silently turns the defense off. Where the family is absent, every operation falls back to the identical full-path sequence. The tests here exercise the module directly, on its own terms. The TOCTOU races these primitives exist to close are proved through the materialization layer, which is what holds a descriptor across a sequence of operations. Testing: `uv run pytest` → 1260 passed, 11 skipped. `ruff check`, `ruff format --check`, and `mypy packages/*/src` all clean. Co-Authored-By: Claude Fable 5 --- packages/client/agents.md | 47 +++ .../src/launchdarkly_ai_server/safe_fs.py | 311 ++++++++++++++++++ packages/client/tests/test_safe_fs.py | 246 ++++++++++++++ 3 files changed, 604 insertions(+) create mode 100644 packages/client/src/launchdarkly_ai_server/safe_fs.py create mode 100644 packages/client/tests/test_safe_fs.py diff --git a/packages/client/agents.md b/packages/client/agents.md index b5733567..152f5771 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,6 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | +| `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | | `src/launchdarkly_ai_server/registry.py` | `Registry`, `global_registry`, `compose`, `resolve_handlers`, `resolve_tools` | | `src/launchdarkly_ai_server/judges.py` | `run_judges`, `build_judge_tasks`, `run_judge` | @@ -334,6 +335,52 @@ The injection path is deliberately narrower than the state's location: `skills.p (`skills._set_store(store)` is the same setter `init_client` uses), and neither should reach into `skills_core` directly. +### Descriptor-pinned filesystem access + +A path check is only as good as the last path resolution after it. Every `lstat`, `realpath` +and containment check validates an *inode*, but a following +`os.replace(tmp, root / key / "SKILL.md")` re-resolves `/` from its *name* — so +anything holding write permission on a managed directory can move the validated directory +aside, leave a symlink in its place, and redirect the write (or an unlink) somewhere else. +Narrowing that window is not a fix; the race is winnable at any width. + +So the checks hand off to a descriptor and nothing re-resolves a path afterwards. The +primitives live in `safe_fs.py`, which knows nothing about skills: + +- `open_directory_nofollow` opens the directory with `O_RDONLY | O_DIRECTORY | O_NOFOLLOW` + and confirms `S_ISDIR` on the `fstat` (the explicit check is what covers platforms with no + `O_DIRECTORY`). `open_or_create_directory` wraps it with `os.mkdir` plus an `lstat` on the + `FileExistsError` path — `Path.mkdir(exist_ok=True)` accepts a symlink-to-directory as + "already there", which would reopen the hole the caller's check just closed. + `pinned_directory` holds either for the duration of a block, so a caller states the + platform split once as `if dir_fd is not None` and cannot forget the `os.close`. +- `atomic_write` creates the temp file with `O_CREAT | O_EXCL | O_NOFOLLOW` **at** that + descriptor (`_mkstemp_at`, since `tempfile` has no `dir_fd` form), `fchmod`s the + descriptor rather than `chmod`ing a path, writes, fsyncs, and renames with + `os.replace(tmp, name, src_dir_fd=fd, dst_dir_fd=fd)`, then fsyncs the directory so the + rename survives a crash. `atomic_write_in` is the same against a directory the caller does + not already hold open. `os.replace` is the single rename call site, reached by attribute + lookup so tests can intercept it, and `os.rename` must not be substituted for it — it is + also the only one with defined overwrite semantics on Windows. +- `unlink_file` probes and unlinks descriptor-relative too. `unlink` never follows a + *trailing* symlink, but it does resolve the directory above it, so the same swap turns a + removal into a delete of an attacker-chosen file. A symlink found where this SDK expects + its own file raises `SymlinkRefused` rather than being tidied away: the state on disk is + not what the caller believes, and that is the caller's to report. + +`safe_fs.SUPPORTS_DIR_FD` gates all of it, and the probe is not the obvious one. +`os.supports_dir_fd` is populated per underlying syscall, and CPython registers `renameat` +under `os.rename` only and `fstatat` under `os.stat` only — even though `os.replace` is the +same `renameat`-backed function and `os.lstat` is `fstatat` with `AT_SYMLINK_NOFOLLOW`. +Probing the names this module actually calls reports "unsupported" on every POSIX platform +and silently turns the defense off, so the probe names the advertised twins +(`{os.rename, os.open, os.unlink, os.stat}`) and a caller's symlink check is spelled +`os.stat(..., follow_symlinks=False)` rather than `os.lstat`. Where the family is absent +(Windows) `open_directory_nofollow` returns `None` after an `lstat` check instead of +attempting the descriptor open — `os.open` cannot open a directory there — and every caller +falls back to the identical full-path sequence, the per-component `lstat` floor. The +residual window on those platforms is documented rather than closed. + --- ## OTel Setup diff --git a/packages/client/src/launchdarkly_ai_server/safe_fs.py b/packages/client/src/launchdarkly_ai_server/safe_fs.py new file mode 100644 index 00000000..3605399e --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/safe_fs.py @@ -0,0 +1,311 @@ +""" +Descriptor-pinned filesystem primitives. + +Split out because none of this knows what a skill is: it is the "write a file +under a directory an attacker may be racing you for" problem, solved once. +``skills_fs.py`` is the only caller today. + +The whole point is that a path check is only as good as the last path +resolution after it. Every operation here therefore runs relative to a +descriptor pinned to a directory the caller has already validated, rather than +re-resolving a name — which is what closes the swap window rather than merely +narrowing it. Where the platform has no ``*at()`` syscall family (Windows) the +identical sequence runs against full paths, the per-component ``lstat`` floor. +""" + +from __future__ import annotations + +import errno +import os +import secrets +import stat +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +_FILE_MODE = 0o644 +"""Mode set explicitly on every written file — never inherited from the umask, +and never executable.""" + +SUPPORTS_DIR_FD = os.supports_dir_fd.issuperset( + # renameat, openat, unlinkat, fstatat — the four this module needs. + {os.rename, os.open, os.unlink, os.stat} +) +""" +Whether the ``*at()`` syscall family is available, so every operation under the +managed root can be performed relative to a descriptor pinned to a directory +this module has already verified rather than re-resolved from its path. + +That is what closes the swap window rather than merely narrowing it: +a descriptor refers to the inode that was checked, so replacing ``/`` +with a symlink after the check cannot redirect a write or an unlink out of the +root. POSIX has these calls; Windows does not, and there the per-component +``lstat`` floor the spec permits applies instead. + +The probe deliberately names ``os.rename`` and ``os.stat`` rather than the +``os.replace`` and ``os.lstat`` this module actually calls. ``os.supports_dir_fd`` +is populated per underlying syscall, and CPython registers ``renameat`` under +``rename`` only and ``fstatat`` under ``stat`` only — even though ``os.replace`` +is the same ``renameat``-backed function and ``os.lstat`` is ``fstatat`` with +``AT_SYMLINK_NOFOLLOW``, and both accept the descriptor keywords wherever their +advertised twin does (verified on CPython 3.12 and 3.13, macOS). Probing the +names this module calls would report "unsupported" on every POSIX platform and +silently disable the defense. +""" + + +def open_directory_nofollow(directory: Path) -> int | None: + """ + Opens *directory* without following a final symlink, and pins it. + + Everything the caller does afterwards goes through the returned descriptor + instead of the path, which is what turns the "narrow window" into no + window at all: the descriptor names the inode that was checked, so swapping + the path for a symlink between the check and the write cannot redirect the + write out of the managed root. + + On a platform without the ``*at()`` family (Windows) this returns ``None`` + after verifying via ``lstat`` that the path is a real, non-symlink + directory — the per-component floor. It must not attempt the descriptor + open there: ``os.open`` goes through the CRT on Windows, which cannot open + a directory at all, so the descriptor path would fail every operation + rather than fall back. + + Raises ``ValueError`` when the path will not open (or inspect) as a real + directory — the caller reports that as a refusal rather than letting it + escape. + """ + if not SUPPORTS_DIR_FD: + try: + mode = os.lstat(directory).st_mode + except OSError as exc: + raise ValueError(f"the directory could not be inspected: {exc}") from exc + if stat.S_ISLNK(mode): + raise ValueError("the directory is a symlink") + if not stat.S_ISDIR(mode): + raise ValueError("the path is not a directory") + return None + + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(directory, flags) + except OSError as exc: + raise ValueError( + f"the directory could not be opened without following links: {exc}" + ) from exc + try: + # O_DIRECTORY already guarantees this wherever the platform defines it; + # the explicit check is what covers the platforms that do not. + if not stat.S_ISDIR(os.fstat(fd).st_mode): + raise ValueError("the path is not a directory") + except BaseException: + os.close(fd) + raise + return fd + + +def open_or_create_directory(directory: Path) -> int | None: + """ + Creates *directory* if absent and returns a descriptor pinned to it. + + ``Path.mkdir(exist_ok=True)`` treats an existing symlink-to-directory as + "already there", which would re-open the very hole the caller's ``lstat`` + check just closed. ``os.mkdir`` plus an ``lstat`` on the ``FileExistsError`` + path does not: a link reports as a link, and is refused. + """ + try: + os.mkdir(directory, 0o755) + except FileExistsError: + mode = os.lstat(directory).st_mode + if stat.S_ISLNK(mode): + raise ValueError("the directory is a symlink") from None + if not stat.S_ISDIR(mode): + raise ValueError("the path is not a directory") from None + return open_directory_nofollow(directory) + + +@contextmanager +def pinned_directory(directory: Path, *, create: bool = False) -> Iterator[int | None]: + """ + Holds *directory* pinned for the duration of the block, then releases it. + + Yields what the two openers above return — a descriptor, or ``None`` on the + ``lstat`` floor — so the caller states the platform split once, as + ``if dir_fd is not None``, and cannot forget the ``os.close``. Raises + ``ValueError`` for a directory that will not pin, exactly as they do. + """ + dir_fd = ( + open_or_create_directory(directory) + if create + else open_directory_nofollow(directory) + ) + try: + yield dir_fd + finally: + if dir_fd is not None: + os.close(dir_fd) + + +class SymlinkRefused(OSError): + """ + Raised instead of removing a symlink found where a real file was expected. + + An ``OSError`` subclass so a caller that only cares that the removal failed + keeps its single ``except``; a distinct type so one that must report *this* + refusal specifically does not have to match on a message. + """ + + +def unlink_file(directory: Path, name: str, *, dir_fd: int | None) -> None: + """ + Removes ``/``, refusing to follow a symlink at *name*. + + The mirror of ``atomic_write``, and descriptor-relative for the same reason: + ``unlink`` never follows a *trailing* symlink, but it does resolve the + directory above it, so a ```` swapped for a symlink after the + caller's checks would otherwise turn this into a delete of an + attacker-chosen file. Given a *dir_fd* the probe and the unlink both run + against it; without one the identical sequence runs against full paths. + + Raises ``SymlinkRefused`` when *name* is a symlink. Note that this refuses + rather than removes: ``unlink`` would happily delete the link itself, but a + link where this SDK expects its own file means the state on disk is not what + the manifest describes, and that is the caller's to report rather than to + tidy away. + """ + if dir_fd is None: + # No ``*at()`` family: the trailing-symlink check and the unlink are both + # path-based, the per-component floor. + target = directory / name + if target.is_symlink(): + raise SymlinkRefused(f"{name} is a symlink") + target.unlink() + return + + # os.stat(follow_symlinks=False), not os.lstat: identical result, and it is + # the spelling os.supports_dir_fd actually advertises. + probe = os.stat(name, dir_fd=dir_fd, follow_symlinks=False) + if stat.S_ISLNK(probe.st_mode): + raise SymlinkRefused(f"{name} is a symlink") + os.unlink(name, dir_fd=dir_fd) + + +def _mkstemp_at(dir_fd: int, prefix: str) -> tuple[int, str]: + """ + ``tempfile.mkstemp`` for a directory descriptor. + + ``tempfile`` has no ``dir_fd`` form, so this reproduces the part that + matters: ``O_CREAT | O_EXCL`` against an unpredictable name, retried on + collision, so an existing temp path is never reused and a planted one is + never written through. + """ + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + for _ in range(tempfile.TMP_MAX): + name = f"{prefix}{secrets.token_hex(8)}.tmp" + try: + return os.open(name, flags, 0o600, dir_fd=dir_fd), name + except FileExistsError: + continue + raise OSError(errno.EEXIST, "no usable temporary file name was found") + + +def atomic_write( + directory: Path, name: str, data: bytes, *, dir_fd: int | None = None +) -> None: + """ + Writes *data* to ``/`` so no partial file is ever + observable. + + The temp file is created exclusively in the target's *own* directory — one + anywhere else would make the rename cross-device, and therefore not atomic — + written, fsynced, renamed over the target, and the directory fsynced so the + rename itself survives a crash. Mode is set explicitly rather than left to + the process umask, and the execute bit is never set. + + Given a *dir_fd* on a platform with the ``*at()`` family, every one of those + steps runs relative to that descriptor and both names are bare filenames. + Without one (Windows) the identical sequence runs against full paths, which + is the per-component ``lstat`` floor. + + ``os.replace`` is the one and only rename call site, reached by attribute + lookup on the ``os`` module so tests can intercept it; ``os.rename`` must + not be substituted for it (it is also the only one with defined overwrite + semantics on Windows). + """ + at_fd = dir_fd if dir_fd is not None and SUPPORTS_DIR_FD else None + prefix = f".{name}." + target: str | Path + + if at_fd is not None: + fd, temp = _mkstemp_at(at_fd, prefix) + target = name + else: + # mkstemp opens with O_CREAT|O_EXCL, so an existing temp path is never + # reused. + fd, temp = tempfile.mkstemp(dir=directory, prefix=prefix, suffix=".tmp") + target = directory / name + + try: + try: + # fchmod, not chmod: operating on the descriptor cannot be redirected + # by anything that swaps the temp path underneath us, and it makes the + # mode independent of the process umask (both creation paths open 0600). + os.fchmod(fd, _FILE_MODE) + view = memoryview(data) + while view: + view = view[os.write(fd, view) :] + os.fsync(fd) + finally: + os.close(fd) + if at_fd is not None: + os.replace(temp, target, src_dir_fd=at_fd, dst_dir_fd=at_fd) + else: + os.replace(temp, target) + except BaseException: + try: + if at_fd is not None: + os.unlink(temp, dir_fd=at_fd) + else: + os.unlink(temp) + except OSError: + pass + raise + + if at_fd is not None: + _fsync_directory_fd(at_fd) + else: + _fsync_directory(directory) + + +def atomic_write_in(directory: Path, name: str, data: bytes) -> None: + """ + ``atomic_write`` against a directory this module does not already hold open. + + Used for the skills manifest, whose directory is the managed root. The + descriptor is taken with ``O_NOFOLLOW``, so a root swapped for a symlink after + ``_resolve_root`` validated it fails the write instead of redirecting it — + the caller turns that into a run-level ``error`` action. + """ + with pinned_directory(directory) as dir_fd: + atomic_write(directory, name, data, dir_fd=dir_fd) + + +def _fsync_directory_fd(fd: int) -> None: + """Best effort — not every platform allows fsync on a directory descriptor.""" + try: + os.fsync(fd) + except OSError: + pass + + +def _fsync_directory(directory: Path) -> None: + """Best effort — not every platform lets a directory be opened for fsync.""" + try: + fd = os.open(directory, os.O_RDONLY) + except OSError: + return + try: + _fsync_directory_fd(fd) + finally: + os.close(fd) diff --git a/packages/client/tests/test_safe_fs.py b/packages/client/tests/test_safe_fs.py new file mode 100644 index 00000000..b148ff9b --- /dev/null +++ b/packages/client/tests/test_safe_fs.py @@ -0,0 +1,246 @@ +""" +Tests for the descriptor-pinned filesystem primitives. + +These exercise ``safe_fs`` directly, on its own terms — the module knows nothing +about skills, and its guarantees are worth asserting without a caller in the way. +The TOCTOU races these primitives exist to close are proved through the +materialization layer, which is what actually holds a descriptor across a +sequence of operations. +""" + +from __future__ import annotations + +import os +import stat +from pathlib import Path + +import pytest + +import launchdarkly_ai_server.safe_fs as safe_fs_module +from launchdarkly_ai_server.safe_fs import ( + SymlinkRefused, + atomic_write, + atomic_write_in, + open_directory_nofollow, + open_or_create_directory, + pinned_directory, + unlink_file, +) + + +class TestOpenDirectory: + """Pinning a directory, and refusing anything that is not one.""" + + def test_opens_a_real_directory(self, tmp_path: Path) -> None: + with pinned_directory(tmp_path) as dir_fd: + if dir_fd is None: + pytest.skip("no *at() family on this platform") + assert stat.S_ISDIR(os.fstat(dir_fd).st_mode) + + def test_refuses_a_symlink_to_a_directory(self, tmp_path: Path) -> None: + real = tmp_path / "real" + real.mkdir() + link = tmp_path / "link" + link.symlink_to(real, target_is_directory=True) + with pytest.raises(ValueError): + open_directory_nofollow(link) + + def test_refuses_a_regular_file(self, tmp_path: Path) -> None: + target = tmp_path / "file" + target.write_text("not a directory") + with pytest.raises(ValueError): + open_directory_nofollow(target) + + def test_refuses_an_absent_path(self, tmp_path: Path) -> None: + with pytest.raises(ValueError): + open_directory_nofollow(tmp_path / "nope") + + def test_create_makes_the_directory(self, tmp_path: Path) -> None: + target = tmp_path / "new" + fd = open_or_create_directory(target) + try: + assert target.is_dir() + finally: + if fd is not None: + os.close(fd) + + def test_create_refuses_an_existing_symlink(self, tmp_path: Path) -> None: + """``Path.mkdir(exist_ok=True)`` would accept this and reopen the hole. + + A symlink-to-directory already present reads as "already there" to + ``exist_ok``, so the caller's containment check would be bypassed by + something that was never checked. ``os.mkdir`` plus an ``lstat`` on the + ``FileExistsError`` path refuses it. + """ + real = tmp_path / "real" + real.mkdir() + link = tmp_path / "link" + link.symlink_to(real, target_is_directory=True) + with pytest.raises(ValueError): + open_or_create_directory(link) + + def test_pinned_directory_closes_the_descriptor(self, tmp_path: Path) -> None: + with pinned_directory(tmp_path) as dir_fd: + if dir_fd is None: + pytest.skip("no *at() family on this platform") + held = dir_fd + with pytest.raises(OSError): + os.fstat(held) + + +class TestAtomicWrite: + """Explicit mode, no observable partial file, and one rename call site.""" + + def test_writes_the_bytes(self, tmp_path: Path) -> None: + with pinned_directory(tmp_path) as dir_fd: + atomic_write(tmp_path, "f.txt", b"hello", dir_fd=dir_fd) + assert (tmp_path / "f.txt").read_bytes() == b"hello" + + def test_mode_is_0644_and_never_executable(self, tmp_path: Path) -> None: + """Set explicitly on the descriptor, so the process umask cannot widen or + narrow it and the execute bit is never inherited.""" + previous = os.umask(0o077) + try: + with pinned_directory(tmp_path) as dir_fd: + atomic_write(tmp_path, "f.txt", b"x", dir_fd=dir_fd) + finally: + os.umask(previous) + mode = (tmp_path / "f.txt").stat().st_mode + assert stat.S_IMODE(mode) == 0o644 + assert not mode & stat.S_IXUSR + + def test_overwrites_an_existing_file(self, tmp_path: Path) -> None: + (tmp_path / "f.txt").write_bytes(b"old") + with pinned_directory(tmp_path) as dir_fd: + atomic_write(tmp_path, "f.txt", b"new", dir_fd=dir_fd) + assert (tmp_path / "f.txt").read_bytes() == b"new" + + def test_leaves_no_temp_file_behind(self, tmp_path: Path) -> None: + with pinned_directory(tmp_path) as dir_fd: + atomic_write(tmp_path, "f.txt", b"x", dir_fd=dir_fd) + assert [p.name for p in tmp_path.iterdir()] == ["f.txt"] + + def test_a_failed_rename_removes_the_temp_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A crash between write and rename must not leave a partial file, and + must not leave the temp file either.""" + + def _boom(*args: object, **kwargs: object) -> None: + raise OSError("injected rename failure") + + monkeypatch.setattr(os, "replace", _boom) + with pinned_directory(tmp_path) as dir_fd: + with pytest.raises(OSError, match="injected"): + atomic_write(tmp_path, "f.txt", b"x", dir_fd=dir_fd) + assert list(tmp_path.iterdir()) == [] + + def test_rename_goes_through_os_replace( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """``os.replace`` is the single rename call site. + + ``os.rename`` must not be substituted for it: it is the only one with + defined overwrite semantics on Windows, and it is the seam the + materialization tests intercept to prove atomicity. + """ + calls: list[object] = [] + real = os.replace + + def _spy(src: object, dst: object, **kwargs: object) -> None: + calls.append(dst) + real(src, dst, **kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(os, "replace", _spy) + with pinned_directory(tmp_path) as dir_fd: + atomic_write(tmp_path, "f.txt", b"x", dir_fd=dir_fd) + assert len(calls) == 1 + assert os.path.basename(str(calls[0])) == "f.txt" + + def test_write_without_a_descriptor_uses_the_path_fallback( + self, tmp_path: Path + ) -> None: + """The no-``*at()`` shape must produce an identical result. + + Windows takes this path for every write, so it is not a degenerate case — + the file, its mode, and the absence of a temp file all have to match. + """ + atomic_write(tmp_path, "f.txt", b"fallback", dir_fd=None) + assert (tmp_path / "f.txt").read_bytes() == b"fallback" + assert stat.S_IMODE((tmp_path / "f.txt").stat().st_mode) == 0o644 + assert [p.name for p in tmp_path.iterdir()] == ["f.txt"] + + def test_write_in_pins_the_directory_itself(self, tmp_path: Path) -> None: + atomic_write_in(tmp_path, "f.txt", b"x") + assert (tmp_path / "f.txt").read_bytes() == b"x" + + def test_write_in_refuses_a_symlinked_directory(self, tmp_path: Path) -> None: + real = tmp_path / "real" + real.mkdir() + link = tmp_path / "link" + link.symlink_to(real, target_is_directory=True) + with pytest.raises(ValueError): + atomic_write_in(link, "f.txt", b"x") + + +class TestUnlinkFile: + """Removing only a real file, and refusing a link found in its place.""" + + def test_removes_a_regular_file(self, tmp_path: Path) -> None: + (tmp_path / "f.txt").write_text("x") + with pinned_directory(tmp_path) as dir_fd: + unlink_file(tmp_path, "f.txt", dir_fd=dir_fd) + assert not (tmp_path / "f.txt").exists() + + def test_refuses_a_symlink_rather_than_removing_it(self, tmp_path: Path) -> None: + """It refuses rather than tidies. + + ``unlink`` would happily delete the link itself, but a link where this SDK + expects its own file means the state on disk is not what the manifest + describes — the caller's to report, not this module's to clean up. + """ + outside = tmp_path / "outside.txt" + outside.write_text("do not touch") + link = tmp_path / "f.txt" + link.symlink_to(outside) + + with pinned_directory(tmp_path) as dir_fd: + with pytest.raises(SymlinkRefused): + unlink_file(tmp_path, "f.txt", dir_fd=dir_fd) + + assert link.is_symlink() + assert outside.read_text() == "do not touch" + + def test_refuses_a_symlink_on_the_path_fallback(self, tmp_path: Path) -> None: + outside = tmp_path / "outside.txt" + outside.write_text("do not touch") + (tmp_path / "f.txt").symlink_to(outside) + with pytest.raises(SymlinkRefused): + unlink_file(tmp_path, "f.txt", dir_fd=None) + assert outside.exists() + + def test_symlink_refused_is_an_oserror(self) -> None: + """A caller that only cares the removal failed keeps its single + ``except OSError``; one that must report this refusal specifically does + not have to match on a message.""" + assert issubclass(SymlinkRefused, OSError) + + +class TestDirFdProbe: + """The capability probe names the advertised twins, not the calls made.""" + + def test_probe_names_the_syscalls_python_advertises(self) -> None: + """``os.supports_dir_fd`` is populated per underlying syscall, and CPython + registers ``renameat`` under ``os.rename`` and ``fstatat`` under + ``os.stat``. Probing ``os.replace`` and ``os.lstat`` — the names this + module actually calls — reports "unsupported" on every POSIX platform and + would silently disable the defense. + """ + expected = os.supports_dir_fd.issuperset( + {os.rename, os.open, os.unlink, os.stat} + ) + assert safe_fs_module.SUPPORTS_DIR_FD is expected + + @pytest.mark.skipif(os.name == "nt", reason="POSIX advertises the *at() family") + def test_posix_has_the_family(self) -> None: + assert safe_fs_module.SUPPORTS_DIR_FD is True From f1aad0afb7629d231fcffde5159ada136f586032 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 25 Aug 2026 16:12:17 -0400 Subject: [PATCH 04/22] =?UTF-8?q?test(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20the=20filesystem=20abuse=20matrix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fifth and last of five slices. The adversary. Every filesystem defense the previous two slices introduced now has a test that fails if the defense is removed, plus the materialization telemetry allowlist. - **Path traversal.** A key that escapes the root, a key that is the manifest filename, an over-long key, and a key that resolves outside after `realpath` — each refused before any filesystem call, and the resolved containment check asserted on inode identity rather than on path strings. - **Symlink attacks.** A symlinked skill directory, a symlinked target file, and the `/` directory swapped for a symlink at the exact instant of the rename and of the unlink — the narrowest version of the window the descriptor pinning exists to close, fired from the interception point rather than from implementation internals. - **The no-`*at()` shape.** The path fallback Windows takes for every operation, exercised with the capability probe forced off, so the platform that cannot pin a descriptor is not the untested one. The TOCTOU tests skip off that same flag deliberately: a probe that wrongly reported "unsupported" cannot also silently skip the tests that would have caught it. - **Non-regular files and clobber protection.** A fifo or a directory where `SKILL.md` belongs, and a file at a managed path with no matching manifest entry — reported and left alone, never overwritten and never removed. - **Corrupt manifests.** Unreadable, unparseable, not an object, malformed entries, and a `manifestVersion` this release cannot read: no overwrites, no prunes, an error action naming the manifest, and the manifest itself left as it was found. - **Atomicity.** A crash injected between the write and the rename leaves neither a partial file nor a temp file, and the one recorded rename is proved to have moved `SKILL.md` within the target's own directory — by descriptor identity where the platform has `renameat`, which also rules out the descriptor having been redirected between the check and the rename. - **Telemetry.** The three signal names are asserted as an allowlist rather than a floor: any other name reaching the emitter fails, the two deliberately excluded names are called out by name, no signal carries a filesystem path or the skill body, an emitter that raises never fails the reconcile, and `client.track()` is never reached. Two of these are worth naming, because the obvious test does not reach the guard. The unencodable-content cases pin `contentHash` to the sha256 of the bytes a non-strict encoder would have fabricated, since an arbitrary wrong hash is rejected by the mismatch check first and never exercises the encoder guard. The redaction cases smuggle the body through `contentHash` and through `key`, since a sweep using a well-formed digest under a valid key reaches neither replacement branch. Testing: `uv run pytest` → 1280 passed. `ruff check`, `ruff format --check`, and `mypy packages/*/src` all clean. Co-Authored-By: Claude Fable 5 --- packages/client/tests/test_skills_fs.py | 1071 ++++++++++++++++++++++- 1 file changed, 1064 insertions(+), 7 deletions(-) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 62d01d59..6502aa5b 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -1,13 +1,9 @@ """ -Tests for ``write_skills`` — filesystem materialization and manifest reconcile -semantics. +Tests for ``write_skills`` — filesystem materialization, manifest reconcile +semantics, and the full security abuse matrix. Every test writes only inside pytest's ``tmp_path``. No network, no real LaunchDarkly client, no real skill transport. - -The security abuse matrix — path traversal, symlink attacks, clobber -protection, corrupt manifests, atomicity under an injected crash, and the -materialization telemetry allowlist — is a separate module. """ from __future__ import annotations @@ -15,24 +11,195 @@ import hashlib import json import os +import stat from pathlib import Path -from typing import Any +from typing import Any, NamedTuple import pytest +import launchdarkly_ai_server.safe_fs as safe_fs_module import launchdarkly_ai_server.skills as skills_module +import launchdarkly_ai_server.skills_fs as skills_fs_module from launchdarkly_ai_server import ( InMemorySkillStore, Skill, SkillReference, + get_skill, + init_client, write_skills, ) MANIFEST_NAME = ".launchdarkly-skills.json" SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" + +MATERIALIZED_SIGNAL = "AgentControl Skill Materialized" +REVOKED_SIGNAL = "AgentControl Skill Revoked Received" INTEGRITY_SIGNAL = "AgentControl Skill Integrity Failure" +# The three signal names are an allowlist, not a floor. +APPROVED_SIGNALS = frozenset({MATERIALIZED_SIGNAL, REVOKED_SIGNAL, INTEGRITY_SIGNAL}) + +# Considered and deliberately excluded from SDK emission — named explicitly +# so the regression is unmissable. +REMOVED_SIGNALS = frozenset( + { + "AgentControl Skill SDK Reference Returned", + "AgentControl Skill Content Retrieved", + } +) + + +pytestmark = pytest.mark.usefixtures("reset_skill_state") + + +_INJECTED = "simulated crash between write and rename" + + +def _dir_id(path: Path) -> tuple[int, int]: + """``(st_dev, st_ino)`` — a directory's identity, independent of its name.""" + info = os.stat(path) + return (info.st_dev, info.st_ino) + + +class _RenameCall(NamedTuple): + """One intercepted ``os.replace`` of a ``SKILL.md``. + + ``src``/``dst`` are exactly what the implementation passed. Where the rename + is ``dir_fd``-relative they are bare filenames and the location lives in the + descriptors, so ``*_dir_id`` carries each descriptor's ``(st_dev, st_ino)`` + resolved *at call time* — the implementation closes the descriptors as soon + as the write returns, so they cannot be resolved from the assertions. + """ + + src: str + dst: str + src_dir_fd: int | None + dst_dir_fd: int | None + src_dir_id: tuple[int, int] | None + dst_dir_id: tuple[int, int] | None + + +class _ReplaceSpy: + """Records — and optionally fails — every atomic rename of a ``SKILL.md``. + + Write/rename interception hook: the implementation performs + the final rename through a single ``os.replace`` call site, so patching the + attribute on the ``os`` module observes it. Destinations other than + ``SKILL.md`` (i.e. the manifest's own atomic write) pass straight through — + the filter holds for both call shapes, since the ``dir_fd``-relative form + passes ``"SKILL.md"`` itself as ``dst``. + + Used two ways: to prove an injected failure is what produced an ``error`` + action (atomicity), and to prove no write was *attempted* for a + rejected key — the OS would reject several hostile keys on its + own, so a failed write is not evidence of a defense. + """ + + def __init__(self, fail: bool = False) -> None: + self.calls: list[_RenameCall] = [] + self._fail = fail + self._real = os.replace + + def __call__(self, src: Any, dst: Any, **kwargs: Any) -> None: + if str(dst).endswith("SKILL.md"): + src_dir_fd = kwargs.get("src_dir_fd") + dst_dir_fd = kwargs.get("dst_dir_fd") + self.calls.append( + _RenameCall( + src=str(src), + dst=str(dst), + src_dir_fd=src_dir_fd, + dst_dir_fd=dst_dir_fd, + src_dir_id=None if src_dir_fd is None else _fd_id(src_dir_fd), + dst_dir_id=None if dst_dir_fd is None else _fd_id(dst_dir_fd), + ) + ) + if self._fail: + raise OSError(_INJECTED) + self._real(src, dst, **kwargs) + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _ReplaceSpy: + # The attribute is set on the shared ``os`` module, so the + # single ``os.replace`` call site in safe_fs is intercepted wherever it is + # reached from. Named through the calling module rather than an arbitrary + # one so the hook documents which code it covers. + monkeypatch.setattr(safe_fs_module.os, "replace", self) + return self + + +def _fd_id(fd: int) -> tuple[int, int]: + info = os.fstat(fd) + return (info.st_dev, info.st_ino) + + +def _assert_atomic_rename_of(spy: _ReplaceSpy, skill_dir: Path) -> None: + """Assert the one recorded rename put ``SKILL.md`` into *skill_dir*. + + The temp file must be created in the target's own + directory, so the rename is atomic rather than cross-device. Two call + shapes prove it. Where the platform has ``renameat`` + the rename is ``dir_fd``-relative and the property is asserted by descriptor + identity — one descriptor for both sides, resolving to *skill_dir*'s inode — + which is stronger than comparing path strings, because it also rules out the + descriptor having been redirected between the check and the rename. On the + ``lstat`` floor (Windows) the names are full paths and share a parent. + """ + assert len(spy.calls) == 1 + call = spy.calls[0] + + if safe_fs_module.SUPPORTS_DIR_FD: + assert call.dst == "SKILL.md" + assert call.src != "SKILL.md" + assert call.src_dir_fd is not None + assert call.src_dir_fd == call.dst_dir_fd + assert call.dst_dir_id == _dir_id(skill_dir) + else: + assert Path(call.dst) == skill_dir / "SKILL.md" + assert Path(call.src).parent == skill_dir + assert Path(call.src).name != "SKILL.md" + + +class _SwapDirectoryDuring: + """Fires the directory-swap race at the exact instant of an operation. + + Renames ``/`` aside and leaves a symlink to *outside* in its + place, then lets the intercepted call proceed — the narrowest possible + version of the window an attacker with write access to the managed root + would otherwise have to hit by timing. Both hooks are the + interception points (``os.replace`` for the write, ``os.unlink`` for the + prune), so no implementation internals are touched. + """ + + def __init__(self, attribute: str, skill_dir: Path, outside: Path) -> None: + self.attribute = attribute + self.skill_dir = skill_dir + self.moved_to = skill_dir.parent / f"{skill_dir.name}.real" + self.outside = outside + self.swapped = False + self._real = getattr(os, attribute) + + def __call__(self, first: Any, *args: Any, **kwargs: Any) -> Any: + named = args[0] if args else first + if str(named).endswith("SKILL.md") and not self.swapped: + os.rename(self.skill_dir, self.moved_to) + os.symlink(self.outside, self.skill_dir, target_is_directory=True) + self.swapped = True + return self._real(first, *args, **kwargs) + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _SwapDirectoryDuring: + # ``os.replace`` is called from safe_fs, ``os.unlink`` from skills_fs; both + # resolve to the same module object, so either name reaches both. + module = safe_fs_module if self.attribute == "replace" else skills_fs_module + monkeypatch.setattr(module.os, self.attribute, self) + return self + + +_needs_dir_fd = pytest.mark.skipif( + not safe_fs_module.SUPPORTS_DIR_FD, + reason="no *at() family on this platform; the per-component lstat floor applies", +) + @pytest.fixture def root(tmp_path: Path) -> Path: @@ -580,3 +747,893 @@ async def test_mismatch_does_not_disturb_existing_managed_file( await write_skills([bad], root) assert target.read_text(encoding="utf-8") == SKILL_BODY + + +class TestAtomicityAndPermissions: + """Atomic writes, no partial files, 0644.""" + + async def test_written_file_is_0644_and_not_executable(self, root: Path) -> None: + await write_skills([_skill("a")], root) + mode = stat.S_IMODE((root / "a" / "SKILL.md").stat().st_mode) + assert mode == 0o644 + assert not mode & stat.S_IXUSR + assert not mode & stat.S_IXGRP + assert not mode & stat.S_IXOTH + + async def test_write_goes_through_a_single_atomic_rename( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Positive control for the interception hook. + + Without this, the ``spy.calls == []`` assertions in the failure tests + below and in the traversal matrix could pass in a suite where the hook + is never reachable at all. + """ + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True + # The temp file is created in the *same* directory + # as the target, so the rename is atomic rather than cross-device. + _assert_atomic_rename_of(spy, root / "a") + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_rename_failure_leaves_prior_content_intact( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + target = _place_managed(root, "a", SKILL_BODY) + spy = _ReplaceSpy(fail=True).install(monkeypatch) + + report = await write_skills([_skill("a", 2, "brand new content\n")], root) + + # The injected failure — not an unrelated rejection, and not an + # implementation that attempted nothing — is what produced the error. + _assert_atomic_rename_of(spy, target.parent) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert _INJECTED in (action.error or "") + + assert target.read_text(encoding="utf-8") == SKILL_BODY + # No temp artifact survives the failed run. + assert sorted(p.name for p in target.parent.iterdir()) == ["SKILL.md"] + + async def test_no_partial_file_at_target_after_failure( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + spy = _ReplaceSpy(fail=True).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + _assert_atomic_rename_of(spy, root / "a") + + assert report.ok is False + assert _INJECTED in (_actions_by_key(report)["a"].error or "") + assert not (root / "a" / "SKILL.md").exists() + # Neither a partial target nor a leaked temp file. + skill_dir = root / "a" + leftovers = ( + sorted(p.name for p in skill_dir.iterdir()) if skill_dir.exists() else [] + ) + assert leftovers == [] + + async def test_manifest_is_valid_json_after_a_run_with_errors( + self, root: Path + ) -> None: + report = await write_skills([_skill("a"), _skill("../evil")], root) + assert report.ok is False + assert isinstance(_read_manifest(root), dict) + + +# --------------------------------------------------------------------------- +# Security abuse matrix +# --------------------------------------------------------------------------- + +HOSTILE_KEYS = [ + "../evil", + "..", + ".", + "", + "/etc/cron.d/x", + "..\\evil", + "c:evil", + "skill:ads", + "sk\0ill", + "-skill", + "Evil", + "a/b", + "x" * 257, + "a/../../b", + "./a", + " leading-space", + "trailing-space ", +] + + +class TestPathTraversal: + """Nothing is ever written outside the root.""" + + @pytest.mark.parametrize("hostile_key", HOSTILE_KEYS) + async def test_hostile_key_is_rejected( + self, tmp_path: Path, hostile_key: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + root = tmp_path / "skills" + root.mkdir() + outside_before = sorted(p.name for p in tmp_path.iterdir()) + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill(hostile_key)], root) + + assert report.ok is False + assert [a.action for a in report.actions if a.key == hostile_key] == ["error"] + + # The SDK's key validation — not the operating system — must be what + # stopped this. An overlong key exceeds NAME_MAX, a null byte raises in + # the path API, and an absolute path outside the root usually fails on + # permissions, so "an error was reported" is not evidence of a defense + # (and the absolute-path verdict would flip on a privileged runner). + # Assert instead that no write was ever attempted. + assert spy.calls == [] + + # Nothing created outside the root, and no skill directory inside it. + assert sorted(p.name for p in tmp_path.iterdir()) == outside_before + assert [p.name for p in root.iterdir() if p.name != MANIFEST_NAME] == [] + assert list(root.rglob("SKILL.md")) == [] + + async def test_interception_hook_fires_for_a_valid_key( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Positive control for the ``spy.calls == []`` assertion above.""" + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("ok-key")], root) + + assert report.ok is True + assert [Path(call.dst).name for call in spy.calls] == ["SKILL.md"] + + async def test_long_but_filesystem_legal_key_is_written(self, root: Path) -> None: + """The ≤ 256 length bound cannot be exercised through ``write_skills``. + + A key becomes a single directory name and NAME_MAX is 255 bytes on Linux + and macOS, so the longest key the data model permits cannot exist on + disk at all. Assert the accepting side at the largest writable length; + the bound itself is covered by the pure layers (config validation and + accessor revalidation). + """ + key = "k" * 255 + report = await write_skills([_skill(key)], root) + + assert report.ok is True + assert (root / key / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_key_at_the_data_model_bound_is_reported_not_raised( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A 256-character key is valid to every pure layer but fits no filesystem. + + Config validation and the accessors must both accept exactly 256 + characters, yet NAME_MAX is 255 + on Linux and macOS, so this key reaches ``write_skills`` legitimately and + cannot become a directory. Every outcome must be visible in + the report, so it must surface as an ``error`` action rather than an + ``OSError`` escaping the call — which would also skip the manifest rewrite + and orphan any file already written in the same run. + """ + spy = _ReplaceSpy().install(monkeypatch) + long_key = "a" * 256 + + report = await write_skills([_skill("good"), _skill(long_key)], root) + + by_key = _actions_by_key(report) + assert by_key[long_key].action == "error" + assert by_key["good"].action == "written" + # The bare-filename ``dst`` of a ``dir_fd``-relative rename carries no + # directory, so "the path does not contain the hostile key" is no longer + # a meaningful check. Assert the stronger thing instead: the only rename + # that happened was into the valid skill's own directory. + assert [call.dst_dir_id for call in spy.calls] == [_dir_id(root / "good")] + # The valid skill is fully reconciled: written AND recorded, not orphaned. + assert (root / "good" / "SKILL.md").exists() + assert "good/SKILL.md" in _read_manifest(root)["entries"] + + async def test_valid_keys_still_write_alongside_rejected_ones( + self, root: Path + ) -> None: + report = await write_skills([_skill("good"), _skill("../evil")], root) + by_key = _actions_by_key(report) + assert by_key["good"].action == "written" + assert by_key["../evil"].action == "error" + assert (root / "good" / "SKILL.md").exists() + + async def test_traversal_key_does_not_create_parent_files( + self, tmp_path: Path + ) -> None: + root = tmp_path / "skills" + root.mkdir() + await write_skills([_skill("../../escaped")], root) + assert not (tmp_path / "escaped").exists() + assert not (tmp_path.parent / "escaped").exists() + + +@pytest.mark.skipif( + not hasattr(os, "symlink"), reason="platform has no symlink support" +) +class TestSymlinkAttacks: + """Never write through a symlink.""" + + async def test_symlinked_root_raises(self, tmp_path: Path) -> None: + real_dir = tmp_path / "real" + real_dir.mkdir() + link_root = tmp_path / "link" + link_root.symlink_to(real_dir, target_is_directory=True) + + with pytest.raises(ValueError): + await write_skills([_skill("a")], link_root) + + assert list(real_dir.iterdir()) == [] + + async def test_symlinked_skill_directory_is_refused(self, tmp_path: Path) -> None: + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (root / "a").symlink_to(outside, target_is_directory=True) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert list(outside.iterdir()) == [] + + async def test_symlinked_target_file_is_refused(self, tmp_path: Path) -> None: + root = tmp_path / "skills" + root.mkdir() + outside_file = tmp_path / "victim.md" + outside_file.write_text("victim content\n", encoding="utf-8") + (root / "a").mkdir() + (root / "a" / "SKILL.md").symlink_to(outside_file) + # Manifest lists the path so clobber protection is not what saves us. + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 1, "victim content\n")}, + }, + ) + + report = await write_skills([_skill("a", 2, "attacker payload\n")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert outside_file.read_text(encoding="utf-8") == "victim content\n" + + async def test_symlinked_target_is_not_pruned(self, tmp_path: Path) -> None: + """A manifest-listed path that is a symlink is refused, not unlinked. + + Asserting only that the victim file survives proves nothing here: + unlinking a symlink never touches its target, so that assertion holds + for an implementation with no symlink check at all. The observable + contract is the refusal itself (prune path). + """ + root = tmp_path / "skills" + root.mkdir() + outside_file = tmp_path / "victim.md" + outside_file.write_text("victim content\n", encoding="utf-8") + (root / "a").mkdir() + link = root / "a" / "SKILL.md" + link.symlink_to(outside_file) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 1, "victim content\n")}, + }, + ) + + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert [a for a in report.actions if a.action == "removed"] == [] + # The symlink itself is left in place and stays managed. + assert link.is_symlink() + assert "a/SKILL.md" in _read_manifest(root)["entries"] + assert outside_file.read_text(encoding="utf-8") == "victim content\n" + + @_needs_dir_fd + async def test_directory_swapped_at_the_rename_cannot_redirect_the_write( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The swap window is closed, not merely narrowed. + + Every check in the world is worthless if the final rename re-resolves + ``/`` from its path: an attacker holding write permission on + the managed root can replace the validated directory with a symlink in + between and redirect the write out of the root. The rename is therefore + performed relative to a descriptor pinned to the directory that was + checked, so it follows the inode rather than the name. + """ + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + race = _SwapDirectoryDuring("replace", root / "a", outside).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert list(outside.iterdir()) == [] + assert (race.moved_to / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + assert report.ok is True + + @_needs_dir_fd + async def test_directory_swapped_at_the_prune_cannot_redirect_the_unlink( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The same window on the destructive side. + + ``unlink`` never follows a *trailing* symlink, but it does resolve the + directory above it, so the swap turns a prune into a delete of an + attacker-chosen outside file. The unlink is descriptor-relative for the + same reason the rename is. + """ + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + victim = outside / "SKILL.md" + victim.write_text("precious\n", encoding="utf-8") + _place_managed(root, "a", SKILL_BODY) + race = _SwapDirectoryDuring("unlink", root / "a", outside).install(monkeypatch) + + report = await write_skills([], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert victim.read_text(encoding="utf-8") == "precious\n" + assert not (race.moved_to / "SKILL.md").exists() + assert [a.action for a in report.actions if a.key == "a"] == ["removed"] + + +class TestWithoutDirFd: + """The full-path fallback for platforms with no ``*at()`` family. + + On Windows ``os.open`` cannot open a directory at all, so acquiring the + descriptor must not even be attempted there — a fallback reached only after + a descriptor open would leave every write, prune and manifest rewrite + failing rather than falling back. These tests force the flag off so the + fallback is exercised on POSIX too. + """ + + @pytest.fixture(autouse=True) + def _no_dir_fd(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Models Windows: no ``*at()`` family, and directories cannot be opened. + + Forcing the flag off alone would not reproduce the platform, because + ``os.open`` on a directory succeeds on POSIX — the fallback would be + reached either way. Making that call raise the ``PermissionError`` + Windows raises is what proves the descriptor open is never attempted. + """ + monkeypatch.setattr(safe_fs_module, "SUPPORTS_DIR_FD", False) + real_open = os.open + + def no_directory_open(path: Any, *args: Any, **kwargs: Any) -> int: + if os.path.isdir(path): + raise PermissionError(13, "Permission denied") + return real_open(path, *args, **kwargs) + + monkeypatch.setattr(safe_fs_module.os, "open", no_directory_open) + + async def test_write_prune_and_manifest_all_succeed(self, root: Path) -> None: + first = await write_skills([_skill("a"), _skill("b")], root) + assert first.ok is True, _error_messages(first) + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + assert _manifest_path(root).exists() + assert stat.S_IMODE((root / "a" / "SKILL.md").stat().st_mode) == 0o644 + + second = await write_skills([_skill("a")], root) + + assert second.ok is True, _error_messages(second) + assert not (root / "b" / "SKILL.md").exists() + assert "b/SKILL.md" not in _read_manifest(root)["entries"] + + async def test_a_symlinked_skill_directory_is_still_refused( + self, root: Path, tmp_path: Path + ) -> None: + """The fallback keeps the ``lstat`` floor: no writing through a link.""" + outside = tmp_path / "outside" + outside.mkdir() + (root / "a").symlink_to(outside, target_is_directory=True) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert list(outside.iterdir()) == [] + + +class TestNonRegularFiles: + """A managed path that is not a regular file is refused, never read.""" + + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="no FIFOs on this platform") + async def test_a_fifo_at_the_managed_path_does_not_block(self, root: Path) -> None: + """Reading a FIFO with no writer blocks forever. + + Same attacker capability the symlink checks defend against: swapping a + managed ``SKILL.md`` for a FIFO would otherwise hang the whole reconcile + — and the caller's event loop with it — well past any ``timeout``, since + the deadline is only consulted between steps. + """ + skill_dir = root / "a" + skill_dir.mkdir() + os.mkfifo(skill_dir / "SKILL.md") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 1, SKILL_BODY)}, + }, + ) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "regular file" in action.error + assert stat.S_ISFIFO(os.lstat(skill_dir / "SKILL.md").st_mode) + + +class TestClobberProtection: + """Destructive ops only on manifest-listed paths.""" + + async def test_unmanaged_file_is_never_overwritten(self, root: Path) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text("user authored\n", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert target.read_text(encoding="utf-8") == "user authored\n" + + async def test_unmanaged_file_is_never_deleted(self, root: Path) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text("user authored\n", encoding="utf-8") + + await write_skills([], root) + + assert target.read_text(encoding="utf-8") == "user authored\n" + + async def test_manifest_entry_with_mismatched_key_does_not_authorize( + self, root: Path + ) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text("user authored\n", encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "a/SKILL.md": _entry("different-key", 1, "user authored\n") + }, + }, + ) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert target.read_text(encoding="utf-8") == "user authored\n" + + +DIVERGENT_CONTENT = "existing content\n" + + +def _live_entries() -> dict[str, Any]: + """A parseable entries map that really does claim ``a/SKILL.md`` as managed.""" + return {"a/SKILL.md": _entry("a", 1, DIVERGENT_CONTENT)} + + +# The first six variants are unparseable: ``entries`` is missing, the wrong type, +# or the whole document is garbage. That makes "performed no destructive action" +# arithmetic rather than a defense — with no entries to act on, a file at a +# managed path is protected by clobber protection and there is nothing to prune, +# so those cases pass against an implementation that simply treats a corrupt +# manifest as an empty one. +# +# The ``*_live_entries`` variants are the ones that actually test round-tripping: corrupt +# ONLY in ``manifestVersion``, with a valid entries map listing the managed path +# under a matching key. The implementation has everything it needs to overwrite +# and to prune, and must refuse anyway. +CORRUPT_MANIFESTS: list[tuple[str, Any]] = [ + ("garbage", "{not json at all"), + ("empty", ""), + ("wrong_types", {"manifestVersion": 1, "entries": ["a/SKILL.md"]}), + ("entries_missing", {"manifestVersion": 1}), + ("future_version", {"manifestVersion": 2, "entries": {}}), + ("version_not_int", {"manifestVersion": "1", "entries": {}}), + ("future_version_live_entries", {"manifestVersion": 2, "entries": _live_entries()}), + ( + "version_not_int_live_entries", + {"manifestVersion": "1", "entries": _live_entries()}, + ), +] + +LIVE_ENTRY_MANIFESTS: list[tuple[str, Any]] = [ + case for case in CORRUPT_MANIFESTS if case[0].endswith("_live_entries") +] + + +class TestCorruptManifest: + """Corrupt manifest fails closed, non-destructively.""" + + @pytest.mark.parametrize( + "raw", + [case[1] for case in CORRUPT_MANIFESTS], + ids=[case[0] for case in CORRUPT_MANIFESTS], + ) + async def test_no_destructive_action_and_error_reported( + self, root: Path, raw: Any + ) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text(DIVERGENT_CONTENT, encoding="utf-8") + _write_manifest(root, raw) + + report = await write_skills([_skill("a", 2, "new content\n")], root) + + assert report.ok is False + # The error must name the manifest. For the unparseable variants the file + # at the managed path is also unmanaged, so a bare "some error happened" + # assertion is satisfied by clobber protection alone and says nothing + # about whether the manifest state was detected at all. + errors = _error_messages(report) + assert any("manifest" in e.lower() for e in errors), errors + assert target.read_text(encoding="utf-8") == DIVERGENT_CONTENT + + async def test_run_level_error_carries_the_empty_key_sentinel( + self, root: Path + ) -> None: + """A run-level error has no skill key to hang off. + + The empty string is public API surface: a caller grouping the report by + key has to know the sentinel exists. Asserted here rather than in the + parametrized cases above so it is a statement about the manifest error + specifically, not about whichever error happens to come first. + """ + _write_manifest(root, "{not json at all") + + report = await write_skills([_skill("a")], root) + + manifest_errors = [ + action + for action in report.errors + if "manifest" in (action.error or "").lower() + ] + assert manifest_errors, _error_messages(report) + assert all(action.key == "" for action in manifest_errors) + # A per-skill error in the same report still carries its real key, so the + # sentinel is not simply "every error action has an empty key". + assert all( + action.key != "" + for action in report.errors + if action not in manifest_errors + ) + + @pytest.mark.parametrize( + "raw", + [case[1] for case in LIVE_ENTRY_MANIFESTS], + ids=[case[0] for case in LIVE_ENTRY_MANIFESTS], + ) + async def test_managed_file_is_not_pruned_when_only_the_version_is_corrupt( + self, root: Path, raw: Any + ) -> None: + """The prune counterpart of the live-entries cases. + + Here the implementation can read the entries map and knows exactly which + file it owns, so refusing to remove it is a real decision rather than an + absence of information. + """ + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text(DIVERGENT_CONTENT, encoding="utf-8") + _write_manifest(root, raw) + + report = await write_skills([], root) + + assert report.ok is False + assert target.read_text(encoding="utf-8") == DIVERGENT_CONTENT + assert [a for a in report.actions if a.action == "removed"] == [] + errors = _error_messages(report) + assert any("manifest" in e.lower() for e in errors), errors + + async def test_nothing_is_pruned_under_a_corrupt_manifest(self, root: Path) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text(DIVERGENT_CONTENT, encoding="utf-8") + _write_manifest(root, "{not json at all") + + report = await write_skills([], root) + + assert report.ok is False + assert target.exists() + assert [a for a in report.actions if a.action == "removed"] == [] + errors = _error_messages(report) + assert any("manifest" in e.lower() for e in errors), errors + + async def test_brand_new_paths_may_still_be_written(self, root: Path) -> None: + _write_manifest(root, "{not json at all") + + report = await write_skills([_skill("fresh")], root) + + actions = _actions_by_key(report) + assert actions["fresh"].action == "written" + assert (root / "fresh" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_corrupt_manifest_file_is_not_destroyed(self, root: Path) -> None: + _write_manifest(root, "{not json at all") + + await write_skills([], root) + + assert _manifest_path(root).exists() + assert _manifest_path(root).read_text(encoding="utf-8") == "{not json at all" + + +class TestWriteSkillsTelemetry: + """Materialized / revoked signals from write_skills.""" + + async def test_materialized_signal_per_action( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + _place_managed(root, "same", SKILL_BODY) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "same/SKILL.md": _entry("same", 1, SKILL_BODY), + "stale/SKILL.md": _entry("stale", 1, "old\n"), + }, + }, + ) + (root / "stale").mkdir() + (root / "stale" / "SKILL.md").write_text("old\n", encoding="utf-8") + + await write_skills( + [_skill("same"), _skill("stale", 2, "fresh\n"), _skill("brand-new")], + root, + ) + + signals = recording_emitter.signals(MATERIALIZED_SIGNAL) + by_key = {s["skill_key"]: s for s in signals} + assert len(signals) == 3 + assert by_key["same"]["reconcile_action"] == "skipped_current" + assert by_key["stale"]["reconcile_action"] == "updated" + assert by_key["brand-new"]["reconcile_action"] == "written" + + async def test_materialized_signal_properties( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + + await write_skills([_skill("a")], root) + + props = recording_emitter.signals(MATERIALIZED_SIGNAL)[0] + assert props["skill_key"] == "a" + assert props["content_bytes"] == len(SKILL_BODY.encode("utf-8")) + assert props["content_hash"] == _hash(SKILL_BODY) + assert props["reconcile_action"] == "written" + assert props["language"] == "python" + + async def test_no_filesystem_paths_in_telemetry( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + + await write_skills([_skill("a")], root) + + for _signal, props in recording_emitter.records: + assert "target_path" not in props + for value in props.values(): + assert str(root) not in str(value) + + async def test_no_skill_body_in_telemetry( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + + await write_skills([_skill("a")], root) + + for _signal, props in recording_emitter.records: + for value in props.values(): + assert "Do the thing." not in str(value) + + async def test_revoked_signal_on_prune( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + _place_managed(root, "gone", SKILL_BODY, version=4) + + await write_skills([], root) + + revoked = recording_emitter.signals(REVOKED_SIGNAL) + assert len(revoked) == 1 + assert revoked[0]["skill_key"] == "gone" + assert revoked[0]["version"] == 4 + assert revoked[0]["removed_from_disk"] is True + assert revoked[0]["language"] == "python" + + async def test_revoked_signal_redacts_an_untrusted_manifest_version( + self, root: Path, recording_emitter: Any + ) -> None: + """The manifest is untrusted, so its version is shape-checked first. + + Anything with write access to the managed root can plant an arbitrary + string here; echoing it verbatim would publish attacker-controlled + content — a skill body, or PII — as a signal property. + """ + skills_module._set_emitter_for_testing(recording_emitter) + target = root / "gone" / "SKILL.md" + target.parent.mkdir(parents=True) + target.write_text(SKILL_BODY, encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "gone/SKILL.md": { + "key": "gone", + "version": "Do the thing. " * 8, + "sha256": _hash(SKILL_BODY), + } + }, + }, + ) + + await write_skills([], root) + + revoked = recording_emitter.signals(REVOKED_SIGNAL) + assert len(revoked) == 1 + assert "version" not in revoked[0] + assert revoked[0]["skill_key"] == "gone" + for value in revoked[0].values(): + assert "Do the thing." not in str(value) + + async def test_no_revoked_signal_when_prune_disabled( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + _place_managed(root, "gone", SKILL_BODY) + + await write_skills([], root, prune=False) + + assert recording_emitter.signals(REVOKED_SIGNAL) == [] + + async def test_write_skills_records_no_signal_outside_the_approved_set( + self, root: Path, recording_emitter: Any + ) -> None: + """Allowlist sweep over a run that exercises all four actions. + + The accessor-side half of this sweep is + ``test_accessors_record_no_signal_outside_the_approved_set`` in + test_skills.py. Asserted over recorded strings, so no module-level + signal-name constant is required of the implementation. + """ + skills_module._set_emitter_for_testing(recording_emitter) + for key, content in (("same", SKILL_BODY), ("stale", "old\n"), ("gone", "g\n")): + (root / key).mkdir() + (root / key / "SKILL.md").write_text(content, encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "same/SKILL.md": _entry("same", 1, SKILL_BODY), + "stale/SKILL.md": _entry("stale", 1, "old\n"), + "gone/SKILL.md": _entry("gone", 1, "g\n"), + }, + }, + ) + + report = await write_skills( + [_skill("same"), _skill("stale", 2, "fresh\n"), _skill("brand-new")], + root, + ) + + # Positive control: the subset assertion is vacuous unless the run + # really did produce all four actions and record for them. + assert {a.action for a in report.actions} == { + "skipped_current", + "updated", + "written", + "removed", + } + recorded = {signal for signal, _props in recording_emitter.records} + assert recorded <= APPROVED_SIGNALS, ( + f"unapproved signal(s): {sorted(recorded - APPROVED_SIGNALS)}" + ) + assert not recorded & REMOVED_SIGNALS + assert recorded == {MATERIALIZED_SIGNAL, REVOKED_SIGNAL} + + async def test_no_ld_track_calls_from_write_skills( + self, root: Path, mock_ld_client: Any + ) -> None: + await init_client(client=mock_ld_client) + + await write_skills([_skill("a"), _skill("../evil")], root) + + mock_ld_client.track.assert_not_called() + + async def test_throwing_emitter_never_breaks_the_reconcile( + self, root: Path, throwing_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(throwing_emitter) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_integrity_signal_property_keys_match_across_layers( + self, root: Path, recording_emitter: Any + ) -> None: + """The same defect, caught at either layer, records + the same property keys. + + Verification runs twice by design: once at the accessor boundary and + again immediately before a write. The signal contract marks ``expected_hash`` + optional, so an implementation that populates it on one path and omits + it on the other passes every other assertion here while making the + signal's shape depend on which internal code path noticed. Oversize + content is the case reachable from both layers with the expected hash in + hand throughout. + """ + skills_module._set_emitter_for_testing(recording_emitter) + oversize = "x" * (64 * 1024 + 1) + content_hash = _hash(oversize) + + # Layer 1 — the accessor boundary. + store = InMemorySkillStore() + store.put( + { + "key": "big", + "version": 1, + "content": oversize, + "contentHash": content_hash, + } + ) + skills_module._set_store(store) + assert await get_skill("big") is None + + # Layer 2 — verify-then-write, on a directly constructed Skill. + report = await write_skills( + [ + Skill( + key="big", + version=1, + content=oversize.encode("utf-8"), + content_hash=content_hash, + ) + ], + root, + ) + assert report.ok is False + + failures = recording_emitter.signals(INTEGRITY_SIGNAL) + assert len(failures) == 2, failures + accessor_keys, write_keys = (set(props) for props in failures) + assert accessor_keys == write_keys, ( + f"accessor-only keys: {sorted(accessor_keys - write_keys)}; " + f"write-only keys: {sorted(write_keys - accessor_keys)}" + ) + assert "expected_hash" in accessor_keys From 568729d3945d1448f4f1a17d066155ad4c269970 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 25 Aug 2026 16:10:53 -0400 Subject: [PATCH 05/22] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20materialize=20onto=20disk=20under=20a=20manifest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fourth of five slices. Adds `write_skills`, which writes `//SKILL.md` and reconciles against a manifest recording what the SDK owns, so it overwrites or removes only files it wrote — a file you placed yourself is reported and left untouched. report = await write_skills(refs, ".claude/skills") `skills` accepts `Skill` values, references, bare keys, or the literal `"*"` for everything the store holds. Every outcome is visible in the returned `ReconcileReport`: one `ReconcileAction` per skill, carrying `written`, `updated`, `skipped_current`, `removed` or `error`, plus `.ok` and `.errors`. A failure belonging to the run rather than to one skill — an unreadable manifest, a retrieval that failed before any key was known — carries the empty string as its key. Writes are atomic, at mode 0644, and every destructive step runs against a descriptor pinned to a directory that was already checked, so a path swapped after the check cannot redirect a write or an unlink out of the managed root. Where the platform has no `*at()` family the identical sequence runs against full paths. The defenses, all of them deliberate and all of them tested: - The key is re-validated here regardless of upstream validation, before any filesystem call, because a key becomes a directory name. The data model allows 256 characters and `NAME_MAX` is 255 bytes, so an over-long key is refused too. - Never write or unlink through a symlink, on the write path or the prune path. - Destruction only on manifest-listed paths whose key matches. - A corrupt manifest fails closed: no overwrites and no prunes, brand-new paths may still be written, an error action names the manifest, and the manifest is not rewritten. - An incomplete retrieval suppresses pruning, so a transport outage cannot read as "everything was revoked". - Content is re-verified immediately before the write, because a `Skill` can also be constructed directly by a caller. Pruning removes formerly-managed skills that are no longer referenced, which is how revocation takes effect. `timeout` bounds retrieval, the writes and the pruning; only the final manifest rewrite runs past it, so files already written are never orphaned. `write_skills` performs synchronous filesystem I/O and does not yield — it is `async` for signature parity with the other accessors. Reconcile one root at a time: a run is atomic against the rest of the loop today, so wrapping it to run concurrently makes two runs against one root race on the manifest. The `"*"` form collapses to one object per key at its newest version, since `//SKILL.md` is a single path and writing it twice in one run is a bug rather than a policy, and it reports a withholding count at WARN for the same reason the accessors do. The security abuse matrix — path traversal, symlink attacks, clobber protection, corrupt manifests, atomicity under an injected crash, and the materialization telemetry allowlist — is the next slice. The guards it exercises are all here; what lands next is the adversary that proves each one fails without them. Testing: `uv run pytest` → 1216 passed. `ruff check`, `ruff format --check`, and `mypy packages/*/src` all clean. Co-Authored-By: Claude Fable 5 --- packages/client/README.md | 63 +- packages/client/agents.md | 114 ++- .../src/launchdarkly_ai_server/__init__.py | 21 + .../src/launchdarkly_ai_server/skills.py | 4 +- .../src/launchdarkly_ai_server/skills_fs.py | 960 ++++++++++++++++++ .../src/launchdarkly_ai_server/types.py | 48 + packages/client/tests/test_skills.py | 123 ++- packages/client/tests/test_skills_fs.py | 582 +++++++++++ 8 files changed, 1889 insertions(+), 26 deletions(-) create mode 100644 packages/client/src/launchdarkly_ai_server/skills_fs.py create mode 100644 packages/client/tests/test_skills_fs.py diff --git a/packages/client/README.md b/packages/client/README.md index ae572b3b..d9c1db15 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -223,15 +223,17 @@ asyncio.run(main()) ### Agent Skills Skills are versioned `SKILL.md` documents managed in LaunchDarkly and attached to AI Config -variations by reference. The SDK surfaces which skills a config references and retrieves -their content. Materializing them onto disk, where agent runtimes discover them, follows. +variations by reference. The SDK surfaces which skills a config references, retrieves their +content, and materializes them onto disk where agent runtimes (Claude Agent SDK, and +anything else following the `//SKILL.md` convention) discover them. ```python import asyncio import hashlib +from pathlib import Path from launchdarkly_ai_server import ( - init_client, inspect_config, skill_refs, get_skill, get_skills, + init_client, inspect_config, skill_refs, get_skill, write_skills, InMemorySkillStore, ) @@ -260,13 +262,18 @@ async def main(): if skill is not None: print(skill.content) - # 3. Or resolve the config's references in one call. - for s in await get_skills(refs): - print(s.key, s.version) + # 3. Write them where the agent runtime will look. Only the leaf directory is + # created, so the parent must already exist. + Path(".claude").mkdir(exist_ok=True) + report = await write_skills(refs, ".claude/skills") + for action in report.errors: + print(f"skill {action.key or ''}: {action.error}") asyncio.run(main()) ``` +Pass `"*"` instead of a reference list to materialize every skill the store holds. + **`skills` is now a validated field.** Config parsing fails closed on a `skills` value that is not a list of `{key, version}` objects (key matching `^[a-z0-9][a-z0-9-]*$`, version an integer ≥ 1): the whole variation is rejected, `inspect_config` returns `config: None`, and @@ -333,7 +340,22 @@ treat `expected_hash` / `observed_hash` as the evidence pair. **Versions are selected, not filtered.** A store may hold several versions of one key at once, because a delivery payload does: the newest version of every skill, plus every version a variation currently pins. `get_skill("k", version=1)` asks the store for version -1 and gets it even when a newer one is also held. +1 and gets it even when a newer one is also held. `all_skills()` and `write_skills("*")` +collapse to one skill per key at its newest version, since `//SKILL.md` is a +single path. + +**The root's parent must exist.** `write_skills` creates the root itself but never its +ancestors, so a typo cannot scatter a directory tree across your project. An absent parent, +a root that is an existing file, and a root that is a symlink each raise `ValueError` — +these are caller errors, distinct from the per-skill `error` actions in the report. + +**`write_skills` is deliberately conservative** about your filesystem. It writes only +`//SKILL.md`, tracks what it owns in a manifest at +`/.launchdarkly-skills.json`, and will overwrite or delete **only** paths that +manifest records. A file you placed yourself is reported as an error and left untouched; it +never writes through a symlink; writes are atomic (temp file, `fsync`, rename) at mode +`0644`; and if the manifest is unreadable it performs no destructive action at all. Removing +a skill from a variation is how revocation works — the next reconcile prunes it. | Export | Description | |---|---| @@ -341,11 +363,34 @@ version a variation currently pins. `get_skill("k", version=1)` asks the store f | `get_skill(key, *, version=None)` | One verified skill, or `None`. `version=None` means newest available; a specific `version` matches exactly. Raises only when no store is configured. | | `get_skills(refs)` | Batch form. Accepts `SkillReference` values and bare key strings (string = latest). Results follow input order; missing or unverifiable entries are omitted. | | `all_skills()` | Every verified skill the store holds, one per key at its newest version. | +| `write_skills(skills, root, *, prune=True, timeout=10.0, on_unavailable="keep")` | Materialize skills under `root`, returning a `ReconcileReport`. `prune` removes formerly-managed skills no longer requested. `on_unavailable="raise"` raises instead of reporting when content cannot be retrieved. Raises `ValueError` for an unusable root, a negative `timeout`, or an unrecognised `on_unavailable`. **Performs synchronous filesystem I/O — see the note below.** | | `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)`. | | `InMemorySkillStore(objects=None)` | A dict-backed store with `put(raw)`, for local development and testing. Holds several versions of a key. | Configure the store with `init_client(options={"skillStore": store})`. With none configured, -the accessors raise `RuntimeError` explaining what to do. `shutdown()` clears it. +the accessors raise `RuntimeError` explaining what to do and `write_skills` reports the +failure in its report (or raises, with `on_unavailable="raise"`). `shutdown()` clears it. + +`ReconcileReport.actions` holds one `ReconcileAction` per outcome — `written`, `updated`, +`skipped_current`, `removed`, or `error` — each carrying `key`, `version`, the resolved +`path`, and `error`. `report.ok` is `True` when no action is an `error`, and +`report.errors` is just the `error` actions, so you rarely need to filter `actions` +yourself. A failure that belongs to the whole run rather than to one skill — an unreadable +manifest, for instance — carries the empty string as its `key`. + +The fixed on-disk values are exported too, so you do not have to hardcode them: +`MANIFEST_FILENAME` (`.launchdarkly-skills.json`, handy for a `.gitignore`), +`SKILL_FILENAME`, and `MANIFEST_VERSION`. So are the two closed-set types, for annotating +your own helpers: `ReconcileActionKind` (`written` / `updated` / `skipped_current` / +`removed` / `error`) and `OnUnavailable` (`keep` / `raise`). + +**`write_skills` blocks.** It is `async` for parity with the other accessors and with the +TypeScript SDK, but it awaits nothing: every read, write, `fsync` and rename runs inline, +so a large reconcile holds the event loop for its duration. Wrap it in +`asyncio.to_thread` if that matters. For the same reason `timeout` is checked between +steps rather than interrupting one already in progress. Reconcile one root at a time, +though: because nothing yields today, a run is atomic against the rest of your loop, and +wrapping it to run concurrently makes two runs against the same root race on the manifest. `all_objects` returns one entry per `(key, version)` under keys that are **opaque** to the SDK — identity is read from each object's own `key` and `version` fields, so a store is free @@ -389,3 +434,5 @@ All types are exported from this package. Handler packages import them from here | `GraphTopology` | The parsed graph flag shape (`root` + `edges`) | | `Skill` | A frozen skill document: `.key`, `.version`, `.content` (verified verbatim `bytes`), `.content_hash`, `.name?`, `.description?` | | `SkillReference` | A frozen version-pinned pointer to a skill: `.key`, `.version` | +| `ReconcileAction` | One `write_skills` outcome: `.key`, `.action`, `.version?`, `.path?`, `.error?` | +| `ReconcileReport` | The `write_skills` result: `.actions`, `.ok`, and `.errors` | diff --git a/packages/client/agents.md b/packages/client/agents.md index 152f5771..feb2e348 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,6 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | +| `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | | `src/launchdarkly_ai_server/registry.py` | `Registry`, `global_registry`, `compose`, `resolve_handlers`, `resolve_tools` | @@ -56,7 +57,7 @@ from launchdarkly_ai_server import ( TrackData, UsageDict, HandlerResult, HandlerStreamEvent, StreamEvent, StreamChunkEvent, StreamDoneEvent, ExecuteStreamEvent, ExecuteStreamDoneEvent, VariationMeta, InitClientOptions, JudgeResult, ParseResult, ParseSuccess, ParseFailure, - Skill, SkillReference, + Skill, SkillReference, ReconcileAction, ReconcileReport, ) # Utilities @@ -76,8 +77,10 @@ from launchdarkly_ai_server import config, graph, resolve_graph # Agent Skills from launchdarkly_ai_server import ( - skill_refs, get_skill, get_skills, all_skills, + skill_refs, get_skill, get_skills, all_skills, write_skills, SkillStore, InMemorySkillStore, + SKILL_FILENAME, MANIFEST_FILENAME, MANIFEST_VERSION, + ReconcileActionKind, OnUnavailable, # the two closed-set unions ) ``` @@ -179,7 +182,7 @@ This is an OTel context value, not W3C baggage, so the id does not leak onto out Versioned `SKILL.md` documents attached to AI Config variations by reference, retrieved through an injectable store, and materialized onto disk for agent runtimes to discover. -Three layers, in increasing order of blast radius. Only the first is implemented here: +Three layers, in increasing order of blast radius: 1. **Reference discovery** — `skill_refs(config)` projects the config's `skills` array into typed `SkillReference` values. Pure: no network, no client, no store, no telemetry. @@ -190,7 +193,8 @@ Three layers, in increasing order of blast radius. Only the first is implemented `init_client(options={"skillStore": store})`; with none configured the accessors raise an actionable `RuntimeError`. A delivery transport can be added behind the seam without touching the public API. -3. **Materialization** — writing skills onto disk under a manifest. +3. **Materialization** — `write_skills(skills, root)` writes `//SKILL.md` and + reconciles against a manifest at `/.launchdarkly-skills.json`. ### The store seam, and why version is part of the lookup @@ -241,6 +245,30 @@ Store data is **untrusted input**; the transport is not part of the trust bounda - **Attacker-controlled strings are never echoed into telemetry.** `contentHash` and `key` come off the wire, so a store could put the skill body in either; both are shape-checked and redacted before they reach a signal or a log line. +- **The key is re-validated inside `write_skills`**, regardless of upstream validation — a + key becomes a directory name. Rejection happens before any filesystem call. +- **Never write through a symlink**, in either the skill directory or the target file, on + the write path *and* the prune path. +- **Destructive operations only on manifest-listed paths whose `key` matches.** A file at a + managed path with no matching manifest entry is reported as `error` and left alone. +- **A corrupt manifest fails closed**: unreadable, unparseable, not an object, malformed + `entries`, or a `manifestVersion` this release cannot read means no overwrites and no + prunes, brand-new paths may still be written, an `error` action names the manifest, and + the manifest file itself is not rewritten. +- **An incomplete retrieval suppresses pruning.** Otherwise a transport outage would read + as "everything was revoked" and delete the customer's managed files. +- **Writes are atomic**: temp file created exclusively in the target's *own* directory, + mode `0644` set explicitly (never inherited from the umask, never executable), write, + fsync, `os.replace`, fsync the directory. `os.replace` is the single rename call site + and must not be swapped for `os.rename`. +- **Every operation under the root goes through a pinned descriptor, not a path.** See + "Descriptor-pinned filesystem access" below. Re-resolving `/` from its path at + write or unlink time reopens a swap window that the checks above cannot cover. +- **A key valid to the data model may still be unrepresentable on disk.** The model allows + 256 characters; `NAME_MAX` is 255 bytes. `write_skills` rejects an over-long key before + any filesystem call, and every per-skill filesystem failure is caught at the loop so it + becomes an `error` action — aborting the loop would skip the manifest rewrite and orphan + files already written in that run. - **A key is untrusted input everywhere it appears.** `skill_key_rejection_reason` is the single canonical explanation, so the config parser and the reference projection reject a key for the same stated reason — and so does every layer added later. A silently @@ -262,9 +290,6 @@ Exactly three signals exist, and the list is an **allowlist, not a floor**: | `AgentControl Skill Materialized` | each `written` / `updated` / `skipped_current` | `skill_key`, `content_bytes`, `content_hash`, `reconcile_action`, `language` | | `AgentControl Skill Revoked Received` | prune removes a formerly managed skill | `skill_key`, `version`, `removed_from_disk`, `language` | -The last two belong to the materialization layer and have no caller yet; they live here -with the first so the allowlist is one section of one file rather than three sites to audit. - ### The integrity-failure log record The signal above is product telemetry; the **log record** beside it is the customer-owned @@ -321,11 +346,12 @@ where they cannot see it. `AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` were considered and **deliberately excluded from SDK emission** — both are observable server-side. Do not add them. The skill body never appears in a signal, a log line, or -an error message, and no signal carries a filesystem path. An emitter that raises is caught -and logged; it never fails the operation. +an error message, and no signal carries a filesystem path (paths belong in the returned +`ReconcileReport`, which is user-facing API). An emitter that raises is caught and logged; +it never fails the operation. -Module state lives in `skills_core.py`, so there is exactly one store and one emitter -however the feature is entered. All three signals are emitted from the `record_*` functions +Module state lives in `skills_core.py`, the module `skills.py` and `skills_fs.py` share, +so there is exactly one store and one emitter however the feature is entered. All three signals are emitted from the `record_*` functions next to the seam there — nothing outside that module calls `emit`, so the allowlist is enforced in one place. @@ -366,7 +392,14 @@ primitives live in `safe_fs.py`, which knows nothing about skills: *trailing* symlink, but it does resolve the directory above it, so the same swap turns a removal into a delete of an attacker-chosen file. A symlink found where this SDK expects its own file raises `SymlinkRefused` rather than being tidied away: the state on disk is - not what the caller believes, and that is the caller's to report. + not what the caller believes, and that is the caller's to report. `_prune_one` goes + through it; `rmdir` stays path-based and is safe that way, since it fails `ENOTDIR` on a + symlink and only ever succeeds on an empty directory. + +Every `lstat`, `realpath` and containment check on the skills side lives in one shared +`_unsafe_path_reason`, so the write and prune paths cannot drift apart on what counts as +unsafe. `skills_fs._prune_one` spells its symlink check `os.stat(..., follow_symlinks=False)` +rather than `os.lstat`, matching the name the capability probe advertises. `safe_fs.SUPPORTS_DIR_FD` gates all of it, and the probe is not the obvious one. `os.supports_dir_fd` is populated per underlying syscall, and CPython registers `renameat` @@ -379,7 +412,43 @@ and silently turns the defense off, so the probe names the advertised twins (Windows) `open_directory_nofollow` returns `None` after an `lstat` check instead of attempting the descriptor open — `os.open` cannot open a directory there — and every caller falls back to the identical full-path sequence, the per-component `lstat` floor. The -residual window on those platforms is documented rather than closed. +residual window on those platforms is documented rather than closed; the TOCTOU tests skip +off this same flag, deliberately, so a probe that wrongly reports "unsupported" cannot also +silently skip the tests that would have caught it. + +Both call shapes are admitted by the test seam. `os.replace` remains the single +interceptable rename call site; under the descriptor-relative shape `dst` is the bare string +`"SKILL.md"`, so an `endswith("SKILL.md")` spy filter still matches, and the +same-directory requirement is proved by descriptor identity (`src_dir_fd == dst_dir_fd`, +resolving to the skill directory's `(st_dev, st_ino)`) instead of by comparing path strings. +A spy must `fstat` the descriptor **inside** the intercepted call — the implementation closes +it as soon as the write returns. + +### Deferred: bounded retries + +`timeout` is implemented — a monotonic deadline, checked before each retrieval, before +each write, and before each prune; only the final manifest rewrite runs past it, so files +already written are never orphaned. Bounded retries inside that deadline are **not** +implemented, and belong to the delivery transport, not to this layer. Three structural +reasons, all of which the transport changes: + +1. **There is nothing transient to retry.** `SkillStore.get_object` is a synchronous + in-process read against already-delivered data, modelled on the LaunchDarkly + data-store API. `InMemorySkillStore` reads a dict. A retry re-invokes customer code and + returns the same answer. +2. **The seam cannot classify a failure.** All it surfaces is "this raised". Retrying a + `PermissionError` or a malformed payload spends the caller's `timeout` on a certainty. + The transient/permanent taxonomy a retry policy needs is the transport's to define. +3. **Backoff has nowhere to sleep.** The retrieval path (`_resolve_requests`, + `_resolve_reference`, `_resolve_all`) is synchronous, called from an async + `write_skills`. Backoff would mean either `time.sleep` — blocking the event loop of every + caller — or async-ifying the whole path for a store that cannot benefit. + +Picking a bound and a backoff now would fix numbers in a cross-language contract with no +transport to calibrate them against, so there is **no** retry test and no assumable attempt +count. When the transport lands it owns the policy; keep both languages retry-free until +then, since the number of times a throwing store is invoked is observable and the two would +otherwise diverge. --- @@ -491,7 +560,7 @@ Install with `pip install "launchdarkly-ai-server[otel]"`; see [OTel Setup](#ote |---|---| | `launchdarkly-server-sdk>=9.0`, and the `otel` extra mirrored (`opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-http`) | Each dynamically-resolved or optional package is repeated in the dev group so the test suite can import it. Something that is *only* optional would not be installed in this workspace and the tests covering its present-and-working path could not run. | | `pytest>=8`, `pytest-asyncio>=0.24` | Test runner and the async support the whole suite relies on. `asyncio_mode = "auto"` is set at the workspace root, which is why no test in this package carries an `@pytest.mark.asyncio`. | -| `mypy>=1.10` (`strict`), `ruff>=0.15` | Type checker and linter/formatter. | +| `mypy>=1.10` (`strict`), `ruff>=0.15` | Type checker and linter/formatter. `mypy` strict mode is the only thing enforcing the `Literal[...]` closed set on `ReconcileAction.action` — unlike `write_skills`'s `on_unavailable`, which is also checked at runtime because the value can arrive from untyped code. | --- @@ -513,6 +582,22 @@ convenience accessor that reads meaning into it — no YAML/frontmatter parsing, verified verbatim byte buffer and nothing more; a consumer who wants structure parses it on their side of the boundary. +### 4. Assuming `write_skills` prunes on every run + +Pruning is suppressed when the manifest is corrupt or any retrieval was incomplete — both +mean the SDK cannot tell what it owns or what is still current, and deleting under that +uncertainty is data loss. A run whose report contains a manifest `error` will not have +pruned anything, so do not read "no `removed` actions" as "nothing is stale". + +### 5. Treating "absent from the resolved set" as always meaning revoked + +Revocation is pruning, but only for a skill the store genuinely no longer serves. An object +that is *present and unverifiable* is a different thing, and `_resolve_all` must emit a +failed `_PendingWrite` for it rather than filtering it out: dropping it silently leaves its +key out of the requested set, so prune deletes the last known-good copy on disk and reports +a routine `removed` with `report.ok` still true. Tampered content must never be able to +trigger deletion. + --- ## Adding a New Export @@ -532,4 +617,5 @@ on their side of the boundary. - Do not route skills telemetry through `client.track()`, and do not introduce an LD context anywhere in the skills path. Signals go through the `skills_core.py` emitter seam, whose default is a no-op, and only via its `record_*` functions. - Do not add a signal name outside the three in the Agent Skills table above — the list is an allowlist. `AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` were considered and deliberately excluded from SDK emission. - Do not rename `ld.skills.integrity_failure`, and do not add a ninth `reason_code` in one language only — both are documented compatibility surfaces. See "The integrity-failure log record" above. +- Do not relax any of the `write_skills` filesystem defenses (local key re-validation, symlink refusal, manifest-authorized destruction, corrupt-manifest fail-closed, atomic `0644` writes). Each is a deliberate security property with abuse-case tests attached. - Do not make `SkillStore` lookups key-only. Version is part of the lookup identity because a payload holds several versions of one key; a key-only seam cannot express a version-pinned reference. diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 02c858f2..37b1570a 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -45,6 +45,13 @@ skill_refs, ) from .skills_core import SkillStore +from .skills_fs import ( + MANIFEST_FILENAME, + MANIFEST_VERSION, + SKILL_FILENAME, + OnUnavailable, + write_skills, +) from .tracking import execute_and_stream, execute_and_track, wrap_tool_handlers from .types import ( NATIVE_TOOL_KEY, @@ -74,6 +81,9 @@ ProviderGraphResponse, ProviderHandler, ProviderResponse, + ReconcileAction, + ReconcileActionKind, + ReconcileReport, Skill, SkillReference, StreamChunkEvent, @@ -136,6 +146,9 @@ "ProviderGraphResponse", "ProviderHandler", "ProviderResponse", + "ReconcileAction", + "ReconcileActionKind", + "ReconcileReport", "Skill", "SkillReference", "StreamChunkEvent", @@ -216,6 +229,14 @@ "get_skill", "get_skills", "all_skills", + "write_skills", "SkillStore", "InMemorySkillStore", + # skills — the two closed-set unions a typed consumer needs to name + "ReconcileActionKind", + "OnUnavailable", + # skills — on-disk constants, identical across languages + "SKILL_FILENAME", + "MANIFEST_FILENAME", + "MANIFEST_VERSION", ] diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py index eb3946c2..9d9b3627 100644 --- a/packages/client/src/launchdarkly_ai_server/skills.py +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -183,8 +183,8 @@ def skill_refs(config: AiConfigRep | None) -> list[SkillReference]: A config that came through ``parse_ai_config`` never contains an invalid entry — parsing fails closed on one. A hand-built dict can, and a silently - shortened projection would leave a caller materializing a skill set it - believes is complete, so every dropped entry is logged. + shortened projection would let ``write_skills`` prune the dropped skill's + on-disk copy, so every dropped entry is logged. """ if not isinstance(config, dict): return [] diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py new file mode 100644 index 00000000..feb8fbc9 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -0,0 +1,960 @@ +""" +Agent Skills — filesystem materialization. + +The highest-blast-radius layer of the feature: this is the part that writes to a +customer's disk. Split out of ``skills.py`` on that boundary — everything here +takes already-verified content and reconciles it against a managed root, while +``skills.py`` owns retrieval and verification and knows nothing about the +filesystem. The dependency runs one way only, and the descriptor-pinned +primitives every destructive step goes through live in ``safe_fs.py``. + +The reconcile is manifest-driven and fails closed: destructive operations only +ever touch paths ``/.launchdarkly-skills.json`` records under a matching +key, a corrupt manifest suppresses every destructive action, and an incomplete +retrieval suppresses pruning. Content is re-verified immediately before the +write, because a ``Skill`` can also be constructed directly by a caller. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import stat +import time +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Literal + +from .safe_fs import ( + SymlinkRefused, + atomic_write, + atomic_write_in, + pinned_directory, + unlink_file, +) +from .skills_core import ( + NO_STORE_MESSAGE, + Resolution, + SkillStore, + VerificationFailure, + get_store, + list_raw_objects, + log_withholding_summary, + newest_by_key, + record_materialized, + record_revoked, + reference_target, + resolve_from_store, + verified_bytes, + verify_raw_skill, +) +from .types import ( + ReconcileAction, + ReconcileActionKind, + ReconcileReport, + Skill, + SkillReference, +) +from .types_validation import ( + is_valid_skill_key, + is_valid_skill_version, + skill_key_rejection_reason, +) + +logger = logging.getLogger(__name__) + +MANIFEST_FILENAME = ".launchdarkly-skills.json" +"""The SDK's record of what it has written under a managed root.""" + +MANIFEST_VERSION = 1 +"""Manifest schema version this release writes, and the highest it can read.""" + +SKILL_FILENAME = "SKILL.md" +"""The single file each skill materializes to, under ``//``.""" + +OnUnavailable = Literal["keep", "raise"] +"""How ``write_skills`` reacts to content it could not retrieve.""" + +_UNAVAILABLE_PREFIX = "skill retrieval unavailable: " +""" +Prefix on every error describing content that could not be retrieved. Callers +assert on it, so it lives in one place. +""" + +_MAX_PATH_COMPONENT_BYTES = 255 +""" +NAME_MAX on Linux and macOS, and the component limit on Windows. A skill key +becomes a single directory name, and the data model permits keys up to 256 +characters — one byte longer than any of those filesystems can represent. Such a +key is rejected before any filesystem call so the caller gets a reported action +rather than an ENAMETOOLONG escaping from a stat deep inside the reconcile. +""" + + +# ------------------------------------------------------------------------- +# The reconcile entry point +# ------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _PendingWrite: + """One skill queued for the reconcile: resolved content, or why there is none.""" + + key: str + skill: Skill | None = None + error: str | None = None + + +async def write_skills( + skills: Sequence[Skill | SkillReference | str] | str, + root: str | os.PathLike[str], + *, + prune: bool = True, + timeout: float = 10.0, + on_unavailable: OnUnavailable = "keep", +) -> ReconcileReport: + """ + Materializes skills under a managed root at ``//SKILL.md``. + + *skills* is a sequence of ``Skill`` / ``SkillReference`` / key strings, or + the literal ``"*"`` meaning everything ``all_skills()`` returns. ``Skill`` + values are used as-is; references and strings resolve through the accessors, + so they need a configured store. + + The reconcile is manifest-driven (``/.launchdarkly-skills.json``): + destructive operations only ever touch paths the manifest records under a + matching key, so a file the SDK did not write is never overwritten or + deleted. ``prune`` removes formerly-managed skills that are no longer in the + requested set — which is also how revocation takes effect. ``timeout`` + bounds retrieval, the writes, and pruning; the final manifest rewrite + always runs, so files already written are never orphaned. ``on_unavailable`` + chooses between reporting a failed retrieval (``"keep"``, leaving existing + managed files alone) and raising (``"raise"``). + + Returns a ``ReconcileReport`` in which every outcome is visible; raises + ``ValueError`` for a caller error such as an unusable root. + + **This call performs synchronous filesystem I/O and does not yield.** It is + ``async`` for signature parity with the other accessors and with the + TypeScript SDK, not because it awaits anything: every read, write, ``fsync`` + and rename runs inline, so a large reconcile blocks the event loop for its + duration. Wrap it in ``asyncio.to_thread`` if that matters on your loop. + ``timeout`` is checked between steps rather than interrupting one in + progress, for the same reason. + + **One root, one reconcile at a time.** Because nothing here yields, a whole + reconcile is atomic against every other task on the loop today. Wrapping it + to run concurrently makes that the caller's problem instead: two runs + against the same root interleave on the manifest, and the loser's entries + are lost — which leaves the files it wrote unmanaged, and a later reconcile + then refuses them as files the SDK did not write. + """ + # Both of these are annotated as closed sets, but the values can still arrive + # from untyped code, so they are checked rather than assumed. + if on_unavailable not in ("keep", "raise"): + raise ValueError( + f'on_unavailable must be "keep" or "raise", got {on_unavailable!r}' + ) + if timeout < 0: + raise ValueError(f"timeout must not be negative, got {timeout!r}") + + deadline = time.monotonic() + timeout + root_path = _resolve_root(root) + manifest, manifest_error = _load_manifest(root_path) + entries: dict[str, Any] = manifest.get("entries", {}) + + actions: list[ReconcileAction] = [] + if manifest_error is not None: + # Run-level failure: there is no single skill key to hang it off. + actions.append(_run_error(manifest_error)) + + requests, incomplete = _resolve_requests(skills, deadline, on_unavailable) + + written, write_timed_out = _write_all(root_path, requests, entries, deadline) + actions.extend(written) + incomplete = incomplete or write_timed_out + + # Pruning is destructive, so it needs a trustworthy picture of both sides: a + # corrupt manifest means we do not know what we own, and an incomplete run — + # a retrieval that failed, or a deadline that expired mid-write — means we do + # not know what is still current. Either way, deleting would be a guess. + if prune and manifest_error is None and not incomplete: + actions.extend( + _prune( + root_path, + entries, + {request.key for request in requests}, + deadline, + ) + ) + + if manifest_error is None: + actions.extend(_rewrite_manifest(root_path, manifest, entries)) + + return ReconcileReport(actions=actions) + + +_RUN_LEVEL_KEY = "" +""" +The documented sentinel for a failure that belongs to no single skill (see +``ReconcileAction``). Spelled once so every path that cannot attribute a +failure to a key agrees with the others. +""" + + +def _run_error(message: str) -> ReconcileAction: + """ + A failure belonging to the run rather than to one skill. + + Uses the run-level sentinel key; it is constructed here so every run-level + error agrees. + """ + return ReconcileAction(key=_RUN_LEVEL_KEY, action="error", error=message) + + +def _write_all( + root: Path, + requests: list[_PendingWrite], + entries: dict[str, Any], + deadline: float, +) -> tuple[list[ReconcileAction], bool]: + """ + Reconciles every pending write. Returns ``(actions, timed out mid-run)``. + + The loop never aborts: a per-skill failure becomes an ``error`` action and the + next skill is attempted, because returning early would skip the caller's + manifest rewrite and orphan every file already written in this run. + """ + actions: list[ReconcileAction] = [] + timed_out = False + + for request in requests: + if request.skill is None: + actions.append( + ReconcileAction( + key=request.key, + action="error", + error=request.error + or f"skill '{request.key}' could not be resolved", + ) + ) + continue + if time.monotonic() >= deadline: + timed_out = True + actions.append( + ReconcileAction( + key=request.key, + action="error", + error=( + "the timeout was exhausted before skill " + f"'{request.key}' could be written" + ), + ) + ) + continue + try: + actions.append(_write_one(root, request.skill, entries)) + except OSError as exc: + # A safety net, not the primary defense. pathlib's stat probes swallow + # only ENOENT/ENOTDIR/EBADF/ELOOP and re-raise every other errno, so an + # unexpected filesystem condition must not abort the loop. + actions.append( + ReconcileAction( + key=request.skill.key, + action="error", + version=request.skill.version, + error=f"skill '{request.skill.key}' could not be reconciled: {exc}", + ) + ) + + return actions, timed_out + + +def _rewrite_manifest( + root: Path, manifest: dict[str, Any], entries: dict[str, Any] +) -> list[ReconcileAction]: + """Writes the updated manifest. Returns an error action, or nothing.""" + manifest["manifestVersion"] = MANIFEST_VERSION + manifest["entries"] = entries + try: + # json.dumps is inside the guard: indent= selects the pure-Python encoder, + # and unknown fields must be round-tripped, so a deeply nested + # planted field can raise RecursionError here — after every skill file is + # already on disk. + serialized = json.dumps(manifest, indent=2, sort_keys=True).encode("utf-8") + atomic_write_in(root, MANIFEST_FILENAME, serialized) + except Exception as exc: + return [_run_error(f"the skills manifest could not be written: {exc}")] + return [] + + +# ------------------------------------------------------------------------- +# Request resolution — content in, or a reason there is none +# ------------------------------------------------------------------------- + + +def _unavailable(reason: str) -> str: + """Wraps *reason* as a retrieval-unavailable message.""" + return f"{_UNAVAILABLE_PREFIX}{reason}" + + +@dataclass(frozen=True) +class _RetrievalBlocked: + """Why retrieval must not be attempted. The reason is caller-facing.""" + + reason: str + + +def _available_store(deadline: float, subject: str) -> SkillStore | _RetrievalBlocked: + """ + The configured store, or why retrieval must not be attempted. + + Written once because this gate is what sets ``unavailable`` and therefore + suppresses pruning. If it were maintained in two places, a condition added + to one and not the other would not merely produce a wrong message — it + would delete the user's files. + """ + if time.monotonic() >= deadline: + return _RetrievalBlocked( + _unavailable( + f"the timeout was exhausted before {subject} could be retrieved" + ) + ) + store = get_store() + if store is None: + return _RetrievalBlocked(_unavailable(NO_STORE_MESSAGE)) + return store + + +def _resolve_requests( + skills: Sequence[Skill | SkillReference | str] | str, + deadline: float, + on_unavailable: OnUnavailable, +) -> tuple[list[_PendingWrite], bool]: + """ + Turns the caller's input into one request per skill. + + Returns the requests plus whether any retrieval was left incomplete — an + absent store, a raising store, or an exhausted timeout. That flag suppresses + pruning: deleting managed files because retrieval failed would turn a + transport outage into data loss. + """ + if isinstance(skills, str): + if skills != "*": + raise ValueError( + 'write_skills takes a sequence of skills or the literal "*"; ' + f"got {skills!r}" + ) + return _resolve_all(deadline, on_unavailable) + + requests: list[_PendingWrite] = [] + incomplete = False + for item in skills: + if isinstance(item, Skill): + requests.append(_PendingWrite(key=item.key, skill=item)) + continue + + key, wanted = reference_target(item) + resolved = _resolve_reference(key, wanted, deadline) + if resolved.unavailable: + incomplete = True + if on_unavailable == "raise": + raise RuntimeError(resolved.error) + requests.append( + _PendingWrite(key=key, skill=resolved.skill, error=resolved.error) + ) + + return requests, incomplete + + +def _resolve_reference( + key: str, wanted_version: int | None, deadline: float +) -> Resolution: + """ + Resolves one reference for the materialization path. + + Same core as the accessors, plus the two conditions only this path treats as + data rather than as an exception: an exhausted deadline and an absent store. + """ + store = _available_store(deadline, f"'{key}'") + if isinstance(store, _RetrievalBlocked): + return Resolution(error=store.reason, unavailable=True) + + resolved = resolve_from_store(store, key, wanted_version) + if resolved.unavailable and resolved.error is not None: + return Resolution(error=_unavailable(resolved.error), unavailable=True) + return resolved + + +def _unavailable_run( + error: str, on_unavailable: OnUnavailable +) -> tuple[list[_PendingWrite], bool]: + """ + One run-level retrieval failure — raised, or reported against the empty key. + + Always reports the run incomplete, which is what suppresses pruning: nothing + was retrieved, so every managed file on disk has to be assumed current. + """ + if on_unavailable == "raise": + raise RuntimeError(error) + return [_PendingWrite(key="", error=error)], True + + +def _pending_for_raw(object_key: str, raw: Any) -> _PendingWrite: + """ + One raw store object as a pending write — verified, or reported as failed. + + Present but unverifiable is NOT the same as revoked. Dropping it silently + would leave the key out of the requested set, so prune would delete the last + known-good copy already on disk and report a routine "removed" with + report.ok still true. A failed request instead gets the same treatment the + reference path already gives (see ``_resolve_reference``): the outcome is + surfaced, and the key stays in the requested set so nothing is pruned. + """ + skill = verify_raw_skill(raw) + if skill is not None: + return _PendingWrite(key=skill.key, skill=skill) + # The on-disk copy lives under the object's *own* key, which a custom store + # may key differently in ``all_objects``. The failure must be recorded under + # the object's key, or the copy written under it on an earlier run would + # fall out of the requested set and be pruned — the very deletion this + # function exists to prevent. + raw_key = raw.get("key") if isinstance(raw, dict) else None + key = raw_key if is_valid_skill_key(raw_key) else object_key + if not is_valid_skill_key(key): + # Neither key is usable, so this failure cannot be attributed to a skill + # — the run-level sentinel is the honest report. + return _PendingWrite( + key=_RUN_LEVEL_KEY, + error="the skill store served an object under an invalid key; " + "it was withheld", + ) + return _PendingWrite( + key=key, + error=f"skill '{key}' failed integrity verification and was " + "withheld; the copy already on disk was left alone", + ) + + +def _resolve_all( + deadline: float, on_unavailable: OnUnavailable +) -> tuple[list[_PendingWrite], bool]: + """Resolves the ``"*"`` form — everything the store currently holds.""" + store = _available_store(deadline, "the skill set") + if isinstance(store, _RetrievalBlocked): + return _unavailable_run(store.reason, on_unavailable) + + # Deliberately not via all_skills(), which reports a raising store as an + # empty result — that would look like "every skill was revoked" and let + # prune delete the lot. + objects, error = list_raw_objects(store) + if error is not None: + return _unavailable_run(_unavailable(error), on_unavailable) + + # One object per key, at its newest version. ``all_objects`` may hold several + # versions of one key, and //SKILL.md is a single path — writing it + # twice in one run is a bug rather than a policy. + candidates = newest_by_key(objects) + requests = [_pending_for_raw(key, raw) for key, raw in candidates] + log_withholding_summary( + "skills held by the store", + len(requests), + sum(1 for request in requests if request.skill is not None), + ) + return requests, False + + +# ------------------------------------------------------------------------- +# The managed root and its manifest +# ------------------------------------------------------------------------- + + +def _resolve_root(root: str | os.PathLike[str]) -> Path: + """ + Resolves the managed root once, up front. + + An unusable root is a caller error rather than a per-skill outcome, so this + raises. Only the leaf directory is ever created — recursively creating + missing ancestors would let a typo scatter a directory tree. + """ + path = Path(os.fspath(root)) + + # pathlib re-raises any errno outside ENOENT/ENOTDIR/EBADF/ELOOP, so an + # unreadable parent would surface as PermissionError where the docs + # promise ValueError. + try: + is_symlink = path.is_symlink() + exists = path.exists() + is_dir = path.is_dir() + except OSError as exc: + raise ValueError(f"the skills root could not be inspected: {exc}") from exc + + if is_symlink: + raise ValueError( + f"the skills root must be a real directory, not a symlink: {path}" + ) + + if exists: + if not is_dir: + raise ValueError(f"the skills root is not a directory: {path}") + else: + parent = path.parent + try: + parent_is_dir = parent.is_dir() + except OSError as exc: + raise ValueError( + f"the parent of the skills root could not be inspected: {exc}" + ) from exc + if not parent_is_dir: + raise ValueError( + f"the parent of the skills root does not exist: {parent}. " + "write_skills creates only the leaf directory." + ) + try: + path.mkdir() + except OSError as exc: + raise ValueError(f"the skills root could not be created: {exc}") from exc + + return Path(os.path.realpath(path)) + + +def _load_manifest(root: Path) -> tuple[dict[str, Any], str | None]: + """ + Loads the manifest. Returns ``(manifest, error)``. + + A manifest that cannot be read, cannot be parsed, is not an object, carries a + ``manifestVersion`` this release does not understand, or has a malformed + ``entries`` map is **corrupt**. The caller then performs no destructive + action and leaves the file itself alone: rewriting it would destroy the only + record of what the SDK owns, and acting on a manifest we cannot read would + mean guessing at which of the customer's files are ours. + + An absent manifest is not corrupt — that is simply a fresh root. + """ + path = root / MANIFEST_FILENAME + fresh: dict[str, Any] = {"manifestVersion": MANIFEST_VERSION, "entries": {}} + + if not path.exists(): + return fresh, None + + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + # UnicodeDecodeError is a ValueError, not an OSError: non-UTF-8 bytes in + # the manifest are corruption, and must fail closed like any other. + return {}, f"the skills manifest {MANIFEST_FILENAME} could not be read: {exc}" + + try: + data = json.loads(text) + except (ValueError, RecursionError) as exc: + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} is not valid JSON ({exc}); " + "refusing every destructive action" + ) + + if not isinstance(data, dict): + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} is not a JSON object; " + "refusing every destructive action" + ) + + version = data.get("manifestVersion") + if ( + not isinstance(version, int) + or isinstance(version, bool) + or version > MANIFEST_VERSION + ): + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} declares manifestVersion " + f"{version!r}, which this SDK cannot read; refusing every destructive " + "action" + ) + + if not isinstance(data.get("entries"), dict): + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} has a malformed 'entries' " + "map; refusing every destructive action" + ) + + return data, None + + +# ------------------------------------------------------------------------- +# Per-skill reconcile +# ------------------------------------------------------------------------- + + +def _unsafe_path_reason( + root: Path, skill_dir: Path, target: Path, key: str, *, require_directory: bool +) -> str | None: + """ + The path defenses, in one place. + + Returns why ``//SKILL.md`` must not be touched, or ``None``. + Shared by the write and prune paths: ``agents.md`` marks these checks + non-relaxable, and maintaining them twice is how they drift. + + *require_directory* is the one genuine difference between the two callers. A + write needs a real directory to write into. A prune only needs to not follow + a link — an entry whose directory has been replaced by a plain file has + already lost the file this SDK owned, so reporting ``removed`` is what lets + the stale manifest entry be dropped rather than pinned forever. + + Note that the containment check is unconditional even though ``skill_dir`` + may not exist yet: ``realpath`` resolves the existing prefix and appends the + rest, so a fresh key under a valid root passes. + """ + if skill_dir.is_symlink(): + return f"{key} is a symlink" + if require_directory and skill_dir.exists() and not skill_dir.is_dir(): + return f"{key} exists and is not a directory" + if target.is_symlink(): + return "the target file is a symlink" + if Path(os.path.realpath(skill_dir)).parent != root: + return f"it resolves outside the managed root {root}" + return None + + +def _key_rejection_reason(key: Any) -> str | None: + """ + Why *key* must not become a directory name under the managed root, or ``None``. + + Re-validated locally whatever any upstream layer already did, and + before any filesystem call, because a key becomes a path component. Shared by + the write and the prune paths so the two cannot disagree about which keys + this SDK could own; ``agents.md`` marks these checks non-relaxable, and + maintaining them twice is how they drift. + + ``key.encode`` is safe here only because it runs *after* the pattern check: + the key grammar admits no surrogate, so there is no unencodable key left to + raise on. Do not reorder these two. + """ + if not is_valid_skill_key(key): + return f"{key!r} is not a valid skill key: it {skill_key_rejection_reason(key)}" + # The data model allows 256 characters; no mainstream filesystem allows a + # 256-byte path component. Catch it here so it is a reported action rather + # than an ENAMETOOLONG raised from the first stat in the caller. + key_bytes = len(key.encode("utf-8")) + if key_bytes > _MAX_PATH_COMPONENT_BYTES: + return ( + f"skill key '{key[:32]}...' is {key_bytes} bytes, over the " + f"{_MAX_PATH_COMPONENT_BYTES}-byte limit for a single directory name" + ) + return None + + +def _write_one(root: Path, skill: Skill, entries: dict[str, Any]) -> ReconcileAction: + """Reconciles one verified skill against the managed root.""" + key = skill.key + + def failed(message: str) -> ReconcileAction: + return ReconcileAction( + key=key, action="error", version=skill.version, error=message + ) + + rejection = _key_rejection_reason(key) + if rejection is not None: + return failed(f"{rejection}; nothing was written") + if not is_valid_skill_version(skill.version): + return failed( + f"skill '{key}' has version {skill.version!r}, which is not an " + "integer >= 1; nothing was written" + ) + + skill_dir = root / key + target = skill_dir / SKILL_FILENAME + relative = f"{key}/{SKILL_FILENAME}" + + unsafe = _unsafe_path_reason(root, skill_dir, target, key, require_directory=True) + if unsafe is not None: + return failed(f"'{relative}' was refused: {unsafe}; nothing was written") + + # Re-verify immediately before writing, through the same core the accessors + # use: a Skill can also be constructed directly by a caller. + verified = verified_bytes(key, skill.content, skill.content_hash, skill.version) + if isinstance(verified, VerificationFailure): + return failed( + f"skill '{key}' failed verification immediately before writing: " + f"{verified.reason}; nothing was written" + ) + encoded, content_hash = verified.encoded, verified.content_hash + + # Overwrite only what the manifest records as ours under this key. + entry = entries.get(relative) + managed = isinstance(entry, dict) and entry.get("key") == key + exists = target.exists() + + if exists and not managed: + return failed( + f"'{relative}' exists but the manifest does not record it as managed " + f"under key '{key}'; refusing to overwrite a file this SDK did not write" + ) + + if exists: + try: + on_disk = _read_regular_file(target) + except OSError as exc: + return failed(f"'{relative}' could not be read: {exc}") + + if hashlib.sha256(on_disk).hexdigest() == content_hash: + _update_entry(entries, relative, skill, content_hash) + record_materialized(key, len(encoded), content_hash, "skipped_current") + return ReconcileAction( + key=key, + action="skipped_current", + version=skill.version, + path=str(target), + ) + # Stale version or local tampering — LD-resolved content wins. + action: ReconcileActionKind = "updated" + else: + action = "written" + + write_error = _write_through_descriptor(skill_dir, encoded, key, relative) + if write_error is not None: + return failed(write_error) + + _update_entry(entries, relative, skill, content_hash) + record_materialized(key, len(encoded), content_hash, action) + return ReconcileAction( + key=key, action=action, version=skill.version, path=str(target) + ) + + +def _read_regular_file(target: Path) -> bytes: + """ + Reads *target*, refusing anything that is not a regular file. + + A plain ``Path.read_bytes`` would ``open()`` by name — and opening a FIFO + with no writer blocks forever, so an attacker who can swap the managed file + for one (the same capability the symlink checks defend against) could hang + the whole reconcile, and the event loop with it. ``O_NONBLOCK`` makes that + open return immediately (it is a no-op for regular files), ``O_NOFOLLOW`` + refuses a trailing symlink, and the ``fstat`` on the descriptor — not the + path — is what the type check trusts. ``O_BINARY`` is what keeps these + bytes the *verbatim* bytes: it is 0 on POSIX, but on Windows a descriptor + without it translates CRLF on read, which would fail the hash comparison + against content that is actually current. + """ + flags = ( + os.O_RDONLY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0) + | getattr(os, "O_BINARY", 0) + ) + fd = os.open(target, flags) + try: + if not stat.S_ISREG(os.fstat(fd).st_mode): + raise OSError("the target file is not a regular file") + chunks: list[bytes] = [] + while True: + chunk = os.read(fd, 65536) + if not chunk: + return b"".join(chunks) + chunks.append(chunk) + finally: + os.close(fd) + + +def _write_through_descriptor( + skill_dir: Path, encoded: bytes, key: str, relative: str +) -> str | None: + """ + Performs the write itself. Returns a failure reason, or ``None`` on success. + + Split out of ``_write_one`` because everything above it decides *whether* to + write and this decides nothing: the directory is pinned to a descriptor and + every remaining step is relative to it, so none of the checks above can be + invalidated by a swap between here and the rename. + """ + try: + with pinned_directory(skill_dir, create=True) as dir_fd: + try: + atomic_write(skill_dir, SKILL_FILENAME, encoded, dir_fd=dir_fd) + except OSError as exc: + return f"'{relative}' could not be written: {exc}" + except OSError as exc: + return f"the directory for skill '{key}' could not be created: {exc}" + except ValueError as exc: + return f"'{relative}' was refused: {exc}" + return None + + +def _update_entry( + entries: dict[str, Any], relative: str, skill: Skill, content_hash: str +) -> None: + """ + Records a managed path in the manifest. + + Merges into any existing entry rather than replacing it, so fields written by + a future SDK release survive this one's rewrite. + + ``sha256`` and ``writtenAt`` are recorded for forensics only: the reconcile + decides currency by hashing the bytes on disk, precisely because the + manifest is untrusted, so neither field is ever read back as a decision + input. + """ + existing = entries.get(relative) + entry = dict(existing) if isinstance(existing, dict) else {} + entry["key"] = skill.key + entry["version"] = skill.version + entry["sha256"] = content_hash + entry["writtenAt"] = _utc_timestamp() + entries[relative] = entry + + +# ------------------------------------------------------------------------- +# Pruning — how revocation takes effect +# ------------------------------------------------------------------------- + + +def _prune_error(key: str, message: str, version: Any = None) -> ReconcileAction: + """ + A prune refusal. Mirrors ``_write_one``'s local ``failed`` helper. + + *version* comes off the manifest, which is untrusted, so it is validated here + rather than at each call site — the same guard the ``removed`` action applies, + so a refusal and a removal report the field identically. + Callers that genuinely do not know a version pass nothing; none of them may + invent one. + """ + return ReconcileAction( + key=key, + action="error", + version=version if is_valid_skill_version(version) else None, + error=message, + ) + + +def _prune( + root: Path, entries: dict[str, Any], requested: set[str], deadline: float +) -> list[ReconcileAction]: + """ + Removes managed skills that are no longer requested. + + This is also how revocation takes effect: a revoked skill is simply absent + from the resolved set, so the next reconcile removes it. There is + deliberately no opt-out. + + The deadline applies here just as it does to the writes: a skill left + unpruned is reported as an error and stays in the manifest, so the next + reconcile picks it up. + """ + actions: list[ReconcileAction] = [] + + for relative, entry in list(entries.items()): + if not isinstance(entry, dict): + continue + key = entry.get("key") + if not isinstance(key, str) or key in requested: + continue + + if time.monotonic() >= deadline: + actions.append( + _prune_error( + key, + f"the timeout was exhausted before '{relative}' could be " + "pruned; it was left in place", + entry.get("version"), + ) + ) + continue + + # Only a manifest path this SDK could have written is removable. + if ( + _key_rejection_reason(key) is not None + or relative != f"{key}/{SKILL_FILENAME}" + ): + actions.append( + _prune_error( + key, + f"manifest entry '{relative}' does not name a path this SDK " + f"could own under key '{key}'; it was left in place", + entry.get("version"), + ) + ) + continue + + try: + actions.append(_prune_one(root, relative, key, entries)) + except OSError as exc: + actions.append( + _prune_error( + key, + f"'{relative}' could not be removed: {exc}", + entry.get("version"), + ) + ) + + return actions + + +def _unlink_through_descriptor(skill_dir: Path, relative: str) -> str | None: + """ + Performs the removal itself. Returns a failure reason, or ``None`` on success. + + The mirror of ``_write_through_descriptor``, and split out for the same + reason: everything above it decides *whether* to remove, and this decides + nothing. The directory is pinned before the unlink because unlink never + follows a trailing symlink but does resolve the directory above it, so a + ``/`` swapped for a symlink between the checks and here would + otherwise delete a file outside the root. + """ + try: + with pinned_directory(skill_dir) as dir_fd: + try: + unlink_file(skill_dir, SKILL_FILENAME, dir_fd=dir_fd) + except SymlinkRefused: + return f"'{relative}' was not removed: the target file is a symlink" + except OSError as exc: + return f"'{relative}' could not be removed: {exc}" + except ValueError as exc: + return f"'{relative}' was not removed: {exc}" + return None + + +def _prune_one( + root: Path, relative: str, key: str, entries: dict[str, Any] +) -> ReconcileAction: + """Removes one managed skill file, and its directory when that empties it.""" + skill_dir = root / key + target = skill_dir / SKILL_FILENAME + version = entries[relative].get("version") + + unsafe = _unsafe_path_reason(root, skill_dir, target, key, require_directory=False) + if unsafe is not None: + return _prune_error(key, f"'{relative}' was not removed: {unsafe}", version) + + removed_from_disk = False + if target.exists(): + failure = _unlink_through_descriptor(skill_dir, relative) + if failure is not None: + return _prune_error(key, failure, version) + removed_from_disk = True + try: + # Path-based, and safe that way: rmdir never follows a trailing + # symlink (it fails ENOTDIR) and only ever succeeds on an empty + # directory. + skill_dir.rmdir() + except OSError: + pass # the customer keeps their own files here too + + entries.pop(relative, None) + + if removed_from_disk: + record_revoked(key, version) + + return ReconcileAction( + key=key, + action="removed", + version=version if is_valid_skill_version(version) else None, + path=str(target), + ) + + +def _utc_timestamp() -> str: + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index 4249d5cb..10c90d8c 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -452,6 +452,54 @@ class Skill: """Description from LaunchDarkly metadata; never parsed from the content.""" +ReconcileActionKind = Literal[ + "written", "updated", "skipped_current", "removed", "error" +] +"""The closed set of outcomes ``write_skills`` reports.""" + + +@dataclass(frozen=True) +class ReconcileAction: + """What ``write_skills`` did — or refused to do — for one skill.""" + + key: str + """ + The skill key, or the **empty string** for a failure that belongs to the run + rather than to one skill — a corrupt manifest, a manifest that could not be + rewritten, a retrieval that failed before any key was known. Callers grouping + a report by key need to expect that sentinel; a report may carry both kinds. + """ + action: ReconcileActionKind + version: int | None = None + path: str | None = None + """Canonical resolved path, when one was determined.""" + error: str | None = None + """Failure detail, set only when ``action == "error"``.""" + + +@dataclass(frozen=True) +class ReconcileReport: + """The result of a ``write_skills`` run — every outcome is visible here.""" + + actions: list[ReconcileAction] = field(default_factory=list) + + @property + def ok(self) -> bool: + """``True`` iff no action is an ``error``.""" + return not self.errors + + @property + def errors(self) -> list[ReconcileAction]: + """ + The ``error`` actions, in ``actions`` order. + + Exposed so callers never re-derive it — filtering ``actions`` is + boilerplate that otherwise reappears in every consumer. ``ok`` is defined + in terms of this, so the two can never disagree. + """ + return [a for a in self.actions if a.action == "error"] + + # --------------------------------------------------------------------------- # Model / graph options # --------------------------------------------------------------------------- diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 203eeb4f..0c656ede 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -17,6 +17,8 @@ import launchdarkly_ai_server.skills as skills_module from launchdarkly_ai_server import ( InMemorySkillStore, + ReconcileAction, + ReconcileReport, Skill, SkillReference, all_skills, @@ -129,7 +131,7 @@ def _fabricated_hash_cases() -> list[Any]: class TestSkillTypes: - """Immutability and optional metadata.""" + """Immutability, optional metadata, and ``ReconcileReport.ok``.""" def test_skill_reference_is_immutable(self) -> None: ref = SkillReference(key="pdf-extraction", version=2) @@ -164,6 +166,70 @@ def test_skill_metadata_defaults_to_none(self) -> None: assert skill.name is None assert skill.description is None + def test_report_ok_true_when_no_error_action(self) -> None: + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="skipped_current", version=2), + ReconcileAction(key="c", action="removed"), + ReconcileAction(key="d", action="updated", version=3), + ] + ) + assert report.ok is True + + def test_report_ok_false_with_error_action(self) -> None: + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="error", error="nope"), + ] + ) + assert report.ok is False + + def test_empty_report_is_ok(self) -> None: + assert ReconcileReport(actions=[]).ok is True + + def test_report_errors_lists_error_actions_in_order(self) -> None: + """The report exposes its error actions itself.""" + first = ReconcileAction(key="b", action="error", error="first") + second = ReconcileAction(key="d", action="error", error="second") + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + first, + ReconcileAction(key="c", action="skipped_current", version=2), + second, + ReconcileAction(key="e", action="removed"), + ] + ) + assert report.errors == [first, second] + + def test_report_errors_empty_when_no_error_action(self) -> None: + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="removed"), + ] + ) + assert report.errors == [] + + def test_empty_report_has_no_errors(self) -> None: + assert ReconcileReport(actions=[]).errors == [] + + def test_report_ok_and_errors_always_agree(self) -> None: + """``ok`` is true iff ``errors`` is empty, on the same objects.""" + clean = ReconcileReport( + actions=[ReconcileAction(key="a", action="written", version=1)] + ) + failed = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="error", error="nope"), + ] + ) + for report in (clean, failed, ReconcileReport(actions=[])): + assert report.ok is (report.errors == []) + class TestSkillRefs: """Pure projection of the config's skills array.""" @@ -267,6 +333,59 @@ def test_object_kind_is_not_public_api(self) -> None: assert "SKILL_OBJECT_KIND" not in package.__all__ assert not hasattr(package, "SKILL_OBJECT_KIND") + def test_constants_are_exported_from_the_package_root(self) -> None: + import launchdarkly_ai_server as package + + assert package.SKILL_FILENAME == "SKILL.md" + assert package.MANIFEST_FILENAME == ".launchdarkly-skills.json" + assert package.MANIFEST_VERSION == 1 + + def test_constants_are_listed_in_dunder_all(self) -> None: + """A name absent from ``__all__`` is not part of the public surface.""" + import launchdarkly_ai_server as package + + expected = { + "SKILL_FILENAME", + "MANIFEST_FILENAME", + "MANIFEST_VERSION", + } + assert expected <= set(package.__all__) + + def test_closed_set_types_are_exported_from_the_package_root(self) -> None: + """The two closed-set unions are public API, not implementation detail. + + ``ReconcileActionKind`` types the ``ReconcileAction.action`` field every + consumer of a report reads and switches on, and ``OnUnavailable`` types + a public keyword argument of ``write_skills``. ``agents.md`` forbids + handler packages from importing sub-path modules, so a name exported + only from the implementation module has no supported import path. + """ + import launchdarkly_ai_server as package + + assert hasattr(package, "ReconcileActionKind") + assert hasattr(package, "OnUnavailable") + assert {"ReconcileActionKind", "OnUnavailable"} <= set(package.__all__) + + def test_exported_action_union_admits_exactly_the_five_actions(self) -> None: + """The union must match the actions a report can actually carry. + + Spelled out rather than imported from the implementation for the same + reason as the constants above: deriving the expectation from the thing + under test would make the assertion circular. + """ + import typing + + import launchdarkly_ai_server as package + + assert set(typing.get_args(package.ReconcileActionKind)) == { + "written", + "updated", + "skipped_current", + "removed", + "error", + } + assert set(typing.get_args(package.OnUnavailable)) == {"keep", "raise"} + def test_retrieval_surface_is_exported_from_the_package_root(self) -> None: """A name absent from ``__all__`` is not part of the public surface.""" import launchdarkly_ai_server as package @@ -811,7 +930,7 @@ class TestWithholdingSummary: """ def _tampered(self, make_raw_skill: Any, key: str = "a") -> dict[str, Any]: - raw = make_raw_skill(key=key) + raw: dict[str, Any] = make_raw_skill(key=key) raw["contentHash"] = "0" * 64 return raw diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py new file mode 100644 index 00000000..62d01d59 --- /dev/null +++ b/packages/client/tests/test_skills_fs.py @@ -0,0 +1,582 @@ +""" +Tests for ``write_skills`` — filesystem materialization and manifest reconcile +semantics. + +Every test writes only inside pytest's ``tmp_path``. No network, no real +LaunchDarkly client, no real skill transport. + +The security abuse matrix — path traversal, symlink attacks, clobber +protection, corrupt manifests, atomicity under an injected crash, and the +materialization telemetry allowlist — is a separate module. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any + +import pytest + +import launchdarkly_ai_server.skills as skills_module +from launchdarkly_ai_server import ( + InMemorySkillStore, + Skill, + SkillReference, + write_skills, +) + +MANIFEST_NAME = ".launchdarkly-skills.json" +SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" + +INTEGRITY_SIGNAL = "AgentControl Skill Integrity Failure" + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + r = tmp_path / "skills" + r.mkdir() + return r + + +pytestmark = pytest.mark.usefixtures("reset_skill_state") +"""Every test in this module runs against freshly cleared module state.""" + + +def _hash(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +def _skill( + key: str = "test-skill", version: int = 1, content: str = SKILL_BODY +) -> Skill: + return Skill( + key=key, + version=version, + content=content.encode("utf-8"), + content_hash=_hash(content), + ) + + +def _manifest_path(root: Path) -> Path: + return root / MANIFEST_NAME + + +def _read_manifest(root: Path) -> dict[str, Any]: + return json.loads(_manifest_path(root).read_text(encoding="utf-8")) + + +def _write_manifest(root: Path, raw: Any) -> None: + root.mkdir(parents=True, exist_ok=True) + _manifest_path(root).write_text( + raw if isinstance(raw, str) else json.dumps(raw), encoding="utf-8" + ) + + +def _entry(key: str, version: int, content: str) -> dict[str, Any]: + return { + "key": key, + "version": version, + "sha256": _hash(content), + "writtenAt": "2026-08-14T19:00:00Z", + } + + +def _place_managed(root: Path, key: str, content: str, version: int = 1) -> Path: + """Pre-create a file AND its manifest entry — i.e. an SDK-managed path.""" + target = root / key / "SKILL.md" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {f"{key}/SKILL.md": _entry(key, version, content)}, + }, + ) + return target + + +def _actions_by_key(report: Any) -> dict[str, Any]: + return {a.key: a for a in report.actions} + + +def _error_messages(report: Any) -> list[str]: + """All ``error`` action messages, regardless of which key they hang off. + + Run-level (manifest) errors have no well-defined ``key`` yet, so assertions + about them scan every error action rather than looking one up by key. + """ + return [a.error or "" for a in report.actions if a.action == "error"] + + +class TestBasicWrites: + """Basic writes and the returned report.""" + + async def test_new_skill_is_written_verbatim(self, root: Path) -> None: + report = await write_skills([_skill("pdf-extraction", 2)], root) + + target = root / "pdf-extraction" / "SKILL.md" + assert target.read_text(encoding="utf-8") == SKILL_BODY + assert report.ok is True + action = _actions_by_key(report)["pdf-extraction"] + assert action.action == "written" + assert action.version == 2 + assert action.path is not None + assert Path(action.path).resolve() == target.resolve() + assert action.error is None + + async def test_skill_inputs_need_no_store(self, root: Path) -> None: + report = await write_skills([_skill("a")], root) + assert report.ok is True + assert (root / "a" / "SKILL.md").exists() + + async def test_reference_inputs_resolve_through_store( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=3)) + report = await write_skills([SkillReference(key="a", version=3)], root) + assert report.ok is True + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_string_inputs_resolve_latest( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=9)) + report = await write_skills(["a"], root) + assert _actions_by_key(report)["a"].version == 9 + + async def test_star_writes_everything_in_the_store( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + for k in ("a", "b", "c"): + store.put(make_raw_skill(key=k)) + report = await write_skills("*", root) + assert report.ok is True + assert len([a for a in report.actions if a.action == "written"]) == 3 + for k in ("a", "b", "c"): + assert (root / k / "SKILL.md").exists() + + async def test_one_action_per_requested_skill(self, root: Path) -> None: + report = await write_skills([_skill("a"), _skill("b")], root) + assert sorted(a.key for a in report.actions) == ["a", "b"] + + async def test_empty_request_on_empty_root_is_ok(self, root: Path) -> None: + report = await write_skills([], root) + assert report.ok is True + assert report.actions == [] + + +class TestManifest: + """Manifest format and forward compatibility.""" + + async def test_manifest_format_is_exact(self, root: Path) -> None: + await write_skills([_skill("pdf-extraction", 2)], root) + + manifest = _read_manifest(root) + assert manifest["manifestVersion"] == 1 + entry = manifest["entries"]["pdf-extraction/SKILL.md"] + assert entry["key"] == "pdf-extraction" + assert entry["version"] == 2 + assert entry["sha256"] == _hash(SKILL_BODY) + assert isinstance(entry["writtenAt"], str) + + async def test_entry_paths_are_forward_slash_relative(self, root: Path) -> None: + await write_skills([_skill("a")], root) + keys = list(_read_manifest(root)["entries"].keys()) + assert keys == ["a/SKILL.md"] + assert "\\" not in keys[0] + assert not keys[0].startswith("/") + + async def test_unknown_fields_are_preserved_on_rewrite(self, root: Path) -> None: + entry = _entry("a", 1, SKILL_BODY) + entry["futureEntryField"] = "keep-me" + _write_manifest( + root, + { + "manifestVersion": 1, + "futureTopLevelField": {"keep": True}, + "entries": {"a/SKILL.md": entry}, + }, + ) + (root / "a").mkdir() + (root / "a" / "SKILL.md").write_text(SKILL_BODY, encoding="utf-8") + + await write_skills([_skill("a", 2, SKILL_BODY + "more\n")], root) + + manifest = _read_manifest(root) + assert manifest["futureTopLevelField"] == {"keep": True} + assert manifest["entries"]["a/SKILL.md"]["futureEntryField"] == "keep-me" + + +class TestReconcileSemantics: + """The reconcile state table.""" + + async def test_unchanged_managed_file_is_skipped_current(self, root: Path) -> None: + target = _place_managed(root, "a", SKILL_BODY) + before = target.stat().st_mtime_ns + + report = await write_skills([_skill("a")], root) + + assert _actions_by_key(report)["a"].action == "skipped_current" + assert target.read_text(encoding="utf-8") == SKILL_BODY + assert target.stat().st_mtime_ns == before + + async def test_new_version_updates(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY, version=1) + new_content = SKILL_BODY + "second version\n" + + report = await write_skills([_skill("a", 2, new_content)], root) + + action = _actions_by_key(report)["a"] + assert action.action == "updated" + assert action.version == 2 + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == new_content + assert _read_manifest(root)["entries"]["a/SKILL.md"]["version"] == 2 + + async def test_local_tampering_is_overwritten(self, root: Path) -> None: + target = _place_managed(root, "a", SKILL_BODY) + target.write_text("locally tampered\n", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert _actions_by_key(report)["a"].action == "updated" + assert target.read_text(encoding="utf-8") == SKILL_BODY + + async def test_prune_removes_formerly_managed_skill(self, root: Path) -> None: + _place_managed(root, "gone", SKILL_BODY) + + report = await write_skills([], root) + + assert _actions_by_key(report)["gone"].action == "removed" + assert not (root / "gone" / "SKILL.md").exists() + assert not (root / "gone").exists() + assert _read_manifest(root)["entries"] == {} + + async def test_prune_false_keeps_the_file(self, root: Path) -> None: + target = _place_managed(root, "gone", SKILL_BODY) + + report = await write_skills([], root, prune=False) + + assert target.exists() + assert [a for a in report.actions if a.action == "removed"] == [] + assert "gone/SKILL.md" in _read_manifest(root)["entries"] + + async def test_prune_does_not_touch_unmanaged_files(self, root: Path) -> None: + _place_managed(root, "gone", SKILL_BODY) + bystander = root / "user-notes.md" + bystander.write_text("mine\n", encoding="utf-8") + user_dir_file = root / "user-skill" / "SKILL.md" + user_dir_file.parent.mkdir() + user_dir_file.write_text("hand written\n", encoding="utf-8") + + await write_skills([], root) + + assert bystander.read_text(encoding="utf-8") == "mine\n" + assert user_dir_file.read_text(encoding="utf-8") == "hand written\n" + + async def test_prune_refusal_for_unownable_path_reports_the_version( + self, root: Path + ) -> None: + """A prune refusal carries the manifest's version. + + A manifest entry whose path is not one this SDK could have written is + refused rather than removed. The entry is in hand at that point, so the + error action must carry its version — otherwise a prune *failure* is + strictly less informative than a prune *success*, which does report it. + """ + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + # Right key, wrong filename — not a path this SDK could own. + "orphan/NOTES.md": _entry("orphan", 7, SKILL_BODY), + }, + }, + ) + + report = await write_skills([], root) + + action = _actions_by_key(report)["orphan"] + assert action.action == "error" + assert action.version == 7 + + async def test_prune_refusal_for_symlinked_target_reports_the_version( + self, root: Path + ) -> None: + """Same contract on the symlink refusal path (prune side).""" + if not hasattr(os, "symlink"): + pytest.skip("platform has no symlink support") + (root / "a").mkdir() + outside_file = root.parent / "victim.md" + outside_file.write_text("victim content\n", encoding="utf-8") + (root / "a" / "SKILL.md").symlink_to(outside_file) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 4, "victim content\n")}, + }, + ) + + report = await write_skills([], root) + + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.version == 4 + + async def test_unresolvable_request_still_reports_no_version( + self, root: Path + ) -> None: + """The other half of the contract: do not invent a version. + + A reference that could not be retrieved has neither a manifest entry + nor a ``Skill``, so there is no version to report and ``version`` stays + ``None``. Without this, "always populate version" would be satisfied by + fabricating one. + """ + report = await write_skills([SkillReference(key="ghost", version=3)], root) + + action = _actions_by_key(report)["ghost"] + assert action.action == "error" + assert action.version is None + + async def test_prune_keeps_directory_when_not_empty(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY) + extra = root / "a" / "user-file.txt" + extra.write_text("keep\n", encoding="utf-8") + + report = await write_skills([], root) + + assert _actions_by_key(report)["a"].action == "removed" + assert not (root / "a" / "SKILL.md").exists() + assert extra.exists() + + +class TestRootHandling: + """Root resolution.""" + + async def test_absent_leaf_root_is_created(self, tmp_path: Path) -> None: + target_root = tmp_path / "skills" + report = await write_skills([_skill("a")], target_root) + assert report.ok is True + assert (target_root / "a" / "SKILL.md").exists() + + async def test_missing_ancestors_raise(self, tmp_path: Path) -> None: + with pytest.raises(ValueError): + await write_skills([_skill("a")], tmp_path / "a" / "b" / "c") + + async def test_root_that_is_a_file_raises(self, tmp_path: Path) -> None: + file_root = tmp_path / "not-a-dir" + file_root.write_text("x", encoding="utf-8") + with pytest.raises(ValueError): + await write_skills([_skill("a")], file_root) + + async def test_accepts_string_root(self, root: Path) -> None: + report = await write_skills([_skill("a")], str(root)) + assert report.ok is True + + +class TestSkillsArgumentErrors: + """A bare string that is not ``"*"`` raises. + + A ``ValueError``, not a ``TypeError``: a string *is* an accepted argument + type here, since ``"*"`` means "everything the store holds", so this is an + acceptable type carrying an invalid value. The accessors' equivalent guard + is a ``TypeError`` because a string is never a valid argument there. + """ + + async def test_bare_non_star_string_raises_value_error(self, root: Path) -> None: + with pytest.raises(ValueError) as excinfo: + await write_skills("pdf-extraction", root) + + # Naming the accepted forms is the actionable half of the message. + assert '"*"' in str(excinfo.value) + + async def test_star_is_accepted(self, root: Path) -> None: + """Positive control — otherwise the guard above could reject every string.""" + store = InMemorySkillStore() + store.put( + { + "key": "a", + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + ) + skills_module._set_store(store) + + report = await write_skills("*", root) + + assert report.ok is True + assert (root / "a" / "SKILL.md").exists() + + async def test_bare_string_writes_nothing(self, root: Path) -> None: + """The raise precedes any filesystem work. + + Asserting only the raise would also pass for an implementation that + created one directory per character before failing. + """ + with pytest.raises(ValueError): + await write_skills("abc", root) + + assert list(root.iterdir()) == [] + + +class TestResilience: + """Unavailable retrieval and timeout.""" + + async def test_keep_is_the_default_and_does_not_raise(self, root: Path) -> None: + existing = _place_managed(root, "a", SKILL_BODY) + + report = await write_skills([SkillReference(key="a", version=1)], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert existing.read_text(encoding="utf-8") == SKILL_BODY + + async def test_raise_mode_propagates(self, root: Path) -> None: + with pytest.raises(RuntimeError, match=r"(?i)(unavailable|skill store)"): + await write_skills( + [SkillReference(key="a", version=1)], root, on_unavailable="raise" + ) + + async def test_store_error_is_reported_not_raised( + self, root: Path, exploding_store: Any + ) -> None: + report = await write_skills([SkillReference(key="a", version=1)], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + + async def test_exhausted_timeout_behaves_as_unavailable( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a")) + report = await write_skills( + [SkillReference(key="a", version=1)], root, timeout=0 + ) + assert report.ok is False + assert not (root / "a" / "SKILL.md").exists() + + async def test_exhausted_timeout_raises_in_raise_mode( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a")) + with pytest.raises(RuntimeError, match=r"(?i)(unavailable|timeout|timed out)"): + await write_skills( + [SkillReference(key="a", version=1)], + root, + timeout=0, + on_unavailable="raise", + ) + + async def test_exhausted_timeout_stops_pruning( + self, root: Path, store: InMemorySkillStore + ) -> None: + """The deadline bounds pruning too, not just retrieval and the writes. + + A run whose writes all land just inside the deadline would otherwise go + on to stat, unlink and rmdir every stale manifest entry unbounded — the + opposite of what a small ``timeout`` asks for. + """ + existing = _place_managed(root, "stale", SKILL_BODY) + + report = await write_skills([], root, timeout=0) + + assert report.ok is False + assert existing.exists(), "prune ran past the exhausted deadline" + assert any("timeout was exhausted" in m for m in _error_messages(report)) + # The entry survives, so the next reconcile picks it up. + assert "stale/SKILL.md" in _read_manifest(root)["entries"] + + async def test_a_verification_failure_never_prunes_the_good_copy( + self, root: Path + ) -> None: + """A store may key ``all_objects`` differently from the object's own key. + + The on-disk copy lives under the object's own key, so a failure recorded + under the *store's* dict key would drop the real key out of the + requested set and let prune delete the last known-good copy. + """ + + class AliasKeyedStore: + """Keys objects by an internal id, not by the skill's own key.""" + + def __init__(self, raw: dict[str, Any]) -> None: + self._raw = raw + + def get_object(self, kind: str, key: str) -> dict[str, Any] | None: + return None + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + return {"internal-uuid-1": self._raw} + + existing = _place_managed(root, "pdf-extraction", SKILL_BODY) + tampered = { + "key": "pdf-extraction", + "version": 1, + "content": "tampered\n", + "contentHash": _hash(SKILL_BODY), # does not match the content + } + skills_module._set_store(AliasKeyedStore(tampered)) + + report = await write_skills("*", root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert [a.action for a in report.actions] == ["error"] + assert _actions_by_key(report)["pdf-extraction"].action == "error" + assert "pdf-extraction/SKILL.md" in _read_manifest(root)["entries"] + + async def test_unavailable_run_does_not_corrupt_manifest(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY) + before = _read_manifest(root) + + await write_skills([SkillReference(key="b", version=1)], root) + + assert ( + _read_manifest(root)["entries"]["a/SKILL.md"] + == (before["entries"]["a/SKILL.md"]) + ) + + +class TestVerifyThenWrite: + """Hash re-verified immediately before writing.""" + + async def test_hash_mismatch_aborts_the_write( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + bad = Skill( + key="a", + version=1, + content=SKILL_BODY.encode("utf-8"), + content_hash="0" * 64, + ) + + report = await write_skills([bad], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert not (root / "a" / "SKILL.md").exists() + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_oversize_skill_aborts_the_write(self, root: Path) -> None: + oversize = "x" * (64 * 1024 + 1) + report = await write_skills([_skill("a", 1, oversize)], root) + assert report.ok is False + assert not (root / "a" / "SKILL.md").exists() + + async def test_mismatch_does_not_disturb_existing_managed_file( + self, root: Path + ) -> None: + target = _place_managed(root, "a", SKILL_BODY) + bad = Skill(key="a", version=2, content=b"new content\n", content_hash="f" * 64) + + await write_skills([bad], root) + + assert target.read_text(encoding="utf-8") == SKILL_BODY From c88d5bf5ce0db1e8ad9f8fa5a5ee782ea768b887 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Wed, 26 Aug 2026 16:26:15 -0400 Subject: [PATCH 06/22] test(client): assert rename containment through the branching helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``test_key_at_the_data_model_bound_is_reported_not_raised`` read ``dst_dir_id`` directly, but that field is only populated when ``os.replace`` is called with ``dir_fd`` kwargs. On the path fallback (``SUPPORTS_DIR_FD`` false — the shape Windows takes) it stays ``None``, so the assertion failed even though the valid skill had been renamed correctly into its own directory. ``_assert_atomic_rename_of`` already branches on both call shapes and asserts the same containment property, plus the single-rename count the list comparison implied. Use it. Verified by forcing the probe off for a whole session: this was the only test in the module that broke under the no-``*at()`` shape, and the helper-based check passes under both. Reported by Cursor Bugbot on #54. Co-Authored-By: Claude Opus 5 --- packages/client/tests/test_skills_fs.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 6502aa5b..5af4d19b 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -932,8 +932,11 @@ async def test_key_at_the_data_model_bound_is_reported_not_raised( # The bare-filename ``dst`` of a ``dir_fd``-relative rename carries no # directory, so "the path does not contain the hostile key" is no longer # a meaningful check. Assert the stronger thing instead: the only rename - # that happened was into the valid skill's own directory. - assert [call.dst_dir_id for call in spy.calls] == [_dir_id(root / "good")] + # that happened was into the valid skill's own directory. Through the + # shared helper, so the check holds on the path fallback too — reading + # ``dst_dir_id`` directly would compare ``None`` there and fail a run + # that had in fact renamed correctly. + _assert_atomic_rename_of(spy, root / "good") # The valid skill is fully reconciled: written AND recorded, not orphaned. assert (root / "good" / "SKILL.md").exists() assert "good/SKILL.md" in _read_manifest(root)["entries"] From 49c2741641051df8b666db78097832b745b2ba6f Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Sun, 30 Aug 2026 04:26:10 -0400 Subject: [PATCH 07/22] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20self-healing=20reconciles,=20and=20keys=20no=20file?= =?UTF-8?q?system=20can=20hold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps from the Agent Skills security design review, both in `skills_fs.py`: AV-3 (partial reconciles are unrecoverable) and DV-2 (the key grammar admits Windows device names). **AV-3 — adopt a file whose bytes already are the resolved content.** `_write_all` reconciles every skill and only then rewrites the manifest, once, last. A process killed in that window leaves a skill file at a managed path with no manifest entry — exactly the condition `_write_one` treats as an unmanaged-file collision, so the skill was wedged permanently: every later reconcile took the same refusal branch. Boot-time execution under a ten-second budget makes the crash window realistic. `_write_one` now reads and hashes first and decides from the bytes. Content byte-identical to what LaunchDarkly resolved is adopted — manifest entry recorded, reported `skipped_current` — and anything else falls through to the same refusal as before. This cannot weaken the clobber guarantee: differing unmanaged bytes are never overwritten, and the existing clobber tests pass unchanged. Three details carry the safety: - A read that fails is a refusal, never an overwrite, with a message distinguishable from the byte-mismatch refusal — it is the comparison that would otherwise authorize the write, and a file that could not be read has not been shown to be ours. - The read stays on `_read_regular_file`. Adoption widens it to genuinely foreign files, so its refusal of FIFOs and other non-regular files is now load-bearing rather than defensive. It gains a `max_bytes` bound of `len(content) + 1` — enough to prove inequality for anything longer, and what keeps a foreign file of arbitrary size out of memory. A bound of exactly `len(content)` would adopt every file that merely begins with the resolved content. - `skipped_current` is reused rather than adding an `adopted` action kind, so `ReconcileActionKind` — public, and owned by an approved PR — does not change. Its documented meaning already fits. Adoption also makes the file prunable later. That is correct rather than a weakening: only byte-identical LaunchDarkly content is ever adopted, so a later prune removes content LaunchDarkly delivered anyway — what would have happened had the crash not occurred. The review also floats a write-intent journal. Assessed as over-engineered; not built. **AV-3, secondary — sweep orphaned temp files.** `atomic_write` unlinks its temp file on any exception but not after a `SIGKILL`, and `_prune` walks manifest entries, which an orphan never has, so nothing would ever notice one. The second-order effect is worse than the disk: `_prune_one`'s `rmdir` only succeeds on an empty directory, so a single orphan pins a skill's directory forever. The sweep runs on both the write and the prune path, and is the one place this SDK removes a file the manifest does not list, so it is bounded on every axis: inside `//` only, for a key that passes `_key_rejection_reason`; only names `safe_fs` itself recognizes, via a new `is_temp_name` beside the naming code rather than a copy of the format string that could drift from it; only regular files, with the type read off the descriptor; unlinked through the pinned descriptor. It never raises and never aborts a run. **DV-2 — reject the 22 Windows reserved device names.** `con`, `prn`, `aux`, `nul`, `com1`–`com9`, `lpt1`–`lpt9` are all valid skill keys and none can be a directory name on Windows. Rejected in `_key_rejection_reason`, which the write and prune paths already share, and *not* in the key grammar: `parse_ai_config` fails closed, so a grammar-level rejection would invalidate an entire AI Config for a Linux customer over a Windows-only constraint, and would silently shrink `skill_refs` — which is what authorizes a prune, turning "fails to write on Windows" into "gets deleted on Linux". The 255-byte component bound is in this layer for the same reason. Unconditional, not platform-gated: a root written from a Linux container is routinely read from a Windows host, and neither repository has a Windows CI runner, so a gated branch would be untestable — the condition that produced the gap. No suffix stripping and no case folding: the grammar admits no `.` and no `$`, so `con.txt` and `CONIN$` are unreachable, and keys are lowercase-only. `com0` and `lpt0` are not reserved and are not included. The trade is real and belongs in the release notes: a customer who legitimately names a skill `aux` now gets a reported `error` action on Linux where it previously worked. Neither gap emits the integrity-failure log record. A key rejection and a clobber refusal are `ReconcileAction` errors, not integrity failures. Tests cover crash-mid-reconcile recovery end to end (adopted, reported, recorded, and the next reconcile an ordinary no-op), byte-differing content still refused untouched, an unmanaged FIFO refused without hanging, a read failure refusing rather than overwriting, the `len + 1` off-by-one, the sweep and the `rmdir` it unblocks, lookalike temp names and a symlink wearing one left alone, all 22 reserved names through both destructive paths, and — the point of the layer choice — each reserved name still valid to `is_valid_skill_key`, `parse_ai_config`, and `skill_refs`. Co-Authored-By: Claude Opus 5 --- packages/client/README.md | 26 ++ packages/client/agents.md | 47 ++- .../src/launchdarkly_ai_server/safe_fs.py | 54 ++- .../src/launchdarkly_ai_server/skills_fs.py | 159 ++++++- packages/client/tests/test_skills_fs.py | 388 ++++++++++++++++++ 5 files changed, 655 insertions(+), 19 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index d9c1db15..711ce5bf 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -357,6 +357,32 @@ never writes through a symlink; writes are atomic (temp file, `fsync`, rename) a `0644`; and if the manifest is unreadable it performs no destructive action at all. Removing a skill from a variation is how revocation works — the next reconcile prunes it. +**One exception, and it is what makes a crashed reconcile recoverable.** A file at a managed +path whose bytes are *already byte-identical* to the content LaunchDarkly resolved is +adopted — recorded in the manifest and reported `skipped_current` — rather than refused. +Without that, a process killed after a skill file lands but before the manifest is rewritten +leaves that file managed-but-unrecorded, which is indistinguishable from a file you wrote +yourself, so every later reconcile would refuse it and the skill would stay wedged until +someone intervened. Adoption cannot weaken the guarantee above, because bytes that differ in +any way are still refused and left untouched. Note that an adopted file becomes prunable +like any other managed file — which is the same outcome the crash pre-empted. + +**A few keys are legal to an AI Config but not to a filesystem.** A key becomes a single +directory name, so `write_skills` applies bounds of its own on top of the key grammar: no +mainstream filesystem allows a 256-byte path component, and Windows reserves 22 MS-DOS +device names (`con`, `prn`, `aux`, `nul`, `com1`–`com9`, `lpt1`–`lpt9`) that cannot be +directory names there. Either one is a reported `error` action for that skill, and the +rejection is unconditional rather than platform-gated — a managed root written from a Linux +container is routinely read from a Windows host, so the on-disk result must not depend on +which OS ran the write. The keys stay valid everywhere else: an AI Config referencing a skill +named `aux` parses, and its other fields are unaffected. If you have a skill named for a +device, rename it. + +**Total path length is yours to bound, not the SDK's.** The 255-byte bound above is per +*component*; the root is your path, so `` + `` + `/SKILL.md` can still exceed +Windows' 260-character `MAX_PATH` with a perfectly legal key. Choose a short managed root on +Windows. + | Export | Description | |---|---| | `skill_refs(config)` | Project a config's `skills` array into `list[SkillReference]`. Pure — no client, store, or network needed. Returns `[]` when absent. | diff --git a/packages/client/agents.md b/packages/client/agents.md index feb2e348..cd99b28f 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -250,7 +250,26 @@ Store data is **untrusted input**; the transport is not part of the trust bounda - **Never write through a symlink**, in either the skill directory or the target file, on the write path *and* the prune path. - **Destructive operations only on manifest-listed paths whose `key` matches.** A file at a - managed path with no matching manifest entry is reported as `error` and left alone. + managed path with no matching manifest entry is reported as `error` and left alone — + *unless its bytes already are the resolved content*, in which case it is adopted (manifest + entry recorded, reported `skipped_current`). That single exception is what makes a + reconcile killed between the content writes and the final manifest rewrite recoverable + instead of permanently wedged, and it cannot be widened: the comparison is over the + verbatim bytes against the resolved `contentHash`, a read that fails is a refusal and + never an overwrite, and the read is bounded at `len(content) + 1` bytes so a file that + merely *begins* with the resolved content is refused too. Do not relax it to a prefix, a + length, an mtime, or the manifest's own recorded `sha256` — that field is untrusted and is + never a decision input. `skipped_current` is reused deliberately rather than adding an + `adopted` action kind; `ReconcileActionKind` is a public closed set. +- **Temp files are swept, within the same bounds as everything else.** `atomic_write` unlinks + its own temp file on any exception, but a `SIGKILL` leaves one behind that no manifest + entry records, and a non-empty directory defeats `_prune_one`'s `rmdir` — so one orphan + pins a skill directory forever. The sweep is the only place this SDK removes a file the + manifest does not list, and it is bounded on every axis: inside `//` only, for a + key that passes `_key_rejection_reason`; only names `safe_fs.is_temp_name` recognizes, + anchored at both ends and asked of `safe_fs` rather than re-spelled (a copy would drift + from the writer); only regular files, with the type read off the descriptor; unlinked + through the pinned descriptor. It never raises and never aborts a run. - **A corrupt manifest fails closed**: unreadable, unparseable, not an object, malformed `entries`, or a `manifestVersion` this release cannot read means no overwrites and no prunes, brand-new paths may still be written, an `error` action names the manifest, and @@ -265,10 +284,28 @@ Store data is **untrusted input**; the transport is not part of the trust bounda "Descriptor-pinned filesystem access" below. Re-resolving `/` from its path at write or unlink time reopens a swap window that the checks above cannot cover. - **A key valid to the data model may still be unrepresentable on disk.** The model allows - 256 characters; `NAME_MAX` is 255 bytes. `write_skills` rejects an over-long key before - any filesystem call, and every per-skill filesystem failure is caught at the loop so it - becomes an `error` action — aborting the loop would skip the manifest rewrite and orphan - files already written in that run. + 256 characters; `NAME_MAX` is 255 bytes. Windows additionally reserves 22 MS-DOS device + names, none of which can be a directory name there: `con`, `prn`, `aux`, `nul`, + `com1`–`com9`, `lpt1`–`lpt9` (`com0` and `lpt0` are *not* reserved; do not add them). + `write_skills` rejects both before any filesystem call, and every per-skill filesystem + failure is caught at the loop so it becomes an `error` action — aborting the loop would + skip the manifest rewrite and orphan files already written in that run. +- **Those two bounds live in `_key_rejection_reason`, not in the key grammar, and must not + move.** `is_valid_skill_key` / `skill_key_rejection_reason` keep admitting an over-long or + reserved key on purpose. `parse_ai_config` fails closed on a bad `skills` entry, so a + grammar-level rejection would invalidate the *entire* AI Config — model, provider, + instructions, tools — for a Linux customer over a Windows-only constraint; and it would + silently shrink `skill_refs`, which is what authorizes a prune, converting "this skill + fails to write on Windows" into "this skill gets deleted on Linux". `_key_rejection_reason` + is shared by the write and prune paths, so one edit covers both destructive paths. + The reserved-name check is unconditional rather than `os.name == "nt"`-gated: a root + written from a Linux container is routinely read from a Windows host, and neither + repository has a Windows CI runner (every matrix job is `ubuntu-latest`), so a gated branch + would be untestable — the exact condition that produced the gap. No suffix stripping and no + case folding are needed, because the grammar admits no `.` and no `$` (so `con.txt` and + `CONIN$` are unreachable) and is lowercase-only. The residual the SDK cannot check is total + path length: the 255-byte bound is per *component*, and the root belongs to the customer, + so `MAX_PATH` overflow is a README note rather than a check. - **A key is untrusted input everywhere it appears.** `skill_key_rejection_reason` is the single canonical explanation, so the config parser and the reference projection reject a key for the same stated reason — and so does every layer added later. A silently diff --git a/packages/client/src/launchdarkly_ai_server/safe_fs.py b/packages/client/src/launchdarkly_ai_server/safe_fs.py index 3605399e..73eef878 100644 --- a/packages/client/src/launchdarkly_ai_server/safe_fs.py +++ b/packages/client/src/launchdarkly_ai_server/safe_fs.py @@ -17,6 +17,7 @@ import errno import os +import re import secrets import stat import tempfile @@ -191,6 +192,53 @@ def unlink_file(directory: Path, name: str, *, dir_fd: int | None) -> None: os.unlink(name, dir_fd=dir_fd) +_TEMP_SUFFIX = ".tmp" +"""Suffix on every temp file this module creates.""" + +_TEMP_TOKEN_BYTES = 8 +"""Bytes of randomness in a temp name, as ``secrets.token_hex`` takes them.""" + +_TEMP_TOKEN_PATTERN = re.compile( + # Two producers, one recognizer. The descriptor path below names its temp + # file with ``secrets.token_hex(_TEMP_TOKEN_BYTES)`` — twice that many + # lowercase hex characters. The fallback path hands naming to + # ``tempfile.mkstemp``, whose sequence is eight characters drawn from + # ``[a-z0-9_]``. Matched with ``fullmatch``, which anchors both branches at + # both ends, so nothing longer or otherwise-shaped is ever recognized. + rf"[0-9a-f]{{{_TEMP_TOKEN_BYTES * 2}}}|[a-z0-9_]{{8}}" +) + + +def temp_name_prefix(name: str) -> str: + """ + The prefix every temp file for *name* is created under. + + Spelled once because two callers need to agree on it: ``atomic_write`` + creates the name, and a caller sweeping orphaned temp files left by a crash + has to recognize it. A copy of the format string in the sweeper would be a + copy that can drift out of step with the writer. + """ + return f".{name}." + + +def is_temp_name(candidate: str, name: str) -> bool: + """ + Whether *candidate* is a name this module could have created for *name*. + + The recognizer for the orphan sweep: ``atomic_write`` unlinks its temp file + on any exception, but a ``SIGKILL`` between the create and the rename leaves + it behind, and nothing else on disk records that it exists. Deliberately + narrow — prefix, random token, and suffix must all match, with nothing + before or after — because the only thing a caller does with a ``True`` here + is delete the file. + """ + prefix = temp_name_prefix(name) + if not candidate.startswith(prefix) or not candidate.endswith(_TEMP_SUFFIX): + return False + token = candidate[len(prefix) : -len(_TEMP_SUFFIX)] + return _TEMP_TOKEN_PATTERN.fullmatch(token) is not None + + def _mkstemp_at(dir_fd: int, prefix: str) -> tuple[int, str]: """ ``tempfile.mkstemp`` for a directory descriptor. @@ -202,7 +250,7 @@ def _mkstemp_at(dir_fd: int, prefix: str) -> tuple[int, str]: """ flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) for _ in range(tempfile.TMP_MAX): - name = f"{prefix}{secrets.token_hex(8)}.tmp" + name = f"{prefix}{secrets.token_hex(_TEMP_TOKEN_BYTES)}{_TEMP_SUFFIX}" try: return os.open(name, flags, 0o600, dir_fd=dir_fd), name except FileExistsError: @@ -234,7 +282,7 @@ def atomic_write( semantics on Windows). """ at_fd = dir_fd if dir_fd is not None and SUPPORTS_DIR_FD else None - prefix = f".{name}." + prefix = temp_name_prefix(name) target: str | Path if at_fd is not None: @@ -243,7 +291,7 @@ def atomic_write( else: # mkstemp opens with O_CREAT|O_EXCL, so an existing temp path is never # reused. - fd, temp = tempfile.mkstemp(dir=directory, prefix=prefix, suffix=".tmp") + fd, temp = tempfile.mkstemp(dir=directory, prefix=prefix, suffix=_TEMP_SUFFIX) target = directory / name try: diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py index feb8fbc9..8a865dd8 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fs.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -33,6 +33,7 @@ SymlinkRefused, atomic_write, atomic_write_in, + is_temp_name, pinned_directory, unlink_file, ) @@ -95,6 +96,27 @@ """ +_WINDOWS_RESERVED_NAMES = frozenset( + {"con", "prn", "aux", "nul"} + | {f"com{digit}" for digit in range(1, 10)} + | {f"lpt{digit}" for digit in range(1, 10)} +) +""" +The 22 MS-DOS device names Windows still reserves, which cannot be directory +names there. The key grammar admits every one of them, so a customer who names a +skill ``con`` gets a working reconcile on Linux and a broken one on Windows — +rejected here instead, on every platform, so the on-disk result never depends on +which OS ran the write. Neither repository has a Windows CI runner, which is the +condition that produced the gap in the first place. + +The bare names are the whole set: no suffix stripping is needed because the key +grammar admits no ``.``, so ``con.txt`` is unreachable, and ``CONIN$`` / +``CONOUT$`` are unreachable for want of a ``$``; no case folding is needed +because the grammar is lowercase-only. ``com0`` and ``lpt0`` are deliberately +absent — those are not reserved. +""" + + # ------------------------------------------------------------------------- # The reconcile entry point # ------------------------------------------------------------------------- @@ -644,6 +666,17 @@ def _key_rejection_reason(key: Any) -> str | None: f"skill key '{key[:32]}...' is {key_bytes} bytes, over the " f"{_MAX_PATH_COMPONENT_BYTES}-byte limit for a single directory name" ) + # Same reasoning as the byte bound above, and it lives at the same layer for + # the same reason: the grammar itself must keep admitting these, because + # rejecting them there would fail the whole AI Config over one skill, and + # would shrink ``skill_refs`` — which is what authorizes a prune, so a + # Windows-only constraint would delete the skill's file on Linux. + if key in _WINDOWS_RESERVED_NAMES: + return ( + f"skill key '{key}' is a name Windows reserves for a device and " + "cannot be a directory name there; it is rejected on every platform " + "so a managed root written on one OS is usable on the other" + ) return None @@ -683,24 +716,45 @@ def failed(message: str) -> ReconcileAction: ) encoded, content_hash = verified.encoded, verified.content_hash + # Sweep before writing rather than after, so a temp file this run is about + # to create can never be a candidate. + _sweep_orphan_temp_files(root, key) + # Overwrite only what the manifest records as ours under this key. entry = entries.get(relative) managed = isinstance(entry, dict) and entry.get("key") == key exists = target.exists() - if exists and not managed: - return failed( - f"'{relative}' exists but the manifest does not record it as managed " - f"under key '{key}'; refusing to overwrite a file this SDK did not write" - ) - if exists: + # Hash first, and decide from the bytes. The manifest check below is what + # protects a customer's own file, but it also refuses the file this SDK + # itself wrote and was killed before recording — the reconcile writes + # every skill and only then rewrites the manifest, so a crash in that + # window leaves a managed path with no entry, and every later reconcile + # takes the refusal branch forever. Comparing the bytes distinguishes the + # two cases without weakening anything: only content byte-identical to + # what LaunchDarkly resolved is ever adopted. try: - on_disk = _read_regular_file(target) + on_disk = _read_regular_file(target, max_bytes=len(encoded)) except OSError as exc: + if not managed: + # A read that failed proves nothing, and must never become an + # overwrite: it is the comparison below that would authorize one. + return failed( + f"'{relative}' exists, the manifest does not record it as " + f"managed under key '{key}', and it could not be read to " + f"compare against the resolved content: {exc}; refusing to " + "overwrite a file this SDK may not have written" + ) return failed(f"'{relative}' could not be read: {exc}") if hashlib.sha256(on_disk).hexdigest() == content_hash: + # ``skipped_current`` covers this deliberately, rather than a new + # action kind: its documented meaning is that the bytes on disk + # already are the resolved content, which is exactly as true for an + # adopted file as for one this SDK wrote and recorded. Adoption does + # add a manifest entry, so the file becomes prunable later — correct, + # because a prune then removes content LaunchDarkly delivered anyway. _update_entry(entries, relative, skill, content_hash) record_materialized(key, len(encoded), content_hash, "skipped_current") return ReconcileAction( @@ -709,6 +763,12 @@ def failed(message: str) -> ReconcileAction: version=skill.version, path=str(target), ) + + if not managed: + return failed( + f"'{relative}' exists but the manifest does not record it as managed " + f"under key '{key}'; refusing to overwrite a file this SDK did not write" + ) # Stale version or local tampering — LD-resolved content wins. action: ReconcileActionKind = "updated" else: @@ -725,7 +785,7 @@ def failed(message: str) -> ReconcileAction: ) -def _read_regular_file(target: Path) -> bytes: +def _read_regular_file(target: Path, *, max_bytes: int) -> bytes: """ Reads *target*, refusing anything that is not a regular file. @@ -739,6 +799,12 @@ def _read_regular_file(target: Path) -> bytes: bytes the *verbatim* bytes: it is 0 on POSIX, but on Windows a descriptor without it translates CRLF on read, which would fail the hash comparison against content that is actually current. + + Reads at most ``max_bytes + 1`` bytes. The only consumer compares a hash, and + anything longer than the resolved content cannot match it, so the one extra + byte is enough to prove inequality — which is what keeps a foreign file of + arbitrary size from being pulled into memory now that adoption reads files + the manifest does not list. """ flags = ( os.O_RDONLY @@ -751,15 +817,82 @@ def _read_regular_file(target: Path) -> bytes: if not stat.S_ISREG(os.fstat(fd).st_mode): raise OSError("the target file is not a regular file") chunks: list[bytes] = [] - while True: - chunk = os.read(fd, 65536) + remaining = max_bytes + 1 + while remaining > 0: + chunk = os.read(fd, min(remaining, 65536)) if not chunk: - return b"".join(chunks) + break chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) finally: os.close(fd) +def _sweep_orphan_temp_files(root: Path, key: str) -> None: + """ + Removes temp files a killed reconcile left behind under ``//``. + + ``atomic_write`` unlinks its own temp file on any exception, but a ``SIGKILL`` + between the create and the rename leaves one on disk, and nothing else + records that it exists: ``_prune`` walks manifest entries, and an orphan + never has one. The second-order effect is what makes this worth doing — + ``_prune_one``'s ``rmdir`` only succeeds on an empty directory, so a single + orphaned temp pins a skill's directory permanently. + + Bounded on every axis, because this is the one place the SDK removes a file + the manifest does not list: only inside a directory named by a key that + passes ``_key_rejection_reason``; only names ``safe_fs`` itself recognizes as + its own temp naming for ``SKILL.md``, anchored at both ends, and asked of + ``safe_fs`` rather than re-spelled here so the recognizer cannot drift from + the writer; only regular files; and every removal relative to a descriptor + pinned with ``O_NOFOLLOW``. It never raises and never aborts the run: the + reconcile itself has succeeded either way, so a sweep that cannot happen is + a warning. + """ + if _key_rejection_reason(key) is not None: + return + skill_dir = root / key + if not skill_dir.is_dir(): + return + + try: + with pinned_directory(skill_dir) as dir_fd: + # Listing by path is safe even though the removals are + # descriptor-relative: a name reaches the unlink only if it matches + # the anchored temp pattern, and the unlink resolves it inside the + # pinned directory, so a listing redirected between the pin and here + # can at worst name a file that is not in it. + for name in sorted(os.listdir(skill_dir)): + if is_temp_name(name, SKILL_FILENAME): + _remove_orphan_temp_file(skill_dir, name, dir_fd) + except (OSError, ValueError) as exc: + logger.warning( + "orphaned temp files under skill '%s' could not be swept: %s", key, exc + ) + + +def _remove_orphan_temp_file(skill_dir: Path, name: str, dir_fd: int | None) -> None: + """ + Removes one recognized orphan. A per-file failure warns and moves on. + + The type check is what keeps the temp naming from being a way to have this + SDK delete something it did not write: a symlink or a FIFO wearing that name + is not a file ``atomic_write`` left behind, so it is not this function's to + remove. It is read off the descriptor, not the path, wherever there is one. + """ + try: + if dir_fd is not None: + mode = os.stat(name, dir_fd=dir_fd, follow_symlinks=False).st_mode + else: + mode = os.lstat(skill_dir / name).st_mode + if not stat.S_ISREG(mode): + return + unlink_file(skill_dir, name, dir_fd=dir_fd) + except (OSError, ValueError) as exc: + logger.warning("an orphaned temp file could not be removed: %s", exc) + + def _write_through_descriptor( skill_dir: Path, encoded: bytes, key: str, relative: str ) -> str | None: @@ -929,6 +1062,10 @@ def _prune_one( if unsafe is not None: return _prune_error(key, f"'{relative}' was not removed: {unsafe}", version) + # Before the removal, so the ``rmdir`` below is not defeated by an orphaned + # temp file that nothing else on disk records. + _sweep_orphan_temp_files(root, key) + removed_from_disk = False if target.exists(): failure = _unlink_through_descriptor(skill_dir, relative) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 5af4d19b..223a3262 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -26,8 +26,11 @@ SkillReference, get_skill, init_client, + parse_ai_config, + skill_refs, write_skills, ) +from launchdarkly_ai_server.types_validation import is_valid_skill_key MANIFEST_NAME = ".launchdarkly-skills.json" SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" @@ -1640,3 +1643,388 @@ async def test_integrity_signal_property_keys_match_across_layers( f"write-only keys: {sorted(write_keys - accessor_keys)}" ) assert "expected_hash" in accessor_keys + + +# --------------------------------------------------------------------------- +# Self-healing partial reconciles +# --------------------------------------------------------------------------- + + +def _place_unmanaged(root: Path, key: str, content: str) -> Path: + """A file at a managed path with **no** manifest entry. + + Exactly the state a reconcile killed between the content writes and the + final manifest rewrite leaves behind — and, indistinguishably on disk, the + state a customer authoring their own file there creates. Which is why the + bytes are the only thing that may decide between them. + """ + target = root / key / "SKILL.md" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return target + + +class TestCrashMidReconcileRecovery: + """A crash between the writes and the manifest rewrite must not wedge a skill.""" + + async def test_byte_identical_unmanaged_file_is_adopted(self, root: Path) -> None: + """The whole point: the second reconcile repairs the first one's crash. + + Without adoption every later reconcile takes the clobber-refusal branch + forever, because the file is at a managed path with no manifest entry. + """ + target = _place_unmanaged(root, "a", SKILL_BODY) + + first = await write_skills([_skill("a")], root) + + assert first.ok is True, _error_messages(first) + action = _actions_by_key(first)["a"] + assert action.action == "skipped_current" + assert action.version == 1 + assert action.path == str(target) + # Adopted, not rewritten, and now recorded. + assert target.read_text(encoding="utf-8") == SKILL_BODY + entry = _read_manifest(root)["entries"]["a/SKILL.md"] + assert entry["key"] == "a" + assert entry["version"] == 1 + assert entry["sha256"] == _hash(SKILL_BODY) + + # And the run after it is an ordinary no-op, through the managed path. + second = await write_skills([_skill("a")], root) + assert second.ok is True, _error_messages(second) + assert _actions_by_key(second)["a"].action == "skipped_current" + + async def test_adoption_writes_nothing( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Adoption is a manifest edit, not a write. Nothing touches the bytes.""" + _place_unmanaged(root, "a", SKILL_BODY) + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert spy.calls == [] + + async def test_adoption_reports_skipped_current_not_a_new_action_kind( + self, root: Path, recording_emitter: Any + ) -> None: + """``skipped_current`` is reused deliberately — no ``adopted`` kind exists.""" + skills_module._set_emitter_for_testing(recording_emitter) + _place_unmanaged(root, "a", SKILL_BODY) + + report = await write_skills([_skill("a")], root) + + assert {a.action for a in report.actions} == {"skipped_current"} + props = recording_emitter.signals(MATERIALIZED_SIGNAL)[0] + assert props["reconcile_action"] == "skipped_current" + assert props["skill_key"] == "a" + + async def test_an_adopted_file_is_prunable_afterwards(self, root: Path) -> None: + """The documented caveat, pinned. + + Adoption records a manifest entry, so a later reconcile may prune the + file. That is correct rather than a weakening: only content byte-identical + to what LaunchDarkly resolved is ever adopted, so the prune removes + content LaunchDarkly delivered — exactly what would have happened had the + crash never occurred. + """ + target = _place_unmanaged(root, "a", SKILL_BODY) + await write_skills([_skill("a")], root) + + report = await write_skills([], root) + + assert report.ok is True, _error_messages(report) + assert _actions_by_key(report)["a"].action == "removed" + assert not target.exists() + + async def test_differing_unmanaged_content_is_still_refused( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The clobber guarantee, restated against the adoption rule. + + Adoption compares bytes, so anything that is not byte-identical to the + resolved content falls through to the same refusal as before. + """ + target = _place_unmanaged(root, "a", "user authored\n") + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "did not write" in action.error + assert target.read_text(encoding="utf-8") == "user authored\n" + assert spy.calls == [] + + async def test_a_longer_file_sharing_the_content_prefix_is_not_adopted( + self, root: Path + ) -> None: + """The read is bounded at ``len(content) + 1``, and that one byte matters. + + A bound of exactly ``len(content)`` would make every file that merely + *begins* with the resolved content hash as current, adopting — and later + pruning — a customer file with the skill body at its head. + """ + longer = SKILL_BODY + "and my own notes below\n" + target = _place_unmanaged(root, "a", longer) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert target.read_text(encoding="utf-8") == longer + + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="no FIFOs on this platform") + async def test_an_unmanaged_fifo_is_refused_and_never_read( + self, root: Path + ) -> None: + """Adoption widened the read to foreign files, so this refusal is load-bearing. + + Opening a FIFO with no writer blocks forever; the descriptor-pinned read + opens ``O_NONBLOCK`` and rejects anything that is not a regular file, so + this returns rather than hanging the reconcile and the event loop with it. + """ + skill_dir = root / "a" + skill_dir.mkdir() + os.mkfifo(skill_dir / "SKILL.md") + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "regular file" in action.error + assert stat.S_ISFIFO(os.lstat(skill_dir / "SKILL.md").st_mode) + + async def test_a_read_failure_on_an_unmanaged_file_refuses( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A failed read proves nothing, so it must never become an overwrite. + + The adoption comparison is what would otherwise authorize the write, and + a file whose bytes could not be read has not been shown to be ours. + """ + target = _place_unmanaged(root, "a", SKILL_BODY) + spy = _ReplaceSpy().install(monkeypatch) + real_open = os.open + + def refuse_the_target(path: Any, *args: Any, **kwargs: Any) -> int: + if isinstance(path, (str, os.PathLike)) and os.fspath(path) == str(target): + raise PermissionError(13, "Permission denied") + return real_open(path, *args, **kwargs) + + monkeypatch.setattr(skills_fs_module.os, "open", refuse_the_target) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + # Distinguishable from the byte-mismatch refusal: this one says the + # comparison could not be made at all. + assert "could not be read to compare" in action.error + assert spy.calls == [] + assert target.read_text(encoding="utf-8") == SKILL_BODY + assert "a/SKILL.md" not in _read_manifest(root)["entries"] + + +# --------------------------------------------------------------------------- +# Orphaned temp files +# --------------------------------------------------------------------------- + + +def _temp_name(token: str = "0123456789abcdef") -> str: + """A name ``atomic_write`` could have created for ``SKILL.md``. + + The prefix comes from ``safe_fs`` itself rather than a copy of its format + string, so a change to the naming breaks this helper instead of silently + making the sweep a no-op. + """ + return f"{safe_fs_module.temp_name_prefix('SKILL.md')}{token}.tmp" + + +class TestOrphanedTempFiles: + """A ``SIGKILL`` mid-write leaves a temp file nothing else records.""" + + async def test_an_orphan_is_swept_on_the_next_write(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY) + orphan = root / "a" / _temp_name() + orphan.write_text("half-written body", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert not orphan.exists() + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_an_orphan_no_longer_blocks_directory_cleanup( + self, root: Path + ) -> None: + """The second-order effect: ``rmdir`` fails on a non-empty directory. + + One orphaned temp file would otherwise pin the skill's directory under + the managed root forever, long after the skill itself was revoked. + """ + _place_managed(root, "a", SKILL_BODY) + orphan = root / "a" / _temp_name() + orphan.write_text("half-written body", encoding="utf-8") + + report = await write_skills([], root) + + assert report.ok is True, _error_messages(report) + assert _actions_by_key(report)["a"].action == "removed" + assert not (root / "a").exists() + + @pytest.mark.parametrize( + "innocent", + [ + "notes.tmp", + "SKILL.md.tmp", + ".SKILL.md.tmp", + ".SKILL.md..tmp", + _temp_name("not-a-token"), + _temp_name("0123456789abcdef") + ".bak", + "x" + _temp_name(), + _temp_name("0123456789abcdefff"), + ], + ) + async def test_a_lookalike_name_is_left_alone( + self, root: Path, innocent: str + ) -> None: + """The recognizer is anchored at both ends, and the sweep deletes files. + + Anything that is not exactly the naming ``safe_fs`` produces belongs to + the customer, whatever it resembles. + """ + _place_managed(root, "a", SKILL_BODY) + bystander = root / "a" / innocent + bystander.write_text("mine\n", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert bystander.read_text(encoding="utf-8") == "mine\n" + + @pytest.mark.skipif( + not hasattr(os, "symlink"), reason="platform has no symlink support" + ) + async def test_a_symlink_wearing_the_temp_name_is_not_removed( + self, root: Path, tmp_path: Path + ) -> None: + """The temp naming must not become a way to have the SDK delete elsewhere. + + Only a regular file is ever swept, and the type comes off the descriptor + rather than a followed path. + """ + outside = tmp_path / "precious.txt" + outside.write_text("do not delete\n", encoding="utf-8") + _place_managed(root, "a", SKILL_BODY) + link = root / "a" / _temp_name() + link.symlink_to(outside) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert outside.read_text(encoding="utf-8") == "do not delete\n" + assert link.is_symlink() + + +# --------------------------------------------------------------------------- +# Windows reserved device names +# --------------------------------------------------------------------------- + +# Spelled out independently of the implementation's own set, so a name dropped +# from that set fails here rather than agreeing with itself. +WINDOWS_RESERVED_KEYS = ( + ["con", "prn", "aux", "nul"] + + [f"com{digit}" for digit in range(1, 10)] + + [f"lpt{digit}" for digit in range(1, 10)] +) + + +class TestWindowsReservedNames: + """Keys Windows cannot hold as directory names, refused on every platform.""" + + def test_the_set_is_exactly_twenty_two_names(self) -> None: + assert len(WINDOWS_RESERVED_KEYS) == len(set(WINDOWS_RESERVED_KEYS)) == 22 + + @pytest.mark.parametrize("reserved", WINDOWS_RESERVED_KEYS) + async def test_a_reserved_key_is_refused_by_write_skills( + self, root: Path, reserved: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill(reserved)], root) + + assert report.ok is False + action = _actions_by_key(report)[reserved] + assert action.action == "error" + assert action.error is not None + assert "Windows reserves" in action.error + assert reserved in action.error + # Rejected before any filesystem call, not by the OS. + assert spy.calls == [] + assert not (root / reserved).exists() + + @pytest.mark.parametrize("reserved", WINDOWS_RESERVED_KEYS) + async def test_a_reserved_key_is_refused_by_the_prune_path( + self, root: Path, reserved: str + ) -> None: + """``_key_rejection_reason`` gates both destructive paths, so both refuse. + + A manifest naming a reserved key is left in place rather than acted on: + the same key check that stops the write stops the delete. + """ + target = _place_managed(root, reserved, SKILL_BODY) + + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)[reserved] + assert action.action == "error" + assert action.error is not None + assert "left in place" in action.error + assert target.read_text(encoding="utf-8") == SKILL_BODY + + @pytest.mark.parametrize("not_reserved", ["com0", "lpt0", "con1", "nul2", "conx"]) + async def test_neighbouring_names_are_not_reserved( + self, root: Path, not_reserved: str + ) -> None: + """``com0`` and ``lpt0`` are not device names, and must still write.""" + report = await write_skills([_skill(not_reserved)], root) + + assert report.ok is True, _error_messages(report) + assert (root / not_reserved / "SKILL.md").exists() + + @pytest.mark.parametrize("reserved", WINDOWS_RESERVED_KEYS) + def test_a_reserved_name_is_still_a_valid_key_to_every_pure_layer( + self, reserved: str + ) -> None: + """The layer choice, asserted — this is the whole point of it. + + The constraint lives in the filesystem layer and must not migrate into + the key grammar. At the grammar level a rejection would fail the *entire* + AI Config — model, provider, instructions, tools — for a Linux customer + over a Windows-only constraint, and would shrink ``skill_refs``, which is + what authorizes a prune: "this skill fails to write on Windows" would + become "this skill gets deleted on Linux". + """ + assert is_valid_skill_key(reserved) is True + + parsed = parse_ai_config( + { + "model": {"name": "claude-3"}, + "provider": {"name": "Anthropic"}, + "instructions": "You are helpful.", + "skills": [{"key": reserved, "version": 1}], + } + ) + assert parsed.success is True + + refs = skill_refs(parsed.data) + assert [ref.key for ref in refs] == [reserved] From eda4ae1f9219a8a3c3b363dcfc05fee566820825 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Mon, 31 Aug 2026 14:04:18 -0400 Subject: [PATCH 08/22] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20a=20distinguishable=20outcome=20for=20integrity=20f?= =?UTF-8?q?ailure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``get_skill`` returns ``None`` for four unrelated outcomes: no such skill, the store raised, the requested version is not the one held, and content that failed hash verification. A caller cannot fail closed on suspected tampering while tolerating a merely-absent skill, so no automated customer-side response is possible — finding LA-2 of the Agent Skills security design review. The information already existed internally, as prose in ``Resolution.error``. This gives it a token: ``Resolution`` grows a typed ``reason``, set explicitly at every construction site and declared without a default so a sixth outcome added later has to choose which public token it maps to. ``get_skill_result`` maps that straight through to a frozen ``SkillOutcome`` (``skill``, ``reason``, ``detail``). Deriving the public reason by matching the error string is the fragility LA-2 is about, so the mapping is readable in one table. ``get_skill`` is untouched — its ``None``-for-every-failure contract is documented in its docstring and in the README, and a test now pins that all four failures still collapse to ``None`` and still never raise. Nothing new is emitted: Gap 1's integrity record already fired inside verification before ``resolve_from_store`` returned, and a test asserts one failed retrieval still produces exactly one record and one signal. Co-Authored-By: Claude Opus 5 --- packages/client/README.md | 56 +++- packages/client/agents.md | 59 +++- .../src/launchdarkly_ai_server/__init__.py | 8 +- .../src/launchdarkly_ai_server/skills.py | 36 ++- .../src/launchdarkly_ai_server/skills_core.py | 39 ++- .../src/launchdarkly_ai_server/skills_fs.py | 10 +- .../src/launchdarkly_ai_server/types.py | 55 ++++ packages/client/tests/test_skills.py | 281 ++++++++++++++++++ 8 files changed, 527 insertions(+), 17 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 711ce5bf..045180b5 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -337,6 +337,55 @@ truncated payload; a mismatch means content was delivered whose bytes are not th LaunchDarkly hashed, which is a possible **active-tampering** signal. Alert on it, and treat `expected_hash` / `observed_hash` as the evidence pair. +#### Failing closed on tampering + +The log record above is the operator's surface. `get_skill_result` is the application's: +same retrieval, same verification, same telemetry as `get_skill`, and it reports which of +five outcomes happened instead of collapsing all of them to `None`. + +```python +from launchdarkly_ai_server import get_skill_result + +outcome = await get_skill_result("pdf-extraction") + +if outcome.reason == "integrity_failure": + # Content was delivered and did not verify. Do not degrade quietly. + raise SystemExit(f"refusing to start: {outcome.detail}") + +if outcome.reason == "store_unavailable": + # The store could not answer at all. Retry, alert, or carry on with what + # you already have — but this is an outage, not a revocation. + print(f"skill retrieval unavailable: {outcome.detail}") +elif outcome.reason in ("absent", "wrong_version"): + # Nothing was tampered with — this skill is simply not available to you. + print(f"continuing without a skill: {outcome.detail}") +elif outcome.skill is not None: + print(outcome.skill.content) +``` + +| `reason` | Meaning | +|---|---| +| `ok` | A verified skill was returned; `.skill` is set and `.detail` is `None`. | +| `absent` | The store answered, and does not hold that key. | +| `integrity_failure` | Content was delivered and failed verification, so it was withheld. **The one to fail closed on.** | +| `store_unavailable` | The store itself could not answer — it raised. An outage, not a deletion. | +| `wrong_version` | The store answered with a version other than the one asked for, so the answer was withheld. | + +`.detail` is human-readable and safe to log or show an operator — it names the key and the +failure mode, and never carries skill content or a filesystem path. Branch on `.reason`, +not on `.detail`. `.skill` is populated only when `.reason == "ok"`. `SkillOutcome` is +frozen, like every other value type here. + +**`get_skill` is unchanged.** It still returns `None` for all four failures and still never +raises for one, so no existing caller has to move. The two accessors run the same code path +and differ only in what they report — `get_skill_result` adds no second log record and no +second signal for a failure that already emitted one, so a caller can switch to it without +double-counting anything. + +`get_skills` and `all_skills` have no reported form: they still omit entries that could not +be resolved, and a run that omitted anything logs a count at WARN. Retrieve individually +with `get_skill_result` when you need the reason per key. + **Versions are selected, not filtered.** A store may hold several versions of one key at once, because a delivery payload does: the newest version of every skill, plus every version a variation currently pins. `get_skill("k", version=1)` asks the store for version @@ -387,6 +436,7 @@ Windows. |---|---| | `skill_refs(config)` | Project a config's `skills` array into `list[SkillReference]`. Pure — no client, store, or network needed. Returns `[]` when absent. | | `get_skill(key, *, version=None)` | One verified skill, or `None`. `version=None` means newest available; a specific `version` matches exactly. Raises only when no store is configured. | +| `get_skill_result(key, *, version=None)` | The same retrieval, reporting **why**: a frozen `SkillOutcome` with `.skill`, `.reason` (`ok` / `absent` / `integrity_failure` / `store_unavailable` / `wrong_version`), and `.detail`. Use it to fail closed on tampering — see *Failing closed on tampering* above. Raises only when no store is configured. | | `get_skills(refs)` | Batch form. Accepts `SkillReference` values and bare key strings (string = latest). Results follow input order; missing or unverifiable entries are omitted. | | `all_skills()` | Every verified skill the store holds, one per key at its newest version. | | `write_skills(skills, root, *, prune=True, timeout=10.0, on_unavailable="keep")` | Materialize skills under `root`, returning a `ReconcileReport`. `prune` removes formerly-managed skills no longer requested. `on_unavailable="raise"` raises instead of reporting when content cannot be retrieved. Raises `ValueError` for an unusable root, a negative `timeout`, or an unrecognised `on_unavailable`. **Performs synchronous filesystem I/O — see the note below.** | @@ -406,9 +456,10 @@ manifest, for instance — carries the empty string as its `key`. The fixed on-disk values are exported too, so you do not have to hardcode them: `MANIFEST_FILENAME` (`.launchdarkly-skills.json`, handy for a `.gitignore`), -`SKILL_FILENAME`, and `MANIFEST_VERSION`. So are the two closed-set types, for annotating +`SKILL_FILENAME`, and `MANIFEST_VERSION`. So are the three closed-set types, for annotating your own helpers: `ReconcileActionKind` (`written` / `updated` / `skipped_current` / -`removed` / `error`) and `OnUnavailable` (`keep` / `raise`). +`removed` / `error`), `OnUnavailable` (`keep` / `raise`), and `SkillOutcomeReason` +(`absent` / `integrity_failure` / `ok` / `store_unavailable` / `wrong_version`). **`write_skills` blocks.** It is `async` for parity with the other accessors and with the TypeScript SDK, but it awaits nothing: every read, write, `fsync` and rename runs inline, @@ -460,5 +511,6 @@ All types are exported from this package. Handler packages import them from here | `GraphTopology` | The parsed graph flag shape (`root` + `edges`) | | `Skill` | A frozen skill document: `.key`, `.version`, `.content` (verified verbatim `bytes`), `.content_hash`, `.name?`, `.description?` | | `SkillReference` | A frozen version-pinned pointer to a skill: `.key`, `.version` | +| `SkillOutcome` | A frozen retrieval outcome: `.skill`, `.reason` (`SkillOutcomeReason`), `.detail` | | `ReconcileAction` | One `write_skills` outcome: `.key`, `.action`, `.version?`, `.path?`, `.error?` | | `ReconcileReport` | The `write_skills` result: `.actions`, `.ok`, and `.errors` | diff --git a/packages/client/agents.md b/packages/client/agents.md index cd99b28f..a7b4ce1b 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -77,10 +77,10 @@ from launchdarkly_ai_server import config, graph, resolve_graph # Agent Skills from launchdarkly_ai_server import ( - skill_refs, get_skill, get_skills, all_skills, write_skills, - SkillStore, InMemorySkillStore, + skill_refs, get_skill, get_skill_result, get_skills, all_skills, write_skills, + SkillStore, InMemorySkillStore, SkillOutcome, SKILL_FILENAME, MANIFEST_FILENAME, MANIFEST_VERSION, - ReconcileActionKind, OnUnavailable, # the two closed-set unions + ReconcileActionKind, OnUnavailable, SkillOutcomeReason, # the three closed-set unions ) ``` @@ -188,8 +188,8 @@ Three layers, in increasing order of blast radius: typed `SkillReference` values. Pure: no network, no client, no store, no telemetry. Validation of the array itself lives in `parse_ai_config` and is **fail closed** — one malformed reference fails the whole config parse. -2. **Content accessors** — `get_skill`, `get_skills`, `all_skills` read through the - `SkillStore` seam. Configure a store with +2. **Content accessors** — `get_skill`, `get_skill_result`, `get_skills`, `all_skills` read + through the `SkillStore` seam. Configure a store with `init_client(options={"skillStore": store})`; with none configured the accessors raise an actionable `RuntimeError`. A delivery transport can be added behind the seam without touching the public API. @@ -217,6 +217,55 @@ one place that collapses the result to one object per key, because both whole-st consumers need it — `all_skills`, since a list holding two versions of one key is not a set of skills, and the `"*"` reconcile, since `//SKILL.md` is a single path. +### The reported outcome vocabulary, and the `Resolution` mapping + +`get_skill` returns `Skill | None`; `get_skill_result` returns a frozen `SkillOutcome` +(`skill`, `reason`, `detail`) naming *which* outcome happened. Both are +`resolve_from_store` — one retrieval, one verification, one telemetry pass — and they differ +only in what they report. `get_skill`'s contract is load-bearing and **frozen**: `None` for +every failure, never raises for one, documented in its docstring and in the README. Change +it and every caller that treats `None` as "no skill" breaks silently. + +`SkillOutcomeReason` is five tokens, listed alphabetically for the same reason +`IntegrityReasonCode` is — so the vocabulary reads identically in the Python and TypeScript +SDKs, where the type name, the accessor name, and the tokens are all deliberately the same. +Do not rename one on one side. + +Internal `Resolution.reason` maps 1:1 onto it, set explicitly at every construction site: + +| `resolve_from_store` outcome | `reason` | +|---|---| +| the store raised (`unavailable=True`) | `store_unavailable` | +| `raw` is not a dict | `absent` | +| `verify_raw_skill` returned `None` | `integrity_failure` | +| `skill.version != wanted_version` | `wrong_version` | +| success | `ok` | + +**Adding a sixth internal outcome means choosing which public token it maps to.** +`Resolution.reason` has no default, so the compiler asks the question; answer it rather than +defaulting to `absent`, which claims the store does not hold the skill. If the new outcome +is genuinely neither of the five, the token set grows — on both sides, in the same commit. + +Two things the reason is deliberately *not*: + +- **Not derived from `Resolution.error`.** That string is prose for a human; recovering a + decision a caller fails closed on by matching it is the fragility the typed token exists + to remove. `detail` *is* that string, passed straight through — safe to surface (key and + failure mode only, never content, never a path), and not for matching on. +- **Not `Resolution.unavailable`.** The flag answers "may prune run?" and the token answers + "what does the caller learn?". They agree by construction — `unavailable` is `True` in + exactly the `store_unavailable` case — and both exist because `store_unavailable` must + stay distinct from `absent`: only a raising store suppresses pruning, since deleting + managed files after a failed lookup turns an outage into data loss. + +`get_skill_result` emits nothing of its own. The integrity log record and signal already +fired inside verification before `resolve_from_store` returned; recording anything here +would double-count one failure in a SIEM and in the product counter. + +There is no `get_skills_result` or `all_skills_result`. The batch accessors keep omitting +unresolved entries and keep logging the run-level WARN count, and a second accessor per +batch form would double the surface for a case nobody has asked for. + ### Security posture — do not relax any of this Store data is **untrusted input**; the transport is not part of the trust boundary. diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 37b1570a..753e1e95 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -41,6 +41,7 @@ InMemorySkillStore, all_skills, get_skill, + get_skill_result, get_skills, skill_refs, ) @@ -85,6 +86,8 @@ ReconcileActionKind, ReconcileReport, Skill, + SkillOutcome, + SkillOutcomeReason, SkillReference, StreamChunkEvent, StreamDoneEvent, @@ -150,6 +153,7 @@ "ReconcileActionKind", "ReconcileReport", "Skill", + "SkillOutcome", "SkillReference", "StreamChunkEvent", "StreamDoneEvent", @@ -227,14 +231,16 @@ # skills "skill_refs", "get_skill", + "get_skill_result", "get_skills", "all_skills", "write_skills", "SkillStore", "InMemorySkillStore", - # skills — the two closed-set unions a typed consumer needs to name + # skills — the three closed-set unions a typed consumer needs to name "ReconcileActionKind", "OnUnavailable", + "SkillOutcomeReason", # skills — on-disk constants, identical across languages "SKILL_FILENAME", "MANIFEST_FILENAME", diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py index 9d9b3627..cb21986b 100644 --- a/packages/client/src/launchdarkly_ai_server/skills.py +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -37,7 +37,7 @@ resolve_from_store, verify_raw_skill, ) -from .types import AiConfigRep, Skill, SkillReference +from .types import AiConfigRep, Skill, SkillOutcome, SkillReference from .types_validation import ( is_valid_skill_key, is_valid_skill_version, @@ -247,6 +247,40 @@ async def get_skill(key: str, *, version: int | None = None) -> Skill | None: return resolve_from_store(require_store(), key, version).skill +async def get_skill_result(key: str, *, version: int | None = None) -> SkillOutcome: + """ + Retrieves one verified skill, reporting *why* when there is none. + + Same retrieval, same verification, same telemetry as ``get_skill`` — the two + differ only in what they report. ``get_skill`` collapses "no such skill", + "the store raised", "that is not the version held", and "the content failed + integrity verification" to one ``None``; this returns a ``SkillOutcome`` + whose ``reason`` names which of them happened, so a caller can fail closed on + suspected tampering while tolerating a merely-absent skill: + + ```python + outcome = await get_skill_result("pdf-extraction") + if outcome.reason == "integrity_failure": + raise SystemExit(f"refusing to run: {outcome.detail}") + if outcome.skill is not None: + print(outcome.skill.content) + ``` + + ``detail`` is human-readable and safe to surface — it names the key and the + failure mode, never any skill content or filesystem path. Branch on + ``reason``, not on ``detail``. + + Emits nothing of its own: an integrity failure has already recorded its log + record and its signal inside verification, and recording a second here would + double-count one failure. Raises ``RuntimeError`` only when no skill store is + configured, exactly as ``get_skill`` does. + """ + resolved = resolve_from_store(require_store(), key, version) + return SkillOutcome( + skill=resolved.skill, reason=resolved.reason, detail=resolved.error + ) + + async def get_skills(refs: Sequence[SkillReference | str]) -> list[Skill]: """ Retrieves a batch of verified skills. diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index 1e517e4e..74cb706b 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -44,7 +44,7 @@ from dataclasses import dataclass from typing import Any, Literal, Protocol, get_args -from .types import Skill, SkillReference +from .types import Skill, SkillOutcomeReason, SkillReference from .types_validation import is_valid_skill_key, is_valid_skill_version logger = logging.getLogger(__name__) @@ -672,6 +672,26 @@ def newest_by_key(objects: dict[str, dict[str, Any]]) -> list[tuple[str, Any]]: class Resolution: """One key resolved against a store: the skill, or why there is none.""" + reason: SkillOutcomeReason + """ + Which of the five public outcomes this resolution is. + + Declared first and **without a default**, so every construction site has to + state it. A default would be the wrong shape twice over: a contributor + adding a sixth internal outcome would inherit whichever token happened to be + the default rather than deciding which public token it maps to, and if that + default were ``"ok"`` a failure would publish ``ok`` with no skill attached. + + Carried as a token rather than derived from ``error`` on the way out: + ``get_skill_result`` publishes this value, and pattern-matching prose to + recover a decision a caller fails closed on is exactly the fragility the + typed outcome exists to remove. A reviewer can read the mapping here. + + Distinct from ``unavailable`` on purpose — that flag answers one question + (may prune run?) and this token answers a different one (what does the + caller learn?) — but the two can only disagree by a bug: ``unavailable`` is + ``True`` in exactly the ``store_unavailable`` case. + """ skill: Skill | None = None error: str | None = None unavailable: bool = False @@ -702,26 +722,33 @@ def resolve_from_store( raw = store.get_object(SKILL_OBJECT_KIND, key, wanted_version) except Exception as exc: logger.error("Skill store raised while retrieving '%s'", key, exc_info=True) - return Resolution(error=store_raised(exc), unavailable=True) + return Resolution( + reason="store_unavailable", + error=store_raised(exc), + unavailable=True, + ) if not isinstance(raw, dict): return Resolution( - error=f"skill '{key}' is not available from the configured skill store" + reason="absent", + error=f"skill '{key}' is not available from the configured skill store", ) skill = verify_raw_skill(raw) if skill is None: return Resolution( - error=f"skill '{key}' failed integrity verification and was withheld" + reason="integrity_failure", + error=f"skill '{key}' failed integrity verification and was withheld", ) if wanted_version is not None and skill.version != wanted_version: return Resolution( + reason="wrong_version", error=( f"skill '{key}' version {wanted_version} is not available " f"(the store holds version {skill.version})" - ) + ), ) - return Resolution(skill=skill) + return Resolution(reason="ok", skill=skill) def reference_target(item: SkillReference | str) -> tuple[str, int | None]: diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py index 8a865dd8..fb163796 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fs.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -404,11 +404,17 @@ def _resolve_reference( """ store = _available_store(deadline, f"'{key}'") if isinstance(store, _RetrievalBlocked): - return Resolution(error=store.reason, unavailable=True) + return Resolution( + reason="store_unavailable", error=store.reason, unavailable=True + ) resolved = resolve_from_store(store, key, wanted_version) if resolved.unavailable and resolved.error is not None: - return Resolution(error=_unavailable(resolved.error), unavailable=True) + return Resolution( + reason="store_unavailable", + error=_unavailable(resolved.error), + unavailable=True, + ) return resolved diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index 10c90d8c..65e4850a 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -452,6 +452,61 @@ class Skill: """Description from LaunchDarkly metadata; never parsed from the content.""" +SkillOutcomeReason = Literal[ + "absent", "integrity_failure", "ok", "store_unavailable", "wrong_version" +] +""" +The closed set of outcomes ``get_skill_result`` reports. + +Alphabetical, as ``reason_code`` is in the integrity log record, so the token +list reads identically in every LaunchDarkly AI SDK. Each token is a distinct +*decision* a caller can make, which is the point of the type: ``absent`` is a +skill the store does not hold, ``integrity_failure`` is content that was +delivered and did not verify, and a caller that wants to fail closed on +suspected tampering while tolerating a merely-absent skill needs the two to be +told apart. + +- ``ok`` — a verified skill was returned. +- ``absent`` — the store answered, and does not hold the key. +- ``integrity_failure`` — content was delivered and failed verification; it was + withheld. The one token worth failing closed on. +- ``store_unavailable`` — the store itself could not answer: it raised. + Deliberately distinct from ``absent``, because an outage is not a deletion. +- ``wrong_version`` — the store answered with a version other than the one + asked for, so the answer was withheld. +""" + + +@dataclass(frozen=True) +class SkillOutcome: + """ + Why one retrieval returned what it did — the reported form of ``get_skill``. + + ``get_skill`` collapses every failure to ``None``, which is the right shape + for a caller that only wants content and cannot act on the difference. This + is the shape for a caller that can: ``reason`` names which of the five + outcomes happened, so an integrity failure is distinguishable from a skill + that simply is not configured. The two accessors differ only in what they + report — the retrieval, the verification, and the telemetry are the same + code path, run once. + + Instances are immutable. + """ + + skill: Skill | None + """The verified skill, and only ever populated when ``reason == "ok"``.""" + reason: SkillOutcomeReason + """Which outcome happened. A closed set — see ``SkillOutcomeReason``.""" + detail: str | None + """ + Human-readable detail, set for every reason except ``ok``. + + Safe to log or surface to an operator: it carries the skill key and the + failure mode, and never any skill content or filesystem path. Intended for a + human, not for matching on — branch on ``reason``. + """ + + ReconcileActionKind = Literal[ "written", "updated", "skipped_current", "removed", "error" ] diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 0c656ede..6470f0c0 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -20,10 +20,12 @@ ReconcileAction, ReconcileReport, Skill, + SkillOutcome, SkillReference, all_skills, get_client, get_skill, + get_skill_result, get_skills, init_client, shutdown, @@ -143,6 +145,16 @@ def test_skill_is_immutable(self) -> None: with pytest.raises(dataclasses.FrozenInstanceError): skill.content = b"tampered" # type: ignore[misc] + def test_skill_outcome_is_immutable(self) -> None: + """A reported outcome is a value, like every other public skills type. + + Matters more here than for the others: a caller that fails closed on + ``reason`` must not be handed something a later layer can rewrite. + """ + outcome = SkillOutcome(skill=None, reason="integrity_failure", detail="nope") + with pytest.raises(dataclasses.FrozenInstanceError): + outcome.reason = "ok" # type: ignore[misc] + def test_skill_content_is_bytes(self) -> None: """Content is the verified verbatim bytes — opaque, never text.""" skill = _skill() @@ -385,6 +397,13 @@ def test_exported_action_union_admits_exactly_the_five_actions(self) -> None: "error", } assert set(typing.get_args(package.OnUnavailable)) == {"keep", "raise"} + assert set(typing.get_args(package.SkillOutcomeReason)) == { + "absent", + "integrity_failure", + "ok", + "store_unavailable", + "wrong_version", + } def test_retrieval_surface_is_exported_from_the_package_root(self) -> None: """A name absent from ``__all__`` is not part of the public surface.""" @@ -393,11 +412,14 @@ def test_retrieval_surface_is_exported_from_the_package_root(self) -> None: expected = { "skill_refs", "get_skill", + "get_skill_result", "get_skills", "all_skills", "SkillStore", "InMemorySkillStore", "Skill", + "SkillOutcome", + "SkillOutcomeReason", "SkillReference", } assert expected <= set(package.__all__) @@ -743,6 +765,265 @@ async def test_multibyte_content_verifies( assert skill.content == content.encode("utf-8") +class _RaisingStore: + """A store whose reads raise — the "the transport is down" case. + + Declared with the full ``get_object`` signature on purpose. A double missing + the ``version`` parameter would also produce a raise here, but a + ``TypeError`` from the call itself rather than from the store, and the test + would then pass without the store ever having been consulted. + """ + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + raise RuntimeError("transport failure") + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + raise RuntimeError("transport failure") + + +class _WrongVersionAnsweringStore: + """A store that answers a pinned lookup with some other version.""" + + def __init__(self, make_raw_skill: Any, answered_version: int = 99) -> None: + self._make = make_raw_skill + self._answered_version = answered_version + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + answer: dict[str, Any] = self._make(key=key, version=self._answered_version) + return answer + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + return {} + + +class TestGetSkillResult: + """ + The reported accessor — one token per outcome a retrieval can have. + + ``get_skill`` collapses four distinct failures to ``None``, which leaves a + caller unable to fail closed on suspected tampering while tolerating a skill + that is merely not configured. These tests pin that the five outcomes are + told apart, and that reporting them changed nothing about ``get_skill``. + """ + + def _tampered(self, make_raw_skill: Any, key: str = "a") -> dict[str, Any]: + raw: dict[str, Any] = make_raw_skill(key=key) + raw["contentHash"] = "0" * 64 + return raw + + async def test_ok_carries_the_skill_and_no_detail( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="pdf-extraction", version=2)) + + outcome = await get_skill_result("pdf-extraction") + + assert outcome.reason == "ok" + assert outcome.detail is None + assert outcome.skill is not None + assert outcome.skill.key == "pdf-extraction" + assert outcome.skill.version == 2 + assert outcome.skill.content == SKILL_BODY.encode("utf-8") + + async def test_absent_when_the_store_does_not_hold_the_key( + self, store: InMemorySkillStore + ) -> None: + outcome = await get_skill_result("nope") + + assert outcome.reason == "absent" + assert outcome.skill is None + assert outcome.detail + assert "'nope'" in outcome.detail + + async def test_integrity_failure_when_content_does_not_verify( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """The one outcome a caller is expected to fail closed on.""" + store.put(self._tampered(make_raw_skill, key="a")) + + outcome = await get_skill_result("a") + + assert outcome.reason == "integrity_failure" + assert outcome.skill is None + assert outcome.detail + # The quoted form, so the assertion is about the key and not about a + # letter that appears in half the words in the message. + assert "'a'" in outcome.detail + + async def test_wrong_version_when_the_store_answers_with_another( + self, make_raw_skill: Any + ) -> None: + skills_module._set_store(_WrongVersionAnsweringStore(make_raw_skill)) + + outcome = await get_skill_result("a", version=1) + + assert outcome.reason == "wrong_version" + assert outcome.skill is None + assert outcome.detail + # The detail is what makes this actionable rather than merely negative: + # it names both the version asked for and the version held. + assert "version 1" in outcome.detail + assert "version 99" in outcome.detail + + async def test_store_unavailable_when_the_store_raises( + self, caplog: pytest.LogCaptureFixture + ) -> None: + skills_module._set_store(_RaisingStore()) + + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + outcome = await get_skill_result("a") + + assert outcome.reason == "store_unavailable" + assert outcome.skill is None + assert outcome.detail + assert "RuntimeError" in outcome.detail + + async def test_store_unavailable_is_distinct_from_absent( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """An outage is not a deletion, and the two must not read alike. + + This is the distinction ``write_skills`` already depends on to decide + whether pruning may run — only a raising store suppresses it — so + collapsing the two tokens here would put the public vocabulary at odds + with a policy the SDK already enforces internally. + """ + skills_module._set_store(_RaisingStore()) + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + raised = await get_skill_result("a") + + skills_module._set_store(InMemorySkillStore()) + empty = await get_skill_result("a") + + # Asserted as two named tokens rather than as an inequality: the type + # checker can already see that these two literals differ, so an + # inequality here would be dead weight. + assert raised.reason == "store_unavailable" + assert empty.reason == "absent" + + async def test_every_non_ok_outcome_carries_a_detail( + self, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """A reason with no detail leaves an operator nothing to act on. + + Swept over all four failures in one test rather than asserted per case + only, so a fifth failure path added later without a message is caught by + a test whose name says what it is about. + """ + stores: list[tuple[str, Any]] = [ + ("absent", InMemorySkillStore()), + ( + "integrity_failure", + InMemorySkillStore({"a": self._tampered(make_raw_skill)}), + ), + ("wrong_version", _WrongVersionAnsweringStore(make_raw_skill)), + ("store_unavailable", _RaisingStore()), + ] + + outcomes: list[SkillOutcome] = [] + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + for _expected, store_double in stores: + skills_module._set_store(store_double) + outcomes.append(await get_skill_result("a", version=1)) + + assert [o.reason for o in outcomes] == [expected for expected, _ in stores] + assert all(o.skill is None for o in outcomes) + assert all(o.detail for o in outcomes) + + async def test_detail_never_carries_the_skill_content( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """``detail`` is safe to log, so the body must not travel in it. + + Same rule the integrity log record follows, asserted separately here + because this string reaches the caller through a different surface. + """ + secret = "---\nname: Secret\n---\nSSN 000-00-0000 and an API key.\n" + store.put(make_raw_skill(key="a", content=secret, contentHash="0" * 64)) + + outcome = await get_skill_result("a") + + assert outcome.reason == "integrity_failure" + assert outcome.detail is not None + assert secret not in outcome.detail + assert "SSN" not in outcome.detail + assert "API key" not in outcome.detail + + async def test_raises_the_same_way_as_get_skill_with_no_store(self) -> None: + """Identical failure mode, down to the message. + + The two accessors differ only in what they report about a retrieval; a + missing store is a configuration error in both, so a caller cannot need + to handle it twice. + """ + with pytest.raises(RuntimeError, match="skill store") as reported: + await get_skill_result("a") + with pytest.raises(RuntimeError, match="skill store") as collapsed: + await get_skill("a") + + assert str(reported.value) == str(collapsed.value) + + async def test_records_no_second_integrity_signal( + self, + store: InMemorySkillStore, + make_raw_skill: Any, + recording_emitter: Any, + caplog: pytest.LogCaptureFixture, + ) -> None: + """One failed retrieval is one failure, on both surfaces. + + Verification already recorded the log record and the signal before + ``resolve_from_store`` returned, so reporting the reason must add + nothing: a second record would double-count one event in a SIEM and + inflate the product counter. ``_integrity_records`` is the shared parser + used by the log-record tests further down this module. + """ + skills_module._set_emitter_for_testing(recording_emitter) + store.put(self._tampered(make_raw_skill, key="a")) + + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + outcome = await get_skill_result("a") + + assert outcome.reason == "integrity_failure" + assert len(_integrity_records(caplog)) == 1 + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_get_skill_still_returns_none_for_every_failure( + self, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """The no-behaviour-change guarantee. + + ``get_skill``'s contract — ``None`` for every failure, and it never + raises for one — is documented in its docstring and in the README, and + every existing caller treats ``None`` as "no skill". Adding a reported + accessor beside it must not move that line, so the four failures are + driven through both accessors in one test: the reason is distinguishable + *and* the collapsed form still collapses. + """ + cases: list[tuple[str, Any]] = [ + ("absent", InMemorySkillStore()), + ( + "integrity_failure", + InMemorySkillStore({"a": self._tampered(make_raw_skill)}), + ), + ("wrong_version", _WrongVersionAnsweringStore(make_raw_skill)), + ("store_unavailable", _RaisingStore()), + ] + + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + for expected_reason, store_double in cases: + skills_module._set_store(store_double) + reported = await get_skill_result("a", version=1) + assert reported.reason == expected_reason + # No pytest.raises wrapper: an escaping exception fails the test + # here, which is the "never raises" half of the contract. + assert await get_skill("a", version=1) is None + + class TestGetSkills: """Batch accessor.""" From 6a0e6aa50f6b528b10174d5834f013090817e962 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 4 Sep 2026 10:19:46 -0400 Subject: [PATCH 09/22] =?UTF-8?q?docs(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20close=20out=20the=20security=20review's=20SDK-side?= =?UTF-8?q?=20remainder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four items left open in the response to the Agent Skills security design review. Three are documentation, one is tests; no behavior changes, and the code halves of rows 9 and 2 are deliberately untouched. **Privilege separation** (row 9 docs half, and the agreed counter-proposal for row 26). The recommended deployment runs the reconcile as a different identity than the agent, which is the whole reason the ``0644``/``0755`` modes deny anything: the agent reads its instructions and cannot rewrite them, or the manifest. That is the mitigation for AZ-1, a prompt-injected agent editing its own skills. Write access to the manifest is the worse half — it is what tells the *next* reconcile which paths the SDK may delete — which is why ``_prune`` re-validates every entry from scratch rather than trusting it. The README documents the pattern and hands the operator the check to run, because the SDK cannot run it: it knows only its own identity, which trivially has write access, having just written there. So ``ReconcileReport`` grows no writability field — the review asked for one and we declined, since any check the SDK could make would answer a different question than the one asked and manufacture false confidence exactly where caution is wanted. ``agents.md`` records that reasoning so the field is not added later by someone reading its absence as an oversight. **Three hostile-manifest prune tests** (row 12 remainder): a well-formed manifest listing ``/etc/passwd``, ``../../../etc/passwd``, and a path under a parent that has since become a symlink. ``_prune`` already refuses all three, so these turn asserted into verified. Two things make them worth more than their line count. They are deliberately *well-formed* — the corrupt-manifest suite above them proves nothing here, because a corrupt manifest suppresses every destructive action wholesale, whereas these manifests give the implementation everything it needs to prune. And "deleted nothing" is asserted through an unlink spy rather than by checking that ``/etc/passwd`` still exists: the test process cannot delete that file anyway, so the obvious assertion would pass against an implementation with no path check at all. **One sentence on** ``"*"`` (row 16 remainder). It materializes the whole project library, so every skill's ``description`` enters the agent's context — including skills no AI Config references and skills belonging to other teams. **The Windows platform bound is now explicit** (row 2 residual), in ``safe_fs.py``, ``agents.md`` and the README. Reparse-point checks (``GetFileAttributesW`` / ``FILE_FLAG_OPEN_REPARSE_POINT``) are not implemented, by decision: Windows is not a supported or tested platform for this release, neither repository has a Windows CI runner so the checks would ship unverified, and the TypeScript SDK could not match them in any case because Node exposes no ``*at()`` family on *any* platform. Implementing them in Python alone would break cross-language parity and trade a documented bound for an unverified one. Two consequences are recorded rather than left to be rediscovered: on Windows, write permission on the managed root is the only boundary, which is what makes privilege separation the mitigation and not merely advice; and this retroactively lowers the priority of the row 25 reserved-device-name work, noted where that code lives. Co-Authored-By: Claude Opus 5 --- packages/client/README.md | 68 ++++++++- packages/client/agents.md | 25 ++++ .../src/launchdarkly_ai_server/safe_fs.py | 23 ++++ packages/client/tests/test_skills_fs.py | 130 ++++++++++++++++++ 4 files changed, 245 insertions(+), 1 deletion(-) diff --git a/packages/client/README.md b/packages/client/README.md index 045180b5..bc8df592 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -272,7 +272,12 @@ async def main(): asyncio.run(main()) ``` -Pass `"*"` instead of a reference list to materialize every skill the store holds. +Pass `"*"` instead of a reference list to materialize every skill the store holds — but know +what you are asking for. `"*"` materializes the **whole project library**, which puts every +skill's `description` into the agent's context, including skills no AI Config references and +skills belonging to other teams. `write_skills(skill_refs(...), root)` is the form used above +because it materializes only what the resolved variation actually asked for; reach for `"*"` +when you genuinely want the whole library on disk. **`skills` is now a validated field.** Config parsing fails closed on a `skills` value that is not a list of `{key, version}` objects (key matching `^[a-z0-9][a-z0-9-]*$`, version an @@ -406,6 +411,19 @@ never writes through a symlink; writes are atomic (temp file, `fsync`, rename) a `0644`; and if the manifest is unreadable it performs no destructive action at all. Removing a skill from a variation is how revocation works — the next reconcile prunes it. +**Platform bound: the descriptor-pinned guarantee is POSIX-only.** On POSIX every destructive +step — the open, the rename, the unlink — runs relative to a directory descriptor opened +`O_RDONLY|O_DIRECTORY|O_NOFOLLOW` and held for the whole reconcile, so a directory swapped for +a symlink *after* its checks cannot redirect a write or a delete: the descriptor names the +inode that was checked, which closes the swap window rather than narrowing it. Windows has no +`*at()` syscall family, so there `write_skills` falls back to a per-component `lstat` check +taken immediately before each step. That floor is a check-then-use race rather than a closed +window: an attacker who already holds **write permission on the managed root** can still win +it. Windows reparse-point checks (`GetFileAttributesW` / `FILE_FLAG_OPEN_REPARSE_POINT`) are +deliberately not implemented in this release, and Windows is not a tested platform for it — +neither SDK repository has a Windows CI runner. Treat write permission on the managed root as +the security boundary on every platform, and on Windows as the *only* one. + **One exception, and it is what makes a crashed reconcile recoverable.** A file at a managed path whose bytes are *already byte-identical* to the content LaunchDarkly resolved is adopted — recorded in the manifest and reported `skipped_current` — rather than refused. @@ -477,6 +495,54 @@ to key its own map however the transport underneath does. > what was hashed. The SDK never parses or interprets them; if you want the frontmatter, > decode and parse the content on your side. +#### Privilege separation: the agent must not be able to rewrite its own skills + +**The recommended deployment runs `write_skills` as a different identity than the agent.** +Reconcile as one user, run the agent as another. Everything the reconcile puts on disk is +owner-write-only, and set explicitly rather than inherited from your umask: skill files and +the manifest at `0644` (via `fchmod` on the descriptor, so it cannot be redirected), the +per-skill `//` directories at `0755`, and the execute bit never set on anything. +Those modes are only a defense if the two identities actually differ — under a single identity +they describe a directory the agent can freely rewrite. + +**What to verify, as the identity that will run the agent.** The SDK cannot check this for you +(see below), so make it a deployment step: confirm the agent's identity has no write access to + +- the managed root itself, +- the per-skill directories `//` and the files `//SKILL.md`, +- the manifest at `/.launchdarkly-skills.json`. + +```bash +# Run as the agent's user. Every line should print DENIED. +root=.claude/skills +for target in "$root" "$root/.launchdarkly-skills.json" "$root"/*/ "$root"/*/SKILL.md; do + [ -e "$target" ] || continue + if [ -w "$target" ]; then echo "WRITABLE — fix this: $target"; else echo "DENIED: $target"; fi +done +``` + +Note that the managed root's own mode is **yours, not the SDK's**: `write_skills` creates only +that one leaf directory and does so with your umask, precisely because the root is a path you +chose. Own it — `chown reconcile-user:agent-group` and `chmod 0755` on the root is the shape +that makes the rest of the tree's modes mean something. + +**Why this is the mitigation that matters.** A `SKILL.md` is agent *instructions*. An agent +that can write its own skills directory can rewrite its own instructions, and an agent +processing untrusted input is exactly the thing that might be induced to do so. Write access +to the manifest is worse than write access to a skill, because the manifest is what tells the +*next* reconcile which paths the SDK owns and may delete: an agent that can edit it can keep a +skill LaunchDarkly has revoked, or aim the SDK's own delete path at something it should not +touch. `write_skills` re-validates every manifest entry from scratch for exactly that reason — +it treats that file as untrusted input, never as authorization — but an agent that cannot edit +it at all is the stronger position, and only your deployment can provide that. + +**The SDK deliberately does not report whether the root is writable.** There is no such field +on `ReconcileReport`, and its absence is a decision rather than an oversight. The SDK knows +only its own identity, which trivially has write access — it just wrote there. It cannot know +which identity will later run the agent, so any check it could make would answer a different +question than the one that matters, and would read as reassurance exactly where caution is +wanted. You know both identities; the SDK knows one. + --- ### Utility Helpers diff --git a/packages/client/agents.md b/packages/client/agents.md index a7b4ce1b..2558f9c3 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -510,6 +510,31 @@ resolving to the skill directory's `(st_dev, st_ino)`) instead of by comparing p A spy must `fstat` the descriptor **inside** the intercepted call — the implementation closes it as soon as the write returns. +**The platform bound is POSIX-only, and that is a decision — do not quietly "fix" it.** +Windows reparse-point checks (`GetFileAttributesW`, `FILE_FLAG_OPEN_REPARSE_POINT`) are not +implemented because Windows is not a supported or tested platform for this release: there is +no Windows CI runner in either repository, so the checks would ship unverified, and the +TypeScript SDK could not match them at all — Node exposes no `*at()` family on *any* +platform, so its racy floor is universal rather than Windows-only. Implementing them in +Python alone would break cross-language parity and trade a documented bound for an unverified +one. Two follow-on facts: on Windows write permission on the managed root is the only +boundary, which is why the privilege-separated deployment is documented as the mitigation +rather than as advice; and this bound retroactively lowers the priority of the reserved-device-name +work above — keep that code, but do not read it as evidence that Windows is hardened. If +Windows becomes a supported platform, revisit both together, and add the CI runner first. + +**Privilege separation is the deployment-side half of this, and `ReconcileReport` must not +grow a writability field.** The recommended deployment runs the reconcile as a different +identity than the agent, so the `0644`/`0755` modes above actually deny something: the agent +reads its instructions and cannot rewrite them or the manifest. That is the mitigation for a +prompt-injected agent editing its own skills. The security review asked for the report to +surface whether the managed root is writable; we declined, and the reasoning is load-bearing +rather than a preference. The SDK knows only its *own* identity, which trivially has write +access — it just wrote there — and cannot know which identity will later run the agent. Any +check it could perform would answer a different question than the one asked and would create +false confidence exactly where caution is wanted. The operator's verification steps live in +the README instead. Do not add the field. + ### Deferred: bounded retries `timeout` is implemented — a monotonic deadline, checked before each retrieval, before diff --git a/packages/client/src/launchdarkly_ai_server/safe_fs.py b/packages/client/src/launchdarkly_ai_server/safe_fs.py index 73eef878..43c21e58 100644 --- a/packages/client/src/launchdarkly_ai_server/safe_fs.py +++ b/packages/client/src/launchdarkly_ai_server/safe_fs.py @@ -11,6 +11,29 @@ re-resolving a name — which is what closes the swap window rather than merely narrowing it. Where the platform has no ``*at()`` syscall family (Windows) the identical sequence runs against full paths, the per-component ``lstat`` floor. + +**Platform bound — this guarantee is POSIX-only, deliberately.** On POSIX the +descriptor walk closes the swap window. On Windows it does not exist: there is no +``*at()`` family, so the ``lstat`` floor is all that runs, and a floor is a +check-then-use race rather than a closed window. The remedy would be +reparse-point checks (``GetFileAttributesW``, or opening with +``FILE_FLAG_OPEN_REPARSE_POINT``) and it is **not implemented, by decision rather +than by oversight**: Windows is not a supported or tested platform for this +release, and neither SDK repository has a Windows CI runner, so the checks would +ship untested — and the TypeScript SDK could not match them in any case, because +Node exposes no ``*at()`` family on *any* platform. Shipping them in Python alone +would break the cross-language parity the two SDKs are held to and would trade a +documented bound for an unverified one. + +Two consequences worth stating plainly rather than discovering later. First, on +Windows write permission on the managed root is the *only* boundary, so the +privilege-separated deployment the README documents is not advice there but the +mitigation. Second, this bound retroactively lowers the priority of the Windows +reserved-device-name work in ``skills_fs.py`` (``_WINDOWS_RESERVED_NAMES``): that +code stays, because it is cheap and it keeps a managed root written on Linux +usable when read from Windows, but it should not be read as evidence that Windows +is a hardened target. It is not. Revisit both together if Windows becomes +supported. """ from __future__ import annotations diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 223a3262..6d5ab2db 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -1394,6 +1394,136 @@ async def test_corrupt_manifest_file_is_not_destroyed(self, root: Path) -> None: assert _manifest_path(root).read_text(encoding="utf-8") == "{not json at all" +# The three literal cases the security review names for the prune path. +# +# The distinction from ``TestCorruptManifest`` above is the whole point: a +# corrupt manifest suppresses every destructive action wholesale, so those tests +# say nothing about these. Each manifest here is *well-formed* — parseable, +# a ``manifestVersion`` this release understands, a real ``entries`` map, and an +# entry whose ``key`` is a perfectly valid skill key that is genuinely absent +# from the requested set. The implementation has every input it needs to prune +# and must refuse anyway, because the recorded *path* is not one this SDK could +# have written. +HOSTILE_RECORDED_PATHS: list[str] = [ + # Absolute: the classic. A recorded path read back and unlinked as-is is a + # delete of an attacker-chosen file with the reconcile's privileges. + "/etc/passwd", + # Traversing: the same attack for an implementation that rejects a leading + # slash and then joins the rest onto the root. + "../../../etc/passwd", +] + + +class _UnlinkSpy: + """Records every ``os.unlink`` while delegating to the real one. + + Asserting only that ``/etc/passwd`` still exists proves nothing: the test + process cannot delete it anyway, so that assertion passes against an + implementation with no path check at all — permissions would be doing the + work. What has teeth is that the removal is never *attempted*: the refusal + happens above the syscall, on a path the SDK recomputes rather than trusts. + """ + + def __init__(self) -> None: + self.targets: list[str] = [] + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _UnlinkSpy: + real = os.unlink + + def spy(path: Any, *args: Any, **kwargs: Any) -> None: + self.targets.append(os.fsdecode(path)) + real(path, *args, **kwargs) + + # ``safe_fs_module.os`` *is* the ``os`` module, so this covers both the + # descriptor-relative ``os.unlink(name, dir_fd=...)`` and the + # ``Path.unlink`` used on the no-``*at()`` floor. + monkeypatch.setattr(safe_fs_module.os, "unlink", spy) + return self + + +class TestHostileManifestPrune: + """A well-formed manifest naming a path this SDK could not have written. + + The manifest is untrusted input. It is a plain file on the customer's disk + that anything with write access to the managed root can edit, and ``prune`` + is the one code path in the SDK that deletes. So a recorded path never + authorizes its own removal: it must match ``/SKILL.md`` for a + re-validated key, and the target is recomputed from the *current* managed + root instead of being read back out of the entry. + """ + + @pytest.mark.parametrize("recorded", HOSTILE_RECORDED_PATHS) + async def test_recorded_path_outside_the_root_is_refused( + self, root: Path, recorded: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + spy = _UnlinkSpy().install(monkeypatch) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {recorded: _entry("a", 1, SKILL_BODY)}, + }, + ) + + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + # The refusal is about ownership of the path, not about the file's state. + assert "could own" in action.error + assert [a for a in report.actions if a.action == "removed"] == [] + # Nothing was even attempted, let alone completed. + assert spy.targets == [] + assert Path("/etc/passwd").exists() + # Left in place rather than tidied away: dropping the entry would let a + # single hostile edit erase the SDK's own record of what it manages. + assert recorded in _read_manifest(root)["entries"] + + async def test_entry_under_a_since_symlinked_parent_is_refused( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The recorded path is the SDK's own, and is still not enough. + + Here the entry is exactly what a legitimate reconcile writes — + ``a/SKILL.md`` under key ``a`` — so the shape check that catches the two + cases above passes. What changed is the disk underneath it: ``/a`` + is now a symlink to somewhere else. This is the case a validate-then-act + implementation fails, because the manifest and the entry are both + entirely legitimate; only the current state of the parent is not. + """ + root = tmp_path / "skills" + root.mkdir() + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + victim = elsewhere / "SKILL.md" + victim.write_text("victim content\n", encoding="utf-8") + + # Managed legitimately first, so the manifest entry is one this SDK + # really did write... + managed = _place_managed(root, "a", SKILL_BODY) + # ...then the parent directory is swapped for a link out of the root. + managed.unlink() + (root / "a").rmdir() + (root / "a").symlink_to(elsewhere, target_is_directory=True) + + spy = _UnlinkSpy().install(monkeypatch) + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "symlink" in action.error + assert [a for a in report.actions if a.action == "removed"] == [] + assert spy.targets == [] + # The file the symlink pointed at is untouched, and so is the link. + assert victim.read_text(encoding="utf-8") == "victim content\n" + assert (root / "a").is_symlink() + assert "a/SKILL.md" in _read_manifest(root)["entries"] + + class TestWriteSkillsTelemetry: """Materialized / revoked signals from write_skills.""" From dd2d55cd680fe5cd56bc937804797cb6943ecdd7 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Thu, 10 Sep 2026 16:02:00 -0400 Subject: [PATCH 10/22] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20re-reconcile=20on=20delivery=20with=20watch=5Fskill?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit write_skills is a one-shot reconcile, so a revocation takes effect at the next process restart. watch_skills runs that reconcile now and again whenever the configured store reports a change, so a revoked skill's files leave the disk within a debounce interval of the store learning about it. It is wired to the SkillStore interface, not to any one transport: it needs a store that implements add_listener and nothing more, and refuses loudly when the store does not, since a watcher that silently never fires looks exactly like one whose skills never changed. Above the interface, remove_listener joins add_listener as the optional second half of change notification, on the SkillStore contract and on InMemorySkillStore. SkillWatcher.close needs it to detach; without it a store held every watcher ever created for the rest of its life. The watcher probes for it, so a store without it keeps working. Co-Authored-By: Claude Fable 5.1 --- packages/client/README.md | 3 +- packages/client/agents.md | 11 +- .../src/launchdarkly_ai_server/__init__.py | 4 + .../src/launchdarkly_ai_server/skills.py | 16 + .../src/launchdarkly_ai_server/skills_core.py | 16 +- .../launchdarkly_ai_server/skills_watch.py | 292 ++++++++++++++++++ packages/client/tests/test_skills.py | 29 ++ packages/client/tests/test_skills_watch.py | 178 +++++++++++ 8 files changed, 538 insertions(+), 11 deletions(-) create mode 100644 packages/client/src/launchdarkly_ai_server/skills_watch.py create mode 100644 packages/client/tests/test_skills_watch.py diff --git a/packages/client/README.md b/packages/client/README.md index bc8df592..41d5c6b3 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -458,8 +458,9 @@ Windows. | `get_skills(refs)` | Batch form. Accepts `SkillReference` values and bare key strings (string = latest). Results follow input order; missing or unverifiable entries are omitted. | | `all_skills()` | Every verified skill the store holds, one per key at its newest version. | | `write_skills(skills, root, *, prune=True, timeout=10.0, on_unavailable="keep")` | Materialize skills under `root`, returning a `ReconcileReport`. `prune` removes formerly-managed skills no longer requested. `on_unavailable="raise"` raises instead of reporting when content cannot be retrieved. Raises `ValueError` for an unusable root, a negative `timeout`, or an unrecognised `on_unavailable`. **Performs synchronous filesystem I/O — see the note below.** | -| `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)`. | +| `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)` / `remove_listener(kind, fn)`. | | `InMemorySkillStore(objects=None)` | A dict-backed store with `put(raw)`, for local development and testing. Holds several versions of a key. | +| `watch_skills(skills, root, …)` | `write_skills` plus a re-reconcile on every delivery change. Returns `(initial report, SkillWatcher)`; close the watcher when done. Revocation then takes effect within `debounce` of arriving rather than at the next restart. | Configure the store with `init_client(options={"skillStore": store})`. With none configured, the accessors raise `RuntimeError` explaining what to do and `write_skills` reports the diff --git a/packages/client/agents.md b/packages/client/agents.md index 2558f9c3..ca56b80e 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,6 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | +| `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | @@ -198,11 +199,11 @@ Three layers, in increasing order of blast radius: ### The store seam, and why version is part of the lookup -`SkillStore` is `get_object(kind, key, version=None)`, `all_objects(kind)`, and an optional -`add_listener(kind, fn)`. Version is part of the **lookup identity**, not a filter applied -to the answer, and that is load-bearing: a delivery payload carries the newest version of -every skill *plus* every version any variation currently pins, so two versions of one key -coexist routinely. A seam keyed by key alone would answer a pinned reference with the newest +`SkillStore` is `get_object(kind, key, version=None)`, `all_objects(kind)`, and the optional +pair `add_listener(kind, fn)` / `remove_listener(kind, fn)`. Version is part of the **lookup +identity**, not a filter applied to the answer, and that is load-bearing: a delivery payload +carries the newest version of every skill *plus* every version any variation currently pins, +so two versions of one key coexist routinely. A store keyed by key alone would answer a pinned reference with the newest object, and the caller would then have to reject it — turning the primary use case, a version-pinned attachment, into a missing skill. `version=None` asks for the newest held. diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 753e1e95..f863d847 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -53,6 +53,7 @@ OnUnavailable, write_skills, ) +from .skills_watch import SkillWatcher, watch_skills from .tracking import execute_and_stream, execute_and_track, wrap_tool_handlers from .types import ( NATIVE_TOOL_KEY, @@ -237,6 +238,9 @@ "write_skills", "SkillStore", "InMemorySkillStore", + # skills — the eager re-reconcile + "watch_skills", + "SkillWatcher", # skills — the three closed-set unions a typed consumer needs to name "ReconcileActionKind", "OnUnavailable", diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py index cb21986b..9016ab9f 100644 --- a/packages/client/src/launchdarkly_ai_server/skills.py +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -167,6 +167,22 @@ def add_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: """ self._listeners.setdefault(kind, []).append(fn) + def remove_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Unregisters *fn* from *kind*, so a subsequent ``put`` no longer calls it. + + Removes one occurrence: a callable registered twice must be removed twice. + Removing a callable that is not registered is a no-op, not an error, so a + consumer that detaches on close can do so unconditionally. + """ + listeners = self._listeners.get(kind) + if listeners is None: + return + try: + listeners.remove(fn) + except ValueError: + return + # --------------------------------------------------------------------------- # Reference discovery diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index 74cb706b..22166799 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -151,11 +151,17 @@ class SkillStore(Protocol): Duck-typed on purpose, mirroring how the LaunchDarkly client interface works in this package: pass any object carrying these methods. - ``add_listener(kind, fn)`` is part of the seam but - **optional**, which is why it is deliberately not declared here: a Protocol - member is required for structural compatibility, so declaring it would reject - every store that does not implement it. Nothing in this module calls it — it - exists for the delivery transport to push updates through. + ``add_listener(kind, fn)`` and ``remove_listener(kind, fn)`` are part of the + interface but **optional**, which is why they are deliberately not declared + here: a Protocol member is required for structural compatibility, so declaring + them would reject every store that does not implement them. Nothing in this + module calls either — they exist for the delivery transport to push updates + through, and for a consumer such as ``watch_skills`` to stop receiving them. + A store that implements ``add_listener`` should implement ``remove_listener`` + too; consumers probe for it and skip detaching when it is absent, so an + older store keeps working at the cost of a listener that lives as long as + the store does. ``remove_listener`` removes one occurrence of *fn* under + *kind* and is a no-op when *fn* is not registered. The raw objects a store serves are wire-shaped, with camelCase field names identical across language implementations:: diff --git a/packages/client/src/launchdarkly_ai_server/skills_watch.py b/packages/client/src/launchdarkly_ai_server/skills_watch.py new file mode 100644 index 00000000..05263057 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_watch.py @@ -0,0 +1,292 @@ +""" +Agent Skills — re-reconcile on delivery, so revocation does not wait for a restart. + +``write_skills`` is a one-shot reconcile: it materializes what the store holds +now. With a hand-populated store that is sufficient, and a revocation takes +effect at the next process restart. + +A streaming FDv2 connection changes the premise. A ``delete-object`` reaches a +live connection in **seconds**, and the store publishes a change listener, so +wiring the two together collapses the gap between "LaunchDarkly revoked this +skill" and "its ``SKILL.md`` is off the agent's disk" from a process lifetime to +a debounce interval. + +``on_unavailable="keep"`` stays the default: an outage must not read as +"everything was revoked". A watcher that pruned on a failed retrieval would +convert every transport failure into deletion of a customer's skill files. + +Layering: this module sits *above* ``skills_fs`` and calls ``write_skills`` +without modifying it. Nothing in the reconcile, the accessors, or verification +knows this file exists. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import threading +from collections.abc import Callable, Sequence +from typing import Any + +from .skills_core import SKILL_OBJECT_KIND, get_store +from .skills_fs import OnUnavailable, write_skills +from .types import ReconcileReport, Skill, SkillReference + +logger = logging.getLogger(__name__) + +DEFAULT_DEBOUNCE_SECONDS = 0.5 +""" +How long a change waits for its neighbours before a reconcile runs. + +A full payload transfer commits many objects at once and the listener fires per +object, so without coalescing a payload of forty skills would run forty +reconciles against one root. Half a second is far below the seconds-scale +latency this feature is trying to achieve and far above the microseconds a +commit's listener calls take. +""" + + +class SkillWatcher: + """ + A running re-reconcile. Returned by ``watch_skills``; stop it with ``close``. + + One watcher owns one root. **Do not point two watchers at the same root**, + and do not run ``write_skills`` against a watched root concurrently: the + reconcile's own contract is one root, one reconcile at a time, because two + interleaved runs lose the loser's manifest entries and leave the files it + wrote unmanaged. This class enforces that for its *own* reconciles — they run + on a single worker thread, serialised — and cannot enforce it against a + caller who reconciles the same root by hand. + + The watcher owns its registration on *store*: it registers ``notify`` when + constructed and unregisters it in ``close``, so a closed watcher is no longer + reachable from the store and can be collected. *store* must implement + ``add_listener``; ``remove_listener`` is probed for and, when the store does + not offer it, the listener stays registered for the store's lifetime. + """ + + def __init__( + self, + request: Sequence[Skill | SkillReference | str] | str, + root: str | os.PathLike[str], + store: Any, + *, + prune: bool, + timeout: float, + on_unavailable: OnUnavailable, + debounce: float, + on_reconcile: Callable[[ReconcileReport], Any] | None, + ) -> None: + self._request = request + self._root = root + self._prune = prune + self._timeout = timeout + self._on_unavailable = on_unavailable + self._debounce = debounce + self._on_reconcile = on_reconcile + + self._wake = threading.Event() + self._stop = threading.Event() + self._reconciles = 0 + self._lock = threading.Lock() + self._thread = threading.Thread( + target=self._run, name="ld-ai-skills-reconcile", daemon=True + ) + + # Register before starting the worker: ``notify`` only sets an event, so a + # change that lands in between is picked up as soon as the worker runs, + # and a store whose ``add_listener`` raises leaves no thread behind. + self._store = store + self._registered = False + store.add_listener(SKILL_OBJECT_KIND, self.notify) + self._registered = True + self._thread.start() + + # -- the listener the store calls ------------------------------------- + + def notify(self, _raw: Any = None) -> None: + """ + The store's change listener. Records that something changed; runs nothing. + + Deliberately trivial. It is called on the delivery thread, where a + reconcile — which does synchronous filesystem I/O, an fsync per file, and + a manifest rewrite — would stall event processing for the duration and, + on a stream, let the connection's read buffer back up behind a disk write. + The argument is ignored: a put's raw object and a revocation's tombstone + both mean the same thing here, which is "the store is not what it was". + """ + self._wake.set() + + # -- the worker -------------------------------------------------------- + + def _run(self) -> None: + while not self._stop.is_set(): + if not self._wake.wait(timeout=0.5): + continue + if self._stop.is_set(): + return + # Coalesce the rest of the burst. Clearing *before* the sleep rather + # than after is what makes a change arriving mid-debounce trigger the + # next pass instead of being swallowed by this one. + self._wake.clear() + if self._stop.wait(self._debounce): + return + self._reconcile_once() + + def _reconcile_once(self) -> None: + try: + report = asyncio.run( + write_skills( + self._request, + self._root, + prune=self._prune, + timeout=self._timeout, + on_unavailable=self._on_unavailable, + ) + ) + except Exception: + # A watcher that died on one bad reconcile would silently stop + # tracking revocations, which is worse than a noisy one. + logger.error( + "A skill re-reconcile raised; the watcher continues", exc_info=True + ) + return + + with self._lock: + self._reconciles += 1 + changed = [ + action + for action in report.actions + if action.action in ("written", "updated", "removed", "error") + ] + if changed: + logger.info( + "Re-reconciled skills after a delivery change: %d action(s) of note", + len(changed), + ) + if self._on_reconcile is not None: + try: + self._on_reconcile(report) + except Exception: + logger.error("A watch_skills callback raised", exc_info=True) + + # -- lifecycle --------------------------------------------------------- + + @property + def reconciles(self) -> int: + """How many re-reconciles have completed since the watcher started. + + Excludes the initial reconcile ``watch_skills`` awaits, which is the + caller's own result.""" + with self._lock: + return self._reconciles + + def close(self, timeout: float = 15.0) -> None: + """ + Stops watching. Idempotent. Does not undo anything already on disk. + + Waits out an in-flight reconcile rather than interrupting one, because a + reconcile killed between its content writes and its manifest rewrite is + the one case the manifest format has to recover from — worth avoiding when + we control the timing. + + Detaches ``notify`` from the store first, so no further change reaches a + watcher that is shutting down and the store no longer holds a reference to + it. A store without the optional ``remove_listener`` is left as it is + rather than failing the close. + """ + self._detach() + self._stop.set() + self._wake.set() + if self._thread.is_alive() and self._thread is not threading.current_thread(): + self._thread.join(timeout=timeout) + + def _detach(self) -> None: + with self._lock: + if not self._registered: + return + self._registered = False + remove_listener = getattr(self._store, "remove_listener", None) + if callable(remove_listener): + remove_listener(SKILL_OBJECT_KIND, self.notify) + + def __enter__(self) -> SkillWatcher: + return self + + def __exit__(self, *_exc: Any) -> None: + self.close() + + +async def watch_skills( + skills: Sequence[Skill | SkillReference | str] | str, + root: str | os.PathLike[str], + *, + prune: bool = True, + timeout: float = 10.0, + on_unavailable: OnUnavailable = "keep", + debounce: float = DEFAULT_DEBOUNCE_SECONDS, + on_reconcile: Callable[[ReconcileReport], Any] | None = None, +) -> tuple[ReconcileReport, SkillWatcher]: + """ + Reconciles now, then re-reconciles whenever delivery changes. + + Every argument that ``write_skills`` takes means the same thing here and is + passed straight through; the reconcile's semantics are untouched. Returns the + initial reconcile's report — so a caller can fail fast on a bad root or a + corrupt manifest exactly as they would with ``write_skills`` — paired with a + ``SkillWatcher`` to close when the process is done:: + + report, watcher = await watch_skills("*", "/etc/agent/skills") + try: + ... + finally: + watcher.close() + + A revocation delivered over a streaming connection then prunes the skill's + files within ``debounce`` of arriving, rather than at the next restart. + + Requires a store that implements the optional ``add_listener`` half of the + ``SkillStore`` interface. Raises ``RuntimeError`` when no store is configured, + and when the configured store has no ``add_listener`` — the second case + failing loudly rather than degrading to a one-shot reconcile, because a + watcher that silently never fires looks exactly like a watcher whose skills + never changed. The optional ``remove_listener`` lets ``SkillWatcher.close`` + detach from the store; a store without it still works, but each closed + watcher then stays registered for the store's lifetime. + """ + store = get_store() + if store is None: + raise RuntimeError( + "watch_skills needs a configured skill store. Configure one with " + 'init_client(options={"skillStore": store}).' + ) + add_listener = getattr(store, "add_listener", None) + if not callable(add_listener): + raise RuntimeError( + "watch_skills needs a skill store that implements add_listener(kind, " + "fn); the configured store does not, so delivery changes cannot be " + "observed. Use write_skills for a one-shot reconcile, or configure a " + "store with a delivery transport." + ) + if debounce < 0: + raise ValueError(f"debounce must not be negative, got {debounce!r}") + + # The initial reconcile runs first and on the caller's thread, so its report + # is the caller's to inspect and a bad root raises out of `watch_skills` + # rather than into a worker thread's log. + report = await write_skills( + skills, root, prune=prune, timeout=timeout, on_unavailable=on_unavailable + ) + + watcher = SkillWatcher( + skills, + root, + store, + prune=prune, + timeout=timeout, + on_unavailable=on_unavailable, + debounce=debounce, + on_reconcile=on_reconcile, + ) + return report, watcher diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 6470f0c0..d3f83095 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -548,6 +548,35 @@ def test_put_does_not_notify_other_kind_listeners( assert seen == [] + def test_remove_listener_stops_put_notifying_it(self, make_raw_skill: Any) -> None: + s = InMemorySkillStore() + seen: list[dict[str, Any]] = [] + s.add_listener("skill", seen.append) + s.remove_listener("skill", seen.append) + + s.put(make_raw_skill(key="a")) + + assert seen == [] + + def test_remove_listener_removes_one_occurrence(self, make_raw_skill: Any) -> None: + s = InMemorySkillStore() + seen: list[dict[str, Any]] = [] + s.add_listener("skill", seen.append) + s.add_listener("skill", seen.append) + s.remove_listener("skill", seen.append) + + s.put(make_raw_skill(key="a")) + + assert len(seen) == 1 + + def test_remove_listener_of_an_unregistered_callable_is_a_no_op(self) -> None: + s = InMemorySkillStore() + s.remove_listener("skill", print) + s.add_listener("skill", print) + s.remove_listener("flag", print) + s.remove_listener("skill", print) + s.remove_listener("skill", print) + class TestStoreConfiguration: """Store wiring on the lifecycle layer.""" diff --git a/packages/client/tests/test_skills_watch.py b/packages/client/tests/test_skills_watch.py new file mode 100644 index 00000000..48a95f07 --- /dev/null +++ b/packages/client/tests/test_skills_watch.py @@ -0,0 +1,178 @@ +""" +Tests for ``watch_skills`` / ``SkillWatcher`` — the eager re-reconcile. + +The watcher is wired to the ``SkillStore`` interface, not to any one transport: +it needs a store that implements ``add_listener``, and nothing more. These tests +therefore drive it from ``InMemorySkillStore``, whose ``put`` notifies its +listeners synchronously, and from small hand-written store doubles. + +Every test writes only inside pytest's ``tmp_path``. The watcher runs a real +worker thread, so tests wait on observable outcomes rather than on fixed sleeps +wherever the outcome is something that *does* happen; a fixed sleep is used only +to assert that something does *not*. +""" + +from __future__ import annotations + +import time +from typing import Any + +import pytest + +from launchdarkly_ai_server import InMemorySkillStore, init_client, watch_skills +from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND + +pytestmark = pytest.mark.usefixtures("reset_skill_state") + + +def wait_until(predicate: Any, timeout: float = 5.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.01) + return predicate() + + +# --------------------------------------------------------------------------- +# Starting a watch, and what it refuses +# --------------------------------------------------------------------------- + + +class TestWatchSkills: + async def test_the_in_memory_store_can_also_drive_a_watch( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + """The watcher is wired to the ``SkillStore`` interface, not to the FDv2 + store.""" + store = InMemorySkillStore() + store.put(make_raw_skill(key="a", version=1, content="body")) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "a" / "SKILL.md" + assert written.read_text() == "body" + store.put(make_raw_skill(key="a", version=2, content="new body")) + assert wait_until(lambda: written.read_text() == "new body", timeout=10) + finally: + watcher.close() + + async def test_a_burst_of_changes_coalesces_into_few_reconciles( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.1) + try: + for i in range(12): + store.put(make_raw_skill(key=f"skill-{i}")) + time.sleep(0.5) + # Twelve objects put back to back fire twelve listener calls; without + # coalescing that is twelve reconciles of one root. + assert watcher.reconciles <= 2 + finally: + watcher.close() + + async def test_a_store_with_no_listener_support_is_refused_loudly( + self, tmp_path: Any + ) -> None: + class NoListeners: + def get_object(self, *_a: Any, **_k: Any) -> None: + return None + + def all_objects(self, _kind: str) -> dict[str, Any]: + return {} + + await init_client(options={"skillStore": NoListeners()}, client=object()) + with pytest.raises(RuntimeError, match="add_listener"): + await watch_skills("*", tmp_path / "s") + + async def test_no_store_configured_raises(self, tmp_path: Any) -> None: + with pytest.raises(RuntimeError, match="configured skill store"): + await watch_skills("*", tmp_path / "s") + + +# --------------------------------------------------------------------------- +# Closing a watch +# --------------------------------------------------------------------------- + + +class TestWatcherDetachesOnClose: + """``SkillWatcher.close`` unregisters ``notify``, so a closed watcher is + neither called nor kept alive by the store.""" + + @staticmethod + def _skill_listeners(store: Any) -> list[Any]: + return list(store._listeners.get(SKILL_OBJECT_KIND, [])) + + async def test_a_closed_watcher_is_no_longer_notified( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="pdf-extraction", version=1, content="first")) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + assert watcher.notify in self._skill_listeners(store) + + watcher.close() + + assert watcher.notify not in self._skill_listeners(store) + store.put(make_raw_skill(key="pdf-extraction", version=4, content="second")) + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + assert wait_until( + lambda: ( + store.get_object(SKILL_OBJECT_KIND, "pdf-extraction", 4) is not None + ), + timeout=10, + ) + time.sleep(0.3) + assert written.read_text() == "first" + assert watcher.reconciles == 0 + + async def test_close_twice_does_not_raise(self, tmp_path: Any) -> None: + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + watcher.close() + watcher.close() + assert self._skill_listeners(store) == [] + + async def test_repeated_watchers_leave_no_listeners_behind( + self, tmp_path: Any + ) -> None: + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + for _ in range(5): + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + assert len(self._skill_listeners(store)) == 1 + watcher.close() + assert self._skill_listeners(store) == [] + + async def test_a_store_without_remove_listener_still_closes( + self, tmp_path: Any + ) -> None: + """``remove_listener`` is optional: an older store keeps working, at the + cost of the listener staying registered.""" + + class AddOnly: + def __init__(self) -> None: + self.listeners: list[Any] = [] + + def get_object(self, *_a: Any, **_k: Any) -> None: + return None + + def all_objects(self, _kind: str) -> dict[str, Any]: + return {} + + def add_listener(self, _kind: str, fn: Any) -> None: + self.listeners.append(fn) + + store = AddOnly() + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + assert store.listeners == [watcher.notify] + + watcher.close() + watcher.close() + + assert store.listeners == [watcher.notify] From 0f797c0b167adf8be462ae8e8b2431705d75dfd7 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Thu, 10 Sep 2026 16:03:21 -0400 Subject: [PATCH 11/22] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20the=20FDv2=20delivery=20protocol,=20without=20the?= =?UTF-8?q?=20network?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The half of the delivery transport that has no I/O: identifying a skill object on the wire by kind inline-resource plus category skill, translating it into the raw object shape the SkillStore interface defines, holding it by (key, objectVersion), and applying a payload's events as one commit at payload-transferred. The store that puts a connection underneath this follows separately, so the three decisions that matter most can be reviewed on their own: - objectVersion is the skill's version; version is the payload's. The translation happens in one place and TestVersionTranslation asserts it in both directions, because confusing them fails silently. - Changes commit at payload-transferred, not per object. A half-applied full transfer would briefly empty the store, which with pruning on is the difference between a reconcile and deleting a customer's files. - A hashless object is held, not dropped, so verification withholds it with a reason code rather than the transport reporting it absent. Flag and segment objects share the connection and are skipped and counted, not rejected. Nothing here is exported yet; the store exports it. Co-Authored-By: Claude Fable 5.1 --- packages/client/agents.md | 46 ++ .../src/launchdarkly_ai_server/skills_fdv2.py | 601 +++++++++++++++++ packages/client/tests/test_skills_fdv2.py | 605 ++++++++++++++++++ 3 files changed, 1252 insertions(+) create mode 100644 packages/client/src/launchdarkly_ai_server/skills_fdv2.py create mode 100644 packages/client/tests/test_skills_fdv2.py diff --git a/packages/client/agents.md b/packages/client/agents.md index ca56b80e..80ec0538 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,6 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | +| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the `objectVersion`/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | | `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | @@ -218,6 +219,51 @@ one place that collapses the result to one object per key, because both whole-st consumers need it — `all_skills`, since a list holding two versions of one key is not a set of skills, and the `"*"` reconcile, since `//SKILL.md` is a single path. +### The delivery transport, and the one field that will bite you + +`skills_fdv2.py` translates LaunchDarkly's FDv2 delivery protocol into raw objects in the +shape `skills_core.SkillStore` documents. It lives below the store interface; **nothing above +that interface knows it exists**. If a transport change ever seems to require editing an +accessor, verification, or `write_skills`, the adapter boundary is wrong. + +**`objectVersion` is the skill's version. `version` is the payload's.** On the wire a skill +`put-object` carries both, and they are not interchangeable: + +```json +{"key":"pdf-extraction","kind":"inline-resource","category":"skill", + "objectVersion":3,"version":42, + "object":{"contentType":"text/markdown","content":"…","contentHash":"…","name":"…"}} +``` + +`objectVersion` (3) is what a `{key, version}` reference pins and what becomes the stored +`version`. `version` (42) is the version of the *payload* the object arrived in — it moves +when anything in the environment moves, including a flag with nothing to do with skills. +Reading it as the skill's version fails **silently**: the object verifies, the hash matches, +and the caller gets content under a version number that means nothing. Flags and segments +carry only `version` and omit both `category` and `objectVersion`, which is exactly why the +two fields look interchangeable. `_store_object_from_put` is the only place the translation +happens, and `TestVersionTranslation` asserts it in both directions. + +**Skills are identified by `kind == "inline-resource" && category == "skill"`; everything else +is ignored, not rejected.** An environment's payload assignment carries its flag payload +alongside its agent-skill payload, so flag and segment objects arrive as a matter of course. +Erroring on an unrecognised kind would turn a normal payload into a permanent reconnect +loop — a flag-delivery outage caused by a skills rollout. + +**Changes commit at `payload-transferred`, not as objects arrive.** A payload version is the +unit of consistency: a half-applied full transfer would publish a state the server never +described, and would briefly empty the store — which, with pruning on, is the difference +between a reconcile and deleting a customer's skill files. An interrupted transfer therefore +leaves last known good intact, and listeners fire once per commit. + +**A hashless object is held, not dropped.** Verification withholds it with +`missing_content_hash`; the transport's job is to make that loud (an error per object, a +summary per wholly-hashless payload, `diagnostics.hashless_objects`) rather than to work +around it. Dropping it at the transport would report `absent` — indistinguishable from "no +such skill" — and would let a prune delete the last known-good copy on disk. Never synthesize +a hash from the delivered content: that certifies the content against itself and verifies +nothing. + ### The reported outcome vocabulary, and the `Resolution` mapping `get_skill` returns `Skill | None`; `get_skill_result` returns a frozen `SkillOutcome` diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py new file mode 100644 index 00000000..80ef1224 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -0,0 +1,601 @@ +""" +Agent Skills — the FDv2 delivery protocol. + +The half of the delivery transport that has no I/O: identifying skill objects +on the wire, translating them into the raw object shape the ``SkillStore`` +interface defines, holding them by ``(key, objectVersion)``, and applying a +payload's events as one consistent commit. ``FDv2SkillStore``, the store that +puts a network connection underneath this, follows in a separate change. + +It sits *below* the ``SkillStore`` interface, and everything above — the +accessors, integrity verification, the ``Skill`` dataclass, materialization — +is unaware of it. + +Layering:: + + launchdarkly_ai_server + └─ SkillStore protocol (skills_core) ── the interface accessors call + └─ FDv2SkillStore (this module) ── deserialise, hold, serve + └─ LaunchDarkly's SDK-facing FDv2 channel + GET /sdk/poll, GET /sdk/stream, authenticated with the + environment's server-side SDK key + +Dependencies run one way: this module imports nothing from the feature beyond +the version validator in ``types_validation``, and nothing in the feature +imports it. It uses only the standard library, so it adds no dependency +to a package whose sole runtime dependency is ``opentelemetry-api``. + +Three things this layer does *not* do, on purpose: + +- **It does not verify content.** Verification lives at the accessor boundary in + ``skills_core`` so that it applies to every store equally, including a + customer's own. +- **It does not work around a missing ``contentHash``.** A hashless object is + held verbatim and *withheld* by verification with ``missing_content_hash``. + This module's job is to make that outcome loud — see ``StoreDiagnostics``. +- **It does not evaluate anything.** Flag and segment objects that share the + connection are skipped and counted, nothing more. + +The design rationale — why ``objectVersion`` is not ``version``, why changes +commit at ``payload-transferred`` — is in ``agents.md`` under *The delivery +transport*. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass, field +from typing import Any + +from .types_validation import is_valid_skill_version + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# The wire contract +# --------------------------------------------------------------------------- + +FDV2_OBJECT_KIND = "inline-resource" +""" +The FDv2 ``kind`` skills are delivered under. Together with +``FDV2_OBJECT_CATEGORY`` it maps onto the single interface value +``skills_core.SKILL_OBJECT_KIND``; that translation is this adapter's job. +""" + +FDV2_OBJECT_CATEGORY = "skill" +"""The ``category`` that narrows ``inline-resource`` to an agent skill.""" + +SDK_DATA_MODEL_VERSION = 1 +""" +The ``mv`` request parameter. The one request parameter whose value could not be +confirmed against a live server, so treat the default as provisional and +override it through the store's ``data_model_version`` if needed. +""" + +_EVENT_SERVER_INTENT = "server-intent" +_EVENT_PUT_OBJECT = "put-object" +_EVENT_DELETE_OBJECT = "delete-object" +_EVENT_PAYLOAD_TRANSFERRED = "payload-transferred" +_EVENT_HEARTBEAT = "heart-beat" +_EVENT_GOODBYE = "goodbye" +_EVENT_ERROR = "error" + +_INTENT_TRANSFER_FULL = "xfer-full" +_INTENT_TRANSFER_CHANGES = "xfer-changes" +_INTENT_TRANSFER_NONE = "none" + +_ENVELOPE_FIELDS = ("contentType", "content", "contentHash", "name", "description") +""" +The skill object envelope's fields, copied through verbatim. Nothing is coerced +or defaulted: a transport that filled in a missing field would be forging the +very thing verification exists to check. +""" + +_MOBILE_KEY_PREFIX = "mob-" +_SERVER_KEY_PREFIX = "sdk-" +_CLIENT_SIDE_ID = re.compile(r"\A[0-9a-f]{20,}\Z") +"""A client-side environment ID: bare lowercase hex. Server-side and mobile keys +both carry a prefix, so this shape is unambiguous rather than heuristic.""" + + +# --------------------------------------------------------------------------- +# Server-side only +# --------------------------------------------------------------------------- + + +def _require_server_side_credential(sdk_key: str) -> None: + """ + Refuses a mobile key or a client-side environment ID. + + Skill content is customer-confidential, and payload assignment is shared + across credential types, so a client-side credential may well *succeed* + against these endpoints. Raises rather than logs: a store built on the wrong + credential should not exist. + """ + if not isinstance(sdk_key, str) or not sdk_key.strip(): + raise ValueError( + "FDv2SkillStore requires a LaunchDarkly server-side SDK key " + "(sdk-...); none was given." + ) + key = sdk_key.strip() + if key.startswith(_MOBILE_KEY_PREFIX): + raise ValueError( + "FDv2SkillStore was given a mobile key (mob-...). Agent Skills are a " + "server-side feature: skill content is customer-confidential and is " + "never delivered to a mobile or client-side process. Use the " + "environment's server-side SDK key (sdk-...)." + ) + if _CLIENT_SIDE_ID.match(key): + raise ValueError( + "FDv2SkillStore was given what looks like a client-side environment " + "ID. Agent Skills are a server-side feature: skill content is " + "customer-confidential and is never delivered to a client-side " + "process. Use the environment's server-side SDK key (sdk-...)." + ) + if not key.startswith(_SERVER_KEY_PREFIX): + # Not rejected: private instances and test doubles issue credentials + # without the public prefix. Only the two unambiguous shapes above are. + logger.warning( + "The credential given to FDv2SkillStore does not look like a " + "LaunchDarkly server-side SDK key (sdk-...). Skills are delivered " + "only to server-side credentials; if this is a client-side or mobile " + "credential the connection will be rejected or will deliver nothing." + ) + + +# --------------------------------------------------------------------------- +# Diagnostics +# --------------------------------------------------------------------------- + + +@dataclass +class StoreDiagnostics: + """ + What the transport has seen. Read-only from a caller's perspective. + + Not part of the ``SkillStore`` interface. It exists because "this environment + has no skills" and "every skill was withheld" are easy to mistake for each + other, and a counter is easier to assert on than a log line. + """ + + payloads_transferred: int = 0 + """Completed ``payload-transferred`` commits since the store started.""" + skill_objects_received: int = 0 + """``put-object`` events identified as skills, across all payloads.""" + objects_ignored: int = 0 + """Objects skipped because they were not skills: flags, segments, and any + future kind. Skipping is the contract, not a failure.""" + objects_revoked: int = 0 + """``delete-object`` events applied to skills.""" + hashless_objects: int = 0 + """ + Skill objects whose envelope carried no ``contentHash``. + + **Nonzero means skills are being withheld**: verification withholds every one + of these with ``missing_content_hash``. + """ + connection_failures: int = 0 + """Recoverable transport failures since the last successful transfer.""" + last_error: str | None = None + """The most recent transport error, if any. Human-readable; do not parse.""" + + +# --------------------------------------------------------------------------- +# Deserialisation — where objectVersion is not version +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _Tombstone: + """A ``delete-object`` narrowed to the identity it revokes.""" + + key: str + object_version: int | None + + +def _is_skill_event(data: Any) -> bool: + """ + Whether one ``put-object`` / ``delete-object`` payload is a skill. + + Both halves are required: ``inline-resource`` may carry other categories, + and flags and segments omit ``category`` entirely. Every other kind is + ignored, not rejected, because flag and segment objects share the connection + and erroring on them would turn a normal payload into a reconnect loop. + """ + if not isinstance(data, dict): + return False + return ( + data.get("kind") == FDV2_OBJECT_KIND + and data.get("category") == FDV2_OBJECT_CATEGORY + ) + + +def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: + """ + Translates one FDv2 skill ``put-object`` into the raw object shape the + ``SkillStore`` interface defines. + + **The one translation this adapter must get right:** + + wire ``objectVersion`` → stored ``version`` (the skill's own version) + wire ``version`` → dropped (the *payload* version) + + ``objectVersion`` is what a ``{key, version}`` reference pins; ``version`` + moves whenever anything in the environment moves. Confusing them fails + silently: the object verifies and the caller gets content under a version + number that means nothing. + + Returns ``None`` only when ``key`` is not a string, since a keyless object + has no identity to store it under. Every other defect is carried through + verbatim so that verification withholds it with a reason code rather than + the transport dropping it into indistinguishable absence. + """ + key = data.get("key") + if not isinstance(key, str) or not key: + logger.warning( + "An FDv2 skill put-object carried no string 'key' and could not be " + "stored under any identity; it was dropped." + ) + return None + + raw: dict[str, Any] = {"key": key} + + # A membership test rather than a `.get` default, so an explicitly-null + # objectVersion stays null and reaches verification as `invalid_version`. + if "objectVersion" in data: + raw["version"] = data["objectVersion"] + + envelope = data.get("object") + if isinstance(envelope, dict): + for wire_field in _ENVELOPE_FIELDS: + if wire_field in envelope: + raw[wire_field] = envelope[wire_field] + return raw + + +def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: + """ + Narrows one FDv2 skill ``delete-object`` to the identity it revokes, with + the same ``objectVersion`` translation as a put. + + An ``object_version`` of ``None`` means the delete named no usable version + and is read as "revoke every version of this key". That is the safe + direction: the alternative is continuing to serve content LaunchDarkly has + withdrawn. + """ + key = data.get("key") + if not isinstance(key, str) or not key: + logger.warning( + "An FDv2 skill delete-object carried no string 'key'; it was ignored." + ) + return None + object_version = data.get("objectVersion") + return _Tombstone( + key=key, + object_version=object_version + if is_valid_skill_version(object_version) + else None, + ) + + +# --------------------------------------------------------------------------- +# The held object set +# --------------------------------------------------------------------------- + + +class _SkillObjectSet: + """ + Raw skill objects held in memory, keyed by ``(key, objectVersion)``. + + Lookup semantics are identical to ``InMemorySkillStore``'s, down to the + fall-through to a version-less entry, so that the store a caller configures + cannot change how a pinned reference resolves; ``TestInterfaceParity`` + asserts it. Reimplemented rather than inherited because the transport needs + ``delete`` and the atomic ``replace_with`` a full transfer requires. + + An object too malformed to carry a usable version is still held, under its + key alone, so verification withholds it with a signal rather than the + transport dropping it. + """ + + def __init__(self) -> None: + self._versions: dict[str, dict[int, dict[str, Any]]] = {} + self._loose: dict[str, dict[str, Any]] = {} + + def put(self, raw: dict[str, Any]) -> None: + key = raw["key"] + version = raw.get("version") + if is_valid_skill_version(version): + self._versions.setdefault(key, {})[version] = raw + else: + self._loose[key] = raw + + def delete(self, tombstone: _Tombstone) -> list[dict[str, Any]]: + """Removes what *tombstone* revokes; returns the raw objects that went away.""" + removed: list[dict[str, Any]] = [] + if tombstone.object_version is None: + held = self._versions.pop(tombstone.key, {}) + removed.extend(held.values()) + loose = self._loose.pop(tombstone.key, None) + if loose is not None: + removed.append(loose) + return removed + + held = self._versions.get(tombstone.key, {}) + gone = held.pop(tombstone.object_version, None) + if gone is not None: + removed.append(gone) + if not held: + self._versions.pop(tombstone.key, None) + return removed + + def get(self, key: str, version: int | None) -> dict[str, Any] | None: + held = self._versions.get(key, {}) + if version is not None: + # Fall through to the version-less entry so a malformed object + # reaches verification rather than reading as simply absent. + return held.get(version) or self._loose.get(key) + if held: + return held[max(held)] + return self._loose.get(key) + + def snapshot(self) -> dict[str, dict[str, Any]]: + """One entry per ``(key, version)``, under keys opaque to the SDK.""" + out: dict[str, dict[str, Any]] = { + f"{key}:{version}": raw + for key, versions in self._versions.items() + for version, raw in versions.items() + } + out.update(self._loose) + return out + + def all_raw(self) -> list[dict[str, Any]]: + return list(self.snapshot().values()) + + def replace_with(self, other: _SkillObjectSet) -> None: + """Adopts *other*'s contents wholesale — how a full transfer commits.""" + self._versions = other._versions + self._loose = other._loose + + def copy(self) -> _SkillObjectSet: + clone = _SkillObjectSet() + clone._versions = {key: dict(v) for key, v in self._versions.items()} + clone._loose = dict(self._loose) + return clone + + def __len__(self) -> int: + return sum(len(v) for v in self._versions.values()) + len(self._loose) + + +# --------------------------------------------------------------------------- +# The protocol state machine — pure, no I/O +# --------------------------------------------------------------------------- + + +@dataclass +class _TransferOutcome: + """What one event did. Aggregated by the caller; nothing here does I/O.""" + + committed: bool = False + changes: list[dict[str, Any]] = field(default_factory=list) + basis: str | None = None + fatal: str | None = None + disconnect: str | None = None + + +class _ProtocolReader: + """ + Applies FDv2 events to an object set. Pure — no sockets, no threads, no clock — + so every wire case is testable without a server. + + **Changes are buffered and committed at ``payload-transferred``.** A payload + version is the unit of consistency: applying half of one would publish a + state the server never described, and on a full transfer would briefly empty + the store. Listeners therefore fire once per commit, not once per object. + """ + + def __init__(self, committed: _SkillObjectSet) -> None: + self._committed = committed + self._intent: str | None = None + self._pending: _SkillObjectSet | None = None + self._changes: list[dict[str, Any]] = [] + self.diagnostics = StoreDiagnostics() + # Identities already reported by ``_warn_hashless``. Per reader, so a + # recreated store reports again and two stores never quieten each other. + # No lock: ``handle`` runs only on its owner's single delivery thread. + self._warned_hashless: set[tuple[str, Any]] = set() + + # -- events ------------------------------------------------------------ + + def handle(self, name: str, data: Any) -> _TransferOutcome: + """Routes one event. Unknown event names are ignored, by contract.""" + if name == _EVENT_SERVER_INTENT: + return self._server_intent(data) + if name == _EVENT_PUT_OBJECT: + return self._put_object(data) + if name == _EVENT_DELETE_OBJECT: + return self._delete_object(data) + if name == _EVENT_PAYLOAD_TRANSFERRED: + return self._payload_transferred(data) + if name == _EVENT_ERROR: + return self._error(data) + if name == _EVENT_GOODBYE: + return self._goodbye(data) + if name == _EVENT_HEARTBEAT: + return _TransferOutcome() + logger.debug("Ignoring unknown FDv2 event '%s'", name) + return _TransferOutcome() + + def _server_intent(self, data: Any) -> _TransferOutcome: + payloads = data.get("payloads") if isinstance(data, dict) else None + if not isinstance(payloads, list) or not payloads: + return _TransferOutcome( + disconnect="server-intent carried no payload description" + ) + first = payloads[0] + intent = first.get("intentCode") if isinstance(first, dict) else None + self._intent = intent + self._changes = [] + if intent == _INTENT_TRANSFER_FULL: + # Built alongside the live set rather than in place, so an + # interrupted transfer leaves last known good intact. + self._pending = _SkillObjectSet() + elif intent == _INTENT_TRANSFER_CHANGES: + self._pending = self._committed.copy() + else: + if intent != _INTENT_TRANSFER_NONE: + logger.debug("Ignoring FDv2 server-intent with intentCode %r", intent) + self._pending = None + return _TransferOutcome() + + def _target_for(self, data: Any) -> _SkillObjectSet | None: + """ + The pending set a skill object event applies to, or ``None`` when the + event is not a skill or the current intent carries no objects. + + An object arriving with no ``server-intent`` at all is treated as a + delta against what is held rather than dropped. + """ + if not _is_skill_event(data): + self.diagnostics.objects_ignored += 1 + return None + if self._pending is None: + if self._intent is None: + self._intent = _INTENT_TRANSFER_CHANGES + if self._intent not in (_INTENT_TRANSFER_FULL, _INTENT_TRANSFER_CHANGES): + return None + self._pending = self._committed.copy() + return self._pending + + def _put_object(self, data: Any) -> _TransferOutcome: + target = self._target_for(data) + if target is None: + return _TransferOutcome() + raw = _store_object_from_put(data) + if raw is None: + return _TransferOutcome() + target.put(raw) + self._changes.append(raw) + self.diagnostics.skill_objects_received += 1 + if not isinstance(raw.get("contentHash"), str): + self.diagnostics.hashless_objects += 1 + self._warn_hashless(raw) + return _TransferOutcome() + + def _delete_object(self, data: Any) -> _TransferOutcome: + target = self._target_for(data) + if target is None: + return _TransferOutcome() + tombstone = _tombstone_from_delete(data) + if tombstone is None: + return _TransferOutcome() + target.delete(tombstone) + self.diagnostics.objects_revoked += 1 + # A tombstone carries identity and no content, so a listener that reads + # content must check for ``content`` rather than assume it. + self._changes.append( + {"key": tombstone.key, "version": tombstone.object_version} + ) + return _TransferOutcome() + + def _payload_transferred(self, data: Any) -> _TransferOutcome: + state = data.get("state") if isinstance(data, dict) else None + version = data.get("version") if isinstance(data, dict) else None + if self._pending is not None: + self._committed.replace_with(self._pending) + _warn_if_nothing_can_verify(self._committed) + self._pending = None + self._intent = None + changes = self._changes + self._changes = [] + self.diagnostics.payloads_transferred += 1 + logger.debug( + "FDv2 payload transferred: payload version %s, %d skill object(s) held", + version, + len(self._committed), + ) + return _TransferOutcome( + committed=True, + changes=changes, + basis=state if isinstance(state, str) and state else None, + ) + + def _abandon_in_flight(self) -> None: + """Drops the in-flight payload and keeps what is committed.""" + self._pending = None + self._intent = None + self._changes = [] + + def _error(self, data: Any) -> _TransferOutcome: + reason = data.get("reason") if isinstance(data, dict) else None + self._abandon_in_flight() + return _TransferOutcome(disconnect=f"server sent error: {reason}") + + def _goodbye(self, data: Any) -> _TransferOutcome: + reason = data.get("reason") if isinstance(data, dict) else None + catastrophe = bool(data.get("catastrophe")) if isinstance(data, dict) else False + silent = bool(data.get("silent")) if isinstance(data, dict) else False + self._abandon_in_flight() + if not silent: + logger.info("FDv2 connection closing: %s", reason) + if catastrophe: + return _TransferOutcome( + fatal=f"server sent a catastrophic goodbye: {reason}" + ) + return _TransferOutcome(disconnect=f"server said goodbye: {reason}") + + # -- diagnostics --------------------------------------------------------- + + def _warn_hashless(self, raw: dict[str, Any]) -> None: + """ + One ERROR per ``(key, version)`` whose envelope had no ``contentHash``. + + ERROR rather than WARN because an empty accessor result is otherwise + indistinguishable from an environment that has no skills. + """ + identity = (raw["key"], raw.get("version")) + if identity in self._warned_hashless: + return + self._warned_hashless.add(identity) + logger.error( + "Skill '%s' version %s arrived without a contentHash and will be withheld. %s", + raw["key"], + raw.get("version"), + _HASHLESS_ADVICE, + extra={"ld_skill_key": raw["key"], "ld_skill_version": raw.get("version")}, + ) + + +_HASHLESS_ADVICE = ( + "The delivered skill object carries no 'contentHash', so integrity " + "verification withholds it with reason_code 'missing_content_hash' and its " + "content will not resolve. The SDK cannot work around this: verification " + "hashes the delivered bytes and compares them against the envelope's " + "'contentHash', and there is nothing to compare against. 'contentHash' is a " + "sha256 over the verbatim UTF-8 content. Contact LaunchDarkly support if " + "skills in your environment arrive without one." +) + + +def _warn_if_nothing_can_verify(committed: _SkillObjectSet) -> None: + """ + One ERROR per committed payload in which *nothing* held can possibly verify. + + ``log_withholding_summary`` reports the same condition at the accessor + boundary, but only once a caller asks. This fires at delivery time, so it is + visible in a process that boots, materializes nothing, and exits. + """ + held = committed.all_raw() + if not held: + return + hashless = [raw for raw in held if not isinstance(raw.get("contentHash"), str)] + if len(hashless) != len(held): + return + logger.error( + "All %d skill object(s) in the delivered payload arrived without a " + "contentHash. No skill content will resolve from this store. %s", + len(held), + _HASHLESS_ADVICE, + ) diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py new file mode 100644 index 00000000..ff5720bc --- /dev/null +++ b/packages/client/tests/test_skills_fdv2.py @@ -0,0 +1,605 @@ +""" +Tests for the FDv2 skill delivery protocol. + +Wire semantics — which objects are skills, ``objectVersion`` versus ``version``, +revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted +against ``_ProtocolReader``, which has no I/O, so each case reads as the contract +it is rather than as a server script. +""" + +from __future__ import annotations + +import hashlib +from typing import Any, ClassVar + +import pytest + +from launchdarkly_ai_server import InMemorySkillStore +from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND +from launchdarkly_ai_server.skills_fdv2 import ( + FDV2_OBJECT_CATEGORY, + FDV2_OBJECT_KIND, + _is_skill_event, + _ProtocolReader, + _SkillObjectSet, + _store_object_from_put, + _tombstone_from_delete, +) + +pytestmark = pytest.mark.usefixtures("reset_skill_state") + +SKILL_BODY = "---\nname: PDF Extraction\n---\nExtract text from PDFs.\n" + + +def _hash(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +# --------------------------------------------------------------------------- +# Wire builders — one place that knows the shape, so a contract change is one edit +# --------------------------------------------------------------------------- + + +def put_skill( + key: str = "pdf-extraction", + *, + object_version: Any = 3, + payload_version: int = 42, + content: str = SKILL_BODY, + content_hash: Any = None, + omit_hash: bool = False, + name: str = "PDF Extraction", +) -> dict[str, Any]: + """One skill ``put-object`` event's data, in the shape the wire delivers it.""" + envelope: dict[str, Any] = { + "contentType": "text/markdown", + "content": content, + "name": name, + "description": "Extracts text", + } + if not omit_hash: + envelope["contentHash"] = ( + content_hash if content_hash is not None else _hash(content) + ) + return { + "key": key, + "kind": FDV2_OBJECT_KIND, + "category": FDV2_OBJECT_CATEGORY, + "objectVersion": object_version, + "version": payload_version, + "object": envelope, + } + + +def delete_skill( + key: str = "pdf-extraction", *, object_version: Any = 3, payload_version: int = 43 +) -> dict[str, Any]: + return { + "key": key, + "kind": FDV2_OBJECT_KIND, + "category": FDV2_OBJECT_CATEGORY, + "objectVersion": object_version, + "version": payload_version, + } + + +def put_flag(key: str = "my-flag", version: int = 17) -> dict[str, Any]: + """A flag ``put-object``: no ``category``, no ``objectVersion``.""" + return { + "key": key, + "kind": "flag", + "version": version, + "object": { + "key": key, + "version": version, + "on": True, + "variations": [True, False], + }, + } + + +def put_segment(key: str = "beta-users", version: int = 4) -> dict[str, Any]: + return { + "key": key, + "kind": "segment", + "version": version, + "object": {"key": key, "version": version, "included": []}, + } + + +def server_intent( + code: str = "xfer-full", payload_id: str = "agent-skill" +) -> dict[str, Any]: + return { + "payloads": [ + {"id": payload_id, "target": 1, "intentCode": code, "reason": "test"} + ] + } + + +def transferred(state: str = "basis-1", version: int = 42) -> dict[str, Any]: + return {"state": state, "version": version} + + +def events(*pairs: tuple[str, Any]) -> list[dict[str, Any]]: + return [{"event": name, "data": data} for name, data in pairs] + + +def full_payload( + *object_events: tuple[str, Any], state: str = "basis-1" +) -> list[dict[str, Any]]: + return events( + ("server-intent", server_intent("xfer-full")), + *object_events, + ("payload-transferred", transferred(state)), + ) + + +# --------------------------------------------------------------------------- +# Identifying skill objects, and ignoring everything else +# --------------------------------------------------------------------------- + + +class TestObjectIdentification: + def test_kind_and_category_together_identify_a_skill(self) -> None: + assert _is_skill_event(put_skill()) is True + + def test_a_flag_is_not_a_skill(self) -> None: + assert _is_skill_event(put_flag()) is False + + def test_a_segment_is_not_a_skill(self) -> None: + assert _is_skill_event(put_segment()) is False + + def test_inline_resource_of_another_category_is_not_a_skill(self) -> None: + """``inline-resource`` is a broad kind, so the category is required too.""" + other = put_skill() + other["category"] = "prompt-template" + assert _is_skill_event(other) is False + + def test_skill_category_under_another_kind_is_not_a_skill(self) -> None: + other = put_skill() + other["kind"] = "some-future-kind" + assert _is_skill_event(other) is False + + def test_a_flag_shaped_object_with_no_category_is_not_a_skill(self) -> None: + """Flags and segments omit ``category`` entirely — the documented shape.""" + assert "category" not in put_flag() + assert "objectVersion" not in put_flag() + + @pytest.mark.parametrize("value", [None, "skill", 3, [], ()]) + def test_non_dict_events_are_not_skills(self, value: Any) -> None: + assert _is_skill_event(value) is False + + +# --------------------------------------------------------------------------- +# objectVersion is not version +# --------------------------------------------------------------------------- + + +class TestVersionTranslation: + def test_object_version_becomes_the_seam_version(self) -> None: + raw = _store_object_from_put(put_skill(object_version=3, payload_version=42)) + assert raw is not None + assert raw["version"] == 3 + + def test_the_payload_version_never_reaches_the_seam(self) -> None: + """ + The failure this asserts against is silent: a store that read ``version`` + would serve verifiable content under a version number that means nothing, + and every pinned reference would resolve to the wrong thing with no error. + """ + raw = _store_object_from_put(put_skill(object_version=3, payload_version=42)) + assert raw is not None + assert raw["version"] != 42 + assert 42 not in raw.values() + + def test_the_two_are_distinguished_even_when_the_payload_version_is_lower( + self, + ) -> None: + raw = _store_object_from_put(put_skill(object_version=99, payload_version=1)) + assert raw is not None + assert raw["version"] == 99 + + def test_a_missing_object_version_is_not_defaulted_from_the_payload(self) -> None: + wire = put_skill() + del wire["objectVersion"] + raw = _store_object_from_put(wire) + assert raw is not None + assert "version" not in raw + + def test_an_explicitly_null_object_version_is_carried_through_as_null(self) -> None: + """Carried, not invented: verification reports ``invalid_version``.""" + raw = _store_object_from_put(put_skill(object_version=None)) + assert raw is not None + assert raw["version"] is None + + def test_a_delete_translates_object_version_too(self) -> None: + tombstone = _tombstone_from_delete( + delete_skill(object_version=3, payload_version=43) + ) + assert tombstone is not None + assert tombstone.object_version == 3 + + def test_a_delete_with_no_usable_object_version_revokes_every_version(self) -> None: + tombstone = _tombstone_from_delete(delete_skill(object_version=None)) + assert tombstone is not None + assert tombstone.object_version is None + + def test_a_keyless_put_is_dropped_because_it_has_no_identity(self) -> None: + wire = put_skill() + del wire["key"] + assert _store_object_from_put(wire) is None + + def test_the_envelope_is_copied_verbatim(self) -> None: + raw = _store_object_from_put(put_skill()) + assert raw is not None + assert raw["content"] == SKILL_BODY + assert raw["contentHash"] == _hash(SKILL_BODY) + assert raw["name"] == "PDF Extraction" + assert raw["contentType"] == "text/markdown" + + def test_an_absent_envelope_field_is_absent_rather_than_defaulted(self) -> None: + wire = put_skill() + del wire["object"]["name"] + raw = _store_object_from_put(wire) + assert raw is not None + assert "name" not in raw + + +# --------------------------------------------------------------------------- +# The protocol reader +# --------------------------------------------------------------------------- + + +def drive(reader: _ProtocolReader, payload_events: list[dict[str, Any]]) -> list[Any]: + return [reader.handle(e["event"], e.get("data")) for e in payload_events] + + +class TestProtocolReader: + def test_a_full_transfer_commits_at_payload_transferred(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + outcomes = drive(reader, full_payload(("put-object", put_skill()))) + assert len(held) == 1 + assert outcomes[-1].committed is True + assert outcomes[-1].basis == "basis-1" + + def test_nothing_is_visible_before_payload_transferred(self) -> None: + """A payload version is the unit of consistency; half of one is not a state.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill()), + ), + ) + assert len(held) == 0 + + def test_an_interrupted_full_transfer_leaves_last_known_good_intact(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill(object_version=1)))) + assert held.get("pdf-extraction", None) is not None + + # A second full transfer starts and never completes. + drive( + reader, + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill(object_version=2)), + ), + ) + still_held = held.get("pdf-extraction", None) + assert still_held is not None + assert still_held["version"] == 1 + + def test_a_full_transfer_replaces_rather_than_merges(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill("first")))) + drive( + reader, full_payload(("put-object", put_skill("second")), state="basis-2") + ) + assert held.get("first", None) is None + assert held.get("second", None) is not None + + def test_a_change_transfer_applies_deltas_over_what_is_held(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill("first")))) + drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("second")), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert held.get("first", None) is not None + assert held.get("second", None) is not None + + def test_a_delete_object_revokes_the_skill(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill(object_version=3)))) + drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill(object_version=3)), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert held.get("pdf-extraction", None) is None + assert reader.diagnostics.objects_revoked == 1 + + def test_a_delete_notifies_with_a_tombstone_carrying_no_content(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill()))) + outcomes = drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ), + ) + (change,) = outcomes[-1].changes + assert change == {"key": "pdf-extraction", "version": 3} + assert "content" not in change + + def test_a_delete_for_one_version_leaves_the_other_held(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + full_payload( + ("put-object", put_skill(object_version=2)), + ("put-object", put_skill(object_version=3)), + ), + ) + drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill(object_version=3)), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert held.get("pdf-extraction", 2) is not None + assert held.get("pdf-extraction", None)["version"] == 2 + + def test_flag_and_segment_objects_are_skipped_cleanly(self) -> None: + """ + The mixed payload is the normal case, not an edge one: an environment's + assignment carries its flag payload alongside its agent-skill payload. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + outcomes = drive( + reader, + full_payload( + ("put-object", put_flag("flag-a")), + ("put-object", put_skill("pdf-extraction")), + ("put-object", put_segment("beta-users")), + ("put-object", put_flag("flag-b")), + ("delete-object", put_flag("flag-c")), + ), + ) + assert len(held) == 1 + assert held.get("pdf-extraction", None) is not None + assert reader.diagnostics.objects_ignored == 4 + assert reader.diagnostics.skill_objects_received == 1 + assert all(o.fatal is None and o.disconnect is None for o in outcomes) + + def test_an_unknown_kind_is_ignored_rather_than_fatal(self) -> None: + """ + Erroring on an unrecognised kind would turn a normal payload into a + permanent reconnect loop — a flag-delivery outage caused by a skills + rollout. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + exotic = { + "key": "x", + "kind": "quantum-widget", + "version": 1, + "object": {"a": 1}, + } + outcomes = drive(reader, full_payload(("put-object", exotic))) + assert len(held) == 0 + assert all(o.fatal is None and o.disconnect is None for o in outcomes) + + def test_an_unknown_event_name_is_ignored(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + outcome = reader.handle("some-future-event", {"anything": True}) + assert outcome.fatal is None + assert outcome.disconnect is None + + def test_a_heartbeat_does_nothing(self) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle("heart-beat", None) + assert outcome == type(outcome)() + + def test_an_error_event_abandons_the_in_flight_payload(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill(object_version=1)))) + outcomes = drive( + reader, + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill(object_version=2)), + ( + "error", + {"payloadId": "agent-skill", "reason": "backend unavailable"}, + ), + ), + ) + assert outcomes[-1].disconnect is not None + assert held.get("pdf-extraction", None)["version"] == 1 + + def test_a_goodbye_asks_for_a_reconnect(self) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle("goodbye", {"reason": "rebalancing", "silent": False}) + assert outcome.disconnect is not None + assert outcome.fatal is None + + def test_a_catastrophic_goodbye_is_fatal(self) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle( + "goodbye", {"reason": "no", "silent": False, "catastrophe": True} + ) + assert outcome.fatal is not None + + def test_transfer_none_holds_everything_and_commits(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill()))) + drive( + reader, + events( + ("server-intent", server_intent("none")), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert len(held) == 1 + + def test_an_object_arriving_with_no_intent_is_treated_as_a_delta(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + events( + ("put-object", put_skill()), + ("payload-transferred", transferred("basis-1")), + ), + ) + assert len(held) == 1 + + +# --------------------------------------------------------------------------- +# Interface parity with InMemorySkillStore +# --------------------------------------------------------------------------- + + +class TestInterfaceParity: + """ + The two stores must resolve identically. ``_SkillObjectSet`` reimplements the + lookup rather than inheriting it — see its docstring for why — so this is the + test that stops the two from drifting. + """ + + RAWS: ClassVar[list[dict[str, Any]]] = [ + {"key": "a", "version": 1, "content": "x", "contentHash": _hash("x")}, + {"key": "a", "version": 4, "content": "y", "contentHash": _hash("y")}, + {"key": "b", "version": 2, "content": "z", "contentHash": _hash("z")}, + {"key": "malformed", "version": "not-a-version", "content": "q"}, + ] + + def _both(self) -> tuple[InMemorySkillStore, _SkillObjectSet]: + memory = InMemorySkillStore() + objects = _SkillObjectSet() + for raw in self.RAWS: + memory.put(dict(raw)) + objects.put(dict(raw)) + return memory, objects + + @pytest.mark.parametrize( + "key,version", + [ + ("a", None), + ("a", 1), + ("a", 4), + ("a", 9), + ("b", 2), + ("b", None), + ("missing", None), + ("missing", 1), + ("malformed", None), + ("malformed", 7), + ], + ) + def test_get_agrees(self, key: str, version: int | None) -> None: + memory, objects = self._both() + assert memory.get_object(SKILL_OBJECT_KIND, key, version) == objects.get( + key, version + ) + + def test_snapshot_agrees(self) -> None: + memory, objects = self._both() + assert memory.all_objects(SKILL_OBJECT_KIND) == objects.snapshot() + + +# --------------------------------------------------------------------------- +# The contentHash gap +# --------------------------------------------------------------------------- + + +def _per_object_hashless_errors(caplog: Any) -> list[Any]: + """The per-object ERROR, as distinct from the whole-payload summary.""" + return [ + r + for r in caplog.records + if r.levelname == "ERROR" + and "arrived without a contentHash" in r.getMessage() + and "No skill content will resolve" not in r.getMessage() + ] + + +class TestMissingContentHash: + """ + A skill delivered without a ``contentHash``, asserted as behaviour. + + An envelope with no ``contentHash`` must produce a *withheld* skill with the + ``missing_content_hash`` reason — loudly, diagnosably, and without a crash. + There is deliberately no fallback that skips verification: a hash the SDK + computed from the content it was handed would certify the content against + itself and verify nothing. + """ + + def test_a_redelivered_hashless_object_logs_once_per_store( + self, caplog: Any + ) -> None: + """Re-delivering the same ``(key, version)`` to one store must not + multiply the ERROR: a polling store sees every object on every poll.""" + reader = _ProtocolReader(_SkillObjectSet()) + payload = full_payload(("put-object", put_skill(omit_hash=True))) + with caplog.at_level("ERROR"): + drive(reader, payload) + drive(reader, payload) + assert len(_per_object_hashless_errors(caplog)) == 1 + + def test_a_recreated_store_reports_the_same_hashless_object_again( + self, caplog: Any + ) -> None: + """ + The dedupe belongs to the store, not the process. A host that rebuilds + its store (reconnect wrapper, config reload, credential rotation) must + get the ERROR again, since it is the loudest signal that a deployment is + broken rather than empty by design. + """ + payload = full_payload(("put-object", put_skill(omit_hash=True))) + with caplog.at_level("ERROR"): + drive(_ProtocolReader(_SkillObjectSet()), payload) + first = len(_per_object_hashless_errors(caplog)) + drive(_ProtocolReader(_SkillObjectSet()), payload) + assert first == 1 + assert len(_per_object_hashless_errors(caplog)) == 2 + + def test_two_live_stores_do_not_suppress_each_other(self, caplog: Any) -> None: + """Two stores in one process (say, two environments) each report.""" + one = _ProtocolReader(_SkillObjectSet()) + two = _ProtocolReader(_SkillObjectSet()) + payload = full_payload(("put-object", put_skill(omit_hash=True))) + with caplog.at_level("ERROR"): + drive(one, payload) + drive(two, payload) + # And each still dedupes its own re-deliveries. + drive(one, payload) + drive(two, payload) + assert len(_per_object_hashless_errors(caplog)) == 2 From e2b54fd990ff94a415e13f9c6b6bdec4a7794410 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 13:43:52 -0400 Subject: [PATCH 12/22] feat(client): an unheld version pin is a miss, not an integrity failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``InMemorySkillStore.get_object`` consulted the version-less entry on any pinned miss, while the unpinned path consulted it only when nothing well-formed was filed under the key. A key holding both well-formed versions and one malformed object therefore answered a pin for an undelivered version with the malformed object, and verification recorded an integrity failure — an alert pointed at a skill whose integrity was never in question — where the honest answer is that the version is not held. Both paths now follow the one rule: the version-less entry answers only when nothing well-formed is filed under the key, which is the case it exists for. A malformed object that is all the store holds still reaches verification and is still withheld with a signal, so tampering cannot read as a skill that was never delivered. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills.py | 17 ++++---- packages/client/tests/test_skills.py | 39 +++++++++++++++++++ 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py index eb3946c2..36ae7dee 100644 --- a/packages/client/src/launchdarkly_ai_server/skills.py +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -131,14 +131,17 @@ def get_object( if kind != SKILL_OBJECT_KIND: return None held = self._versions.get(key, {}) + if not held: + # Nothing well-formed is filed under this key, so the version-less + # entry is all there is: serve it, and let verification withhold it + # with a signal rather than have it read as simply absent. A pin that + # misses while well-formed versions do exist is a plain miss, and + # answering it with a leftover malformed object would record an + # integrity failure for a skill whose integrity is not in question. + return self._loose.get(key) if version is not None: - # Fall through to the version-less entry when the pin does not match - # anything well-formed, so a malformed object reaches verification and - # is withheld with a signal rather than reading as simply absent. - return held.get(version) or self._loose.get(key) - if held: - return held[max(held)] - return self._loose.get(key) + return held.get(version) + return held[max(held)] def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: """ diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 203eeb4f..6b9dfe34 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -378,6 +378,23 @@ def test_get_object_unknown_version_falls_back_to_a_malformed_object( assert s.get_object("skill", "a", 2) == malformed assert s.get_object("skill", "a") == malformed + def test_get_object_unknown_version_beside_well_formed_ones_is_absent( + self, make_raw_skill: Any + ) -> None: + """A pin miss is a miss, not an integrity failure. + + The fall-back above applies only when nothing well-formed is filed under + the key. Once well-formed versions are held, a leftover malformed object + must not answer for a version that was never delivered: verification + would withhold it and record an integrity failure against a skill whose + integrity is not in question. + """ + s = InMemorySkillStore() + s.put(make_raw_skill(key="a", version=1, content="one\n")) + s.put(make_raw_skill(key="a", version=2, content="two\n")) + s.put(make_raw_skill(key="a", version="two")) + assert s.get_object("skill", "a", 5) is None + def test_all_objects_unknown_kind_is_empty(self, make_raw_skill: Any) -> None: s = InMemorySkillStore() s.put(make_raw_skill(key="a")) @@ -769,6 +786,28 @@ async def test_pin_to_a_version_the_store_does_not_hold_returns_none( await self._two_versions(store, make_raw_skill) assert await get_skill("a", version=9) is None + async def test_pin_miss_beside_a_malformed_object_records_no_failure( + self, + store: Any, + make_raw_skill: Any, + recording_emitter: Any, + ) -> None: + """An undelivered version must not raise an integrity alarm. + + A malformed object is filed under its key alone, and a pinned lookup + serves it when that is all the store holds — so verification withholds it + with a signal rather than letting tampering read as a skill that was + never delivered. Once well-formed versions are held, that reasoning no + longer applies: the pin is simply not there, and reporting an integrity + failure would point an alert at the wrong skill. + """ + skills_module._set_emitter_for_testing(recording_emitter) + await self._two_versions(store, make_raw_skill) + store.put(make_raw_skill(key="a", version="two")) + + assert await get_skill("a", version=9) is None + assert recording_emitter.signals(INTEGRITY_SIGNAL) == [] + async def test_all_skills_returns_one_entry_per_key_at_the_newest_version( self, store: Any, make_raw_skill: Any ) -> None: From 5817d89136b74dd2dbdd36b4cafdf71161c47275 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:16:17 -0400 Subject: [PATCH 13/22] fix(client): a broken store answer is not an absent skill Two untrusted-store answers were read as data rather than as failures. ``list_raw_objects`` collapsed a non-mapping listing to ``{}`` with no error, so a store that served nothing usable was indistinguishable from one holding no skills. ``resolve_from_store`` read identity off the object without checking it against the key that was asked for, so an answer served under a different key came back under the caller's key while carrying its own. Both are now withheld and reported, alongside the version check that already guarded the same way. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_core.py | 28 ++++++++++- packages/client/tests/test_skills.py | 50 +++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index 1e517e4e..a3894335 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -621,13 +621,27 @@ def list_raw_objects( worded identically. Letting the exception out instead would make each of them re-derive the log line and the message, which is the drift this module exists to prevent. + + An answer that is not a mapping is a broken store, on the same footing as + one that raised — **not** an empty one. Collapsing it to ``{}`` would make a + store that served nothing usable indistinguishable from a store that holds + no skills, which reads downstream as "every skill was revoked". """ try: objects = store.all_objects(SKILL_OBJECT_KIND) except Exception as exc: logger.error("Skill store raised while listing skills", exc_info=True) return {}, store_raised(exc) - return (objects if isinstance(objects, dict) else {}), None + if not isinstance(objects, dict): + logger.error( + "Skill store listed skills as %s rather than an object", + type(objects).__name__, + ) + return {}, ( + f"the skill store listed skills as {type(objects).__name__} " + "rather than an object" + ) + return objects, None def newest_by_key(objects: dict[str, dict[str, Any]]) -> list[tuple[str, Any]]: @@ -696,7 +710,10 @@ def resolve_from_store( versions of one key and only it can pick between them; ``None`` asks for the newest. The equality check afterwards is kept as a **defense**, not as the selection mechanism: the store is untrusted, so an answer that is not the - version that was asked for is withheld rather than returned. + version that was asked for is withheld rather than returned. The key is + checked the same way and for the same reason: identity is read off the + object itself, so an answer served under a different key would otherwise be + returned under the caller's key while carrying its own. """ try: raw = store.get_object(SKILL_OBJECT_KIND, key, wanted_version) @@ -714,6 +731,13 @@ def resolve_from_store( return Resolution( error=f"skill '{key}' failed integrity verification and was withheld" ) + if skill.key != key: + return Resolution( + error=( + f"skill '{key}' is not available: the store answered under " + f"key '{skill.key}'" + ) + ) if wanted_version is not None and skill.version != wanted_version: return Resolution( error=( diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 6b9dfe34..0b0cadef 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -27,6 +27,7 @@ shutdown, skill_refs, ) +from launchdarkly_ai_server.skills_core import list_raw_objects, require_store SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" @@ -631,6 +632,28 @@ async def test_other_version_returns_none( async def test_missing_key_returns_none(self, store: InMemorySkillStore) -> None: assert await get_skill("nope") is None + async def test_a_store_answering_under_a_different_key_is_withheld( + self, make_raw_skill: Any + ) -> None: + """The key needs the same post-fetch defense the version already has. + + Identity is read off the object itself, and the store is untrusted. An + answer served under a different key would otherwise be handed back + under the key the caller asked for while carrying its own. + """ + + class _AliasingStore: + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return make_raw_skill(key="other-key") + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + skills_module._set_store(_AliasingStore()) + assert await get_skill("asked-for") is None + async def test_multibyte_content_verifies( self, store: InMemorySkillStore, make_raw_skill: Any ) -> None: @@ -724,6 +747,33 @@ async def test_omits_skills_that_fail_verification( assert {s.key for s in result} == {"good"} assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + async def test_a_non_mapping_listing_is_reported_as_a_broken_store(self) -> None: + """A listing that is not a mapping is a broken store, not an empty one. + + ``all_skills`` has no way to report the difference, so it returns an + empty list either way — but the reason has to reach the caller that + does act on it. Collapsing the answer to "no skills" reads downstream + as "every skill was revoked". + """ + + class _BrokenListingStore: + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return None + + def all_objects(self, kind: str) -> Any: + return None + + skills_module._set_store(_BrokenListingStore()) + + assert await all_skills() == [] + + objects, error = list_raw_objects(require_store()) + assert objects == {} + assert error is not None + assert "rather than an object" in error + class TestVersionPinning: """ From 29ad3feda3c50808d0673e4b21171cdf2b250230 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:17:57 -0400 Subject: [PATCH 14/22] fix(client): attach the skill watcher's listener before its first reconcile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit watch_skills awaited write_skills and only then constructed SkillWatcher, which is where the store listener attaches. The reconcile snapshots the store as its first step and then spends the rest of its time on the filesystem, so every write, fsync, prune and manifest rewrite in that first pass ran with nothing listening. A change delivered in that window was never seen, and since nothing re-reconciles on a timer, a revocation that landed there waited for the next unrelated change — on a quiet root, the next restart. Exactly the gap watch_skills exists to close. The watcher now attaches its listener before the initial reconcile and starts its worker after. notify only sets an event, so a change arriving mid-reconcile is recorded and picked up by the worker's first pass, while holding the thread back keeps write_skills's one-root-one-reconcile contract: the worker cannot race the caller's own reconcile over the same manifest. A reconcile that raises detaches the listener on the way out, since the caller is handed an exception rather than a watcher to close. Co-Authored-By: Claude Opus 5 --- .../launchdarkly_ai_server/skills_watch.py | 51 ++++++++++++--- packages/client/tests/test_skills_watch.py | 62 +++++++++++++++++++ 2 files changed, 103 insertions(+), 10 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_watch.py b/packages/client/src/launchdarkly_ai_server/skills_watch.py index 05263057..94b31d46 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_watch.py +++ b/packages/client/src/launchdarkly_ai_server/skills_watch.py @@ -94,13 +94,26 @@ def __init__( target=self._run, name="ld-ai-skills-reconcile", daemon=True ) - # Register before starting the worker: ``notify`` only sets an event, so a - # change that lands in between is picked up as soon as the worker runs, - # and a store whose ``add_listener`` raises leaves no thread behind. + # Register before the initial reconcile, and leave the worker unstarted + # until ``start``. ``notify`` only sets an event, so a change that lands + # while that reconcile is still running is recorded rather than lost, and + # the worker cannot reconcile the root while the caller's own reconcile is + # in flight. A store whose ``add_listener`` raises leaves no thread behind. self._store = store self._registered = False store.add_listener(SKILL_OBJECT_KIND, self.notify) self._registered = True + + def _start(self) -> None: + """ + Starts the worker. ``watch_skills`` calls this once, after the initial + reconcile; it is not part of the caller-facing interface. + + Split from construction so registration and reconciling can be ordered + independently: the listener attaches first, so no change is missed, while + the first re-reconcile waits for the initial one to finish, so a root only + ever has one reconcile running at a time. + """ self._thread.start() # -- the listener the store calls ------------------------------------- @@ -272,13 +285,13 @@ async def watch_skills( if debounce < 0: raise ValueError(f"debounce must not be negative, got {debounce!r}") - # The initial reconcile runs first and on the caller's thread, so its report - # is the caller's to inspect and a bad root raises out of `watch_skills` - # rather than into a worker thread's log. - report = await write_skills( - skills, root, prune=prune, timeout=timeout, on_unavailable=on_unavailable - ) - + # The watcher attaches its listener before the initial reconcile, not after. + # The reconcile snapshots the store as its first step and then spends the + # rest of its time on the filesystem — a write and an fsync per skill, the + # prune, the manifest rewrite — so a change delivered after that snapshot + # needs something already listening to be seen at all. Nothing re-reconciles + # on a timer, so a revocation that landed unobserved would wait for the next + # unrelated change, which on a quiet root means the next restart. watcher = SkillWatcher( skills, root, @@ -289,4 +302,22 @@ async def watch_skills( debounce=debounce, on_reconcile=on_reconcile, ) + try: + # The initial reconcile runs on the caller's thread, so its report is the + # caller's to inspect and a bad root raises out of `watch_skills` rather + # than into a worker thread's log. + report = await write_skills( + skills, root, prune=prune, timeout=timeout, on_unavailable=on_unavailable + ) + except BaseException: + # The listener is already attached, so a reconcile that raises must not + # leave it on the store: the caller has no watcher to close. + watcher.close() + raise + + # Only now start the worker. A change that arrived during the reconcile has + # already set the wake event, so the worker's first pass picks it up; one that + # arrived before the reconcile's snapshot is already on disk, and the + # redundant pass it triggers converges on the same state. + watcher._start() return report, watcher diff --git a/packages/client/tests/test_skills_watch.py b/packages/client/tests/test_skills_watch.py index 48a95f07..dada2015 100644 --- a/packages/client/tests/test_skills_watch.py +++ b/packages/client/tests/test_skills_watch.py @@ -92,6 +92,68 @@ async def test_no_store_configured_raises(self, tmp_path: Any) -> None: await watch_skills("*", tmp_path / "s") +# --------------------------------------------------------------------------- +# Changes that land while the initial reconcile is running +# --------------------------------------------------------------------------- + + +class TestChangesDuringTheInitialReconcile: + """The listener attaches before the initial reconcile, so a change delivered + while that reconcile is still running is acted on rather than lost.""" + + async def test_a_revocation_landing_mid_reconcile_is_not_missed( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + class RevokesAfterSnapshot(InMemorySkillStore): + """Revokes everything the moment the reconcile has taken its + snapshot — where a ``delete-object`` lands when it arrives a fraction + of a second into startup, with nothing after it.""" + + def __init__(self) -> None: + super().__init__() + self.snapshots = 0 + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + objects = super().all_objects(kind) + self.snapshots += 1 + if self.snapshots == 1: + self._versions.clear() + self._loose.clear() + for listener in self._listeners.get(SKILL_OBJECT_KIND, []): + listener({"key": "pdf-extraction"}) + return objects + + store = RevokesAfterSnapshot() + store.put(make_raw_skill(key="pdf-extraction", version=1, content="body")) + await init_client(options={"skillStore": store}, client=object()) + + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + # The initial reconcile wrote what its snapshot held, so the file is + # on disk and the revocation that followed it is the only change + # left to act on. + assert written.read_text() == "body" + assert wait_until(lambda: not written.exists(), timeout=10) + finally: + watcher.close() + + async def test_a_failed_initial_reconcile_leaves_no_listener_behind( + self, tmp_path: Any + ) -> None: + """Registering first means a reconcile that raises has to detach: the + caller is handed an exception, not a watcher to close.""" + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + not_a_directory = tmp_path / "file" + not_a_directory.write_text("") + + with pytest.raises(ValueError, match="not a directory"): + await watch_skills("*", not_a_directory) + + assert store._listeners.get(SKILL_OBJECT_KIND, []) == [] + + # --------------------------------------------------------------------------- # Closing a watch # --------------------------------------------------------------------------- From 4c6d965b066941fa1d836a5b53ec6ddfe5e513e4 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:20:57 -0400 Subject: [PATCH 15/22] test(client): a broken store answer never deletes managed files Covers the reconcile side of the withheld-answer fix: a listing that is not a mapping leaves every managed file alone rather than reading as a full revocation, and an answer served under a different key writes nothing, is reported against the key that was asked for, and does not reach that other key's file. Each one previously deleted a file and reported a clean run. Co-Authored-By: Claude Opus 5 --- packages/client/tests/test_skills_fs.py | 114 ++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 62d01d59..4aa9ed8e 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -532,6 +532,120 @@ def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: assert _actions_by_key(report)["pdf-extraction"].action == "error" assert "pdf-extraction/SKILL.md" in _read_manifest(root)["entries"] + async def test_a_non_mapping_listing_never_prunes(self, root: Path) -> None: + """A store that cannot list is not a store holding nothing. + + A listing collapsed to "no skills" is indistinguishable from every + skill having been revoked, and prune would then delete every managed + file and report a clean run. The listing failure has to reach the + prune gate as an incomplete run. + """ + + class NoListingStore: + """Answers the listing with something that is not a mapping.""" + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return None + + def all_objects(self, kind: str) -> Any: + return None + + existing = _place_managed(root, "pdf-extraction", SKILL_BODY) + skills_module._set_store(NoListingStore()) + + report = await write_skills("*", root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert [a.action for a in report.actions] == ["error"] + assert any("rather than an object" in m for m in _error_messages(report)) + # The entry survives, so the next reconcile picks it up. + assert "pdf-extraction/SKILL.md" in _read_manifest(root)["entries"] + + async def test_an_answer_under_another_key_writes_nothing(self, root: Path) -> None: + """The file is named after the key the object carries, so a store + answering under a different key would write one path and prune another. + + Left unchecked, the run wrote the aliased key, then deleted it in the + same pass because prune keys off the request — and reported ok. The + requested key has to be the one the outcome is reported against. + """ + + class AliasingStore: + """Answers every lookup with an object carrying its own key.""" + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return { + "key": "other-key", + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + skills_module._set_store(AliasingStore()) + + report = await write_skills(["requested-key"], root) + + assert report.ok is False + assert [a.action for a in report.actions] == ["error"] + # Reported against the key that was asked for, not the one served. + assert _actions_by_key(report)["requested-key"].action == "error" + assert not (root / "other-key").exists() + assert _read_manifest(root)["entries"] == {} + + async def test_an_answer_under_another_key_does_not_overwrite_that_key( + self, root: Path + ) -> None: + """The aliased answer must not reach the real key's file. + + Both keys are requested here, so nothing is prunable and the write + itself is what is under test: unchecked, the object served under the + alias is written to the *other* key's path, clobbering the content that + key's own lookup resolved — and the run still reports ok. + """ + aliased = "aliased\n" + + class AliasingStore: + """Answers one key honestly and the other under that same key.""" + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + if key == "other-key": + return { + "key": "other-key", + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + return { + "key": "other-key", + "version": 2, + "content": aliased, + "contentHash": _hash(aliased), + } + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + existing = _place_managed(root, "other-key", SKILL_BODY) + skills_module._set_store(AliasingStore()) + + # The alias is resolved last, so an unchecked write lands on top. + report = await write_skills(["other-key", "requested-key"], root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert _actions_by_key(report)["requested-key"].action == "error" + assert _actions_by_key(report)["other-key"].action == "skipped_current" + async def test_unavailable_run_does_not_corrupt_manifest(self, root: Path) -> None: _place_managed(root, "a", SKILL_BODY) before = _read_manifest(root) From 1cf523566fdfd0a6fc70ecc9eac5220ac52e2a45 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:46:56 -0400 Subject: [PATCH 16/22] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20name=20the=20payload=20a=20transfer=20completed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The protocol reader took payloads[0]'s intentCode and applied it to the skill object set, which is what the delivery protocol requires — one payload per credential, read the first intent, tolerate the rest — but it left the assumption behind that rule undocumented and unguarded. If the one-payload guarantee ever widens, an xfer-full for another payload would start an empty pending set and the next payload-transferred would publish it: every skill reported revoked, and with pruning on, a customer's files deleted. The first payload is still the payload that is read. What is new is that the reader now knows which payload skills actually arrive on — learnt from the intent's id, or from the (p::) selector, since no object or transfer event carries a payload id of its own — and declines to apply a transfer of any other, holding last known good, warning once, and counting it in diagnostics.payloads_ignored. An intent describing more than one payload warns once on its own, because that is the one case the comparison cannot catch: another payload's transfer arriving before any skill has been seen has nothing to be compared against. Behaviour under one-payload delivery is unchanged, and a full transfer of the skill payload still empties it — every skill deleted is a real state the guard must not mask. Co-Authored-By: Claude Opus 5 --- packages/client/agents.md | 13 + .../src/launchdarkly_ai_server/skills_fdv2.py | 132 +++++++++- packages/client/tests/test_skills_fdv2.py | 226 ++++++++++++++++++ 3 files changed, 370 insertions(+), 1 deletion(-) diff --git a/packages/client/agents.md b/packages/client/agents.md index 80ec0538..e73c96d4 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -256,6 +256,19 @@ described, and would briefly empty the store — which, with pruning on, is the between a reconcile and deleting a customer's skill files. An interrupted transfer therefore leaves last known good intact, and listeners fire once per commit. +**The first payload intent is read, and is assumed to be the skill payload.** Delivery +provides one payload per credential and the protocol requires a client to ignore all but the +first payload intent, so `payloads[0]` is both what arrives and what the protocol says to +read. The cost of that assumption is that an `xfer-full` for somebody *else's* payload would +start an empty pending set, and the next `payload-transferred` would publish it — every skill +reported revoked, and with pruning on, a customer's files deleted. `_ProtocolReader` +therefore learns which payload skills arrive on, from the intent's `id` or from the +`(p::)` selector, and declines to apply a transfer of any other: once at +WARNING, counted in `diagnostics.payloads_ignored`, holding last known good. A transfer that +names no payload is applied, since one-payload delivery is the common case. The residual is +the first transfer of a connection — before a skill has arrived there is nothing to compare +against — which is what the separate WARNING on a multi-payload intent is for. + **A hashless object is held, not dropped.** Verification withholds it with `missing_content_hash`; the transport's job is to make that loud (an error per object, a summary per wholly-hashless payload, `diagnostics.hashless_objects`) rather than to work diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 80ef1224..f644a349 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -36,6 +36,12 @@ - **It does not evaluate anything.** Flag and segment objects that share the connection are skipped and counted, nothing more. +One assumption it *does* make, and states: **the payload intent it reads is the +payload skills arrive on.** Delivery sends one payload per credential and the +protocol tells a client to read only the first payload intent, so today those are +the same payload. ``_ProtocolReader`` keeps the pair apart anyway, because the +cost of conflating them is an emptied skill set. + The design rationale — why ``objectVersion`` is not ``version``, why changes commit at ``payload-transferred`` — is in ``agents.md`` under *The delivery transport*. @@ -93,6 +99,16 @@ very thing verification exists to check. """ +_PAYLOAD_SELECTOR = re.compile(r"\(p:([^:()]+):\d+\)") +""" +The payload identity inside a transfer's selector, ``(p::)``. + +The selector is the only place a completed transfer names its own payload: +``put-object``, ``delete-object`` and ``payload-transferred`` carry no payload id +of their own. ``_ProtocolReader`` reads it as a fallback for an intent that named +no ``id``. +""" + _MOBILE_KEY_PREFIX = "mob-" _SERVER_KEY_PREFIX = "sdk-" _CLIENT_SIDE_ID = re.compile(r"\A[0-9a-f]{20,}\Z") @@ -169,6 +185,11 @@ class StoreDiagnostics: future kind. Skipping is the contract, not a failure.""" objects_revoked: int = 0 """``delete-object`` events applied to skills.""" + payloads_ignored: int = 0 + """ + Transfers not applied because they completed a payload other than the one + skills arrive on. Zero while delivery sends one payload per connection. + """ hashless_objects: int = 0 """ Skill objects whose envelope carried no ``contentHash``. @@ -255,6 +276,22 @@ def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: return raw +def _payload_id_of(intent: Any) -> str | None: + """The payload id one payload intent names, when it names a usable one.""" + if not isinstance(intent, dict): + return None + value = intent.get("id") + return value if isinstance(value, str) and value else None + + +def _payload_id_from_selector(state: Any) -> str | None: + """The payload id inside a transfer's selector, when it carries one.""" + if not isinstance(state, str): + return None + match = _PAYLOAD_SELECTOR.search(state) + return match.group(1) if match else None + + def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: """ Narrows one FDv2 skill ``delete-object`` to the identity it revokes, with @@ -394,6 +431,18 @@ class _ProtocolReader: version is the unit of consistency: applying half of one would publish a state the server never described, and on a full transfer would briefly empty the store. Listeners therefore fire once per commit, not once per object. + + **The first payload intent is read, and is assumed to be the skill payload.** + Delivery provides one payload per credential and the protocol requires a + client to ignore all but the first payload intent, so ``payloads[0]`` is both + what arrives and what the protocol says to read. If that ever widens, an + ``xfer-full`` for somebody else's payload would empty the skill set and the + next ``payload-transferred`` would publish it empty — with pruning on, the + difference between a reconcile and deleting a customer's files. This layer + therefore learns which payload skills arrive on and declines to apply a + transfer of any other, once at WARNING and counted. The residual is the first + transfer of a connection: before a skill has arrived there is nothing to + compare a payload against. """ def __init__(self, committed: _SkillObjectSet) -> None: @@ -406,6 +455,14 @@ def __init__(self, committed: _SkillObjectSet) -> None: # recreated store reports again and two stores never quieten each other. # No lock: ``handle`` runs only on its owner's single delivery thread. self._warned_hashless: set[tuple[str, Any]] = set() + # The payload the current intent describes, and the payload skills have + # actually arrived on. One payload per connection makes these the same + # payload; the class docstring says why they are kept apart regardless. + self._intent_payload_id: str | None = None + self._skill_payload_id: str | None = None + self._skills_in_payload = 0 + self._warned_multiple_payloads = False + self._warned_foreign_payload = False # -- events ------------------------------------------------------------ @@ -434,10 +491,15 @@ def _server_intent(self, data: Any) -> _TransferOutcome: return _TransferOutcome( disconnect="server-intent carried no payload description" ) + if len(payloads) > 1: + self._warn_multiple_payloads(payloads) + # The first payload only, as the protocol requires. first = payloads[0] intent = first.get("intentCode") if isinstance(first, dict) else None self._intent = intent + self._intent_payload_id = _payload_id_of(first) self._changes = [] + self._skills_in_payload = 0 if intent == _INTENT_TRANSFER_FULL: # Built alongside the live set rather than in place, so an # interrupted transfer leaves last known good intact. @@ -479,6 +541,7 @@ def _put_object(self, data: Any) -> _TransferOutcome: target.put(raw) self._changes.append(raw) self.diagnostics.skill_objects_received += 1 + self._skills_in_payload += 1 if not isinstance(raw.get("contentHash"), str): self.diagnostics.hashless_objects += 1 self._warn_hashless(raw) @@ -493,6 +556,8 @@ def _delete_object(self, data: Any) -> _TransferOutcome: return _TransferOutcome() target.delete(tombstone) self.diagnostics.objects_revoked += 1 + # A revocation identifies the payload as ours just as a put does. + self._skills_in_payload += 1 # A tombstone carries identity and no content, so a listener that reads # content must check for ``content`` rather than assume it. self._changes.append( @@ -503,11 +568,23 @@ def _delete_object(self, data: Any) -> _TransferOutcome: def _payload_transferred(self, data: Any) -> _TransferOutcome: state = data.get("state") if isinstance(data, dict) else None version = data.get("version") if isinstance(data, dict) else None - if self._pending is not None: + payload_id = self._intent_payload_id or _payload_id_from_selector(state) + if self._pending is not None and self._is_foreign_payload(payload_id): + self._warn_foreign_payload(payload_id) + self.diagnostics.payloads_ignored += 1 + self._changes = [] + elif self._pending is not None: self._committed.replace_with(self._pending) _warn_if_nothing_can_verify(self._committed) + if self._skills_in_payload and payload_id is not None: + # Learnt, not configured: nothing below the interface is told + # which payload is which, so the payload that carried a skill + # put or revocation is the payload skills arrive on. + self._skill_payload_id = payload_id self._pending = None self._intent = None + self._intent_payload_id = None + self._skills_in_payload = 0 changes = self._changes self._changes = [] self.diagnostics.payloads_transferred += 1 @@ -526,6 +603,8 @@ def _abandon_in_flight(self) -> None: """Drops the in-flight payload and keeps what is committed.""" self._pending = None self._intent = None + self._intent_payload_id = None + self._skills_in_payload = 0 self._changes = [] def _error(self, data: Any) -> _TransferOutcome: @@ -546,8 +625,59 @@ def _goodbye(self, data: Any) -> _TransferOutcome: ) return _TransferOutcome(disconnect=f"server said goodbye: {reason}") + # -- payload identity ---------------------------------------------------- + + def _is_foreign_payload(self, payload_id: str | None) -> bool: + """ + Whether a transfer completes a payload other than the one skills arrive on. + + ``False`` unless both payloads are known, so one-payload delivery and the + first transfer of a connection behave exactly as they did before this + check existed. + """ + return ( + self._skill_payload_id is not None + and payload_id is not None + and payload_id != self._skill_payload_id + ) + # -- diagnostics --------------------------------------------------------- + def _warn_multiple_payloads(self, payloads: list[Any]) -> None: + """ + One WARNING per reader for an intent describing more than one payload. + + Not an error: reading only the first is what the protocol asks for. But it + means the first payload is no longer *guaranteed* to be the skill payload, + and an intent for another payload arriving before any skill has been seen + is the one case ``_is_foreign_payload`` cannot catch. + """ + if self._warned_multiple_payloads: + return + self._warned_multiple_payloads = True + logger.warning( + "An FDv2 server-intent described %d payloads (%s). Only the first is " + "read, as the protocol requires, and it is taken to be the payload " + "skills arrive on. If skills stop resolving from this point, that is " + "the assumption that broke; contact LaunchDarkly support.", + len(payloads), + ", ".join(str(_payload_id_of(p)) for p in payloads), + ) + + def _warn_foreign_payload(self, payload_id: str | None) -> None: + """One WARNING per reader for a transfer this layer declined to apply.""" + if self._warned_foreign_payload: + return + self._warned_foreign_payload = True + logger.warning( + "An FDv2 transfer of payload %s was not applied to the skills held, " + "which arrive on payload %s. Applying it would have replaced them " + "with whatever that payload carried — nothing, in the case of a flag " + "payload. The skills held are unchanged.", + payload_id, + self._skill_payload_id, + ) + def _warn_hashless(self, raw: dict[str, Any]) -> None: """ One ERROR per ``(key, version)`` whose envelope had no ``contentHash``. diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index ff5720bc..87596609 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -482,6 +482,232 @@ def test_an_object_arriving_with_no_intent_is_treated_as_a_delta(self) -> None: assert len(held) == 1 +# --------------------------------------------------------------------------- +# Which payload a transfer completed +# --------------------------------------------------------------------------- + + +def _payload_warnings(caplog: Any, fragment: str) -> list[Any]: + return [ + r + for r in caplog.records + if r.levelname == "WARNING" and fragment in r.getMessage() + ] + + +def skill_payload( + *object_events: tuple[str, Any], + payload_id: str = "agent-skill", + code: str = "xfer-full", + state: str = "basis-1", +) -> list[dict[str, Any]]: + """One payload's events, with the payload it belongs to named explicitly.""" + return events( + ("server-intent", server_intent(code, payload_id)), + *object_events, + ("payload-transferred", transferred(state)), + ) + + +class TestPayloadIdentity: + """ + Which payload a transfer completed, and why this layer tracks it at all. + + Delivery provides one payload per credential and the protocol requires a + client to read only the first payload intent, so today the payload read is + the payload skills arrive on. These assert the behaviour that survives if + the first of those stops holding: another payload's ``xfer-full`` must not + publish an empty skill set, because with pruning on that deletes a + customer's materialized files. + """ + + def test_only_the_first_payload_intent_is_read(self) -> None: + """Reading only the first is what the protocol asks for, however many + arrive — the point of the rest of this class is to make that safe.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + events( + ( + "server-intent", + { + "payloads": [ + { + "id": "agent-skill", + "target": 1, + "intentCode": "xfer-full", + }, + {"id": "env-flags", "target": 2, "intentCode": "none"}, + ] + }, + ), + ("put-object", put_skill()), + ("payload-transferred", transferred()), + ), + ) + assert len(held) == 1 + + def test_more_than_one_payload_intent_warns_once(self, caplog: Any) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + intent = { + "payloads": [ + {"id": "env-flags", "target": 1, "intentCode": "xfer-changes"}, + {"id": "agent-skill", "target": 2, "intentCode": "xfer-changes"}, + ] + } + with caplog.at_level("WARNING"): + reader.handle("server-intent", intent) + reader.handle("server-intent", intent) + assert len(_payload_warnings(caplog, "described 2 payloads")) == 1 + + def test_one_payload_intent_warns_about_nothing(self, caplog: Any) -> None: + with caplog.at_level("WARNING"): + drive( + _ProtocolReader(_SkillObjectSet()), + skill_payload(("put-object", put_skill())), + ) + assert _payload_warnings(caplog, "payload") == [] + + def test_another_payloads_full_transfer_does_not_empty_the_skills_held( + self, caplog: Any + ) -> None: + """ + The case this guard exists for. A flag payload's ``xfer-full`` starts an + empty pending set; applying it at ``payload-transferred`` would publish + every skill as revoked, which a reconcile with pruning on reads as + "delete these files". + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, skill_payload(("put-object", put_skill()))) + with caplog.at_level("WARNING"): + outcomes = drive( + reader, + skill_payload( + ("put-object", put_flag()), payload_id="env-flags", state="basis-2" + ), + ) + assert held.get("pdf-extraction", None) is not None + assert reader.diagnostics.payloads_ignored == 1 + assert len(_payload_warnings(caplog, "was not applied")) == 1 + # Nothing changed, so no listener is woken to reconcile against it. + assert outcomes[-1].changes == [] + + def test_a_declined_transfer_warns_once_however_often_it_repeats( + self, caplog: Any + ) -> None: + """A polling connection sees the other payload on every poll.""" + reader = _ProtocolReader(_SkillObjectSet()) + drive(reader, skill_payload(("put-object", put_skill()))) + foreign = skill_payload(("put-object", put_flag()), payload_id="env-flags") + with caplog.at_level("WARNING"): + drive(reader, foreign) + drive(reader, foreign) + assert len(_payload_warnings(caplog, "was not applied")) == 1 + assert reader.diagnostics.payloads_ignored == 2 + + def test_a_full_transfer_of_the_skill_payload_still_empties_it(self) -> None: + """Every skill deleted is a real state, and the guard must not mask it.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, skill_payload(("put-object", put_skill()))) + drive(reader, skill_payload(state="basis-2")) + assert len(held) == 0 + assert reader.diagnostics.payloads_ignored == 0 + + def test_a_revocation_identifies_the_payload_as_the_skill_payload(self) -> None: + """A payload that only revokes is still a payload skills arrive on.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + skill_payload(("delete-object", delete_skill()), code="xfer-changes"), + ) + drive( + reader, skill_payload(("put-object", put_skill()), payload_id="env-flags") + ) + assert reader.diagnostics.payloads_ignored == 1 + + def test_the_payload_is_identified_from_the_selector_when_no_id_is_named( + self, + ) -> None: + """``payload-transferred``'s selector is the only other place a completed + transfer names its payload.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + unnamed = {"payloads": [{"target": 1, "intentCode": "xfer-full"}]} + drive( + reader, + events( + ("server-intent", unnamed), + ("put-object", put_skill()), + ("payload-transferred", transferred("(p:agent-skill:53)")), + ), + ) + drive( + reader, + events( + ("server-intent", unnamed), + ("put-object", put_flag()), + ("payload-transferred", transferred("(p:env-flags:12)")), + ), + ) + assert held.get("pdf-extraction", None) is not None + assert reader.diagnostics.payloads_ignored == 1 + + def test_an_unidentifiable_payload_is_applied_rather_than_withheld(self) -> None: + """ + A transfer naming no payload at all is the store's own, since delivery + sends it one payload. Withholding it would break the common case to + defend against a hypothetical one. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, skill_payload(("put-object", put_skill()))) + drive( + reader, + events( + ("server-intent", {"payloads": [{"intentCode": "xfer-full"}]}), + ("put-object", put_skill(object_version=4)), + ("payload-transferred", {"version": 44}), + ), + ) + assert held.get("pdf-extraction", None)["version"] == 4 + assert reader.diagnostics.payloads_ignored == 0 + + def test_the_first_transfer_of_a_connection_is_the_residual( + self, caplog: Any + ) -> None: + """ + Before a skill has arrived there is nothing to compare a payload + against, so another payload's ``xfer-full`` arriving first cannot be + told apart. The multiple-payload WARNING is the only signal there is, + which is why it exists. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + with caplog.at_level("WARNING"): + drive( + reader, + events( + ( + "server-intent", + { + "payloads": [ + {"id": "env-flags", "intentCode": "xfer-full"}, + {"id": "agent-skill", "intentCode": "xfer-full"}, + ] + }, + ), + ("put-object", put_flag()), + ("payload-transferred", transferred()), + ), + ) + assert len(held) == 0 + assert len(_payload_warnings(caplog, "described 2 payloads")) == 1 + + # --------------------------------------------------------------------------- # Interface parity with InMemorySkillStore # --------------------------------------------------------------------------- From efc4ca77daf50a878b5988a591613cd96b778fcb Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 15:24:45 -0400 Subject: [PATCH 17/22] =?UTF-8?q?fix(client):=20Agent=20Skills=20=E2=80=94?= =?UTF-8?q?=20read=20the=20skill's=20version=20off=20the=20wire=20key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SDK-facing FDv2 channel now delivers skills the way streamer #4681 and gonfalon #70638 spell them: object kinds are open strings, the agent-skill payload is classified `generic`, and every generic object carries only `key`, `kind`, `version` and `object`, exactly like a flag. A skill arrives under kind `skill` with its own version folded into the key as `:`. There is no `category` field and no `objectVersion` field; both came from an earlier streamer draft that never shipped. Identification is now the kind alone. The wire key is split in one place, `_split_wire_key`, and both the put and the delete translation go through it. A key that will not split cleanly is held rather than dropped — version-less, or with the offending text as its version — so verification withholds it with `invalid_version` under a key the caller recognises; only a key with nothing before the delimiter is dropped, since there is no identity to hold it under. `SDK_DATA_MODEL_VERSION` goes with it: the connection's `mv` parameter only accepts flag model versions, and generic payloads ignore it. The transport stops sending it in the following change. Co-Authored-By: Claude Fable 5.1 --- packages/client/agents.md | 46 ++--- .../src/launchdarkly_ai_server/skills_core.py | 6 +- .../src/launchdarkly_ai_server/skills_fdv2.py | 159 ++++++++++++------ packages/client/tests/test_skills_fdv2.py | 130 ++++++++++---- 4 files changed, 233 insertions(+), 108 deletions(-) diff --git a/packages/client/agents.md b/packages/client/agents.md index e73c96d4..978dc8ad 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,7 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | -| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the `objectVersion`/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | +| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the wire-key/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | | `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | @@ -226,29 +226,37 @@ shape `skills_core.SkillStore` documents. It lives below the store interface; ** that interface knows it exists**. If a transport change ever seems to require editing an accessor, verification, or `write_skills`, the adapter boundary is wrong. -**`objectVersion` is the skill's version. `version` is the payload's.** On the wire a skill -`put-object` carries both, and they are not interchangeable: +**The skill's version is in the object's `key`. `version` is the payload's.** Each version +of a skill is its own object on the wire, identified as `:`: ```json -{"key":"pdf-extraction","kind":"inline-resource","category":"skill", - "objectVersion":3,"version":42, +{"key":"pdf-extraction:3","kind":"skill","version":42, "object":{"contentType":"text/markdown","content":"…","contentHash":"…","name":"…"}} ``` -`objectVersion` (3) is what a `{key, version}` reference pins and what becomes the stored -`version`. `version` (42) is the version of the *payload* the object arrived in — it moves -when anything in the environment moves, including a flag with nothing to do with skills. -Reading it as the skill's version fails **silently**: the object verifies, the hash matches, -and the caller gets content under a version number that means nothing. Flags and segments -carry only `version` and omit both `category` and `objectVersion`, which is exactly why the -two fields look interchangeable. `_store_object_from_put` is the only place the translation -happens, and `TestVersionTranslation` asserts it in both directions. - -**Skills are identified by `kind == "inline-resource" && category == "skill"`; everything else -is ignored, not rejected.** An environment's payload assignment carries its flag payload -alongside its agent-skill payload, so flag and segment objects arrive as a matter of course. -Erroring on an unrecognised kind would turn a normal payload into a permanent reconnect -loop — a flag-delivery outage caused by a skills rollout. +The `3` after the delimiter is what a `{key, version}` reference pins and what becomes the +stored `version`, under the stored key `pdf-extraction`. `version` (42) is the version of the +*payload* the object arrived in — it moves when anything in the environment moves, +including a flag with nothing to do with skills. Reading it as the skill's version fails +**silently**: the object verifies, the hash matches, and the caller gets content under a +version number that means nothing. There is no separate field for the skill's version: the +agent-skill payload is a *generic* payload, and generic objects carry only `key`, `kind`, +`version` and `object`, exactly like a flag. `_split_wire_key` is the only place the wire key +is read, `_store_object_from_put` and `_tombstone_from_delete` both go through it, and +`TestVersionTranslation` asserts the translation in both directions. A wire key that will +not split cleanly is *held*, not dropped — version-less, or with the offending text as its +version — so verification withholds it with `invalid_version` under a key the caller +recognises; only a key with nothing before the delimiter is dropped, since there is no +identity to hold it under. + +**Skills are identified by `kind == "skill"`; everything else is ignored, not rejected.** +Object kinds on the SDK-facing channel are open strings, and the agent-skill payload is +classified `generic`, so a skill arrives under the kind its producer registered — the bare +category name — not under a broader wrapper kind with a narrowing field. An environment's +payload assignment carries its flag payload alongside its agent-skill payload, so flag and +segment objects arrive as a matter of course. Erroring on an unrecognised kind would turn a +normal payload into a permanent reconnect loop — a flag-delivery outage caused by a skills +rollout. **Changes commit at `payload-transferred`, not as objects arrive.** A payload version is the unit of consistency: a half-applied full transfer would publish a state the server never diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index 22166799..d2d870c4 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -56,9 +56,9 @@ An **internal seam value**, deliberately not exported from the package root. It is the string ``skills.py`` and ``skills_fs.py`` pass to ``SkillStore.get_object`` and ``SkillStore.all_objects``, and a store adapter is free to map it onto -whatever the transport underneath actually uses — a delivery payload may well -carry skills under a broader kind with a narrower category, in which case -translating that pair to this one value is the adapter's job. +whatever the transport underneath actually uses — the value happens to match +the kind LaunchDarkly's delivery channel uses today, but a transport that spelt +it differently would translate, and that translation is the adapter's job. Exporting it would publish an SDK-side seam string as though it were the wire contract, which is a claim this side cannot make and would be hard to walk back diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index f644a349..d20a0e2f 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -3,7 +3,7 @@ The half of the delivery transport that has no I/O: identifying skill objects on the wire, translating them into the raw object shape the ``SkillStore`` -interface defines, holding them by ``(key, objectVersion)``, and applying a +interface defines, holding them by ``(key, version)``, and applying a payload's events as one consistent commit. ``FDv2SkillStore``, the store that puts a network connection underneath this, follows in a separate change. @@ -42,9 +42,9 @@ the same payload. ``_ProtocolReader`` keeps the pair apart anyway, because the cost of conflating them is an emptied skill set. -The design rationale — why ``objectVersion`` is not ``version``, why changes -commit at ``payload-transferred`` — is in ``agents.md`` under *The delivery -transport*. +The design rationale — why the skill's version is read from the object's +``key`` and never from ``version``, why changes commit at +``payload-transferred`` — is in ``agents.md`` under *The delivery transport*. """ from __future__ import annotations @@ -63,21 +63,27 @@ # The wire contract # --------------------------------------------------------------------------- -FDV2_OBJECT_KIND = "inline-resource" +FDV2_OBJECT_KIND = "skill" """ -The FDv2 ``kind`` skills are delivered under. Together with -``FDV2_OBJECT_CATEGORY`` it maps onto the single interface value -``skills_core.SKILL_OBJECT_KIND``; that translation is this adapter's job. +The FDv2 ``kind`` skills are delivered under. + +Object kinds on the SDK-facing channel are open strings: the agent-skill payload +is classified ``generic`` and every object in it carries the kind its producer +registered, which for skills is the bare category name. Delivery lower-cases the +kind, so an exact comparison is the whole test. The kind happens to equal +``skills_core.SKILL_OBJECT_KIND`` today; they are still separate constants, +because one is a wire value LaunchDarkly owns and the other is an SDK seam. """ -FDV2_OBJECT_CATEGORY = "skill" -"""The ``category`` that narrows ``inline-resource`` to an agent skill.""" - -SDK_DATA_MODEL_VERSION = 1 +FDV2_KEY_DELIMITER = ":" """ -The ``mv`` request parameter. The one request parameter whose value could not be -confirmed against a live server, so treat the default as provisional and -override it through the store's ``data_model_version`` if needed. +What separates a skill's key from its version inside the object's wire ``key``. + +A generic object is identified on the wire as ``:`` — the skill's +own key, one delimiter, the skill's own version — because each version of a +skill is a distinct object in the payload. Delivery forbids the delimiter inside +a registered category and skill keys cannot contain it, so a well-formed wire key +has exactly one. """ _EVENT_SERVER_INTENT = "server-intent" @@ -204,7 +210,7 @@ class StoreDiagnostics: # --------------------------------------------------------------------------- -# Deserialisation — where objectVersion is not version +# Deserialisation — where the skill's version lives in the key, not in version # --------------------------------------------------------------------------- @@ -220,17 +226,58 @@ def _is_skill_event(data: Any) -> bool: """ Whether one ``put-object`` / ``delete-object`` payload is a skill. - Both halves are required: ``inline-resource`` may carry other categories, - and flags and segments omit ``category`` entirely. Every other kind is - ignored, not rejected, because flag and segment objects share the connection - and erroring on them would turn a normal payload into a reconnect loop. + The kind alone decides it. Every other kind is ignored, not rejected, + because flag and segment objects share the connection and erroring on them + would turn a normal payload into a reconnect loop. """ if not isinstance(data, dict): return False - return ( - data.get("kind") == FDV2_OBJECT_KIND - and data.get("category") == FDV2_OBJECT_CATEGORY - ) + return data.get("kind") == FDV2_OBJECT_KIND + + +@dataclass(frozen=True) +class _WireIdentity: + """A skill object's wire ``key``, split into the skill's key and version.""" + + key: str + version: Any + """``int`` when the wire carried one; the offending text when it did not; + absent (``_NO_VERSION``) when the wire key had no delimiter at all.""" + + +_NO_VERSION = object() + + +def _split_wire_key(wire_key: Any) -> _WireIdentity | None: + """ + Reads ``:`` off one object's wire ``key``. + + Lenient where leniency keeps the object diagnosable and strict only where + there is nothing to diagnose: + + - No delimiter: the whole wire key is the skill key and there is no version, + so the object is held version-less and verification reports + ``invalid_version`` under a key the caller can recognise. + - A version that is not a run of digits (``"pdf:latest"``, ``"pdf:"``, + ``"a:1:2"``): the text is carried through *as the version*, for the same + reason — the caller learns that ``pdf`` arrived broken, not that it is + absent. + - An empty key before the delimiter (``":3"``): there is no identity to hold + it under, so ``None``, and the caller drops it. + + Leading zeros are accepted (``"pdf:03"`` is version 3) since ``int`` is the + identity a reference pins, not the spelling. + """ + if not isinstance(wire_key, str) or not wire_key: + return None + key, delimiter, version_text = wire_key.partition(FDV2_KEY_DELIMITER) + if not key: + return None + if not delimiter: + return _WireIdentity(key=key, version=_NO_VERSION) + if version_text.isascii() and version_text.isdigit(): + return _WireIdentity(key=key, version=int(version_text)) + return _WireIdentity(key=key, version=version_text) def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: @@ -240,33 +287,36 @@ def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: **The one translation this adapter must get right:** - wire ``objectVersion`` → stored ``version`` (the skill's own version) - wire ``version`` → dropped (the *payload* version) + wire ``key`` → stored ``key`` and ``version`` (split on ``:``) + wire ``version`` → dropped (the *payload* version) - ``objectVersion`` is what a ``{key, version}`` reference pins; ``version`` - moves whenever anything in the environment moves. Confusing them fails - silently: the object verifies and the caller gets content under a version - number that means nothing. + Each version of a skill is its own object on the wire, identified as + ``:``; that version is what a ``{key, version}`` reference + pins. The event's ``version`` field is the version of the payload the object + arrived in and moves whenever anything in the environment moves. Confusing + them fails silently: the object verifies and the caller gets content under a + version number that means nothing. - Returns ``None`` only when ``key`` is not a string, since a keyless object - has no identity to store it under. Every other defect is carried through - verbatim so that verification withholds it with a reason code rather than - the transport dropping it into indistinguishable absence. + Returns ``None`` only when the wire ``key`` carries no skill key at all, + since such an object has no identity to store it under. Every other defect + is carried through so that verification withholds it with a reason code + rather than the transport dropping it into indistinguishable absence. """ - key = data.get("key") - if not isinstance(key, str) or not key: + identity = _split_wire_key(data.get("key")) + if identity is None: logger.warning( - "An FDv2 skill put-object carried no string 'key' and could not be " - "stored under any identity; it was dropped." + "An FDv2 skill put-object carried no usable 'key' (%r) and could not " + "be stored under any identity; it was dropped.", + data.get("key"), ) return None - raw: dict[str, Any] = {"key": key} + raw: dict[str, Any] = {"key": identity.key} - # A membership test rather than a `.get` default, so an explicitly-null - # objectVersion stays null and reaches verification as `invalid_version`. - if "objectVersion" in data: - raw["version"] = data["objectVersion"] + # Absent stays absent and malformed stays malformed, so verification sees + # what arrived (as `invalid_version`) rather than something invented here. + if identity.version is not _NO_VERSION: + raw["version"] = identity.version envelope = data.get("object") if isinstance(envelope, dict): @@ -294,25 +344,26 @@ def _payload_id_from_selector(state: Any) -> str | None: def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: """ - Narrows one FDv2 skill ``delete-object`` to the identity it revokes, with - the same ``objectVersion`` translation as a put. + Narrows one FDv2 skill ``delete-object`` to the identity it revokes, reading + the wire ``key`` the same way a put does. An ``object_version`` of ``None`` means the delete named no usable version and is read as "revoke every version of this key". That is the safe direction: the alternative is continuing to serve content LaunchDarkly has - withdrawn. + withdrawn. It also removes whatever a malformed put of the same wire key + left held, since that was stored version-less under the same skill key. """ - key = data.get("key") - if not isinstance(key, str) or not key: + identity = _split_wire_key(data.get("key")) + if identity is None: logger.warning( - "An FDv2 skill delete-object carried no string 'key'; it was ignored." + "An FDv2 skill delete-object carried no usable 'key' (%r); it was ignored.", + data.get("key"), ) return None - object_version = data.get("objectVersion") return _Tombstone( - key=key, - object_version=object_version - if is_valid_skill_version(object_version) + key=identity.key, + object_version=identity.version + if is_valid_skill_version(identity.version) else None, ) @@ -324,7 +375,7 @@ def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: class _SkillObjectSet: """ - Raw skill objects held in memory, keyed by ``(key, objectVersion)``. + Raw skill objects held in memory, keyed by ``(key, version)``. Lookup semantics are identical to ``InMemorySkillStore``'s, down to the fall-through to a version-less entry, so that the store a caller configures diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index 87596609..b88d14d9 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -1,8 +1,8 @@ """ Tests for the FDv2 skill delivery protocol. -Wire semantics — which objects are skills, ``objectVersion`` versus ``version``, -revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted +Wire semantics — which objects are skills, the skill's version in the wire ``key`` +versus the payload's in ``version``, revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted against ``_ProtocolReader``, which has no I/O, so each case reads as the contract it is rather than as a server script. """ @@ -17,7 +17,7 @@ from launchdarkly_ai_server import InMemorySkillStore from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND from launchdarkly_ai_server.skills_fdv2 import ( - FDV2_OBJECT_CATEGORY, + FDV2_KEY_DELIMITER, FDV2_OBJECT_KIND, _is_skill_event, _ProtocolReader, @@ -40,6 +40,18 @@ def _hash(content: str) -> str: # --------------------------------------------------------------------------- +def wire_key(key: str, object_version: Any) -> str: + """ + The wire ``key`` of one skill object: ``:``. + + ``None`` builds a key with no version at all, which is how the tests spell a + malformed object; anything else is spelled after the delimiter verbatim. + """ + if object_version is None: + return key + return f"{key}{FDV2_KEY_DELIMITER}{object_version}" + + def put_skill( key: str = "pdf-extraction", *, @@ -62,10 +74,8 @@ def put_skill( content_hash if content_hash is not None else _hash(content) ) return { - "key": key, + "key": wire_key(key, object_version), "kind": FDV2_OBJECT_KIND, - "category": FDV2_OBJECT_CATEGORY, - "objectVersion": object_version, "version": payload_version, "object": envelope, } @@ -75,16 +85,14 @@ def delete_skill( key: str = "pdf-extraction", *, object_version: Any = 3, payload_version: int = 43 ) -> dict[str, Any]: return { - "key": key, + "key": wire_key(key, object_version), "kind": FDV2_OBJECT_KIND, - "category": FDV2_OBJECT_CATEGORY, - "objectVersion": object_version, "version": payload_version, } def put_flag(key: str = "my-flag", version: int = 17) -> dict[str, Any]: - """A flag ``put-object``: no ``category``, no ``objectVersion``.""" + """A flag ``put-object``: the same envelope fields, a different ``kind``.""" return { "key": key, "kind": "flag", @@ -141,30 +149,37 @@ def full_payload( class TestObjectIdentification: - def test_kind_and_category_together_identify_a_skill(self) -> None: + def test_the_kind_alone_identifies_a_skill(self) -> None: assert _is_skill_event(put_skill()) is True + def test_the_kind_is_the_bare_category_name(self) -> None: + """ + Object kinds on the channel are open strings and the agent-skill payload + is ``generic``, so a skill arrives under the kind its producer + registered — ``skill`` — not under a broader wrapper kind. + """ + assert FDV2_OBJECT_KIND == "skill" + def test_a_flag_is_not_a_skill(self) -> None: assert _is_skill_event(put_flag()) is False def test_a_segment_is_not_a_skill(self) -> None: assert _is_skill_event(put_segment()) is False - def test_inline_resource_of_another_category_is_not_a_skill(self) -> None: - """``inline-resource`` is a broad kind, so the category is required too.""" + def test_another_generic_kind_is_not_a_skill(self) -> None: + """A generic payload may carry other registered kinds one day.""" other = put_skill() - other["category"] = "prompt-template" + other["kind"] = "prompt-template" assert _is_skill_event(other) is False - def test_skill_category_under_another_kind_is_not_a_skill(self) -> None: + def test_a_skill_shaped_envelope_under_another_kind_is_not_a_skill(self) -> None: other = put_skill() other["kind"] = "some-future-kind" assert _is_skill_event(other) is False - def test_a_flag_shaped_object_with_no_category_is_not_a_skill(self) -> None: - """Flags and segments omit ``category`` entirely — the documented shape.""" - assert "category" not in put_flag() - assert "objectVersion" not in put_flag() + def test_nothing_but_the_kind_is_consulted(self) -> None: + """No secondary field narrows the kind, and none may be required.""" + assert set(put_skill()) == {"key", "kind", "version", "object"} @pytest.mark.parametrize("value", [None, "skill", 3, [], ()]) def test_non_dict_events_are_not_skills(self, value: Any) -> None: @@ -172,15 +187,27 @@ def test_non_dict_events_are_not_skills(self, value: Any) -> None: # --------------------------------------------------------------------------- -# objectVersion is not version +# The skill's version is in the wire key; `version` is the payload's # --------------------------------------------------------------------------- class TestVersionTranslation: - def test_object_version_becomes_the_seam_version(self) -> None: + def test_the_wire_key_is_key_colon_version(self) -> None: + assert ( + put_skill("pdf-extraction", object_version=3)["key"] == "pdf-extraction:3" + ) + + def test_the_version_after_the_delimiter_becomes_the_seam_version(self) -> None: raw = _store_object_from_put(put_skill(object_version=3, payload_version=42)) assert raw is not None assert raw["version"] == 3 + assert isinstance(raw["version"], int) + + def test_the_key_before_the_delimiter_becomes_the_seam_key(self) -> None: + """A caller asks for ``pdf-extraction``, never for ``pdf-extraction:3``.""" + raw = _store_object_from_put(put_skill("pdf-extraction", object_version=3)) + assert raw is not None + assert raw["key"] == "pdf-extraction" def test_the_payload_version_never_reaches_the_seam(self) -> None: """ @@ -200,36 +227,75 @@ def test_the_two_are_distinguished_even_when_the_payload_version_is_lower( assert raw is not None assert raw["version"] == 99 - def test_a_missing_object_version_is_not_defaulted_from_the_payload(self) -> None: - wire = put_skill() - del wire["objectVersion"] - raw = _store_object_from_put(wire) + def test_a_key_with_no_delimiter_is_held_version_less(self) -> None: + """Not defaulted from the payload version, and not dropped: verification + reports ``invalid_version`` under a key the caller recognises.""" + raw = _store_object_from_put(put_skill(object_version=None)) assert raw is not None + assert raw["key"] == "pdf-extraction" assert "version" not in raw - def test_an_explicitly_null_object_version_is_carried_through_as_null(self) -> None: - """Carried, not invented: verification reports ``invalid_version``.""" - raw = _store_object_from_put(put_skill(object_version=None)) + @pytest.mark.parametrize("spelling", ["latest", "", "3.0", "-1", "1:2", "３"]) + def test_a_version_that_is_not_digits_is_carried_through_as_invalid( + self, spelling: str + ) -> None: + """Carried, not invented: verification reports ``invalid_version`` for + the object rather than the transport reporting it absent.""" + raw = _store_object_from_put(put_skill(object_version=spelling)) + assert raw is not None + assert raw["key"] == "pdf-extraction" + assert raw["version"] == spelling + + def test_leading_zeros_spell_the_same_version(self) -> None: + raw = _store_object_from_put(put_skill(object_version="03")) assert raw is not None - assert raw["version"] is None + assert raw["version"] == 3 - def test_a_delete_translates_object_version_too(self) -> None: + def test_a_delete_reads_the_wire_key_the_same_way(self) -> None: tombstone = _tombstone_from_delete( delete_skill(object_version=3, payload_version=43) ) assert tombstone is not None + assert tombstone.key == "pdf-extraction" assert tombstone.object_version == 3 - def test_a_delete_with_no_usable_object_version_revokes_every_version(self) -> None: - tombstone = _tombstone_from_delete(delete_skill(object_version=None)) + @pytest.mark.parametrize("spelling", [None, "latest", "0"]) + def test_a_delete_with_no_usable_version_revokes_every_version( + self, spelling: Any + ) -> None: + tombstone = _tombstone_from_delete(delete_skill(object_version=spelling)) assert tombstone is not None + assert tombstone.key == "pdf-extraction" assert tombstone.object_version is None + @pytest.mark.parametrize("bad_key", [":3", "", None, 3]) + def test_a_put_with_no_skill_key_is_dropped_because_it_has_no_identity( + self, bad_key: Any + ) -> None: + wire = put_skill() + wire["key"] = bad_key + assert _store_object_from_put(wire) is None + def test_a_keyless_put_is_dropped_because_it_has_no_identity(self) -> None: wire = put_skill() del wire["key"] assert _store_object_from_put(wire) is None + def test_a_delete_with_no_skill_key_is_ignored(self) -> None: + wire = delete_skill() + wire["key"] = ":3" + assert _tombstone_from_delete(wire) is None + + def test_the_stored_identity_round_trips_to_the_wire_key(self) -> None: + """``_SkillObjectSet.snapshot`` spells its opaque keys the way the wire + does, so a held object can be matched back to the event that carried it.""" + held = _SkillObjectSet() + wire = put_skill("pdf-extraction", object_version=3) + raw = _store_object_from_put(wire) + assert raw is not None + held.put(raw) + assert set(held.snapshot()) == {wire["key"]} + def test_the_envelope_is_copied_verbatim(self) -> None: raw = _store_object_from_put(put_skill()) assert raw is not None From b6a25f9f455258dabb8d13b13978123239327376 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Thu, 10 Sep 2026 16:05:04 -0400 Subject: [PATCH 18/22] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20the=20FDv2=20delivery=20transport?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FDv2SkillStore puts LaunchDarkly's SDK-facing FDv2 channel underneath the protocol layer: GET /sdk/poll and GET /sdk/stream, authenticated with the environment's server-side SDK key, streaming by default. It carries basis across requests, sends If-None-Match and treats 304 as a current answer, retries with capped jittered backoff, honours Retry-After only up to max_backoff, gives up after a bounded run of consecutive failures where a committed payload resets the count, and keeps serving last known good through every failure. A mobile key or client-side environment ID is refused in the constructor. Standard library only. close interrupts the socket rather than only setting a flag, because the delivery thread lives in a read no flag can reach; without that every shutdown of a healthy stream waited out the full join timeout. The no-store message now names FDv2SkillStore first, and watch_skills points at it as the store with a delivery transport. Co-Authored-By: Claude Fable 5.1 --- packages/client/README.md | 62 + packages/client/agents.md | 25 +- .../src/launchdarkly_ai_server/__init__.py | 5 +- .../src/launchdarkly_ai_server/skills_core.py | 13 +- .../src/launchdarkly_ai_server/skills_fdv2.py | 748 ++++++++- .../launchdarkly_ai_server/skills_watch.py | 2 +- packages/client/tests/test_skills.py | 12 + packages/client/tests/test_skills_fdv2.py | 1332 ++++++++++++++++- packages/client/tests/test_skills_watch.py | 5 +- 9 files changed, 2162 insertions(+), 42 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 41d5c6b3..42b1f40d 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -445,6 +445,66 @@ which OS ran the write. The keys stay valid everywhere else: an AI Config refere named `aux` parses, and its other fields are unaffected. If you have a skill named for a device, rename it. +#### Receiving skills from LaunchDarkly + +`InMemorySkillStore` is for tests and bring-your-own-content. In production, skill content +arrives through `FDv2SkillStore`, which speaks LaunchDarkly's SDK-facing FDv2 delivery +channel — the same `GET /sdk/poll` and `GET /sdk/stream` endpoints the base SDK's FDv2 data +source uses, authenticated with the environment's server-side SDK key. + +```python +import os + +from launchdarkly_ai_server import FDv2SkillStore, init_client, watch_skills + +store = FDv2SkillStore(os.environ["LD_SDK_KEY"]).start() +store.wait_for_skills(timeout=10) +await init_client(options={"skillStore": store}) + +# Materialize now, and re-materialize whenever delivery changes. +report, watcher = await watch_skills("*", ".claude/skills") +try: + ... +finally: + watcher.close() + store.close() +``` + +**Nothing above the store changes.** The accessors, verification, and `write_skills` see raw +objects through the `SkillStore` interface and cannot tell which store produced them. + +**Server-side only.** Skills are for server-side agent runtimes and skill content is +customer-confidential. A mobile key (`mob-…`) or a client-side environment ID raises from the +constructor. + +**Streaming is the default, and it is what makes revocation fast.** A `delete-object` reaches +a live stream in seconds; with `mode="poll"` it arrives within one `poll_interval`. Paired +with `watch_skills`, a revoked skill's `SKILL.md` leaves the disk without a restart. During an +outage the store keeps serving the last content it received and `write_skills`' default +`on_unavailable="keep"` leaves managed files alone — an outage must not read as "everything +was revoked". + +**One network timeout, and its default depends on the mode.** `read_timeout` bounds every +socket operation of a request, connecting included. In `mode="poll"` it bounds the whole +request and defaults to 10 seconds; in `mode="stream"` it bounds each wait for the next bytes +and defaults to 300 seconds, well beyond LaunchDarkly's heartbeat interval. + +**The connection also carries your flags.** A client cannot request only the skill payload, +so a skills-enabled environment delivers flag and segment objects on the same connection. +They are skipped, not evaluated — this store does no evaluation of any kind — and +`diagnostics.objects_ignored` counts them. + +> **Beta caveats, worth knowing before you deploy.** Payload signing does not exist on this +> channel yet, so delivery is TLS-only and the content hash establishes self-consistency, not +> origin authenticity. The FDv2 protocol is opt-in per account: without it the endpoints +> return HTTP 403, which the store reports as a fatal error explaining what to do. `ld-relay` +> does not speak the FDv2 endpoints, so relay-only deployments cannot receive skills. + +**If every skill comes back empty, check `diagnostics.hashless_objects`.** Verification +withholds any delivered object without a `contentHash`, so a nonzero count means skills are +being withheld rather than that the environment has none. The store also logs an error per +hashless object naming the reason. There is deliberately no fallback that skips verification. + **Total path length is yours to bound, not the SDK's.** The 255-byte bound above is per *component*; the root is your path, so `` + `` + `/SKILL.md` can still exceed Windows' 260-character `MAX_PATH` with a perfectly legal key. Choose a short managed root on @@ -460,7 +520,9 @@ Windows. | `write_skills(skills, root, *, prune=True, timeout=10.0, on_unavailable="keep")` | Materialize skills under `root`, returning a `ReconcileReport`. `prune` removes formerly-managed skills no longer requested. `on_unavailable="raise"` raises instead of reporting when content cannot be retrieved. Raises `ValueError` for an unusable root, a negative `timeout`, or an unrecognised `on_unavailable`. **Performs synchronous filesystem I/O — see the note below.** | | `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)` / `remove_listener(kind, fn)`. | | `InMemorySkillStore(objects=None)` | A dict-backed store with `put(raw)`, for local development and testing. Holds several versions of a key. | +| `FDv2SkillStore(sdk_key, *, base_uri=…, mode="stream", …)` | The delivery transport: a store fed by LaunchDarkly over the SDK-facing FDv2 channel. `start()`, `wait_for_skills(timeout)`, `close()`, `diagnostics`, `failed`; also a context manager. **Server-side only** — a mobile key or client-side environment ID raises. See *Receiving skills from LaunchDarkly* above. | | `watch_skills(skills, root, …)` | `write_skills` plus a re-reconcile on every delivery change. Returns `(initial report, SkillWatcher)`; close the watcher when done. Revocation then takes effect within `debounce` of arriving rather than at the next restart. | +| `StoreDiagnostics` | What the transport has seen: `payloads_transferred`, `skill_objects_received`, `objects_ignored`, `objects_revoked`, `hashless_objects`, `connection_failures`, `last_error`. | Configure the store with `init_client(options={"skillStore": store})`. With none configured, the accessors raise `RuntimeError` explaining what to do and `write_skills` reports the diff --git a/packages/client/agents.md b/packages/client/agents.md index 978dc8ad..2a19c273 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,7 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | -| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the wire-key/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | +| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery transport — the FDv2 protocol, the wire-key/`version` translation, the held object set, and `FDv2SkillStore`. Sits **below** the store interface; imports `skills_core` only, and nothing imports it | | `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | @@ -221,10 +221,12 @@ of skills, and the `"*"` reconcile, since `//SKILL.md` is a single pa ### The delivery transport, and the one field that will bite you -`skills_fdv2.py` translates LaunchDarkly's FDv2 delivery protocol into raw objects in the -shape `skills_core.SkillStore` documents. It lives below the store interface; **nothing above -that interface knows it exists**. If a transport change ever seems to require editing an -accessor, verification, or `write_skills`, the adapter boundary is wrong. +`FDv2SkillStore` speaks LaunchDarkly's SDK-facing FDv2 channel (`GET /sdk/poll`, +`GET /sdk/stream`, server-side SDK key in `Authorization`, `basis` + `mv` params, +`If-None-Match`/304). It lives below the store interface and produces raw objects in the +shape `skills_core.SkillStore` documents; **nothing above that interface knows it exists**. If a transport +change ever seems to require editing an accessor, verification, or `write_skills`, the adapter +boundary is wrong. **The skill's version is in the object's `key`. `version` is the payload's.** Each version of a skill is its own object on the wire, identified as `:`: @@ -285,6 +287,19 @@ such skill" — and would let a prune delete the last known-good copy on disk. N a hash from the delivered content: that certifies the content against itself and verifies nothing. +**There is one network timeout, not two.** `urllib`'s `timeout` is the socket timeout for the +whole operation, so connect, headers and each read share it, and the module cannot bound the +connect separately without a custom connection class it should not carry. `read_timeout` is +therefore the only knob, and its default is per mode (`DEFAULT_POLL_TIMEOUT` for a whole poll +request, `DEFAULT_STREAM_READ_TIMEOUT` for the gap between reads on a stream). Do not add a +parameter that the standard library cannot honour; `TestTimeouts` measures the bound against a +socket that accepts and never answers. + +**`close` interrupts the socket, it does not just set a flag.** The delivery thread spends its +life blocked in a read that no flag can reach, and closing a response from another thread does +not unblock CPython's buffered reader. `_interrupt_read` shuts the socket down underneath it. +Without that, every shutdown of a *healthy* stream blocks for the full join timeout. + ### The reported outcome vocabulary, and the `Resolution` mapping `get_skill` returns `Skill | None`; `get_skill_result` returns a frozen `SkillOutcome` diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index f863d847..0f987117 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -46,6 +46,7 @@ skill_refs, ) from .skills_core import SkillStore +from .skills_fdv2 import FDv2SkillStore, StoreDiagnostics from .skills_fs import ( MANIFEST_FILENAME, MANIFEST_VERSION, @@ -238,7 +239,9 @@ "write_skills", "SkillStore", "InMemorySkillStore", - # skills — the eager re-reconcile + # skills — the FDv2 delivery transport, and the eager re-reconcile it enables + "FDv2SkillStore", + "StoreDiagnostics", "watch_skills", "SkillWatcher", # skills — the three closed-set unions a typed consumer needs to name diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index d2d870c4..b8043012 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -134,9 +134,18 @@ NO_STORE_MESSAGE = ( "No skill store is configured, so skill content cannot be retrieved. Configure " - 'one with init_client(options={"skillStore": store}) — InMemorySkillStore is ' - "available for local development and testing." + 'one with init_client(options={"skillStore": store}) — FDv2SkillStore receives ' + "content from LaunchDarkly, and InMemorySkillStore is available for local " + "development and testing." ) +""" +The first thing a user sees when no store is configured, so it names both stores. + +``FDv2SkillStore`` comes first because it is the answer in production, and a +message that offered only ``InMemorySkillStore`` would point a deployment at the +development store. Callers match on "skill store"; keep that phrase if the +wording changes. +""" # --------------------------------------------------------------------------- diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index d20a0e2f..da84a3b4 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -1,15 +1,10 @@ """ -Agent Skills — the FDv2 delivery protocol. +Agent Skills — the FDv2 delivery transport. -The half of the delivery transport that has no I/O: identifying skill objects -on the wire, translating them into the raw object shape the ``SkillStore`` -interface defines, holding them by ``(key, version)``, and applying a -payload's events as one consistent commit. ``FDv2SkillStore``, the store that -puts a network connection underneath this, follows in a separate change. - -It sits *below* the ``SkillStore`` interface, and everything above — the -accessors, integrity verification, the ``Skill`` dataclass, materialization — -is unaware of it. +The store implementation that talks to LaunchDarkly. It sits *below* the +``SkillStore`` interface: it produces raw wire objects in the shape +``skills_core`` documents, and everything above — the accessors, integrity +verification, the ``Skill`` dataclass, materialization — is unaware of it. Layering:: @@ -20,12 +15,12 @@ GET /sdk/poll, GET /sdk/stream, authenticated with the environment's server-side SDK key -Dependencies run one way: this module imports nothing from the feature beyond -the version validator in ``types_validation``, and nothing in the feature -imports it. It uses only the standard library, so it adds no dependency +Dependencies run one way: this module imports ``skills_core`` for the +interface's kind constant and nothing else from the feature, and nothing in the +feature imports it. It uses only the standard library, so it adds no dependency to a package whose sole runtime dependency is ``opentelemetry-api``. -Three things this layer does *not* do, on purpose: +Three things this module does *not* do, on purpose: - **It does not verify content.** Verification lives at the accessor boundary in ``skills_core`` so that it applies to every store equally, including a @@ -36,24 +31,29 @@ - **It does not evaluate anything.** Flag and segment objects that share the connection are skipped and counted, nothing more. -One assumption it *does* make, and states: **the payload intent it reads is the -payload skills arrive on.** Delivery sends one payload per credential and the -protocol tells a client to read only the first payload intent, so today those are -the same payload. ``_ProtocolReader`` keeps the pair apart anyway, because the -cost of conflating them is an emptied skill set. - The design rationale — why the skill's version is read from the object's ``key`` and never from ``version``, why changes commit at -``payload-transferred`` — is in ``agents.md`` under *The delivery transport*. +``payload-transferred``, why there is one network timeout — is in +``agents.md`` under *The delivery transport*. """ from __future__ import annotations +import json import logging +import math +import random import re +import socket +import threading +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable from dataclasses import dataclass, field -from typing import Any +from typing import Any, Literal +from .skills_core import SKILL_OBJECT_KIND from .types_validation import is_valid_skill_version logger = logging.getLogger(__name__) @@ -86,6 +86,20 @@ has exactly one. """ +DEFAULT_BASE_URI = "https://sdk.launchdarkly.com" +"""Where the SDK-facing FDv2 endpoints live. Overridable for Federal and private +instances.""" + +POLL_PATH = "/sdk/poll" +STREAM_PATH = "/sdk/stream" + +DEFAULT_POLL_TIMEOUT = 10.0 +"""Default ``read_timeout`` in ``"poll"`` mode: the bound on one whole request.""" + +DEFAULT_STREAM_READ_TIMEOUT = 300.0 +"""Default ``read_timeout`` in ``"stream"`` mode: the longest gap tolerated +between two reads. LaunchDarkly's heartbeats arrive well inside this.""" + _EVENT_SERVER_INTENT = "server-intent" _EVENT_PUT_OBJECT = "put-object" _EVENT_DELETE_OBJECT = "delete-object" @@ -115,6 +129,8 @@ no ``id``. """ +Mode = Literal["stream", "poll"] + _MOBILE_KEY_PREFIX = "mob-" _SERVER_KEY_PREFIX = "sdk-" _CLIENT_SIDE_ID = re.compile(r"\A[0-9a-f]{20,}\Z") @@ -504,7 +520,6 @@ def __init__(self, committed: _SkillObjectSet) -> None: self.diagnostics = StoreDiagnostics() # Identities already reported by ``_warn_hashless``. Per reader, so a # recreated store reports again and two stores never quieten each other. - # No lock: ``handle`` runs only on its owner's single delivery thread. self._warned_hashless: set[tuple[str, Any]] = set() # The payload the current intent describes, and the payload skills have # actually arrived on. One payload per connection makes these the same @@ -609,8 +624,8 @@ def _delete_object(self, data: Any) -> _TransferOutcome: self.diagnostics.objects_revoked += 1 # A revocation identifies the payload as ours just as a put does. self._skills_in_payload += 1 - # A tombstone carries identity and no content, so a listener that reads - # content must check for ``content`` rather than assume it. + # A tombstone carries identity and no content; see + # ``FDv2SkillStore.add_listener`` for what listeners should expect. self._changes.append( {"key": tombstone.key, "version": tombstone.object_version} ) @@ -780,3 +795,686 @@ def _warn_if_nothing_can_verify(committed: _SkillObjectSet) -> None: len(held), _HASHLESS_ADVICE, ) + + +# --------------------------------------------------------------------------- +# HTTP +# --------------------------------------------------------------------------- + + +class _FatalTransportError(Exception): + """A failure retrying cannot fix: bad credential, forbidden, wrong URI.""" + + +class _RecoverableTransportError(Exception): + """A failure worth retrying. Carries a server-requested delay when given one.""" + + def __init__(self, message: str, retry_after: float | None = None) -> None: + super().__init__(message) + self.retry_after = retry_after + + +_FORBIDDEN_ADVICE = ( + "The FDv2 protocol is opt-in per LaunchDarkly account and is served as HTTP " + "403 while it is off. Skill delivery needs it enabled; contact LaunchDarkly " + "support to enable it for your account." +) + + +def _retry_after_seconds(headers: Any) -> float | None: + """ + ``Retry-After`` in seconds, when the server sent a usable one. + + The HTTP-date form, and non-finite values such as ``inf`` or ``1e309`` that + ``float`` accepts, fall back to our own backoff: none of them is a delay, + and an infinite one would overflow the wait that honours it. + """ + if headers is None: + return None + try: + raw = headers.get("Retry-After") + except AttributeError: + return None + if raw is None: + return None + try: + seconds: float = float(str(raw).strip()) + except ValueError: + return None + if not math.isfinite(seconds): + return None + return max(0.0, seconds) + + +def _classify_status(status: int, headers: Any) -> Exception: + """Turns an HTTP error status into the right exception type.""" + if status == 401: + return _FatalTransportError( + "LaunchDarkly rejected the SDK key (HTTP 401). Skill delivery cannot " + "start. Check that the key is the environment's server-side SDK key." + ) + if status == 403: + return _FatalTransportError( + f"LaunchDarkly returned HTTP 403. {_FORBIDDEN_ADVICE}" + ) + if status in (400, 405, 406, 414, 501): + return _FatalTransportError( + f"LaunchDarkly returned HTTP {status}, which retrying will not fix. " + "The request this adapter sent was not understood. It carries only " + "the SDK key and, after the first payload, a 'basis' selector, so " + "check the base URI and that the endpoint speaks FDv2." + ) + return _RecoverableTransportError( + f"LaunchDarkly returned HTTP {status}", _retry_after_seconds(headers) + ) + + +def _interrupt_read(response: Any) -> None: + """ + Best-effort interruption of a read blocked on *response*, from another thread. + + Closing the response is not enough: CPython's buffered reader stays parked in + ``readline`` until bytes arrive. Shutting the *socket* down underneath it + unblocks it immediately. Reaching the socket means walking urllib's private + attribute chain, so every step is guarded and failure is silent: the + delivery thread is a daemon and ``close``'s join timeout is the backstop. + """ + for path in (("fp", "raw", "_sock"), ("fp", "_sock"), ("_sock",)): + found: Any = response + for name in path: + found = getattr(found, name, None) + if found is None: + break + if found is not None and hasattr(found, "shutdown"): + try: + found.shutdown(socket.SHUT_RDWR) + except OSError: + pass + return + + +class _StreamConnection: + """ + One open streaming connection: an event iterator plus a way to interrupt it + from another thread, which is what ``FDv2SkillStore.close`` needs. + """ + + def __init__(self, response: Any) -> None: + self._response = response + self.events = _iter_sse(response) + + def close(self) -> None: + """Interrupts the read. Safe to call from any thread, and twice.""" + _interrupt_read(self._response) + try: + self._response.close() + except Exception: + pass + + +@dataclass(frozen=True) +class _PollResult: + not_modified: bool + events: list[tuple[str, Any]] + etag: str | None + + +class _Requester: + """ + The only place this module opens a socket. Standard library only, on purpose. + + *read_timeout* is applied to every socket operation of a request. ``urllib`` + has no separate connect timeout: its ``timeout`` becomes the socket timeout + for the whole operation, so connecting, waiting for headers and each body + read are all bounded by the same value. + """ + + def __init__( + self, + sdk_key: str, + base_uri: str, + *, + read_timeout: float, + opener: Any = None, + ) -> None: + self._sdk_key = sdk_key + self._base_uri = base_uri.rstrip("/") + self._read_timeout = read_timeout + # Injectable so tests can drive a fake endpoint without a socket. + self._opener = opener or urllib.request.build_opener() + + def _url(self, path: str, basis: str | None) -> str: + """ + The request URL: the path, plus ``basis`` once a payload has committed. + + Deliberately no ``mv`` (data model version). That parameter selects the + *flag* data model and the connection rejects any value but the flag + default; the agent-skill payload is generic, is served regardless of it, + and has no model version of its own to ask for. + """ + if not basis: + return f"{self._base_uri}{path}" + return f"{self._base_uri}{path}?{urllib.parse.urlencode({'basis': basis})}" + + def _request( + self, path: str, basis: str | None, headers: dict[str, str] + ) -> urllib.request.Request: + all_headers = {"Authorization": self._sdk_key, **headers} + return urllib.request.Request( + self._url(path, basis), headers=all_headers, method="GET" + ) + + def poll(self, basis: str | None, etag: str | None) -> _PollResult: + """One ``GET /sdk/poll``. A 304 is a first-class outcome, not an error.""" + headers = {"Accept": "application/json"} + if etag: + headers["If-None-Match"] = etag + request = self._request(POLL_PATH, basis, headers) + try: + with self._opener.open(request, timeout=self._read_timeout) as response: + status = getattr(response, "status", None) or response.getcode() + if status == 304: + return _PollResult(not_modified=True, events=[], etag=etag) + body = response.read() + new_etag = response.headers.get("ETag") or etag + except urllib.error.HTTPError as exc: + if exc.code == 304: + # urllib raises on 304 when no redirect handler swallows it. + return _PollResult(not_modified=True, events=[], etag=etag) + raise _classify_status(exc.code, exc.headers) from exc + except Exception as exc: + raise _RecoverableTransportError( + f"polling request failed: {type(exc).__name__}: {exc}" + ) from exc + + return _PollResult( + not_modified=False, events=_decode_poll_body(body), etag=new_etag + ) + + def stream(self, basis: str | None) -> _StreamConnection: + """Opens ``GET /sdk/stream``.""" + request = self._request( + STREAM_PATH, + basis, + {"Accept": "text/event-stream", "Cache-Control": "no-cache"}, + ) + try: + response = self._opener.open(request, timeout=self._read_timeout) + except urllib.error.HTTPError as exc: + raise _classify_status(exc.code, exc.headers) from exc + except Exception as exc: + raise _RecoverableTransportError( + f"streaming request failed: {type(exc).__name__}: {exc}" + ) from exc + return _StreamConnection(response) + + +def _decode_poll_body(body: bytes) -> list[tuple[str, Any]]: + """ + Unwraps ``{"events": [...]}``. Polling and streaming carry identical event + objects, which is why the protocol reader is shared between the two modes. + """ + try: + parsed = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise _RecoverableTransportError( + f"polling response was not valid JSON: {exc}" + ) from exc + if not isinstance(parsed, dict) or not isinstance(parsed.get("events"), list): + raise _RecoverableTransportError("polling response had no 'events' array") + events: list[tuple[str, Any]] = [] + for entry in parsed["events"]: + if not isinstance(entry, dict): + continue + name = entry.get("event") + if isinstance(name, str): + events.append((name, entry.get("data"))) + return events + + +def _iter_sse(response: Any) -> Any: + """ + Decodes an SSE body into ``(event name, data)`` pairs. + + Minimal on purpose: ``event:``/``data:`` fields, multi-line ``data`` joined + with newlines, a blank line dispatching, and ``:`` comments skipped. + """ + try: + name: str | None = None + data_lines: list[str] = [] + for raw_line in response: + line = raw_line.decode("utf-8", errors="replace").rstrip("\r\n") + if line == "": + if name is not None: + payload = "\n".join(data_lines) + try: + parsed = json.loads(payload) if payload else None + except json.JSONDecodeError: + logger.warning( + "Discarding FDv2 '%s' event whose data was not JSON", name + ) + parsed = None + else: + yield name, parsed + name = None + data_lines = [] + continue + if line.startswith(":"): + continue + field_name, _, value = line.partition(":") + value = value[1:] if value.startswith(" ") else value + if field_name == "event": + name = value + elif field_name == "data": + data_lines.append(value) + finally: + try: + response.close() + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Backoff +# --------------------------------------------------------------------------- + + +def _backoff_delay( + attempt: int, *, base: float, maximum: float, jitter: float = 0.5 +) -> float: + """ + Exponential backoff with jitter, capped at *maximum*. + + Jitter is subtractive over the whole range rather than added on top, so the + cap is a real ceiling: a fleet restarted together must not reconnect in + lockstep, and must not exceed the interval the cap promises. + """ + # float(2 ** n): the integer power is untyped to mypy. + ceiling: float = min(maximum, base * float(2 ** max(0, attempt - 1))) + return ceiling * (1.0 - jitter * random.random()) + + +# --------------------------------------------------------------------------- +# The store +# --------------------------------------------------------------------------- + + +class FDv2SkillStore: + """ + A ``SkillStore`` fed by LaunchDarkly's SDK-facing FDv2 delivery channel. + + Constructed with the environment's server-side SDK key, started explicitly, + and passed to ``init_client``:: + + store = FDv2SkillStore(sdk_key=os.environ["LD_SDK_KEY"]) + store.start() + store.wait_for_skills(timeout=10) + await init_client(options={"skillStore": store}) + + skill = await get_skill("pdf-extraction") + ... + store.close() + + It also works as a context manager. + + **Server-side only.** A mobile key or a client-side environment ID is + refused in the constructor. + + **Delivery is in the background; retrieval is not.** A daemon thread owns + the connection and fills memory, and ``get_object`` only ever reads what has + already arrived. A process that calls ``get_skill`` immediately after + ``start()`` may see an empty store; ``wait_for_skills`` orders boot against + the first payload. + + **Last known good survives an outage.** A transport failure never empties + the store and never makes ``get_object`` raise, which is what makes + ``write_skills(on_unavailable="keep")`` correct. ``diagnostics`` and + ``failed`` report the degradation. + + **What arrives is untrusted.** Raw wire objects are held verbatim and + verified at the accessor boundary, not here. In particular an object with no + ``contentHash`` is held and then *withheld*; see + ``StoreDiagnostics.hashless_objects``. + """ + + def __init__( + self, + sdk_key: str, + *, + base_uri: str = DEFAULT_BASE_URI, + mode: Mode = "stream", + poll_interval: float = 30.0, + read_timeout: float | None = None, + initial_backoff: float = 1.0, + max_backoff: float = 30.0, + max_consecutive_failures: int = 10, + _requester: Any = None, + ) -> None: + """ + *mode* is ``"stream"`` by default. Prefer it: a ``delete-object`` reaches + a live stream in seconds. ``"poll"`` exists for environments that cannot + hold a long-lived connection, and revocation there is one + ``poll_interval`` late. + + *read_timeout* is the only network timeout and bounds every socket + operation of a request, so its meaning and default follow the mode: in + ``"poll"`` it bounds the whole request (``DEFAULT_POLL_TIMEOUT``); in + ``"stream"`` it bounds each wait for the next bytes + (``DEFAULT_STREAM_READ_TIMEOUT``). Must be positive when given. + + *max_backoff* caps every delay between retries, including one the server + asks for with ``Retry-After``. + + *max_consecutive_failures* bounds the retry loop. On exceeding it the + transport stops, logs an error, and the store keeps serving last known + good; ``failed`` reports it. Only failures in a row count: a committed + payload resets the count. + """ + _require_server_side_credential(sdk_key) + if mode not in ("stream", "poll"): + raise ValueError(f'mode must be "stream" or "poll", got {mode!r}') + if poll_interval <= 0: + raise ValueError(f"poll_interval must be positive, got {poll_interval!r}") + if read_timeout is None: + read_timeout = ( + DEFAULT_STREAM_READ_TIMEOUT + if mode == "stream" + else DEFAULT_POLL_TIMEOUT + ) + elif not (math.isfinite(read_timeout) and read_timeout > 0): + raise ValueError(f"read_timeout must be positive, got {read_timeout!r}") + + self._mode: Mode = mode + self._poll_interval = poll_interval + self._initial_backoff = initial_backoff + self._max_backoff = max_backoff + self._max_consecutive_failures = max_consecutive_failures + + self._objects = _SkillObjectSet() + self._reader = _ProtocolReader(self._objects) + self._lock = threading.RLock() + self._listeners: dict[str, list[Callable[[dict[str, Any]], Any]]] = {} + + self._basis: str | None = None + self._etag: str | None = None + + self._requester = _requester or _Requester( + sdk_key.strip(), + base_uri, + read_timeout=read_timeout, + ) + + self._stop = threading.Event() + self._first_payload = threading.Event() + self._thread: threading.Thread | None = None + self._failed_reason: str | None = None + # The open streaming connection, so ``close`` can interrupt its read. + self._connection: Any = None + # Recoverable failures since the last committed payload. Reset at the + # commit rather than when a connection returns: a stream only ever ends + # by being dropped, so resetting on return would count every healthy, + # server-recycled connection as a failure. + self._failures = 0 + + # -- lifecycle --------------------------------------------------------- + + def start(self) -> FDv2SkillStore: + """ + Starts the delivery thread. Idempotent; returns ``self`` so it chains. + + Does not block: use ``wait_for_skills`` when boot ordering matters. + """ + with self._lock: + if self._thread is not None and self._thread.is_alive(): + return self + self._stop.clear() + self._thread = threading.Thread( + target=self._run, name="ld-ai-skills-fdv2", daemon=True + ) + self._thread.start() + return self + + def close(self, timeout: float = 5.0) -> None: + """ + Stops delivery. Idempotent, and safe to call from any thread. + + Held content is *not* dropped: a closed store still answers from what it + received. Detaching the store from the accessors is the job of the + package-level ``launchdarkly_ai_server.shutdown()`` coroutine. + """ + self._stop.set() + # The delivery thread is normally blocked in a socket read that no flag + # can reach; without this the join waits out its full timeout. + with self._lock: + connection = self._connection + if connection is not None: + connection.close() + thread = self._thread + if ( + thread is not None + and thread.is_alive() + and thread is not threading.current_thread() + ): + thread.join(timeout=timeout) + + def __enter__(self) -> FDv2SkillStore: + return self.start() + + def __exit__(self, *_exc: Any) -> None: + self.close() + + def wait_for_skills(self, timeout: float = 10.0) -> bool: + """ + Blocks until the first payload has been committed, or *timeout* elapses. + + ``True`` means a payload arrived — not that any skill in it verified, and + not that the environment has any skills. ``diagnostics`` answers the rest. + """ + return self._first_payload.wait(timeout=timeout) + + @property + def failed(self) -> str | None: + """Why delivery stopped for good, or ``None`` while it is running.""" + with self._lock: + return self._failed_reason + + @property + def diagnostics(self) -> StoreDiagnostics: + """A snapshot of what the transport has seen. See ``StoreDiagnostics``.""" + with self._lock: + return StoreDiagnostics(**vars(self._reader.diagnostics)) + + # -- the SkillStore interface ----------------------------------------- + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + if kind != SKILL_OBJECT_KIND: + return None + with self._lock: + return self._objects.get(key, version) + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + if kind != SKILL_OBJECT_KIND: + return {} + with self._lock: + return self._objects.snapshot() + + def add_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Registers *fn* to be called once per changed object, at + ``payload-transferred`` rather than as objects stream in. + + A put notifies with the raw skill object. A revocation notifies with a + ``{"key", "version"}`` tombstone carrying no content, so a listener that + reads content must check for ``content`` rather than assume it. + + *fn* runs on the delivery thread. Keep it cheap and non-blocking. An + exception it raises is logged and swallowed, because a broken listener + must not be able to kill delivery. + """ + with self._lock: + self._listeners.setdefault(kind, []).append(fn) + + def remove_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Unregisters *fn* from *kind*. Safe to call from any thread, including + from inside a listener: a removal during one commit takes effect from + the next. + + Removes one occurrence; removing a callable that is not registered is a + no-op, so ``SkillWatcher.close`` can detach unconditionally. + """ + with self._lock: + listeners = self._listeners.get(kind) + if listeners is None: + return + try: + listeners.remove(fn) + except ValueError: + return + + def _notify(self, changes: list[dict[str, Any]]) -> None: + with self._lock: + listeners = list(self._listeners.get(SKILL_OBJECT_KIND, [])) + for raw in changes: + for listener in listeners: + try: + listener(raw) + except Exception: + logger.error( + "A skill store change listener raised; delivery continues", + exc_info=True, + ) + + # -- the delivery loop ------------------------------------------------- + + def _run(self) -> None: + while not self._stop.is_set(): + try: + if self._mode == "stream": + self._stream_once() + else: + self._poll_once() + # A poll that returned is a current answer even when it committed + # nothing (HTTP 304). A stream never returns normally; its + # successes are counted at each commit in ``_apply``. + self._record_success() + except _FatalTransportError as exc: + self._give_up(str(exc)) + return + except _RecoverableTransportError as exc: + with self._lock: + self._failures += 1 + failures = self._failures + self._reader.diagnostics.connection_failures = failures + self._reader.diagnostics.last_error = str(exc) + if failures > self._max_consecutive_failures: + self._give_up( + f"gave up after {failures} consecutive failures; " + f"last error: {exc}" + ) + return + delay = exc.retry_after + if delay is None or not math.isfinite(delay): + delay = _backoff_delay( + failures, base=self._initial_backoff, maximum=self._max_backoff + ) + # ``Retry-After`` is a request and ``max_backoff`` is a promise. + # The header may come from a proxy rather than LaunchDarkly, and + # a value in the hours would park revocation for that long. + delay = min(delay, self._max_backoff) + logger.warning( + "Skill delivery failed (%s); retrying in %.1fs", exc, delay + ) + if self._stop.wait(delay): + return + continue + except Exception as exc: # pragma: no cover - defensive + self._give_up(f"unexpected error in skill delivery: {exc!r}") + logger.error("Unexpected error in skill delivery", exc_info=True) + return + + if self._mode == "poll" and self._stop.wait(self._poll_interval): + return + + def _record_success(self) -> None: + with self._lock: + self._failures = 0 + self._reader.diagnostics.connection_failures = 0 + + def _give_up(self, reason: str) -> None: + with self._lock: + self._failed_reason = reason + self._reader.diagnostics.last_error = reason + logger.error( + "Skill delivery has stopped and will not retry: %s. The store keeps " + "serving the last content it received; skills will not update until " + "the process restarts with a working connection.", + reason, + ) + # Unblock anyone waiting on a first payload that is never coming. + self._first_payload.set() + + def _apply(self, name: str, data: Any) -> None: + """ + Feeds one event to the reader, publishes a commit, and raises the + transport error the event calls for, if any. + """ + with self._lock: + outcome = self._reader.handle(name, data) + if outcome.committed and outcome.basis is not None: + self._basis = outcome.basis + if outcome.committed: + # A commit breaks the row of consecutive failures. + self._record_success() + self._first_payload.set() + if outcome.changes: + self._notify(outcome.changes) + if outcome.fatal: + raise _FatalTransportError(outcome.fatal) + if outcome.disconnect: + raise _RecoverableTransportError(outcome.disconnect) + + def _poll_once(self) -> None: + with self._lock: + basis, etag = self._basis, self._etag + result = self._requester.poll(basis, etag) + with self._lock: + self._etag = result.etag + if result.not_modified: + logger.debug("Skill payload unchanged (HTTP 304)") + # A 304 counts as a first payload, so a boot that reconnects with a + # cached basis is not blocked on a transfer the server will not send. + self._first_payload.set() + return + for name, data in result.events: + self._apply(name, data) + + def _stream_once(self) -> None: + with self._lock: + basis = self._basis + connection = self._requester.stream(basis) + with self._lock: + self._connection = connection + try: + # ``close`` may have run while the connect was in flight and found + # no connection to interrupt; this is the last chance to notice + # before the read below blocks. + if self._stop.is_set(): + return + for name, data in connection.events: + if self._stop.is_set(): + return + self._apply(name, data) + except Exception: + if self._stop.is_set(): + # ``close`` interrupted the read on purpose. + return + raise + finally: + connection.close() + with self._lock: + self._connection = None + # A stream that ends without a goodbye is a dropped connection. + raise _RecoverableTransportError("the FDv2 stream closed unexpectedly") diff --git a/packages/client/src/launchdarkly_ai_server/skills_watch.py b/packages/client/src/launchdarkly_ai_server/skills_watch.py index 05263057..e421df6a 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_watch.py +++ b/packages/client/src/launchdarkly_ai_server/skills_watch.py @@ -267,7 +267,7 @@ async def watch_skills( "watch_skills needs a skill store that implements add_listener(kind, " "fn); the configured store does not, so delivery changes cannot be " "observed. Use write_skills for a one-shot reconcile, or configure a " - "store with a delivery transport." + "store with a delivery transport (FDv2SkillStore)." ) if debounce < 0: raise ValueError(f"debounce must not be negative, got {debounce!r}") diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index d3f83095..a07ea6ae 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -603,6 +603,18 @@ async def test_all_skills_raises_actionably_when_no_store(self) -> None: with pytest.raises(RuntimeError, match="skill store"): await all_skills() + async def test_the_no_store_message_names_the_delivery_store_first(self) -> None: + """ + A deployment that hits this message must be pointed at the store that + receives content from LaunchDarkly, not only at the development one. + """ + with pytest.raises(RuntimeError) as reported: + await get_skill("a") + message = str(reported.value) + assert "FDv2SkillStore" in message + assert "InMemorySkillStore" in message + assert message.index("FDv2SkillStore") < message.index("InMemorySkillStore") + async def test_shutdown_clears_the_store( self, make_raw_skill: Any, mock_ld_client: Any ) -> None: diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index b88d14d9..0cce2f0c 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -1,26 +1,55 @@ """ -Tests for the FDv2 skill delivery protocol. - -Wire semantics — which objects are skills, the skill's version in the wire ``key`` -versus the payload's in ``version``, revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted -against ``_ProtocolReader``, which has no I/O, so each case reads as the contract -it is rather than as a server script. +Tests for the FDv2 skill delivery transport. + +Two layers, deliberately: + +- **A real fake endpoint.** ``_FakeFDv2Endpoint`` is an in-process + ``ThreadingHTTPServer`` that implements the wire contract — the ``basis`` + query parameter, ``Authorization``, ``If-None-Match``/304, the + ``{"events": [...]}`` polling envelope, and SSE for streaming. The store under + test opens real sockets against it, so request construction and header + handling are exercised rather than mocked. +- **The protocol reader driven directly.** Wire semantics — which objects are + skills, the skill's version in the wire ``key`` versus the payload's in + ``version``, revocation, mixed payloads — are + asserted against ``_ProtocolReader``, which has no I/O, so those cases read as + the contract they are instead of as a server script. """ from __future__ import annotations import hashlib +import json +import socket +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, ClassVar +from urllib.parse import parse_qs, urlparse import pytest -from launchdarkly_ai_server import InMemorySkillStore +from launchdarkly_ai_server import ( + FDv2SkillStore, + InMemorySkillStore, + all_skills, + get_skill, + get_skill_result, + init_client, + watch_skills, +) from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND from launchdarkly_ai_server.skills_fdv2 import ( + DEFAULT_POLL_TIMEOUT, + DEFAULT_STREAM_READ_TIMEOUT, FDV2_KEY_DELIMITER, FDV2_OBJECT_KIND, + _backoff_delay, _is_skill_event, _ProtocolReader, + _RecoverableTransportError, + _Requester, + _retry_after_seconds, _SkillObjectSet, _store_object_from_put, _tombstone_from_delete, @@ -28,6 +57,7 @@ pytestmark = pytest.mark.usefixtures("reset_skill_state") +SDK_KEY = "sdk-00000000-0000-4000-8000-000000000000" SKILL_BODY = "---\nname: PDF Extraction\n---\nExtract text from PDFs.\n" @@ -143,6 +173,182 @@ def full_payload( ) +# --------------------------------------------------------------------------- +# The fake endpoint +# --------------------------------------------------------------------------- + + +class _FakeFDv2Endpoint: + """ + An in-process server implementing the SDK-facing FDv2 contract. + + Scripted per request: ``queue_poll`` appends a response for the next + ``/sdk/poll``, ``queue_stream`` appends a sequence of SSE events for the next + ``/sdk/stream``. Every request's method, path, query and headers are recorded + in ``requests`` so the tests can assert on what the store actually sent — + which is the only way ``basis`` round-tripping and ``If-None-Match`` can be + checked at all. + """ + + def __init__(self) -> None: + self.requests: list[dict[str, Any]] = [] + self._polls: list[dict[str, Any]] = [] + self._streams: list[list[dict[str, Any]]] = [] + self._lock = threading.Lock() + self.hold_stream_open = False + self._release = threading.Event() + + endpoint = self + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args: Any) -> None: + return + + def do_GET(self) -> None: + parsed = urlparse(self.path) + query = {k: v[0] for k, v in parse_qs(parsed.query).items()} + with endpoint._lock: + endpoint.requests.append( + { + "path": parsed.path, + "query": query, + "authorization": self.headers.get("Authorization"), + "if_none_match": self.headers.get("If-None-Match"), + "accept": self.headers.get("Accept"), + } + ) + if parsed.path == "/sdk/poll": + endpoint._serve_poll(self) + elif parsed.path == "/sdk/stream": + endpoint._serve_stream(self) + else: + self.send_response(404) + self.send_header("Content-Length", "0") + self.end_headers() + + class Server(ThreadingHTTPServer): + # Handler threads are not joined on shutdown: a test that ends while + # a stream is deliberately held open should not pay for the hold. + daemon_threads = True + + self._server = Server(("127.0.0.1", 0), Handler) + # A short poll interval so `shutdown` is prompt: the default 0.5s is + # paid at the teardown of every test that touches the endpoint. + self._thread = threading.Thread( + target=lambda: self._server.serve_forever(poll_interval=0.01), daemon=True + ) + self._thread.start() + + @property + def base_uri(self) -> str: + host, port = self._server.server_address[:2] + return f"http://{host}:{port}" + + # -- scripting --------------------------------------------------------- + + def queue_poll( + self, + payload_events: list[dict[str, Any]] | None = None, + *, + status: int = 200, + etag: str | None = None, + retry_after: str | None = None, + ) -> None: + with self._lock: + self._polls.append( + { + "status": status, + "events": payload_events or [], + "etag": etag, + "retry_after": retry_after, + } + ) + + def queue_stream(self, payload_events: list[dict[str, Any]]) -> None: + with self._lock: + self._streams.append(payload_events) + + # -- serving ----------------------------------------------------------- + + def _serve_poll(self, handler: BaseHTTPRequestHandler) -> None: + with self._lock: + response = ( + self._polls.pop(0) if self._polls else {"status": 304, "events": []} + ) + status = response["status"] + handler.send_response(status) + if response.get("etag"): + handler.send_header("ETag", response["etag"]) + if response.get("retry_after"): + handler.send_header("Retry-After", response["retry_after"]) + if status in (200,): + body = json.dumps({"events": response["events"]}).encode("utf-8") + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + return + handler.send_header("Content-Length", "0") + handler.end_headers() + + def _serve_stream(self, handler: BaseHTTPRequestHandler) -> None: + with self._lock: + payload_events = self._streams.pop(0) if self._streams else [] + handler.send_response(200) + handler.send_header("Content-Type", "text/event-stream") + handler.send_header("Cache-Control", "no-cache") + handler.send_header("Transfer-Encoding", "chunked") + handler.end_headers() + for event in payload_events: + chunk = ( + f"event: {event['event']}\ndata: {json.dumps(event.get('data'))}\n\n" + ).encode() + handler.wfile.write(f"{len(chunk):X}\r\n".encode() + chunk + b"\r\n") + handler.wfile.flush() + if self.hold_stream_open: + # Keeps the connection up so a test can assert on the store's state + # without racing the reconnect path. Released on ``close`` so the + # hold costs the suite nothing once the test is done with it. + self._release.wait(timeout=10) + handler.wfile.write(b"0\r\n\r\n") + + def close(self) -> None: + self._release.set() + self._server.shutdown() + self._server.server_close() + + +@pytest.fixture +def endpoint() -> Any: + server = _FakeFDv2Endpoint() + yield server + server.close() + + +def poll_store(endpoint: Any, **kwargs: Any) -> FDv2SkillStore: + return FDv2SkillStore( + SDK_KEY, + base_uri=endpoint.base_uri, + mode="poll", + poll_interval=kwargs.pop("poll_interval", 0.05), + initial_backoff=kwargs.pop("initial_backoff", 0.01), + max_backoff=kwargs.pop("max_backoff", 0.05), + read_timeout=kwargs.pop("read_timeout", 5.0), + **kwargs, + ) + + +def wait_until(predicate: Any, timeout: float = 5.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.01) + return predicate() + + # --------------------------------------------------------------------------- # Identifying skill objects, and ignoring everything else # --------------------------------------------------------------------------- @@ -827,6 +1033,608 @@ def test_snapshot_agrees(self) -> None: assert memory.all_objects(SKILL_OBJECT_KIND) == objects.snapshot() +# --------------------------------------------------------------------------- +# The store against the fake endpoint +# --------------------------------------------------------------------------- + + +class TestPollingAgainstTheEndpoint: + def test_a_polled_skill_becomes_retrievable_through_the_accessors( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + raw = store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") + assert raw is not None + assert raw["version"] == 3 + + def test_the_request_carries_the_sdk_key_and_no_data_model_version( + self, endpoint: Any + ) -> None: + """ + No ``mv``: that parameter selects the *flag* data model, the connection + rejects any value but the flag default, and the generic agent-skill + payload is served regardless of it. Sending ``mv=1`` — the skill + payload's own model version — gets the whole connection refused. + """ + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + first = endpoint.requests[0] + assert first["path"] == "/sdk/poll" + assert first["authorization"] == SDK_KEY + assert "mv" not in first["query"] + + def test_the_first_request_sends_no_basis(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert "basis" not in endpoint.requests[0]["query"] + + def test_the_basis_from_payload_transferred_is_echoed_on_the_next_request( + self, endpoint: Any + ) -> None: + endpoint.queue_poll( + full_payload(("put-object", put_skill()), state="selector-abc") + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert endpoint.requests[1]["query"]["basis"] == "selector-abc" + + def test_the_basis_advances_across_successive_payloads(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()), state="basis-1")) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("second")), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint): + assert wait_until(lambda: len(endpoint.requests) >= 3) + bases = [r["query"].get("basis") for r in endpoint.requests[:3]] + assert bases == [None, "basis-1", "basis-2"] + + def test_an_etag_is_returned_as_if_none_match(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill())), etag='W/"v1"') + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert endpoint.requests[1]["if_none_match"] == 'W/"v1"' + + def test_a_304_keeps_the_held_content(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill())), etag='W/"v1"') + endpoint.queue_poll(status=304) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert wait_until(lambda: len(endpoint.requests) >= 3) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + assert store.diagnostics.payloads_transferred == 1 + assert store.failed is None + + def test_a_304_before_any_payload_still_releases_wait_for_skills( + self, endpoint: Any + ) -> None: + """A reconnect with a cached basis has nothing to transfer; boot must not + block on a payload the server has no reason to send.""" + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + + def test_a_mixed_payload_over_the_wire_yields_only_the_skill( + self, endpoint: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_flag("flag-a")), + ("put-object", put_segment("beta")), + ("put-object", put_skill("pdf-extraction")), + ("put-object", put_flag("flag-b")), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + held = store.all_objects(SKILL_OBJECT_KIND) + assert len(held) == 1 + assert next(iter(held.values()))["key"] == "pdf-extraction" + assert store.diagnostics.objects_ignored == 3 + + def test_a_revocation_over_the_wire_removes_the_skill(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + assert wait_until( + lambda: ( + store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is None + and store.diagnostics.objects_revoked == 1 + ) + ) + + def test_the_store_asks_for_only_the_kind_it_serves(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert store.get_object("flag", "pdf-extraction") is None + assert store.all_objects("flag") == {} + + +class TestStreamingAgainstTheEndpoint: + def test_a_streamed_payload_lands(self, endpoint: Any) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore( + SDK_KEY, base_uri=endpoint.base_uri, mode="stream", initial_backoff=0.01 + ) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + finally: + store.close() + + def test_the_stream_request_advertises_event_stream(self, endpoint: Any) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + try: + store.start() + store.wait_for_skills(timeout=5) + finally: + store.close() + assert endpoint.requests[0]["path"] == "/sdk/stream" + assert endpoint.requests[0]["accept"] == "text/event-stream" + + def test_a_streamed_revocation_arrives_without_a_restart( + self, endpoint: Any + ) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream( + full_payload(("put-object", put_skill())) + + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ) + ) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + try: + store.start() + assert wait_until( + lambda: ( + store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is None + and store.diagnostics.objects_revoked == 1 + ) + ) + finally: + store.close() + + def test_a_dropped_stream_reconnects_with_the_basis_it_reached( + self, endpoint: Any + ) -> None: + endpoint.queue_stream( + full_payload(("put-object", put_skill()), state="basis-1") + ) + endpoint.queue_stream(events(("heart-beat", None))) + store = FDv2SkillStore( + SDK_KEY, + base_uri=endpoint.base_uri, + mode="stream", + initial_backoff=0.01, + max_backoff=0.05, + ) + try: + store.start() + assert wait_until(lambda: len(endpoint.requests) >= 2) + finally: + store.close() + assert endpoint.requests[1]["query"]["basis"] == "basis-1" + + def test_close_returns_promptly_while_a_stream_is_open(self, endpoint: Any) -> None: + """ + The delivery thread is blocked in a socket read that no stop flag can + reach, so ``close`` closes the connection under it. Without that, every + shutdown of a healthy stream waits out the join timeout. + """ + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + store.start() + assert store.wait_for_skills(timeout=5) is True + started = time.monotonic() + store.close(timeout=5.0) + assert time.monotonic() - started < 1.0 + + def test_an_interrupted_stream_is_not_reported_as_a_failure( + self, endpoint: Any + ) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + store.start() + store.wait_for_skills(timeout=5) + store.close() + assert store.failed is None + + def test_content_survives_a_reconnect(self, endpoint: Any) -> None: + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + endpoint.queue_stream(events(("heart-beat", None))) + store = FDv2SkillStore( + SDK_KEY, base_uri=endpoint.base_uri, mode="stream", initial_backoff=0.01 + ) + try: + store.start() + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + finally: + store.close() + + +# --------------------------------------------------------------------------- +# Failure handling +# --------------------------------------------------------------------------- + + +class _ScriptedConnection: + """Stands in for ``_StreamConnection``: an event iterator plus a close.""" + + def __init__(self, payload_events: Any) -> None: + self.events = iter(payload_events) + self.closed = False + + def close(self) -> None: + self.closed = True + + +class _ScriptedRequester: + """Raises a scripted sequence, so backoff is asserted without real sockets.""" + + def __init__(self, *outcomes: Any) -> None: + self.outcomes = list(outcomes) + self.calls: list[tuple[str | None, str | None]] = [] + + def poll(self, basis: str | None, etag: str | None) -> Any: + self.calls.append((basis, etag)) + outcome = ( + self.outcomes.pop(0) if self.outcomes else _RecoverableTransportError("x") + ) + if isinstance(outcome, Exception): + raise outcome + return outcome + + def stream(self, basis: str | None) -> Any: + self.calls.append((basis, None)) + outcome = ( + self.outcomes.pop(0) if self.outcomes else _RecoverableTransportError("x") + ) + if isinstance(outcome, Exception): + raise outcome + return _ScriptedConnection(outcome) + + +class _RecyclingRequester: + """ + A healthy server that recycles connections: every ``stream`` call succeeds, + transfers a full payload, and then ends the connection, as LaunchDarkly and + any proxy in between do to a long-lived stream. + """ + + def __init__(self) -> None: + self.connections = 0 + + def stream(self, basis: str | None) -> Any: + self.connections += 1 + return _ScriptedConnection( + [ + (e["event"], e["data"]) + for e in full_payload( + ("put-object", put_skill()), state=f"basis-{self.connections}" + ) + ] + ) + + +class _BlockingConnection: + """A stream that never produces an event until it is closed.""" + + def __init__(self) -> None: + self._closed = threading.Event() + + @property + def events(self) -> Any: + self._closed.wait() + return iter(()) + + def close(self) -> None: + self._closed.set() + + +class _SlowConnectRequester: + """ + A ``stream`` whose connect does not return until the test releases it, + standing in for a slow TLS handshake, followed by a read that never yields. + """ + + def __init__(self) -> None: + self.entered = threading.Event() + self.release = threading.Event() + + def stream(self, basis: str | None) -> Any: + self.entered.set() + self.release.wait(timeout=10) + return _BlockingConnection() + + +def stream_store(**kwargs: Any) -> FDv2SkillStore: + return FDv2SkillStore( + SDK_KEY, + mode="stream", + initial_backoff=kwargs.pop("initial_backoff", 0.001), + max_backoff=kwargs.pop("max_backoff", 0.002), + **kwargs, + ) + + +class TestFailureHandling: + def test_a_403_stops_delivery_and_explains_why( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll(status=403) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert "403" in store.failed + assert "opt-in" in store.failed + assert any("opt-in" in r.getMessage() for r in caplog.records) + + def test_a_401_stops_delivery(self, endpoint: Any) -> None: + endpoint.queue_poll(status=401) + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert "401" in store.failed + + def test_a_fatal_failure_releases_wait_for_skills_rather_than_hanging( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(status=401) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + assert store.failed is not None + + def test_a_fatal_failure_keeps_last_known_good_servable( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll(status=403) + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + + def test_a_500_is_retried(self, endpoint: Any) -> None: + endpoint.queue_poll(status=500) + endpoint.queue_poll(status=503) + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + assert store.failed is None + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + + def test_a_retry_resets_the_failure_count_on_success(self, endpoint: Any) -> None: + endpoint.queue_poll(status=500) + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) + assert wait_until(lambda: store.diagnostics.connection_failures == 0) + + def test_retries_are_bounded(self) -> None: + store = FDv2SkillStore( + SDK_KEY, + mode="poll", + poll_interval=0.01, + initial_backoff=0.001, + max_backoff=0.002, + max_consecutive_failures=3, + _requester=_ScriptedRequester(), + ) + try: + store.start() + assert wait_until(lambda: store.failed is not None) + # Four, not three: the bound is the number of failures *tolerated*, + # so the run that exceeds it is the one that gives up. + assert "gave up after 4 consecutive failures" in store.failed + finally: + store.close() + + def test_recycled_stream_connections_are_not_failures(self) -> None: + # A streaming connection only ever ends by being dropped, so a loop + # that counted every drop as a failure would give up on a healthy + # server after max_consecutive_failures + 1 recycles, and delivery + # (including revocation) would silently stop for the process lifetime. + requester = _RecyclingRequester() + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert wait_until(lambda: requester.connections >= 8) + assert store.failed is None + assert store.diagnostics.payloads_transferred >= 8 + # A drop is a failure until the next commit clears it, so the count + # may read 1 mid-reconnect. What it must never do is climb. + assert store.diagnostics.connection_failures <= 1 + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + finally: + store.close() + + def test_a_stream_commit_resets_the_failure_count(self) -> None: + payload = [ + (e["event"], e["data"]) for e in full_payload(("put-object", put_skill())) + ] + requester = _ScriptedRequester( + _RecoverableTransportError("x"), + _RecoverableTransportError("x"), + _RecoverableTransportError("x"), + payload, + ) + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=5) + # Three failures reach the bound, then a commit, then the exhausted + # requester fails on every reconnect. The count must start again at + # the commit: the stream's own drop is failure one, and three more + # connects are owed before giving up. Carrying the three over would + # give up on the drop itself, with no further connect at all. + assert wait_until(lambda: store.failed is not None) + assert "gave up after 4 consecutive failures" in store.failed + assert "last error: x" in store.failed + assert len(requester.calls) == 7 + finally: + store.close() + + def test_stream_retries_are_bounded(self) -> None: + store = stream_store( + max_consecutive_failures=3, _requester=_ScriptedRequester() + ) + try: + store.start() + assert wait_until(lambda: store.failed is not None) + assert "gave up after 4 consecutive failures" in store.failed + finally: + store.close() + + def test_a_retry_after_header_is_honoured(self) -> None: + requester = _ScriptedRequester( + _RecoverableTransportError("slow down", retry_after=0.25), + ) + store = FDv2SkillStore( + SDK_KEY, + mode="poll", + poll_interval=10.0, + initial_backoff=5.0, + _requester=requester, + ) + try: + started = time.monotonic() + store.start() + assert wait_until(lambda: len(requester.calls) >= 2, timeout=3) + elapsed = time.monotonic() - started + # The server asked for 0.25s; our own backoff would have been 5s. + assert 0.2 <= elapsed < 3.0 + finally: + store.close() + + def test_a_retry_after_header_is_parsed_off_the_wire(self, endpoint: Any) -> None: + endpoint.queue_poll(status=429, retry_after="0") + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint, initial_backoff=5.0) as store: + # If Retry-After were ignored the 5s backoff would blow the timeout. + assert store.wait_for_skills(timeout=3) is True + + @pytest.mark.parametrize("raw", ["inf", "Infinity", "-inf", "nan", "1e309"]) + def test_a_non_finite_retry_after_is_ignored(self, raw: str) -> None: + assert _retry_after_seconds({"Retry-After": raw}) is None + + def test_retry_after_parsing_keeps_its_edges(self) -> None: + assert _retry_after_seconds({"Retry-After": "0"}) == 0.0 + assert _retry_after_seconds({"Retry-After": "-5"}) == 0.0 + assert ( + _retry_after_seconds({"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}) + is None + ) + assert _retry_after_seconds({"Retry-After": "2.5"}) == 2.5 + + @pytest.mark.parametrize("retry_after", [float("inf"), float("nan"), 86400.0]) + def test_an_unreasonable_retry_after_neither_kills_delivery_nor_parks_it( + self, retry_after: float + ) -> None: + # An infinite wait would overflow inside the retry handler and kill the + # thread with `failed` still None; a day-long one would be honoured to + # the second. Both must fall back to the max_backoff cap and carry on. + requester = _ScriptedRequester( + _RecoverableTransportError("slow down", retry_after=retry_after), + [ + (e["event"], e["data"]) + for e in full_payload(("put-object", put_skill())) + ], + ) + store = stream_store(max_backoff=0.05, _requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=3) is True + assert store.failed is None + assert store._thread is not None and store._thread.is_alive() + finally: + store.close() + + def test_a_non_finite_retry_after_off_the_wire_falls_back_to_backoff( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(status=429, retry_after="inf") + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=3) is True + assert store.failed is None + + def test_backoff_is_exponential_and_capped(self) -> None: + assert _backoff_delay(1, base=1.0, maximum=30.0, jitter=0.0) == 1.0 + assert _backoff_delay(2, base=1.0, maximum=30.0, jitter=0.0) == 2.0 + assert _backoff_delay(3, base=1.0, maximum=30.0, jitter=0.0) == 4.0 + assert _backoff_delay(20, base=1.0, maximum=30.0, jitter=0.0) == 30.0 + + def test_jitter_never_exceeds_the_cap(self) -> None: + for attempt in range(1, 12): + for _ in range(50): + assert 0.0 <= _backoff_delay(attempt, base=1.0, maximum=5.0) <= 5.0 + + def test_a_malformed_polling_envelope_is_recoverable_not_fatal( + self, endpoint: Any + ) -> None: + store = FDv2SkillStore( + SDK_KEY, + mode="poll", + poll_interval=0.01, + initial_backoff=0.001, + _requester=_ScriptedRequester( + _RecoverableTransportError("polling response had no 'events' array") + ), + ) + try: + store.start() + assert wait_until(lambda: store.diagnostics.connection_failures >= 1) + assert store.failed is None + finally: + store.close() + + def test_a_listener_that_raises_does_not_kill_delivery(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill("first")))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("second")), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.add_listener(SKILL_OBJECT_KIND, lambda _raw: 1 / 0) + assert wait_until( + lambda: store.get_object(SKILL_OBJECT_KIND, "second") is not None + ) + assert store.failed is None + + # --------------------------------------------------------------------------- # The contentHash gap # --------------------------------------------------------------------------- @@ -854,6 +1662,63 @@ class TestMissingContentHash: itself and verify nothing. """ + async def test_a_hashless_skill_is_withheld_with_the_right_reason( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill(omit_hash=True)))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + outcome = await get_skill_result("pdf-extraction") + assert outcome.skill is None + assert outcome.reason == "integrity_failure" + assert await get_skill("pdf-extraction") is None + assert await all_skills() == [] + + async def test_the_object_is_still_held_so_the_outcome_is_not_absent( + self, endpoint: Any + ) -> None: + """ + Holding it is what makes the failure diagnosable. Dropping it at the + transport would report ``absent`` — indistinguishable from "no such + skill" — and would additionally let a prune delete the last known-good + copy already on disk. + """ + endpoint.queue_poll(full_payload(("put-object", put_skill(omit_hash=True)))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + raw = store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") + assert raw is not None + assert "contentHash" not in raw + await init_client(options={"skillStore": store}, client=object()) + assert (await get_skill_result("pdf-extraction")).reason != "absent" + + def test_the_store_counts_hashless_objects(self, endpoint: Any) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill("a", omit_hash=True)), + ("put-object", put_skill("b", omit_hash=True)), + ("put-object", put_skill("c")), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert store.diagnostics.hashless_objects == 2 + assert store.diagnostics.skill_objects_received == 3 + + def test_a_hashless_object_logs_an_error_naming_the_reason_code( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill(omit_hash=True)))) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + rendered = "\n".join(r.getMessage() for r in caplog.records) + assert "missing_content_hash" in rendered + assert "pdf-extraction" in rendered + assert "contentHash" in rendered + def test_a_redelivered_hashless_object_logs_once_per_store( self, caplog: Any ) -> None: @@ -895,3 +1760,456 @@ def test_two_live_stores_do_not_suppress_each_other(self, caplog: Any) -> None: drive(one, payload) drive(two, payload) assert len(_per_object_hashless_errors(caplog)) == 2 + + def test_a_wholly_hashless_payload_says_so_once( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill("a", omit_hash=True)), + ("put-object", put_skill("b", omit_hash=True)), + ) + ) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + summaries = [ + r + for r in caplog.records + if "No skill content will resolve" in r.getMessage() + ] + assert len(summaries) == 1 + assert "All 2 skill object(s)" in summaries[0].getMessage() + + def test_a_partly_hashed_payload_does_not_claim_total_failure( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill("a", omit_hash=True)), + ("put-object", put_skill("b")), + ) + ) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + rendered = "\n".join(r.getMessage() for r in caplog.records) + assert "No skill content will resolve" not in rendered + + async def test_a_hash_that_does_not_match_is_a_different_failure( + self, endpoint: Any + ) -> None: + """``missing_content_hash`` and ``hash_mismatch`` must not collapse: one + means the envelope carried no hash, the other means the content did not + match the hash it carried.""" + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(content_hash=_hash("something else"))) + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert store.diagnostics.hashless_objects == 0 + await init_client(options={"skillStore": store}, client=object()) + assert ( + await get_skill_result("pdf-extraction") + ).reason == "integrity_failure" + + async def test_a_hashed_skill_resolves_end_to_end(self, endpoint: Any) -> None: + """The positive control: a well-formed envelope resolves end to end.""" + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + skill = await get_skill("pdf-extraction") + assert skill is not None + assert skill.key == "pdf-extraction" + assert skill.version == 3 + assert skill.content == SKILL_BODY.encode("utf-8") + assert skill.content_hash == _hash(SKILL_BODY) + assert skill.name == "PDF Extraction" + + async def test_a_pinned_reference_resolves_to_the_pinned_object_version( + self, endpoint: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(object_version=2, content="v2 body")), + ("put-object", put_skill(object_version=5, content="v5 body")), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + pinned = await get_skill("pdf-extraction", version=2) + assert pinned is not None + assert pinned.content == b"v2 body" + newest = await get_skill("pdf-extraction") + assert newest is not None + assert newest.version == 5 + + async def test_the_payload_version_is_not_resolvable_as_a_skill_version( + self, endpoint: Any + ) -> None: + """ + The end-to-end form of the wire-key/``version`` assertion. + + Asking for the payload version resolves nothing — reported ``absent``, + because the store answers "I hold no such version" rather than answering + with the wrong one. The version that *does* resolve is the one after the + delimiter in the object's wire ``key``. + """ + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(object_version=3, payload_version=42)) + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + by_payload_version = await get_skill_result("pdf-extraction", version=42) + assert by_payload_version.skill is None + assert by_payload_version.reason == "absent" + assert await get_skill("pdf-extraction", version=3) is not None + + +# --------------------------------------------------------------------------- +# Server-side only +# --------------------------------------------------------------------------- + + +class TestServerSideOnly: + def test_a_mobile_key_is_refused(self) -> None: + with pytest.raises(ValueError, match="mobile key"): + FDv2SkillStore("mob-00000000-0000-4000-8000-000000000000") + + def test_a_client_side_environment_id_is_refused(self) -> None: + with pytest.raises(ValueError, match="client-side"): + FDv2SkillStore("0123456789abcdef01234567") + + def test_an_empty_credential_is_refused(self) -> None: + with pytest.raises(ValueError, match="server-side SDK key"): + FDv2SkillStore(" ") + + def test_a_server_side_key_is_accepted(self) -> None: + assert FDv2SkillStore(SDK_KEY) is not None + + def test_an_unrecognised_credential_shape_warns_but_is_allowed( + self, caplog: Any + ) -> None: + """Private instances and test doubles issue keys without the public prefix.""" + with caplog.at_level("WARNING"): + FDv2SkillStore("my-private-instance-credential") + assert any("server-side SDK key" in r.message for r in caplog.records) + + def test_an_unknown_mode_is_refused(self) -> None: + with pytest.raises(ValueError, match="stream"): + FDv2SkillStore(SDK_KEY, mode="mobile") # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# The eager re-reconcile, end to end over the transport +# --------------------------------------------------------------------------- + + +class TestWatchSkillsOverTheTransport: + """ + ``watch_skills`` against a live ``FDv2SkillStore``. The watcher's own + behaviour — debounce, refusal of a store without ``add_listener``, detaching + on close — is covered in ``test_skills_watch.py`` against the in-memory + store; these are the cases that only mean something with a transport + underneath: a wire-level revocation, a new skill version, and an outage. + """ + + async def test_a_revocation_prunes_without_a_restart( + self, endpoint: Any, tmp_path: Any + ) -> None: + """ + The store's change listener drives the reconcile, so the file goes away + seconds after the ``delete-object`` rather than at the next process start. + """ + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + + with poll_store(endpoint, poll_interval=0.2) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + report, watcher = await watch_skills( + "*", tmp_path / "skills", debounce=0.05 + ) + try: + written = tmp_path / "skills" / "pdf-extraction" / "SKILL.md" + assert written.exists() + assert any(a.action == "written" for a in report.actions) + assert wait_until(lambda: not written.exists(), timeout=10) + finally: + watcher.close() + + async def test_a_new_version_is_rewritten_without_a_restart( + self, endpoint: Any, tmp_path: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill(content="first")))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill(object_version=4, content="second")), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + + with poll_store(endpoint, poll_interval=0.2) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + assert written.read_text() == "first" + assert wait_until(lambda: written.read_text() == "second", timeout=10) + finally: + watcher.close() + + async def test_the_default_keeps_last_known_good_during_an_outage( + self, endpoint: Any, tmp_path: Any + ) -> None: + """``on_unavailable="keep"`` is the default: an outage must not read as + "everything was revoked".""" + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll(status=500) + with poll_store(endpoint, poll_interval=0.05) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + assert written.exists() + # ``last_error`` rather than ``connection_failures``: the counter + # resets on the next successful poll, so asserting on it races + # the retry that is supposed to happen. + assert wait_until( + lambda: store.diagnostics.last_error is not None, timeout=10 + ) + time.sleep(0.3) + assert written.exists() + finally: + watcher.close() + + +# --------------------------------------------------------------------------- +# Listener registration +# --------------------------------------------------------------------------- + + +class TestListenerRegistration: + @staticmethod + def _skill_listeners(store: Any) -> list[Any]: + return list(store._listeners.get(SKILL_OBJECT_KIND, [])) + + def test_fdv2_remove_listener_of_an_unregistered_callable_is_a_no_op( + self, endpoint: Any + ) -> None: + with poll_store(endpoint) as store: + store.remove_listener(SKILL_OBJECT_KIND, print) + store.add_listener(SKILL_OBJECT_KIND, print) + store.remove_listener("flag", print) + store.remove_listener(SKILL_OBJECT_KIND, print) + store.remove_listener(SKILL_OBJECT_KIND, print) + assert self._skill_listeners(store) == [] + + +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + + +class TestLifecycle: + def test_start_is_idempotent(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store = poll_store(endpoint) + try: + assert store.start() is store + assert store.start() is store + assert store.wait_for_skills(timeout=5) + finally: + store.close() + + def test_close_is_idempotent(self, endpoint: Any) -> None: + store = poll_store(endpoint) + store.start() + store.close() + store.close() + + def test_close_during_a_slow_connect_returns_promptly(self) -> None: + # Before the connect returns there is no connection for close() to + # interrupt. If the delivery thread then enters the read anyway, close() + # sits out its whole join timeout on a stream that will never speak. + requester = _SlowConnectRequester() + store = stream_store(_requester=requester) + store.start() + assert requester.entered.wait(timeout=5) + threading.Timer(0.1, requester.release.set).start() + started = time.monotonic() + store.close(timeout=5.0) + elapsed = time.monotonic() - started + assert elapsed < 2.0 + assert store._thread is not None + assert not store._thread.is_alive() + + def test_a_closed_store_still_answers_from_what_it_received( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store = poll_store(endpoint) + store.start() + store.wait_for_skills(timeout=5) + store.close() + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + + def test_wait_for_skills_times_out_rather_than_hanging(self) -> None: + store = FDv2SkillStore( + SDK_KEY, mode="poll", poll_interval=60, _requester=_ScriptedRequester() + ) + try: + assert store.wait_for_skills(timeout=0.05) is False + finally: + store.close() + + def test_the_store_satisfies_the_seam_before_it_starts(self) -> None: + store = FDv2SkillStore(SDK_KEY) + assert store.get_object(SKILL_OBJECT_KIND, "anything") is None + assert store.all_objects(SKILL_OBJECT_KIND) == {} + + +# --------------------------------------------------------------------------- +# Timeouts +# --------------------------------------------------------------------------- + + +class _BlackHole: + """ + A listening socket that accepts connections and never sends a byte. + + This is the host ``read_timeout`` exists for: the TCP handshake completes, so + nothing fails fast, and then no response ever comes. A request against it can + only end by timing out, which makes the elapsed time a direct measurement of + the timeout actually applied. + """ + + def __init__(self) -> None: + self._listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._listener.bind(("127.0.0.1", 0)) + self._listener.listen(8) + self._accepted: list[socket.socket] = [] + self._stop = threading.Event() + self._thread = threading.Thread(target=self._accept_forever, daemon=True) + self._thread.start() + host, port = self._listener.getsockname() + self.base_uri = f"http://{host}:{port}" + + def _accept_forever(self) -> None: + self._listener.settimeout(0.05) + while not self._stop.is_set(): + try: + conn, _ = self._listener.accept() + except OSError: + continue + self._accepted.append(conn) + + def close(self) -> None: + self._stop.set() + self._thread.join(timeout=2) + for conn in self._accepted: + conn.close() + self._listener.close() + + +@pytest.fixture +def black_hole() -> Any: + server = _BlackHole() + yield server + server.close() + + +class TestTimeouts: + """ + ``read_timeout`` is the only network timeout, and every request honours it. + + The bounds asserted here are loose on purpose: the point is that a request + against an unresponsive host fails in roughly ``read_timeout`` rather than in + minutes, and that a regression back to a much longer default fails this + suite quickly instead of hanging it. + """ + + def test_a_poll_against_an_unresponsive_host_fails_within_read_timeout( + self, black_hole: Any + ) -> None: + requester = _Requester(SDK_KEY, black_hole.base_uri, read_timeout=0.3) + started = time.monotonic() + with pytest.raises(_RecoverableTransportError) as excinfo: + requester.poll(None, None) + elapsed = time.monotonic() - started + assert 0.2 <= elapsed < 2.0 + assert "timed out" in str(excinfo.value) + + def test_a_stream_against_an_unresponsive_host_fails_within_read_timeout( + self, black_hole: Any + ) -> None: + requester = _Requester(SDK_KEY, black_hole.base_uri, read_timeout=0.3) + started = time.monotonic() + with pytest.raises(_RecoverableTransportError): + requester.stream(None) + assert time.monotonic() - started < 2.0 + + def test_the_store_reports_the_timeout_and_keeps_going( + self, black_hole: Any + ) -> None: + store = FDv2SkillStore( + SDK_KEY, + base_uri=black_hole.base_uri, + mode="poll", + poll_interval=0.05, + initial_backoff=0.01, + max_backoff=0.05, + read_timeout=0.3, + ) + try: + store.start() + assert wait_until(lambda: store.diagnostics.connection_failures >= 1) + assert store.failed is None + assert "timed out" in (store.diagnostics.last_error or "") + finally: + store.close() + + def test_the_default_bound_depends_on_the_mode(self) -> None: + assert DEFAULT_POLL_TIMEOUT == 10.0 + assert DEFAULT_STREAM_READ_TIMEOUT == 300.0 + polling = FDv2SkillStore(SDK_KEY, mode="poll") + streaming = FDv2SkillStore(SDK_KEY, mode="stream") + assert polling._requester._read_timeout == DEFAULT_POLL_TIMEOUT + assert streaming._requester._read_timeout == DEFAULT_STREAM_READ_TIMEOUT + + @pytest.mark.parametrize("mode", ["poll", "stream"]) + def test_an_explicit_read_timeout_overrides_the_default(self, mode: Any) -> None: + store = FDv2SkillStore(SDK_KEY, mode=mode, read_timeout=42.0) + assert store._requester._read_timeout == 42.0 + + @pytest.mark.parametrize("value", [0.0, -1.0, float("inf"), float("nan")]) + def test_a_non_positive_read_timeout_is_rejected(self, value: float) -> None: + with pytest.raises(ValueError, match="read_timeout"): + FDv2SkillStore(SDK_KEY, read_timeout=value) + + def test_there_is_no_separate_connect_timeout(self) -> None: + # ``urllib`` cannot bound the connect separately from the reads, so the + # constructor does not offer a parameter that would only pretend to. + with pytest.raises(TypeError): + FDv2SkillStore(SDK_KEY, connect_timeout=2.0) # type: ignore[call-arg] diff --git a/packages/client/tests/test_skills_watch.py b/packages/client/tests/test_skills_watch.py index 48a95f07..b44ffea0 100644 --- a/packages/client/tests/test_skills_watch.py +++ b/packages/client/tests/test_skills_watch.py @@ -4,7 +4,10 @@ The watcher is wired to the ``SkillStore`` interface, not to any one transport: it needs a store that implements ``add_listener``, and nothing more. These tests therefore drive it from ``InMemorySkillStore``, whose ``put`` notifies its -listeners synchronously, and from small hand-written store doubles. +listeners synchronously, and from small hand-written store doubles. The +end-to-end path — a ``delete-object`` arriving over a live FDv2 connection and +pruning a skill's files — is exercised in ``test_skills_fdv2.py``, where the fake +endpoint lives. Every test writes only inside pytest's ``tmp_path``. The watcher runs a real worker thread, so tests wait on observable outcomes rather than on fixed sleeps From c285ff533991b3e74a2d0b976a91de1008302596 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:04:26 -0400 Subject: [PATCH 19/22] fix(client): retry an FDv2 stream that dies mid-read _Requester.stream wrapped only the connect as recoverable, so a read timeout, reset or truncated chunk in the body reached the delivery loop as whatever the socket raised. The loop read that as a bug and gave up: delivery stopped for the process lifetime, taking updates and revocations with it, the first time a socket died. read_timeout exists to bound a stream that has gone quiet so the loop can reconnect, and tripping it did the opposite. The body now carries the same promise the connect already did. Wrapping the line source rather than the whole read keeps protocol reader errors out of it: those are raised from the consumer's loop body, where they still surface as the bugs they are. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fdv2.py | 21 ++++++- packages/client/tests/test_skills_fdv2.py | 61 +++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index da84a3b4..18b81842 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -1032,6 +1032,25 @@ def _decode_poll_body(body: bytes) -> list[tuple[str, Any]]: return events +def _iter_stream_lines(response: Any) -> Any: + """ + Yields a streaming body's raw lines, presenting a read failure as retryable. + + A live stream dies mid-body far more often than it refuses to open: a read + timeout on a stream that went quiet, a reset, a truncated chunk. Each of + those arrives as whatever the socket raised, and the delivery loop retries + only the transport errors this module defines — anything else it reads as a + bug and stops for the process lifetime. Connecting is already wrapped in + ``_Requester.stream``; this is the same promise for the body. + """ + try: + yield from response + except Exception as exc: + raise _RecoverableTransportError( + f"reading the FDv2 stream failed: {type(exc).__name__}: {exc}" + ) from exc + + def _iter_sse(response: Any) -> Any: """ Decodes an SSE body into ``(event name, data)`` pairs. @@ -1042,7 +1061,7 @@ def _iter_sse(response: Any) -> Any: try: name: str | None = None data_lines: list[str] = [] - for raw_line in response: + for raw_line in _iter_stream_lines(response): line = raw_line.decode("utf-8", errors="replace").rstrip("\r\n") if line == "": if name is not None: diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index 0cce2f0c..af5d1812 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -23,6 +23,7 @@ import socket import threading import time +from http.client import IncompleteRead from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, ClassVar from urllib.parse import parse_qs, urlparse @@ -52,6 +53,7 @@ _retry_after_seconds, _SkillObjectSet, _store_object_from_put, + _StreamConnection, _tombstone_from_delete, ) @@ -1287,6 +1289,41 @@ def test_content_survives_a_reconnect(self, endpoint: Any) -> None: # --------------------------------------------------------------------------- +class _DyingResponse: + """ + A streaming body that transfers a payload and then fails mid-read. + + This is how a live stream actually ends: not with a clean end of body but + with a read timeout on a stream that went quiet, or a reset from the server + or a proxy in between. + """ + + def __init__(self, exc: BaseException) -> None: + self._exc = exc + + def __iter__(self) -> Any: + for event in full_payload(("put-object", put_skill())): + yield f"event: {event['event']}\n".encode() + yield f"data: {json.dumps(event['data'])}\n".encode() + yield b"\n" + raise self._exc + + def close(self) -> None: + pass + + +class _DyingStreamRequester: + """Every connection transfers a payload, then dies with *exc* mid-read.""" + + def __init__(self, exc: BaseException) -> None: + self.connections = 0 + self._exc = exc + + def stream(self, basis: str | None) -> Any: + self.connections += 1 + return _StreamConnection(_DyingResponse(self._exc)) + + class _ScriptedConnection: """Stands in for ``_StreamConnection``: an event iterator plus a close.""" @@ -1477,6 +1514,30 @@ def test_recycled_stream_connections_are_not_failures(self) -> None: finally: store.close() + @pytest.mark.parametrize( + "exc", + [ + TimeoutError("timed out"), + ConnectionResetError(54, "Connection reset by peer"), + IncompleteRead(b"partial"), + ], + ids=["read timeout", "reset", "truncated body"], + ) + def test_a_stream_that_dies_mid_read_reconnects(self, exc: BaseException) -> None: + # A stream fails in its body far more often than at its connect, and + # ``read_timeout`` exists to bound one that has gone quiet. Treating + # such a failure as unexpected would stop delivery — including + # revocation — for the process lifetime the first time a socket died. + requester = _DyingStreamRequester(exc) + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + assert wait_until(lambda: requester.connections >= 5) + assert store.failed is None + finally: + store.close() + def test_a_stream_commit_resets_the_failure_count(self) -> None: payload = [ (e["event"], e["data"]) for e in full_payload(("put-object", put_skill())) From daf04a6e29720e9a20755f5768752e0bfe71187a Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 16:14:39 -0400 Subject: [PATCH 20/22] fix(client): raise the skill content cap to 10 MiB The client-side cap was 64 KiB, close enough to the platform's own limit that any backend increase would force an SDK release. Raise it to 10 MiB so the guard stays a backstop against absurd input rather than a second enforcement of a bound this side does not own, and the real limit can grow without the SDKs moving. Co-Authored-By: Claude Opus 5 --- packages/client/README.md | 2 +- packages/client/agents.md | 2 +- .../client/src/launchdarkly_ai_server/skills_core.py | 8 +++++++- packages/client/tests/test_skills.py | 10 +++++----- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index ae572b3b..8ad2985c 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -275,7 +275,7 @@ field of a different shape must rename it before upgrading. **Integrity is not optional.** Content is only returned after its sha256 (lowercase hex, over the verbatim UTF-8 bytes) matches the delivered `contentHash`, its key and version -revalidate, and its size is within 64 KiB. Anything that fails is withheld and treated as +revalidate, and its size is within 10 MiB. Anything that fails is withheld and treated as missing — no unverified content ever reaches your code. A retrieval that withheld anything logs a count at WARN, so a run that resolved nothing is not silent. diff --git a/packages/client/agents.md b/packages/client/agents.md index b5733567..2c28e64f 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -224,7 +224,7 @@ Store data is **untrusted input**; the transport is not part of the trust bounda themselves. - **Integrity is mandatory and doubled, through one implementation.** Every raw object is verified at the accessor boundary (key pattern and length, integer version >= 1, content - at most 64 KiB, sha256 lowercase hex over the verbatim bytes against `contentHash`) + at most 10 MiB, sha256 lowercase hex over the verbatim bytes against `contentHash`) and the hash is re-verified immediately before a write, both through `skills_core.verified_bytes`, so the integrity signal's property set cannot depend on which layer caught the defect. A `Skill` is only ever constructed from content that diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index a3894335..3e7fabe3 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -67,11 +67,17 @@ ``launchdarkly_ai_server.skills_core``. """ -MAX_SKILL_CONTENT_BYTES = 64 * 1024 +MAX_SKILL_CONTENT_BYTES = 10 * 1024 * 1024 """ Hard cap on skill content. Legitimately delivered skills are well under this bound, so anything larger is withheld regardless of whether its hash checks out. +Set well above the platform's own limit on purpose. This is a backstop against +absurd input, not a second enforcement of the real bound: the platform refuses +oversized skills before they are ever delivered, and a client-side number sitting +just above that one would turn every backend increase into an SDK release. The +headroom lets the real limit grow without this constant moving. + Deliberately **not** exported from the package root, unlike the on-disk and on-the-wire constants beside it. Those are values this SDK defines and a caller may need to agree with; this one is a local enforcement bound on content the diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 0b0cadef..51e2217b 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -248,7 +248,7 @@ def test_content_cap_is_not_public_api(self) -> None: import launchdarkly_ai_server as package from launchdarkly_ai_server import skills_core - assert skills_core.MAX_SKILL_CONTENT_BYTES == 65536 + assert skills_core.MAX_SKILL_CONTENT_BYTES == 10485760 assert "MAX_SKILL_CONTENT_BYTES" not in package.__all__ assert not hasattr(package, "MAX_SKILL_CONTENT_BYTES") @@ -969,7 +969,7 @@ async def test_oversize_content_rejected( self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any ) -> None: skills_module._set_emitter_for_testing(recording_emitter) - oversize = "x" * (64 * 1024 + 1) + oversize = "x" * (10 * 1024 * 1024 + 1) store.put(make_raw_skill(key="a", content=oversize)) assert await get_skill("a") is None assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 @@ -977,11 +977,11 @@ async def test_oversize_content_rejected( async def test_content_at_size_cap_is_accepted( self, store: InMemorySkillStore, make_raw_skill: Any ) -> None: - at_cap = "x" * (64 * 1024) + at_cap = "x" * (10 * 1024 * 1024) store.put(make_raw_skill(key="a", content=at_cap)) skill = await get_skill("a") assert skill is not None - assert len(skill.content) == 64 * 1024 + assert len(skill.content) == 10 * 1024 * 1024 async def test_key_at_length_bound_from_store_accepted( self, store: InMemorySkillStore, make_raw_skill: Any @@ -1132,7 +1132,7 @@ def _raw_without(field: str) -> dict[str, Any]: return raw -_OVERSIZE = "x" * (64 * 1024 + 1) +_OVERSIZE = "x" * (10 * 1024 * 1024 + 1) REASON_CODE_CASES = [ # Not a dict at all. Reachable through ``all_skills`` and not through From b7ee71a3477bad38d31123bed63e17a6203f3cbd Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Mon, 14 Sep 2026 14:36:58 -0400 Subject: [PATCH 21/22] fix(client): keep FDv2 delivery alive on an idle stream, and shut down promptly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three faults in the delivery loop, all of which left the store reporting itself healthy while doing less than it claimed. **An up-to-date stream tripped the failure cap.** The consecutive-failure count reset only at a commit, and an environment whose skills are not changing answers every reconnect with `intentCode: "none"` and transfers nothing. A stream only ever ends by being dropped, so each recycle of a perfectly healthy idle connection counted as a failure — announced with a `goodbye` or not — and `max_consecutive_failures + 1` of them stopped delivery for the process lifetime, revocations included. The reset on commit covered only the case where content had changed, which is the case that was easy to test and not the case that runs in production. `_TransferOutcome` now reports `up_to_date`, and a complete answer that transfers nothing breaks the row of failures exactly as a commit does. An intent this module does not recognise is still not an answer. **`close` could not interrupt a poll.** The interrupt reached the streaming connection only, so polling parked in its request with nothing to reach and `close` returned when its join timed out — on a 300s-class request, long after the process meant to exit. `_Requester` now tracks the response of a poll in flight and offers `interrupt`, which `close` calls alongside the stream's own. A request still inside its connect has no response to reach; that one is bounded by `read_timeout`, and `start` no longer leaves the store inert when a join times out around it. An interrupt we asked for is no longer recorded as a delivery failure. **`close` left a waiter parked.** `wait_for_skills` waited on the first payload alone, so a shutdown racing a waiter added the waiter's whole timeout to it. Delivery ending is now its own event: a waiter is released by a payload, a give-up or a close, and reports whether a payload actually arrived rather than merely that it was let go. That also settles what `_give_up` had been quietly asserting — it set the first-payload flag to unblock waiters, which made `wait_for_skills` answer `True` for a store holding nothing. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fdv2.py | 123 ++++++- packages/client/tests/test_skills_fdv2.py | 315 +++++++++++++++++- 2 files changed, 420 insertions(+), 18 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 18b81842..6a84092e 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -487,6 +487,14 @@ class _TransferOutcome: basis: str | None = None fatal: str | None = None disconnect: str | None = None + up_to_date: bool = False + """ + The server said what we hold is current and it has nothing to transfer. + + A complete answer that commits nothing, which is exactly what a 304 is to a + poll. The delivery loop counts it as a healthy connection; see + ``FDv2SkillStore._apply``. + """ class _ProtocolReader: @@ -576,6 +584,10 @@ def _server_intent(self, data: Any) -> _TransferOutcome: if intent != _INTENT_TRANSFER_NONE: logger.debug("Ignoring FDv2 server-intent with intentCode %r", intent) self._pending = None + # ``none`` is a complete answer that carries nothing. An intent this + # module does not recognise is not an answer at all, so only the + # former reports itself up to date. + return _TransferOutcome(up_to_date=intent == _INTENT_TRANSFER_NONE) return _TransferOutcome() def _target_for(self, data: Any) -> _SkillObjectSet | None: @@ -942,6 +954,25 @@ def __init__( self._read_timeout = read_timeout # Injectable so tests can drive a fake endpoint without a socket. self._opener = opener or urllib.request.build_opener() + self._lock = threading.Lock() + # The response of a poll in flight, so ``interrupt`` can reach its + # socket from another thread. Polling only: a stream's response is + # handed straight to the caller as a ``_StreamConnection``, which + # carries an interrupt of its own. + self._in_flight: Any = None + + def interrupt(self) -> None: + """ + Unblocks a poll parked in its body read, from another thread. + + Best effort, and safe to call when nothing is in flight. A request still + inside its connect has no response to reach yet and is bounded only by + ``read_timeout``; ``FDv2SkillStore.start`` covers what that leaves. + """ + with self._lock: + response = self._in_flight + if response is not None: + _interrupt_read(response) def _url(self, path: str, basis: str | None) -> str: """ @@ -972,11 +1003,17 @@ def poll(self, basis: str | None, etag: str | None) -> _PollResult: request = self._request(POLL_PATH, basis, headers) try: with self._opener.open(request, timeout=self._read_timeout) as response: - status = getattr(response, "status", None) or response.getcode() - if status == 304: - return _PollResult(not_modified=True, events=[], etag=etag) - body = response.read() - new_etag = response.headers.get("ETag") or etag + with self._lock: + self._in_flight = response + try: + status = getattr(response, "status", None) or response.getcode() + if status == 304: + return _PollResult(not_modified=True, events=[], etag=etag) + body = response.read() + new_etag = response.headers.get("ETag") or etag + finally: + with self._lock: + self._in_flight = None except urllib.error.HTTPError as exc: if exc.code == 304: # urllib raises on 304 when no redirect handler swallows it. @@ -1225,6 +1262,18 @@ def __init__( self._stop = threading.Event() self._first_payload = threading.Event() + """A payload has committed. The fact ``wait_for_skills`` reports.""" + self._delivery_ended = threading.Event() + """ + Delivery has stopped, by ``close`` or by ``_give_up``. Kept apart from + ``_first_payload`` because it is not one: a waiter has to be let go + either way, but only a payload makes ``wait_for_skills`` true. + """ + self._released = threading.Event() + """ + Either of the two above, and what a waiter actually parks on: an + ``Event`` cannot wait on two, so the setters funnel through here. + """ self._thread: threading.Thread | None = None self._failed_reason: str | None = None # The open streaming connection, so ``close`` can interrupt its read. @@ -1244,7 +1293,13 @@ def start(self) -> FDv2SkillStore: Does not block: use ``wait_for_skills`` when boot ordering matters. """ with self._lock: + self._rearm_waiters() if self._thread is not None and self._thread.is_alive(): + # A ``close`` whose join timed out leaves the previous thread + # running with the stop flag still set. Clearing it lets that + # thread carry on delivering, rather than leaving a store that + # reports itself started and never delivers again. + self._stop.clear() return self self._stop.clear() self._thread = threading.Thread( @@ -1253,6 +1308,16 @@ def start(self) -> FDv2SkillStore: self._thread.start() return self + def _rearm_waiters(self) -> None: + """ + Re-arms ``wait_for_skills`` for a store being started again after a + ``close``. A payload already held stays an answer; an ended delivery + does not, or the next waiter would be released before it began. + """ + self._delivery_ended.clear() + if not self._first_payload.is_set(): + self._released.clear() + def close(self, timeout: float = 5.0) -> None: """ Stops delivery. Idempotent, and safe to call from any thread. @@ -1262,12 +1327,17 @@ def close(self, timeout: float = 5.0) -> None: package-level ``launchdarkly_ai_server.shutdown()`` coroutine. """ self._stop.set() + # A waiter parked in ``wait_for_skills`` is owed an answer now rather + # than at the end of its timeout; delivery is over either way. + self._end_delivery() # The delivery thread is normally blocked in a socket read that no flag - # can reach; without this the join waits out its full timeout. + # can reach; without this the join waits out its full timeout. Streaming + # parks in the connection, polling in the request, so interrupt both. with self._lock: connection = self._connection if connection is not None: connection.close() + self._requester.interrupt() thread = self._thread if ( thread is not None @@ -1288,8 +1358,24 @@ def wait_for_skills(self, timeout: float = 10.0) -> bool: ``True`` means a payload arrived — not that any skill in it verified, and not that the environment has any skills. ``diagnostics`` answers the rest. + + Returns early, ``False``, when delivery ends before any payload does: + a ``close`` from another thread, or a failure delivery cannot retry. + Waiting out the full timeout for an answer that has already arrived + would delay every shutdown that raced a waiter. """ - return self._first_payload.wait(timeout=timeout) + self._released.wait(timeout=timeout) + return self._first_payload.is_set() + + def _publish_first_payload(self) -> None: + """Records the first committed payload and lets any waiter go.""" + self._first_payload.set() + self._released.set() + + def _end_delivery(self) -> None: + """Records that delivery has stopped and lets any waiter go.""" + self._delivery_ended.set() + self._released.set() @property def failed(self) -> str | None: @@ -1383,6 +1469,11 @@ def _run(self) -> None: self._give_up(str(exc)) return except _RecoverableTransportError as exc: + if self._stop.is_set(): + # ``close`` interrupted the request on purpose. Counting it + # would spend a retry from the bounded budget and leave a + # misleading ``last_error`` on a healthy store. + return with self._lock: self._failures += 1 failures = self._failures @@ -1432,8 +1523,8 @@ def _give_up(self, reason: str) -> None: "the process restarts with a working connection.", reason, ) - # Unblock anyone waiting on a first payload that is never coming. - self._first_payload.set() + # Let go of anyone waiting on a first payload that is never coming. + self._end_delivery() def _apply(self, name: str, data: Any) -> None: """ @@ -1444,10 +1535,16 @@ def _apply(self, name: str, data: Any) -> None: outcome = self._reader.handle(name, data) if outcome.committed and outcome.basis is not None: self._basis = outcome.basis - if outcome.committed: - # A commit breaks the row of consecutive failures. + if outcome.committed or outcome.up_to_date: + # Both break the row of consecutive failures: a commit is a payload + # delivered, and ``up_to_date`` is the server confirming we already + # hold it. Counting only the commit would give up on a healthy + # stream serving an environment whose skills are not changing: + # nothing to transfer means no commit, while every recycled + # connection still ends in a drop. self._record_success() - self._first_payload.set() + if outcome.committed: + self._publish_first_payload() if outcome.changes: self._notify(outcome.changes) if outcome.fatal: @@ -1465,7 +1562,7 @@ def _poll_once(self) -> None: logger.debug("Skill payload unchanged (HTTP 304)") # A 304 counts as a first payload, so a boot that reconnects with a # cached basis is not blocked on a transfer the server will not send. - self._first_payload.set() + self._publish_first_payload() return for name, data in result.events: self._apply(name, data) diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index af5d1812..d063e451 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -538,6 +538,17 @@ def test_a_full_transfer_commits_at_payload_transferred(self) -> None: assert outcomes[-1].committed is True assert outcomes[-1].basis == "basis-1" + def test_an_up_to_date_intent_is_reported_as_such(self) -> None: + """``intentCode: "none"`` is the stream's 304: current, nothing to send.""" + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle("server-intent", server_intent("none")) + assert outcome.up_to_date is True + assert outcome.committed is False + assert outcome.disconnect is None + # A transfer intent is a promise of content, not an up-to-date answer. + transfer = reader.handle("server-intent", server_intent("xfer-full")) + assert transfer.up_to_date is False + def test_nothing_is_visible_before_payload_transferred(self) -> None: """A payload version is the unit of consistency; half of one is not a state.""" held = _SkillObjectSet() @@ -1312,7 +1323,17 @@ def close(self) -> None: pass -class _DyingStreamRequester: +class _FakeRequester: + """ + Base for the requester fakes: supplies the ``interrupt`` the store calls on + ``close``, so each fake only scripts the part it is about. + """ + + def interrupt(self) -> None: + """No real socket to reach; these fakes end their own connections.""" + + +class _DyingStreamRequester(_FakeRequester): """Every connection transfers a payload, then dies with *exc* mid-read.""" def __init__(self, exc: BaseException) -> None: @@ -1335,7 +1356,7 @@ def close(self) -> None: self.closed = True -class _ScriptedRequester: +class _ScriptedRequester(_FakeRequester): """Raises a scripted sequence, so backoff is asserted without real sockets.""" def __init__(self, *outcomes: Any) -> None: @@ -1361,7 +1382,7 @@ def stream(self, basis: str | None) -> Any: return _ScriptedConnection(outcome) -class _RecyclingRequester: +class _RecyclingRequester(_FakeRequester): """ A healthy server that recycles connections: every ``stream`` call succeeds, transfers a full payload, and then ends the connection, as LaunchDarkly and @@ -1383,6 +1404,53 @@ def stream(self, basis: str | None) -> Any: ) +class _UpToDateRecyclingRequester(_FakeRequester): + """ + A healthy server with nothing new to say: every connection answers + ``intentCode: "none"`` — the stream's equivalent of a 304 — transfers + nothing, and is then recycled. This is the steady state of an environment + whose skills are not changing, which is most environments most of the time. + """ + + def __init__(self, farewell: bool = False) -> None: + self.connections = 0 + self._farewell = farewell + + def stream(self, basis: str | None) -> Any: + self.connections += 1 + script: list[tuple[str, Any]] = [ + ("server-intent", server_intent("none")), + ("heart-beat", {}), + ] + if self._farewell: + # A recycle is often announced rather than abrupt. + script.append(("goodbye", {"reason": "connection recycled"})) + return _ScriptedConnection(script) + + +class _SlowPollRequester(_FakeRequester): + """ + A poll whose request does not return until the test releases it, standing in + for one blocked where no interrupt can reach: inside its connect. + """ + + def __init__(self) -> None: + self.entered = threading.Event() + self.release = threading.Event() + + def poll(self, basis: str | None, etag: str | None) -> Any: + self.entered.set() + self.release.wait(timeout=10) + raise _RecoverableTransportError("released") + + +class _SilentStreamRequester(_FakeRequester): + """A stream that connects and then delivers nothing until it is closed.""" + + def stream(self, basis: str | None) -> Any: + return _BlockingConnection() + + class _BlockingConnection: """A stream that never produces an event until it is closed.""" @@ -1398,7 +1466,7 @@ def close(self) -> None: self._closed.set() -class _SlowConnectRequester: +class _SlowConnectRequester(_FakeRequester): """ A ``stream`` whose connect does not return until the test releases it, standing in for a slow TLS handshake, followed by a read that never yields. @@ -1447,7 +1515,11 @@ def test_a_fatal_failure_releases_wait_for_skills_rather_than_hanging( ) -> None: endpoint.queue_poll(status=401) with poll_store(endpoint) as store: - assert store.wait_for_skills(timeout=5) is True + started = time.monotonic() + # Released promptly, and ``False``: no payload arrived, and saying + # otherwise would send a caller on to read a store holding nothing. + assert store.wait_for_skills(timeout=5) is False + assert time.monotonic() - started < 2.0 assert store.failed is not None def test_a_fatal_failure_keeps_last_known_good_servable( @@ -1514,6 +1586,28 @@ def test_recycled_stream_connections_are_not_failures(self) -> None: finally: store.close() + @pytest.mark.parametrize("farewell", [False, True], ids=["dropped", "goodbye"]) + def test_an_up_to_date_recycled_stream_is_not_a_failure( + self, farewell: bool + ) -> None: + # Resetting at a commit covers only a connection that carried new + # content. An environment whose skills are not changing answers every + # reconnect with ``intentCode: "none"`` and transfers nothing, so a loop + # that counted those drops would give up on a *healthy* idle stream + # after max_consecutive_failures + 1 recycles — and revocation, the one + # thing streaming exists to deliver promptly, would never arrive again. + requester = _UpToDateRecyclingRequester(farewell=farewell) + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert wait_until(lambda: requester.connections >= 8) + assert store.failed is None + # As with a payload-carrying recycle, the count may read 1 + # mid-reconnect. What it must never do is climb. + assert store.diagnostics.connection_failures <= 1 + finally: + store.close() + @pytest.mark.parametrize( "exc", [ @@ -2194,6 +2288,64 @@ def close(self) -> None: self._listener.close() +class _StalledBody: + """ + A listening socket that answers with headers and then stalls the body. + + Distinct from ``_BlackHole``: here the request succeeds far enough to hand + urllib a response, and the caller then parks in ``read``. That is the state + ``close`` has to interrupt — and, unlike a request still inside its connect, + the state an interrupt can actually reach. + """ + + def __init__(self) -> None: + self._listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._listener.bind(("127.0.0.1", 0)) + self._listener.listen(8) + self._accepted: list[socket.socket] = [] + self.serving = threading.Event() + self._stop = threading.Event() + self._thread = threading.Thread(target=self._serve_forever, daemon=True) + self._thread.start() + host, port = self._listener.getsockname() + self.base_uri = f"http://{host}:{port}" + + def _serve_forever(self) -> None: + self._listener.settimeout(0.05) + while not self._stop.is_set(): + try: + conn, _ = self._listener.accept() + except OSError: + continue + self._accepted.append(conn) + try: + conn.recv(4096) + # A length far longer than the body that follows, so the read + # blocks rather than seeing the end of the message. + conn.sendall( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" + b"Content-Length: 4096\r\n\r\n" + ) + except OSError: + continue + self.serving.set() + + def close(self) -> None: + self._stop.set() + self._thread.join(timeout=2) + for conn in self._accepted: + conn.close() + self._listener.close() + + +@pytest.fixture +def stalled_body() -> Any: + server = _StalledBody() + yield server + server.close() + + @pytest.fixture def black_hole() -> Any: server = _BlackHole() @@ -2274,3 +2426,156 @@ def test_there_is_no_separate_connect_timeout(self) -> None: # constructor does not offer a parameter that would only pretend to. with pytest.raises(TypeError): FDv2SkillStore(SDK_KEY, connect_timeout=2.0) # type: ignore[call-arg] + + +class TestWaitingForSkills: + """ + ``wait_for_skills`` answers with what happened, and never outlives it. + + Its budget is a boot-ordering allowance, not a delay to spend: a store that + already knows no payload is coming owes the caller that answer immediately. + """ + + def test_close_releases_a_waiter_rather_than_leaving_it_parked(self) -> None: + # A shutdown racing a waiter is the ordinary case, not an exotic one: + # ``close`` on the main thread while a worker is still waiting for its + # first payload. Parking that worker for the rest of its timeout adds + # the whole budget to a process that has already decided to stop. + store = stream_store(_requester=_SilentStreamRequester()) + store.start() + answers: list[bool] = [] + waiter = threading.Thread( + target=lambda: answers.append(store.wait_for_skills(timeout=10)), + daemon=True, + ) + waiter.start() + time.sleep(0.2) + started = time.monotonic() + store.close() + waiter.join(timeout=5) + assert not waiter.is_alive() + assert time.monotonic() - started < 2.0 + assert answers == [False] + + def test_a_payload_already_held_still_answers_true_after_close(self) -> None: + # ``close`` does not drop content, so it must not turn the answer about + # that content into a lie either. + requester = _RecyclingRequester() + store = stream_store(_requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + finally: + store.close() + assert store.wait_for_skills(timeout=5) is True + + def test_a_restarted_store_waits_again(self) -> None: + # The released flag is sticky by design, so a store closed before any + # payload and then started again has to re-arm: otherwise the next + # waiter is let go before delivery has had a chance to begin. + store = stream_store(_requester=_SilentStreamRequester()) + store.start() + store.close() + assert store.wait_for_skills(timeout=0.1) is False + store.start() + try: + started = time.monotonic() + assert store.wait_for_skills(timeout=0.5) is False + # Waited, rather than being released by the previous close. + assert time.monotonic() - started >= 0.4 + finally: + store.close() + + +class TestPollShutdown: + """ + ``close`` has to interrupt a poll in flight, as it already does a stream. + + Without it the delivery thread stays parked in its request and ``close`` + returns only when the join times out — on a 300s-class request, long after + the process meant to exit. The bound is loose on purpose: the point is + promptly rather than a particular number of milliseconds. + """ + + def test_interrupt_unblocks_a_poll_stalled_in_its_body( + self, stalled_body: Any + ) -> None: + requester = _Requester(SDK_KEY, stalled_body.base_uri, read_timeout=30.0) + raised: list[BaseException] = [] + + def poll_until_interrupted() -> None: + try: + requester.poll(None, None) + except BaseException as exc: + raised.append(exc) + + thread = threading.Thread(target=poll_until_interrupted, daemon=True) + thread.start() + assert stalled_body.serving.wait(timeout=5) + # The response is in hand; give the read a moment to park in it. + time.sleep(0.2) + started = time.monotonic() + requester.interrupt() + thread.join(timeout=5) + assert not thread.is_alive() + assert time.monotonic() - started < 2.0 + assert raised and isinstance(raised[0], _RecoverableTransportError) + + def test_close_during_a_stalled_poll_returns_promptly( + self, stalled_body: Any + ) -> None: + store = FDv2SkillStore( + SDK_KEY, + base_uri=stalled_body.base_uri, + mode="poll", + poll_interval=0.05, + read_timeout=30.0, + ) + store.start() + assert stalled_body.serving.wait(timeout=5) + time.sleep(0.2) + started = time.monotonic() + store.close(timeout=5.0) + assert time.monotonic() - started < 2.0 + assert store._thread is not None and not store._thread.is_alive() + + def test_a_poll_we_interrupted_is_not_a_delivery_failure( + self, stalled_body: Any + ) -> None: + # Our own shutdown is not an outage: counting it would spend a retry + # from the bounded budget and leave a misleading ``last_error`` behind + # on a store whose content is still perfectly good. + store = FDv2SkillStore( + SDK_KEY, + base_uri=stalled_body.base_uri, + mode="poll", + poll_interval=0.05, + read_timeout=30.0, + ) + store.start() + assert stalled_body.serving.wait(timeout=5) + time.sleep(0.2) + store.close(timeout=5.0) + assert store.diagnostics.connection_failures == 0 + assert store.diagnostics.last_error is None + assert store.failed is None + + def test_a_close_that_timed_out_leaves_the_store_restartable(self) -> None: + # A request blocked inside its connect is beyond any interrupt, so + # ``close`` can still return with the thread alive. ``start`` must not + # then find that thread and return with the stop flag set: the store + # would report itself started and never deliver again. + requester = _SlowPollRequester() + store = FDv2SkillStore( + SDK_KEY, mode="poll", poll_interval=0.01, _requester=requester + ) + try: + store.start() + assert requester.entered.wait(timeout=5) + store.close(timeout=0.2) + assert store._thread is not None and store._thread.is_alive() + store.start() + assert store._stop.is_set() is False + finally: + requester.release.set() + store.close(timeout=2) From 88c225ebc296613713d32fcfa3f7fb474ffe6495 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Mon, 14 Sep 2026 14:47:44 -0400 Subject: [PATCH 22/22] fix(client): log a recycled FDv2 stream at debug rather than warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A stream only ever ends by being dropped, and LaunchDarkly — and any proxy in between — recycles a long-lived one. Every reconnect therefore logged "Skill delivery failed" at WARNING, for as long as the process ran. Until the previous commit that noise was bounded, because an idle stream gave up after eleven recycles and went quiet; now that delivery correctly survives them, it would run forever and describe a healthy store as failing. A connection that got a complete answer before it ended — a committed payload, or an up-to-date intent — delivered everything it was asked for, so its reconnect is now DEBUG and says so. A connection that ended without answering is the case the warning exists for and still gets it: a connect that never landed, or a transfer that died part-way through. Filling a customer's logs with a fault they do not have is not merely untidy; it teaches them that the level which means something can be ignored. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fdv2.py | 27 ++++++++++++++-- packages/client/tests/test_skills_fdv2.py | 32 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 6a84092e..71411c8a 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -1283,6 +1283,10 @@ def __init__( # by being dropped, so resetting on return would count every healthy, # server-recycled connection as a failure. self._failures = 0 + # Whether the current attempt got a complete answer before it ended. + # A stream only ever ends by being dropped, so this is what separates + # a recycled healthy connection from one that failed. + self._attempt_answered = False # -- lifecycle --------------------------------------------------------- @@ -1456,6 +1460,8 @@ def _notify(self, changes: list[dict[str, Any]]) -> None: def _run(self) -> None: while not self._stop.is_set(): + with self._lock: + self._attempt_answered = False try: if self._mode == "stream": self._stream_once() @@ -1477,6 +1483,7 @@ def _run(self) -> None: with self._lock: self._failures += 1 failures = self._failures + answered = self._attempt_answered self._reader.diagnostics.connection_failures = failures self._reader.diagnostics.last_error = str(exc) if failures > self._max_consecutive_failures: @@ -1494,9 +1501,22 @@ def _run(self) -> None: # The header may come from a proxy rather than LaunchDarkly, and # a value in the hours would park revocation for that long. delay = min(delay, self._max_backoff) - logger.warning( - "Skill delivery failed (%s); retrying in %.1fs", exc, delay - ) + if answered: + # LaunchDarkly, and any proxy in between, recycles a + # long-lived stream. A connection that answered before it + # ended delivered everything it was asked for, so the + # reconnect is routine rather than a fault worth warning + # about for as long as the process runs. + logger.debug( + "The FDv2 stream ended after a complete answer (%s); " + "reconnecting in %.1fs", + exc, + delay, + ) + else: + logger.warning( + "Skill delivery failed (%s); retrying in %.1fs", exc, delay + ) if self._stop.wait(delay): return continue @@ -1511,6 +1531,7 @@ def _run(self) -> None: def _record_success(self) -> None: with self._lock: self._failures = 0 + self._attempt_answered = True self._reader.diagnostics.connection_failures = 0 def _give_up(self, reason: str) -> None: diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index d063e451..b0c6aa86 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -1608,6 +1608,38 @@ def test_an_up_to_date_recycled_stream_is_not_a_failure( finally: store.close() + def test_a_recycled_connection_reconnects_quietly(self, caplog: Any) -> None: + # A healthy idle stream reconnects for as long as the process runs, so + # warning on each one would fill a customer's logs with a fault they do + # not have and teach them to ignore the level that means something. + requester = _UpToDateRecyclingRequester() + store = stream_store(_requester=requester) + with caplog.at_level("DEBUG", logger="launchdarkly_ai_server.skills_fdv2"): + try: + store.start() + assert wait_until(lambda: requester.connections >= 5) + finally: + store.close() + assert store.failed is None + assert not [r for r in caplog.records if r.levelname == "WARNING"] + assert [r for r in caplog.records if "reconnecting in" in r.getMessage()] + + def test_a_connection_that_never_answered_still_warns(self, caplog: Any) -> None: + # The quiet path is earned by answering. A connection that failed before + # it told us anything is the case the warning exists for. + store = stream_store( + max_consecutive_failures=10, _requester=_ScriptedRequester() + ) + with caplog.at_level("DEBUG", logger="launchdarkly_ai_server.skills_fdv2"): + try: + store.start() + assert wait_until(lambda: store.diagnostics.connection_failures >= 3) + finally: + store.close() + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert warnings + assert all("Skill delivery failed" in r.getMessage() for r in warnings) + @pytest.mark.parametrize( "exc", [