From 0f797c0b167adf8be462ae8e8b2431705d75dfd7 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Thu, 10 Sep 2026 16:03:21 -0400 Subject: [PATCH 1/7] =?UTF-8?q?feat(client):=20Agent=20Skills=20=E2=80=94?= =?UTF-8?q?=20the=20FDv2=20delivery=20protocol,=20without=20the=20network?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The half of the delivery transport that has no I/O: identifying a skill object on the wire by kind inline-resource plus category skill, translating it into the raw object shape the SkillStore interface defines, holding it by (key, objectVersion), and applying a payload's events as one commit at payload-transferred. The store that puts a connection underneath this follows separately, so the three decisions that matter most can be reviewed on their own: - objectVersion is the skill's version; version is the payload's. The translation happens in one place and TestVersionTranslation asserts it in both directions, because confusing them fails silently. - Changes commit at payload-transferred, not per object. A half-applied full transfer would briefly empty the store, which with pruning on is the difference between a reconcile and deleting a customer's files. - A hashless object is held, not dropped, so verification withholds it with a reason code rather than the transport reporting it absent. Flag and segment objects share the connection and are skipped and counted, not rejected. Nothing here is exported yet; the store exports it. Co-Authored-By: Claude Fable 5.1 --- packages/client/agents.md | 46 ++ .../src/launchdarkly_ai_server/skills_fdv2.py | 601 +++++++++++++++++ packages/client/tests/test_skills_fdv2.py | 605 ++++++++++++++++++ 3 files changed, 1252 insertions(+) create mode 100644 packages/client/src/launchdarkly_ai_server/skills_fdv2.py create mode 100644 packages/client/tests/test_skills_fdv2.py diff --git a/packages/client/agents.md b/packages/client/agents.md index ca56b80e..80ec0538 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,6 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | +| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the `objectVersion`/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | | `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | @@ -218,6 +219,51 @@ one place that collapses the result to one object per key, because both whole-st consumers need it — `all_skills`, since a list holding two versions of one key is not a set of skills, and the `"*"` reconcile, since `//SKILL.md` is a single path. +### The delivery transport, and the one field that will bite you + +`skills_fdv2.py` translates LaunchDarkly's FDv2 delivery protocol into raw objects in the +shape `skills_core.SkillStore` documents. It lives below the store interface; **nothing above +that interface knows it exists**. If a transport change ever seems to require editing an +accessor, verification, or `write_skills`, the adapter boundary is wrong. + +**`objectVersion` is the skill's version. `version` is the payload's.** On the wire a skill +`put-object` carries both, and they are not interchangeable: + +```json +{"key":"pdf-extraction","kind":"inline-resource","category":"skill", + "objectVersion":3,"version":42, + "object":{"contentType":"text/markdown","content":"…","contentHash":"…","name":"…"}} +``` + +`objectVersion` (3) is what a `{key, version}` reference pins and what becomes the stored +`version`. `version` (42) is the version of the *payload* the object arrived in — it moves +when anything in the environment moves, including a flag with nothing to do with skills. +Reading it as the skill's version fails **silently**: the object verifies, the hash matches, +and the caller gets content under a version number that means nothing. Flags and segments +carry only `version` and omit both `category` and `objectVersion`, which is exactly why the +two fields look interchangeable. `_store_object_from_put` is the only place the translation +happens, and `TestVersionTranslation` asserts it in both directions. + +**Skills are identified by `kind == "inline-resource" && category == "skill"`; everything else +is ignored, not rejected.** An environment's payload assignment carries its flag payload +alongside its agent-skill payload, so flag and segment objects arrive as a matter of course. +Erroring on an unrecognised kind would turn a normal payload into a permanent reconnect +loop — a flag-delivery outage caused by a skills rollout. + +**Changes commit at `payload-transferred`, not as objects arrive.** A payload version is the +unit of consistency: a half-applied full transfer would publish a state the server never +described, and would briefly empty the store — which, with pruning on, is the difference +between a reconcile and deleting a customer's skill files. An interrupted transfer therefore +leaves last known good intact, and listeners fire once per commit. + +**A hashless object is held, not dropped.** Verification withholds it with +`missing_content_hash`; the transport's job is to make that loud (an error per object, a +summary per wholly-hashless payload, `diagnostics.hashless_objects`) rather than to work +around it. Dropping it at the transport would report `absent` — indistinguishable from "no +such skill" — and would let a prune delete the last known-good copy on disk. Never synthesize +a hash from the delivered content: that certifies the content against itself and verifies +nothing. + ### The reported outcome vocabulary, and the `Resolution` mapping `get_skill` returns `Skill | None`; `get_skill_result` returns a frozen `SkillOutcome` diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py new file mode 100644 index 00000000..80ef1224 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -0,0 +1,601 @@ +""" +Agent Skills — the FDv2 delivery protocol. + +The half of the delivery transport that has no I/O: identifying skill objects +on the wire, translating them into the raw object shape the ``SkillStore`` +interface defines, holding them by ``(key, objectVersion)``, and applying a +payload's events as one consistent commit. ``FDv2SkillStore``, the store that +puts a network connection underneath this, follows in a separate change. + +It sits *below* the ``SkillStore`` interface, and everything above — the +accessors, integrity verification, the ``Skill`` dataclass, materialization — +is unaware of it. + +Layering:: + + launchdarkly_ai_server + └─ SkillStore protocol (skills_core) ── the interface accessors call + └─ FDv2SkillStore (this module) ── deserialise, hold, serve + └─ LaunchDarkly's SDK-facing FDv2 channel + GET /sdk/poll, GET /sdk/stream, authenticated with the + environment's server-side SDK key + +Dependencies run one way: this module imports nothing from the feature beyond +the version validator in ``types_validation``, and nothing in the feature +imports it. It uses only the standard library, so it adds no dependency +to a package whose sole runtime dependency is ``opentelemetry-api``. + +Three things this layer does *not* do, on purpose: + +- **It does not verify content.** Verification lives at the accessor boundary in + ``skills_core`` so that it applies to every store equally, including a + customer's own. +- **It does not work around a missing ``contentHash``.** A hashless object is + held verbatim and *withheld* by verification with ``missing_content_hash``. + This module's job is to make that outcome loud — see ``StoreDiagnostics``. +- **It does not evaluate anything.** Flag and segment objects that share the + connection are skipped and counted, nothing more. + +The design rationale — why ``objectVersion`` is not ``version``, why changes +commit at ``payload-transferred`` — is in ``agents.md`` under *The delivery +transport*. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass, field +from typing import Any + +from .types_validation import is_valid_skill_version + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# The wire contract +# --------------------------------------------------------------------------- + +FDV2_OBJECT_KIND = "inline-resource" +""" +The FDv2 ``kind`` skills are delivered under. Together with +``FDV2_OBJECT_CATEGORY`` it maps onto the single interface value +``skills_core.SKILL_OBJECT_KIND``; that translation is this adapter's job. +""" + +FDV2_OBJECT_CATEGORY = "skill" +"""The ``category`` that narrows ``inline-resource`` to an agent skill.""" + +SDK_DATA_MODEL_VERSION = 1 +""" +The ``mv`` request parameter. The one request parameter whose value could not be +confirmed against a live server, so treat the default as provisional and +override it through the store's ``data_model_version`` if needed. +""" + +_EVENT_SERVER_INTENT = "server-intent" +_EVENT_PUT_OBJECT = "put-object" +_EVENT_DELETE_OBJECT = "delete-object" +_EVENT_PAYLOAD_TRANSFERRED = "payload-transferred" +_EVENT_HEARTBEAT = "heart-beat" +_EVENT_GOODBYE = "goodbye" +_EVENT_ERROR = "error" + +_INTENT_TRANSFER_FULL = "xfer-full" +_INTENT_TRANSFER_CHANGES = "xfer-changes" +_INTENT_TRANSFER_NONE = "none" + +_ENVELOPE_FIELDS = ("contentType", "content", "contentHash", "name", "description") +""" +The skill object envelope's fields, copied through verbatim. Nothing is coerced +or defaulted: a transport that filled in a missing field would be forging the +very thing verification exists to check. +""" + +_MOBILE_KEY_PREFIX = "mob-" +_SERVER_KEY_PREFIX = "sdk-" +_CLIENT_SIDE_ID = re.compile(r"\A[0-9a-f]{20,}\Z") +"""A client-side environment ID: bare lowercase hex. Server-side and mobile keys +both carry a prefix, so this shape is unambiguous rather than heuristic.""" + + +# --------------------------------------------------------------------------- +# Server-side only +# --------------------------------------------------------------------------- + + +def _require_server_side_credential(sdk_key: str) -> None: + """ + Refuses a mobile key or a client-side environment ID. + + Skill content is customer-confidential, and payload assignment is shared + across credential types, so a client-side credential may well *succeed* + against these endpoints. Raises rather than logs: a store built on the wrong + credential should not exist. + """ + if not isinstance(sdk_key, str) or not sdk_key.strip(): + raise ValueError( + "FDv2SkillStore requires a LaunchDarkly server-side SDK key " + "(sdk-...); none was given." + ) + key = sdk_key.strip() + if key.startswith(_MOBILE_KEY_PREFIX): + raise ValueError( + "FDv2SkillStore was given a mobile key (mob-...). Agent Skills are a " + "server-side feature: skill content is customer-confidential and is " + "never delivered to a mobile or client-side process. Use the " + "environment's server-side SDK key (sdk-...)." + ) + if _CLIENT_SIDE_ID.match(key): + raise ValueError( + "FDv2SkillStore was given what looks like a client-side environment " + "ID. Agent Skills are a server-side feature: skill content is " + "customer-confidential and is never delivered to a client-side " + "process. Use the environment's server-side SDK key (sdk-...)." + ) + if not key.startswith(_SERVER_KEY_PREFIX): + # Not rejected: private instances and test doubles issue credentials + # without the public prefix. Only the two unambiguous shapes above are. + logger.warning( + "The credential given to FDv2SkillStore does not look like a " + "LaunchDarkly server-side SDK key (sdk-...). Skills are delivered " + "only to server-side credentials; if this is a client-side or mobile " + "credential the connection will be rejected or will deliver nothing." + ) + + +# --------------------------------------------------------------------------- +# Diagnostics +# --------------------------------------------------------------------------- + + +@dataclass +class StoreDiagnostics: + """ + What the transport has seen. Read-only from a caller's perspective. + + Not part of the ``SkillStore`` interface. It exists because "this environment + has no skills" and "every skill was withheld" are easy to mistake for each + other, and a counter is easier to assert on than a log line. + """ + + payloads_transferred: int = 0 + """Completed ``payload-transferred`` commits since the store started.""" + skill_objects_received: int = 0 + """``put-object`` events identified as skills, across all payloads.""" + objects_ignored: int = 0 + """Objects skipped because they were not skills: flags, segments, and any + future kind. Skipping is the contract, not a failure.""" + objects_revoked: int = 0 + """``delete-object`` events applied to skills.""" + hashless_objects: int = 0 + """ + Skill objects whose envelope carried no ``contentHash``. + + **Nonzero means skills are being withheld**: verification withholds every one + of these with ``missing_content_hash``. + """ + connection_failures: int = 0 + """Recoverable transport failures since the last successful transfer.""" + last_error: str | None = None + """The most recent transport error, if any. Human-readable; do not parse.""" + + +# --------------------------------------------------------------------------- +# Deserialisation — where objectVersion is not version +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _Tombstone: + """A ``delete-object`` narrowed to the identity it revokes.""" + + key: str + object_version: int | None + + +def _is_skill_event(data: Any) -> bool: + """ + Whether one ``put-object`` / ``delete-object`` payload is a skill. + + Both halves are required: ``inline-resource`` may carry other categories, + and flags and segments omit ``category`` entirely. Every other kind is + ignored, not rejected, because flag and segment objects share the connection + and erroring on them would turn a normal payload into a reconnect loop. + """ + if not isinstance(data, dict): + return False + return ( + data.get("kind") == FDV2_OBJECT_KIND + and data.get("category") == FDV2_OBJECT_CATEGORY + ) + + +def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: + """ + Translates one FDv2 skill ``put-object`` into the raw object shape the + ``SkillStore`` interface defines. + + **The one translation this adapter must get right:** + + wire ``objectVersion`` → stored ``version`` (the skill's own version) + wire ``version`` → dropped (the *payload* version) + + ``objectVersion`` is what a ``{key, version}`` reference pins; ``version`` + moves whenever anything in the environment moves. Confusing them fails + silently: the object verifies and the caller gets content under a version + number that means nothing. + + Returns ``None`` only when ``key`` is not a string, since a keyless object + has no identity to store it under. Every other defect is carried through + verbatim so that verification withholds it with a reason code rather than + the transport dropping it into indistinguishable absence. + """ + key = data.get("key") + if not isinstance(key, str) or not key: + logger.warning( + "An FDv2 skill put-object carried no string 'key' and could not be " + "stored under any identity; it was dropped." + ) + return None + + raw: dict[str, Any] = {"key": key} + + # A membership test rather than a `.get` default, so an explicitly-null + # objectVersion stays null and reaches verification as `invalid_version`. + if "objectVersion" in data: + raw["version"] = data["objectVersion"] + + envelope = data.get("object") + if isinstance(envelope, dict): + for wire_field in _ENVELOPE_FIELDS: + if wire_field in envelope: + raw[wire_field] = envelope[wire_field] + return raw + + +def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: + """ + Narrows one FDv2 skill ``delete-object`` to the identity it revokes, with + the same ``objectVersion`` translation as a put. + + An ``object_version`` of ``None`` means the delete named no usable version + and is read as "revoke every version of this key". That is the safe + direction: the alternative is continuing to serve content LaunchDarkly has + withdrawn. + """ + key = data.get("key") + if not isinstance(key, str) or not key: + logger.warning( + "An FDv2 skill delete-object carried no string 'key'; it was ignored." + ) + return None + object_version = data.get("objectVersion") + return _Tombstone( + key=key, + object_version=object_version + if is_valid_skill_version(object_version) + else None, + ) + + +# --------------------------------------------------------------------------- +# The held object set +# --------------------------------------------------------------------------- + + +class _SkillObjectSet: + """ + Raw skill objects held in memory, keyed by ``(key, objectVersion)``. + + Lookup semantics are identical to ``InMemorySkillStore``'s, down to the + fall-through to a version-less entry, so that the store a caller configures + cannot change how a pinned reference resolves; ``TestInterfaceParity`` + asserts it. Reimplemented rather than inherited because the transport needs + ``delete`` and the atomic ``replace_with`` a full transfer requires. + + An object too malformed to carry a usable version is still held, under its + key alone, so verification withholds it with a signal rather than the + transport dropping it. + """ + + def __init__(self) -> None: + self._versions: dict[str, dict[int, dict[str, Any]]] = {} + self._loose: dict[str, dict[str, Any]] = {} + + def put(self, raw: dict[str, Any]) -> None: + key = raw["key"] + version = raw.get("version") + if is_valid_skill_version(version): + self._versions.setdefault(key, {})[version] = raw + else: + self._loose[key] = raw + + def delete(self, tombstone: _Tombstone) -> list[dict[str, Any]]: + """Removes what *tombstone* revokes; returns the raw objects that went away.""" + removed: list[dict[str, Any]] = [] + if tombstone.object_version is None: + held = self._versions.pop(tombstone.key, {}) + removed.extend(held.values()) + loose = self._loose.pop(tombstone.key, None) + if loose is not None: + removed.append(loose) + return removed + + held = self._versions.get(tombstone.key, {}) + gone = held.pop(tombstone.object_version, None) + if gone is not None: + removed.append(gone) + if not held: + self._versions.pop(tombstone.key, None) + return removed + + def get(self, key: str, version: int | None) -> dict[str, Any] | None: + held = self._versions.get(key, {}) + if version is not None: + # Fall through to the version-less entry so a malformed object + # reaches verification rather than reading as simply absent. + return held.get(version) or self._loose.get(key) + if held: + return held[max(held)] + return self._loose.get(key) + + def snapshot(self) -> dict[str, dict[str, Any]]: + """One entry per ``(key, version)``, under keys opaque to the SDK.""" + out: dict[str, dict[str, Any]] = { + f"{key}:{version}": raw + for key, versions in self._versions.items() + for version, raw in versions.items() + } + out.update(self._loose) + return out + + def all_raw(self) -> list[dict[str, Any]]: + return list(self.snapshot().values()) + + def replace_with(self, other: _SkillObjectSet) -> None: + """Adopts *other*'s contents wholesale — how a full transfer commits.""" + self._versions = other._versions + self._loose = other._loose + + def copy(self) -> _SkillObjectSet: + clone = _SkillObjectSet() + clone._versions = {key: dict(v) for key, v in self._versions.items()} + clone._loose = dict(self._loose) + return clone + + def __len__(self) -> int: + return sum(len(v) for v in self._versions.values()) + len(self._loose) + + +# --------------------------------------------------------------------------- +# The protocol state machine — pure, no I/O +# --------------------------------------------------------------------------- + + +@dataclass +class _TransferOutcome: + """What one event did. Aggregated by the caller; nothing here does I/O.""" + + committed: bool = False + changes: list[dict[str, Any]] = field(default_factory=list) + basis: str | None = None + fatal: str | None = None + disconnect: str | None = None + + +class _ProtocolReader: + """ + Applies FDv2 events to an object set. Pure — no sockets, no threads, no clock — + so every wire case is testable without a server. + + **Changes are buffered and committed at ``payload-transferred``.** A payload + version is the unit of consistency: applying half of one would publish a + state the server never described, and on a full transfer would briefly empty + the store. Listeners therefore fire once per commit, not once per object. + """ + + def __init__(self, committed: _SkillObjectSet) -> None: + self._committed = committed + self._intent: str | None = None + self._pending: _SkillObjectSet | None = None + self._changes: list[dict[str, Any]] = [] + self.diagnostics = StoreDiagnostics() + # Identities already reported by ``_warn_hashless``. Per reader, so a + # recreated store reports again and two stores never quieten each other. + # No lock: ``handle`` runs only on its owner's single delivery thread. + self._warned_hashless: set[tuple[str, Any]] = set() + + # -- events ------------------------------------------------------------ + + def handle(self, name: str, data: Any) -> _TransferOutcome: + """Routes one event. Unknown event names are ignored, by contract.""" + if name == _EVENT_SERVER_INTENT: + return self._server_intent(data) + if name == _EVENT_PUT_OBJECT: + return self._put_object(data) + if name == _EVENT_DELETE_OBJECT: + return self._delete_object(data) + if name == _EVENT_PAYLOAD_TRANSFERRED: + return self._payload_transferred(data) + if name == _EVENT_ERROR: + return self._error(data) + if name == _EVENT_GOODBYE: + return self._goodbye(data) + if name == _EVENT_HEARTBEAT: + return _TransferOutcome() + logger.debug("Ignoring unknown FDv2 event '%s'", name) + return _TransferOutcome() + + def _server_intent(self, data: Any) -> _TransferOutcome: + payloads = data.get("payloads") if isinstance(data, dict) else None + if not isinstance(payloads, list) or not payloads: + return _TransferOutcome( + disconnect="server-intent carried no payload description" + ) + first = payloads[0] + intent = first.get("intentCode") if isinstance(first, dict) else None + self._intent = intent + self._changes = [] + if intent == _INTENT_TRANSFER_FULL: + # Built alongside the live set rather than in place, so an + # interrupted transfer leaves last known good intact. + self._pending = _SkillObjectSet() + elif intent == _INTENT_TRANSFER_CHANGES: + self._pending = self._committed.copy() + else: + if intent != _INTENT_TRANSFER_NONE: + logger.debug("Ignoring FDv2 server-intent with intentCode %r", intent) + self._pending = None + return _TransferOutcome() + + def _target_for(self, data: Any) -> _SkillObjectSet | None: + """ + The pending set a skill object event applies to, or ``None`` when the + event is not a skill or the current intent carries no objects. + + An object arriving with no ``server-intent`` at all is treated as a + delta against what is held rather than dropped. + """ + if not _is_skill_event(data): + self.diagnostics.objects_ignored += 1 + return None + if self._pending is None: + if self._intent is None: + self._intent = _INTENT_TRANSFER_CHANGES + if self._intent not in (_INTENT_TRANSFER_FULL, _INTENT_TRANSFER_CHANGES): + return None + self._pending = self._committed.copy() + return self._pending + + def _put_object(self, data: Any) -> _TransferOutcome: + target = self._target_for(data) + if target is None: + return _TransferOutcome() + raw = _store_object_from_put(data) + if raw is None: + return _TransferOutcome() + target.put(raw) + self._changes.append(raw) + self.diagnostics.skill_objects_received += 1 + if not isinstance(raw.get("contentHash"), str): + self.diagnostics.hashless_objects += 1 + self._warn_hashless(raw) + return _TransferOutcome() + + def _delete_object(self, data: Any) -> _TransferOutcome: + target = self._target_for(data) + if target is None: + return _TransferOutcome() + tombstone = _tombstone_from_delete(data) + if tombstone is None: + return _TransferOutcome() + target.delete(tombstone) + self.diagnostics.objects_revoked += 1 + # A tombstone carries identity and no content, so a listener that reads + # content must check for ``content`` rather than assume it. + self._changes.append( + {"key": tombstone.key, "version": tombstone.object_version} + ) + return _TransferOutcome() + + def _payload_transferred(self, data: Any) -> _TransferOutcome: + state = data.get("state") if isinstance(data, dict) else None + version = data.get("version") if isinstance(data, dict) else None + if self._pending is not None: + self._committed.replace_with(self._pending) + _warn_if_nothing_can_verify(self._committed) + self._pending = None + self._intent = None + changes = self._changes + self._changes = [] + self.diagnostics.payloads_transferred += 1 + logger.debug( + "FDv2 payload transferred: payload version %s, %d skill object(s) held", + version, + len(self._committed), + ) + return _TransferOutcome( + committed=True, + changes=changes, + basis=state if isinstance(state, str) and state else None, + ) + + def _abandon_in_flight(self) -> None: + """Drops the in-flight payload and keeps what is committed.""" + self._pending = None + self._intent = None + self._changes = [] + + def _error(self, data: Any) -> _TransferOutcome: + reason = data.get("reason") if isinstance(data, dict) else None + self._abandon_in_flight() + return _TransferOutcome(disconnect=f"server sent error: {reason}") + + def _goodbye(self, data: Any) -> _TransferOutcome: + reason = data.get("reason") if isinstance(data, dict) else None + catastrophe = bool(data.get("catastrophe")) if isinstance(data, dict) else False + silent = bool(data.get("silent")) if isinstance(data, dict) else False + self._abandon_in_flight() + if not silent: + logger.info("FDv2 connection closing: %s", reason) + if catastrophe: + return _TransferOutcome( + fatal=f"server sent a catastrophic goodbye: {reason}" + ) + return _TransferOutcome(disconnect=f"server said goodbye: {reason}") + + # -- diagnostics --------------------------------------------------------- + + def _warn_hashless(self, raw: dict[str, Any]) -> None: + """ + One ERROR per ``(key, version)`` whose envelope had no ``contentHash``. + + ERROR rather than WARN because an empty accessor result is otherwise + indistinguishable from an environment that has no skills. + """ + identity = (raw["key"], raw.get("version")) + if identity in self._warned_hashless: + return + self._warned_hashless.add(identity) + logger.error( + "Skill '%s' version %s arrived without a contentHash and will be withheld. %s", + raw["key"], + raw.get("version"), + _HASHLESS_ADVICE, + extra={"ld_skill_key": raw["key"], "ld_skill_version": raw.get("version")}, + ) + + +_HASHLESS_ADVICE = ( + "The delivered skill object carries no 'contentHash', so integrity " + "verification withholds it with reason_code 'missing_content_hash' and its " + "content will not resolve. The SDK cannot work around this: verification " + "hashes the delivered bytes and compares them against the envelope's " + "'contentHash', and there is nothing to compare against. 'contentHash' is a " + "sha256 over the verbatim UTF-8 content. Contact LaunchDarkly support if " + "skills in your environment arrive without one." +) + + +def _warn_if_nothing_can_verify(committed: _SkillObjectSet) -> None: + """ + One ERROR per committed payload in which *nothing* held can possibly verify. + + ``log_withholding_summary`` reports the same condition at the accessor + boundary, but only once a caller asks. This fires at delivery time, so it is + visible in a process that boots, materializes nothing, and exits. + """ + held = committed.all_raw() + if not held: + return + hashless = [raw for raw in held if not isinstance(raw.get("contentHash"), str)] + if len(hashless) != len(held): + return + logger.error( + "All %d skill object(s) in the delivered payload arrived without a " + "contentHash. No skill content will resolve from this store. %s", + len(held), + _HASHLESS_ADVICE, + ) diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py new file mode 100644 index 00000000..ff5720bc --- /dev/null +++ b/packages/client/tests/test_skills_fdv2.py @@ -0,0 +1,605 @@ +""" +Tests for the FDv2 skill delivery protocol. + +Wire semantics — which objects are skills, ``objectVersion`` versus ``version``, +revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted +against ``_ProtocolReader``, which has no I/O, so each case reads as the contract +it is rather than as a server script. +""" + +from __future__ import annotations + +import hashlib +from typing import Any, ClassVar + +import pytest + +from launchdarkly_ai_server import InMemorySkillStore +from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND +from launchdarkly_ai_server.skills_fdv2 import ( + FDV2_OBJECT_CATEGORY, + FDV2_OBJECT_KIND, + _is_skill_event, + _ProtocolReader, + _SkillObjectSet, + _store_object_from_put, + _tombstone_from_delete, +) + +pytestmark = pytest.mark.usefixtures("reset_skill_state") + +SKILL_BODY = "---\nname: PDF Extraction\n---\nExtract text from PDFs.\n" + + +def _hash(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +# --------------------------------------------------------------------------- +# Wire builders — one place that knows the shape, so a contract change is one edit +# --------------------------------------------------------------------------- + + +def put_skill( + key: str = "pdf-extraction", + *, + object_version: Any = 3, + payload_version: int = 42, + content: str = SKILL_BODY, + content_hash: Any = None, + omit_hash: bool = False, + name: str = "PDF Extraction", +) -> dict[str, Any]: + """One skill ``put-object`` event's data, in the shape the wire delivers it.""" + envelope: dict[str, Any] = { + "contentType": "text/markdown", + "content": content, + "name": name, + "description": "Extracts text", + } + if not omit_hash: + envelope["contentHash"] = ( + content_hash if content_hash is not None else _hash(content) + ) + return { + "key": key, + "kind": FDV2_OBJECT_KIND, + "category": FDV2_OBJECT_CATEGORY, + "objectVersion": object_version, + "version": payload_version, + "object": envelope, + } + + +def delete_skill( + key: str = "pdf-extraction", *, object_version: Any = 3, payload_version: int = 43 +) -> dict[str, Any]: + return { + "key": key, + "kind": FDV2_OBJECT_KIND, + "category": FDV2_OBJECT_CATEGORY, + "objectVersion": object_version, + "version": payload_version, + } + + +def put_flag(key: str = "my-flag", version: int = 17) -> dict[str, Any]: + """A flag ``put-object``: no ``category``, no ``objectVersion``.""" + return { + "key": key, + "kind": "flag", + "version": version, + "object": { + "key": key, + "version": version, + "on": True, + "variations": [True, False], + }, + } + + +def put_segment(key: str = "beta-users", version: int = 4) -> dict[str, Any]: + return { + "key": key, + "kind": "segment", + "version": version, + "object": {"key": key, "version": version, "included": []}, + } + + +def server_intent( + code: str = "xfer-full", payload_id: str = "agent-skill" +) -> dict[str, Any]: + return { + "payloads": [ + {"id": payload_id, "target": 1, "intentCode": code, "reason": "test"} + ] + } + + +def transferred(state: str = "basis-1", version: int = 42) -> dict[str, Any]: + return {"state": state, "version": version} + + +def events(*pairs: tuple[str, Any]) -> list[dict[str, Any]]: + return [{"event": name, "data": data} for name, data in pairs] + + +def full_payload( + *object_events: tuple[str, Any], state: str = "basis-1" +) -> list[dict[str, Any]]: + return events( + ("server-intent", server_intent("xfer-full")), + *object_events, + ("payload-transferred", transferred(state)), + ) + + +# --------------------------------------------------------------------------- +# Identifying skill objects, and ignoring everything else +# --------------------------------------------------------------------------- + + +class TestObjectIdentification: + def test_kind_and_category_together_identify_a_skill(self) -> None: + assert _is_skill_event(put_skill()) is True + + def test_a_flag_is_not_a_skill(self) -> None: + assert _is_skill_event(put_flag()) is False + + def test_a_segment_is_not_a_skill(self) -> None: + assert _is_skill_event(put_segment()) is False + + def test_inline_resource_of_another_category_is_not_a_skill(self) -> None: + """``inline-resource`` is a broad kind, so the category is required too.""" + other = put_skill() + other["category"] = "prompt-template" + assert _is_skill_event(other) is False + + def test_skill_category_under_another_kind_is_not_a_skill(self) -> None: + other = put_skill() + other["kind"] = "some-future-kind" + assert _is_skill_event(other) is False + + def test_a_flag_shaped_object_with_no_category_is_not_a_skill(self) -> None: + """Flags and segments omit ``category`` entirely — the documented shape.""" + assert "category" not in put_flag() + assert "objectVersion" not in put_flag() + + @pytest.mark.parametrize("value", [None, "skill", 3, [], ()]) + def test_non_dict_events_are_not_skills(self, value: Any) -> None: + assert _is_skill_event(value) is False + + +# --------------------------------------------------------------------------- +# objectVersion is not version +# --------------------------------------------------------------------------- + + +class TestVersionTranslation: + def test_object_version_becomes_the_seam_version(self) -> None: + raw = _store_object_from_put(put_skill(object_version=3, payload_version=42)) + assert raw is not None + assert raw["version"] == 3 + + def test_the_payload_version_never_reaches_the_seam(self) -> None: + """ + The failure this asserts against is silent: a store that read ``version`` + would serve verifiable content under a version number that means nothing, + and every pinned reference would resolve to the wrong thing with no error. + """ + raw = _store_object_from_put(put_skill(object_version=3, payload_version=42)) + assert raw is not None + assert raw["version"] != 42 + assert 42 not in raw.values() + + def test_the_two_are_distinguished_even_when_the_payload_version_is_lower( + self, + ) -> None: + raw = _store_object_from_put(put_skill(object_version=99, payload_version=1)) + assert raw is not None + assert raw["version"] == 99 + + def test_a_missing_object_version_is_not_defaulted_from_the_payload(self) -> None: + wire = put_skill() + del wire["objectVersion"] + raw = _store_object_from_put(wire) + assert raw is not None + assert "version" not in raw + + def test_an_explicitly_null_object_version_is_carried_through_as_null(self) -> None: + """Carried, not invented: verification reports ``invalid_version``.""" + raw = _store_object_from_put(put_skill(object_version=None)) + assert raw is not None + assert raw["version"] is None + + def test_a_delete_translates_object_version_too(self) -> None: + tombstone = _tombstone_from_delete( + delete_skill(object_version=3, payload_version=43) + ) + assert tombstone is not None + assert tombstone.object_version == 3 + + def test_a_delete_with_no_usable_object_version_revokes_every_version(self) -> None: + tombstone = _tombstone_from_delete(delete_skill(object_version=None)) + assert tombstone is not None + assert tombstone.object_version is None + + def test_a_keyless_put_is_dropped_because_it_has_no_identity(self) -> None: + wire = put_skill() + del wire["key"] + assert _store_object_from_put(wire) is None + + def test_the_envelope_is_copied_verbatim(self) -> None: + raw = _store_object_from_put(put_skill()) + assert raw is not None + assert raw["content"] == SKILL_BODY + assert raw["contentHash"] == _hash(SKILL_BODY) + assert raw["name"] == "PDF Extraction" + assert raw["contentType"] == "text/markdown" + + def test_an_absent_envelope_field_is_absent_rather_than_defaulted(self) -> None: + wire = put_skill() + del wire["object"]["name"] + raw = _store_object_from_put(wire) + assert raw is not None + assert "name" not in raw + + +# --------------------------------------------------------------------------- +# The protocol reader +# --------------------------------------------------------------------------- + + +def drive(reader: _ProtocolReader, payload_events: list[dict[str, Any]]) -> list[Any]: + return [reader.handle(e["event"], e.get("data")) for e in payload_events] + + +class TestProtocolReader: + def test_a_full_transfer_commits_at_payload_transferred(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + outcomes = drive(reader, full_payload(("put-object", put_skill()))) + assert len(held) == 1 + assert outcomes[-1].committed is True + assert outcomes[-1].basis == "basis-1" + + def test_nothing_is_visible_before_payload_transferred(self) -> None: + """A payload version is the unit of consistency; half of one is not a state.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill()), + ), + ) + assert len(held) == 0 + + def test_an_interrupted_full_transfer_leaves_last_known_good_intact(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill(object_version=1)))) + assert held.get("pdf-extraction", None) is not None + + # A second full transfer starts and never completes. + drive( + reader, + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill(object_version=2)), + ), + ) + still_held = held.get("pdf-extraction", None) + assert still_held is not None + assert still_held["version"] == 1 + + def test_a_full_transfer_replaces_rather_than_merges(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill("first")))) + drive( + reader, full_payload(("put-object", put_skill("second")), state="basis-2") + ) + assert held.get("first", None) is None + assert held.get("second", None) is not None + + def test_a_change_transfer_applies_deltas_over_what_is_held(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill("first")))) + drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("second")), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert held.get("first", None) is not None + assert held.get("second", None) is not None + + def test_a_delete_object_revokes_the_skill(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill(object_version=3)))) + drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill(object_version=3)), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert held.get("pdf-extraction", None) is None + assert reader.diagnostics.objects_revoked == 1 + + def test_a_delete_notifies_with_a_tombstone_carrying_no_content(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill()))) + outcomes = drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ), + ) + (change,) = outcomes[-1].changes + assert change == {"key": "pdf-extraction", "version": 3} + assert "content" not in change + + def test_a_delete_for_one_version_leaves_the_other_held(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + full_payload( + ("put-object", put_skill(object_version=2)), + ("put-object", put_skill(object_version=3)), + ), + ) + drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill(object_version=3)), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert held.get("pdf-extraction", 2) is not None + assert held.get("pdf-extraction", None)["version"] == 2 + + def test_flag_and_segment_objects_are_skipped_cleanly(self) -> None: + """ + The mixed payload is the normal case, not an edge one: an environment's + assignment carries its flag payload alongside its agent-skill payload. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + outcomes = drive( + reader, + full_payload( + ("put-object", put_flag("flag-a")), + ("put-object", put_skill("pdf-extraction")), + ("put-object", put_segment("beta-users")), + ("put-object", put_flag("flag-b")), + ("delete-object", put_flag("flag-c")), + ), + ) + assert len(held) == 1 + assert held.get("pdf-extraction", None) is not None + assert reader.diagnostics.objects_ignored == 4 + assert reader.diagnostics.skill_objects_received == 1 + assert all(o.fatal is None and o.disconnect is None for o in outcomes) + + def test_an_unknown_kind_is_ignored_rather_than_fatal(self) -> None: + """ + Erroring on an unrecognised kind would turn a normal payload into a + permanent reconnect loop — a flag-delivery outage caused by a skills + rollout. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + exotic = { + "key": "x", + "kind": "quantum-widget", + "version": 1, + "object": {"a": 1}, + } + outcomes = drive(reader, full_payload(("put-object", exotic))) + assert len(held) == 0 + assert all(o.fatal is None and o.disconnect is None for o in outcomes) + + def test_an_unknown_event_name_is_ignored(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + outcome = reader.handle("some-future-event", {"anything": True}) + assert outcome.fatal is None + assert outcome.disconnect is None + + def test_a_heartbeat_does_nothing(self) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle("heart-beat", None) + assert outcome == type(outcome)() + + def test_an_error_event_abandons_the_in_flight_payload(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill(object_version=1)))) + outcomes = drive( + reader, + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill(object_version=2)), + ( + "error", + {"payloadId": "agent-skill", "reason": "backend unavailable"}, + ), + ), + ) + assert outcomes[-1].disconnect is not None + assert held.get("pdf-extraction", None)["version"] == 1 + + def test_a_goodbye_asks_for_a_reconnect(self) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle("goodbye", {"reason": "rebalancing", "silent": False}) + assert outcome.disconnect is not None + assert outcome.fatal is None + + def test_a_catastrophic_goodbye_is_fatal(self) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle( + "goodbye", {"reason": "no", "silent": False, "catastrophe": True} + ) + assert outcome.fatal is not None + + def test_transfer_none_holds_everything_and_commits(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill()))) + drive( + reader, + events( + ("server-intent", server_intent("none")), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert len(held) == 1 + + def test_an_object_arriving_with_no_intent_is_treated_as_a_delta(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + events( + ("put-object", put_skill()), + ("payload-transferred", transferred("basis-1")), + ), + ) + assert len(held) == 1 + + +# --------------------------------------------------------------------------- +# Interface parity with InMemorySkillStore +# --------------------------------------------------------------------------- + + +class TestInterfaceParity: + """ + The two stores must resolve identically. ``_SkillObjectSet`` reimplements the + lookup rather than inheriting it — see its docstring for why — so this is the + test that stops the two from drifting. + """ + + RAWS: ClassVar[list[dict[str, Any]]] = [ + {"key": "a", "version": 1, "content": "x", "contentHash": _hash("x")}, + {"key": "a", "version": 4, "content": "y", "contentHash": _hash("y")}, + {"key": "b", "version": 2, "content": "z", "contentHash": _hash("z")}, + {"key": "malformed", "version": "not-a-version", "content": "q"}, + ] + + def _both(self) -> tuple[InMemorySkillStore, _SkillObjectSet]: + memory = InMemorySkillStore() + objects = _SkillObjectSet() + for raw in self.RAWS: + memory.put(dict(raw)) + objects.put(dict(raw)) + return memory, objects + + @pytest.mark.parametrize( + "key,version", + [ + ("a", None), + ("a", 1), + ("a", 4), + ("a", 9), + ("b", 2), + ("b", None), + ("missing", None), + ("missing", 1), + ("malformed", None), + ("malformed", 7), + ], + ) + def test_get_agrees(self, key: str, version: int | None) -> None: + memory, objects = self._both() + assert memory.get_object(SKILL_OBJECT_KIND, key, version) == objects.get( + key, version + ) + + def test_snapshot_agrees(self) -> None: + memory, objects = self._both() + assert memory.all_objects(SKILL_OBJECT_KIND) == objects.snapshot() + + +# --------------------------------------------------------------------------- +# The contentHash gap +# --------------------------------------------------------------------------- + + +def _per_object_hashless_errors(caplog: Any) -> list[Any]: + """The per-object ERROR, as distinct from the whole-payload summary.""" + return [ + r + for r in caplog.records + if r.levelname == "ERROR" + and "arrived without a contentHash" in r.getMessage() + and "No skill content will resolve" not in r.getMessage() + ] + + +class TestMissingContentHash: + """ + A skill delivered without a ``contentHash``, asserted as behaviour. + + An envelope with no ``contentHash`` must produce a *withheld* skill with the + ``missing_content_hash`` reason — loudly, diagnosably, and without a crash. + There is deliberately no fallback that skips verification: a hash the SDK + computed from the content it was handed would certify the content against + itself and verify nothing. + """ + + def test_a_redelivered_hashless_object_logs_once_per_store( + self, caplog: Any + ) -> None: + """Re-delivering the same ``(key, version)`` to one store must not + multiply the ERROR: a polling store sees every object on every poll.""" + reader = _ProtocolReader(_SkillObjectSet()) + payload = full_payload(("put-object", put_skill(omit_hash=True))) + with caplog.at_level("ERROR"): + drive(reader, payload) + drive(reader, payload) + assert len(_per_object_hashless_errors(caplog)) == 1 + + def test_a_recreated_store_reports_the_same_hashless_object_again( + self, caplog: Any + ) -> None: + """ + The dedupe belongs to the store, not the process. A host that rebuilds + its store (reconnect wrapper, config reload, credential rotation) must + get the ERROR again, since it is the loudest signal that a deployment is + broken rather than empty by design. + """ + payload = full_payload(("put-object", put_skill(omit_hash=True))) + with caplog.at_level("ERROR"): + drive(_ProtocolReader(_SkillObjectSet()), payload) + first = len(_per_object_hashless_errors(caplog)) + drive(_ProtocolReader(_SkillObjectSet()), payload) + assert first == 1 + assert len(_per_object_hashless_errors(caplog)) == 2 + + def test_two_live_stores_do_not_suppress_each_other(self, caplog: Any) -> None: + """Two stores in one process (say, two environments) each report.""" + one = _ProtocolReader(_SkillObjectSet()) + two = _ProtocolReader(_SkillObjectSet()) + payload = full_payload(("put-object", put_skill(omit_hash=True))) + with caplog.at_level("ERROR"): + drive(one, payload) + drive(two, payload) + # And each still dedupes its own re-deliveries. + drive(one, payload) + drive(two, payload) + assert len(_per_object_hashless_errors(caplog)) == 2 From 1cf523566fdfd0a6fc70ecc9eac5220ac52e2a45 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:46:56 -0400 Subject: [PATCH 2/7] =?UTF-8?q?feat(client):=20Agent=20Skills=20=E2=80=94?= =?UTF-8?q?=20name=20the=20payload=20a=20transfer=20completed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The protocol reader took payloads[0]'s intentCode and applied it to the skill object set, which is what the delivery protocol requires — one payload per credential, read the first intent, tolerate the rest — but it left the assumption behind that rule undocumented and unguarded. If the one-payload guarantee ever widens, an xfer-full for another payload would start an empty pending set and the next payload-transferred would publish it: every skill reported revoked, and with pruning on, a customer's files deleted. The first payload is still the payload that is read. What is new is that the reader now knows which payload skills actually arrive on — learnt from the intent's id, or from the (p::) selector, since no object or transfer event carries a payload id of its own — and declines to apply a transfer of any other, holding last known good, warning once, and counting it in diagnostics.payloads_ignored. An intent describing more than one payload warns once on its own, because that is the one case the comparison cannot catch: another payload's transfer arriving before any skill has been seen has nothing to be compared against. Behaviour under one-payload delivery is unchanged, and a full transfer of the skill payload still empties it — every skill deleted is a real state the guard must not mask. Co-Authored-By: Claude Opus 5 --- packages/client/agents.md | 13 + .../src/launchdarkly_ai_server/skills_fdv2.py | 132 +++++++++- packages/client/tests/test_skills_fdv2.py | 226 ++++++++++++++++++ 3 files changed, 370 insertions(+), 1 deletion(-) diff --git a/packages/client/agents.md b/packages/client/agents.md index 80ec0538..e73c96d4 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -256,6 +256,19 @@ described, and would briefly empty the store — which, with pruning on, is the between a reconcile and deleting a customer's skill files. An interrupted transfer therefore leaves last known good intact, and listeners fire once per commit. +**The first payload intent is read, and is assumed to be the skill payload.** Delivery +provides one payload per credential and the protocol requires a client to ignore all but the +first payload intent, so `payloads[0]` is both what arrives and what the protocol says to +read. The cost of that assumption is that an `xfer-full` for somebody *else's* payload would +start an empty pending set, and the next `payload-transferred` would publish it — every skill +reported revoked, and with pruning on, a customer's files deleted. `_ProtocolReader` +therefore learns which payload skills arrive on, from the intent's `id` or from the +`(p::)` selector, and declines to apply a transfer of any other: once at +WARNING, counted in `diagnostics.payloads_ignored`, holding last known good. A transfer that +names no payload is applied, since one-payload delivery is the common case. The residual is +the first transfer of a connection — before a skill has arrived there is nothing to compare +against — which is what the separate WARNING on a multi-payload intent is for. + **A hashless object is held, not dropped.** Verification withholds it with `missing_content_hash`; the transport's job is to make that loud (an error per object, a summary per wholly-hashless payload, `diagnostics.hashless_objects`) rather than to work diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 80ef1224..f644a349 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -36,6 +36,12 @@ - **It does not evaluate anything.** Flag and segment objects that share the connection are skipped and counted, nothing more. +One assumption it *does* make, and states: **the payload intent it reads is the +payload skills arrive on.** Delivery sends one payload per credential and the +protocol tells a client to read only the first payload intent, so today those are +the same payload. ``_ProtocolReader`` keeps the pair apart anyway, because the +cost of conflating them is an emptied skill set. + The design rationale — why ``objectVersion`` is not ``version``, why changes commit at ``payload-transferred`` — is in ``agents.md`` under *The delivery transport*. @@ -93,6 +99,16 @@ very thing verification exists to check. """ +_PAYLOAD_SELECTOR = re.compile(r"\(p:([^:()]+):\d+\)") +""" +The payload identity inside a transfer's selector, ``(p::)``. + +The selector is the only place a completed transfer names its own payload: +``put-object``, ``delete-object`` and ``payload-transferred`` carry no payload id +of their own. ``_ProtocolReader`` reads it as a fallback for an intent that named +no ``id``. +""" + _MOBILE_KEY_PREFIX = "mob-" _SERVER_KEY_PREFIX = "sdk-" _CLIENT_SIDE_ID = re.compile(r"\A[0-9a-f]{20,}\Z") @@ -169,6 +185,11 @@ class StoreDiagnostics: future kind. Skipping is the contract, not a failure.""" objects_revoked: int = 0 """``delete-object`` events applied to skills.""" + payloads_ignored: int = 0 + """ + Transfers not applied because they completed a payload other than the one + skills arrive on. Zero while delivery sends one payload per connection. + """ hashless_objects: int = 0 """ Skill objects whose envelope carried no ``contentHash``. @@ -255,6 +276,22 @@ def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: return raw +def _payload_id_of(intent: Any) -> str | None: + """The payload id one payload intent names, when it names a usable one.""" + if not isinstance(intent, dict): + return None + value = intent.get("id") + return value if isinstance(value, str) and value else None + + +def _payload_id_from_selector(state: Any) -> str | None: + """The payload id inside a transfer's selector, when it carries one.""" + if not isinstance(state, str): + return None + match = _PAYLOAD_SELECTOR.search(state) + return match.group(1) if match else None + + def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: """ Narrows one FDv2 skill ``delete-object`` to the identity it revokes, with @@ -394,6 +431,18 @@ class _ProtocolReader: version is the unit of consistency: applying half of one would publish a state the server never described, and on a full transfer would briefly empty the store. Listeners therefore fire once per commit, not once per object. + + **The first payload intent is read, and is assumed to be the skill payload.** + Delivery provides one payload per credential and the protocol requires a + client to ignore all but the first payload intent, so ``payloads[0]`` is both + what arrives and what the protocol says to read. If that ever widens, an + ``xfer-full`` for somebody else's payload would empty the skill set and the + next ``payload-transferred`` would publish it empty — with pruning on, the + difference between a reconcile and deleting a customer's files. This layer + therefore learns which payload skills arrive on and declines to apply a + transfer of any other, once at WARNING and counted. The residual is the first + transfer of a connection: before a skill has arrived there is nothing to + compare a payload against. """ def __init__(self, committed: _SkillObjectSet) -> None: @@ -406,6 +455,14 @@ def __init__(self, committed: _SkillObjectSet) -> None: # recreated store reports again and two stores never quieten each other. # No lock: ``handle`` runs only on its owner's single delivery thread. self._warned_hashless: set[tuple[str, Any]] = set() + # The payload the current intent describes, and the payload skills have + # actually arrived on. One payload per connection makes these the same + # payload; the class docstring says why they are kept apart regardless. + self._intent_payload_id: str | None = None + self._skill_payload_id: str | None = None + self._skills_in_payload = 0 + self._warned_multiple_payloads = False + self._warned_foreign_payload = False # -- events ------------------------------------------------------------ @@ -434,10 +491,15 @@ def _server_intent(self, data: Any) -> _TransferOutcome: return _TransferOutcome( disconnect="server-intent carried no payload description" ) + if len(payloads) > 1: + self._warn_multiple_payloads(payloads) + # The first payload only, as the protocol requires. first = payloads[0] intent = first.get("intentCode") if isinstance(first, dict) else None self._intent = intent + self._intent_payload_id = _payload_id_of(first) self._changes = [] + self._skills_in_payload = 0 if intent == _INTENT_TRANSFER_FULL: # Built alongside the live set rather than in place, so an # interrupted transfer leaves last known good intact. @@ -479,6 +541,7 @@ def _put_object(self, data: Any) -> _TransferOutcome: target.put(raw) self._changes.append(raw) self.diagnostics.skill_objects_received += 1 + self._skills_in_payload += 1 if not isinstance(raw.get("contentHash"), str): self.diagnostics.hashless_objects += 1 self._warn_hashless(raw) @@ -493,6 +556,8 @@ def _delete_object(self, data: Any) -> _TransferOutcome: return _TransferOutcome() target.delete(tombstone) self.diagnostics.objects_revoked += 1 + # A revocation identifies the payload as ours just as a put does. + self._skills_in_payload += 1 # A tombstone carries identity and no content, so a listener that reads # content must check for ``content`` rather than assume it. self._changes.append( @@ -503,11 +568,23 @@ def _delete_object(self, data: Any) -> _TransferOutcome: def _payload_transferred(self, data: Any) -> _TransferOutcome: state = data.get("state") if isinstance(data, dict) else None version = data.get("version") if isinstance(data, dict) else None - if self._pending is not None: + payload_id = self._intent_payload_id or _payload_id_from_selector(state) + if self._pending is not None and self._is_foreign_payload(payload_id): + self._warn_foreign_payload(payload_id) + self.diagnostics.payloads_ignored += 1 + self._changes = [] + elif self._pending is not None: self._committed.replace_with(self._pending) _warn_if_nothing_can_verify(self._committed) + if self._skills_in_payload and payload_id is not None: + # Learnt, not configured: nothing below the interface is told + # which payload is which, so the payload that carried a skill + # put or revocation is the payload skills arrive on. + self._skill_payload_id = payload_id self._pending = None self._intent = None + self._intent_payload_id = None + self._skills_in_payload = 0 changes = self._changes self._changes = [] self.diagnostics.payloads_transferred += 1 @@ -526,6 +603,8 @@ def _abandon_in_flight(self) -> None: """Drops the in-flight payload and keeps what is committed.""" self._pending = None self._intent = None + self._intent_payload_id = None + self._skills_in_payload = 0 self._changes = [] def _error(self, data: Any) -> _TransferOutcome: @@ -546,8 +625,59 @@ def _goodbye(self, data: Any) -> _TransferOutcome: ) return _TransferOutcome(disconnect=f"server said goodbye: {reason}") + # -- payload identity ---------------------------------------------------- + + def _is_foreign_payload(self, payload_id: str | None) -> bool: + """ + Whether a transfer completes a payload other than the one skills arrive on. + + ``False`` unless both payloads are known, so one-payload delivery and the + first transfer of a connection behave exactly as they did before this + check existed. + """ + return ( + self._skill_payload_id is not None + and payload_id is not None + and payload_id != self._skill_payload_id + ) + # -- diagnostics --------------------------------------------------------- + def _warn_multiple_payloads(self, payloads: list[Any]) -> None: + """ + One WARNING per reader for an intent describing more than one payload. + + Not an error: reading only the first is what the protocol asks for. But it + means the first payload is no longer *guaranteed* to be the skill payload, + and an intent for another payload arriving before any skill has been seen + is the one case ``_is_foreign_payload`` cannot catch. + """ + if self._warned_multiple_payloads: + return + self._warned_multiple_payloads = True + logger.warning( + "An FDv2 server-intent described %d payloads (%s). Only the first is " + "read, as the protocol requires, and it is taken to be the payload " + "skills arrive on. If skills stop resolving from this point, that is " + "the assumption that broke; contact LaunchDarkly support.", + len(payloads), + ", ".join(str(_payload_id_of(p)) for p in payloads), + ) + + def _warn_foreign_payload(self, payload_id: str | None) -> None: + """One WARNING per reader for a transfer this layer declined to apply.""" + if self._warned_foreign_payload: + return + self._warned_foreign_payload = True + logger.warning( + "An FDv2 transfer of payload %s was not applied to the skills held, " + "which arrive on payload %s. Applying it would have replaced them " + "with whatever that payload carried — nothing, in the case of a flag " + "payload. The skills held are unchanged.", + payload_id, + self._skill_payload_id, + ) + def _warn_hashless(self, raw: dict[str, Any]) -> None: """ One ERROR per ``(key, version)`` whose envelope had no ``contentHash``. diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index ff5720bc..87596609 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -482,6 +482,232 @@ def test_an_object_arriving_with_no_intent_is_treated_as_a_delta(self) -> None: assert len(held) == 1 +# --------------------------------------------------------------------------- +# Which payload a transfer completed +# --------------------------------------------------------------------------- + + +def _payload_warnings(caplog: Any, fragment: str) -> list[Any]: + return [ + r + for r in caplog.records + if r.levelname == "WARNING" and fragment in r.getMessage() + ] + + +def skill_payload( + *object_events: tuple[str, Any], + payload_id: str = "agent-skill", + code: str = "xfer-full", + state: str = "basis-1", +) -> list[dict[str, Any]]: + """One payload's events, with the payload it belongs to named explicitly.""" + return events( + ("server-intent", server_intent(code, payload_id)), + *object_events, + ("payload-transferred", transferred(state)), + ) + + +class TestPayloadIdentity: + """ + Which payload a transfer completed, and why this layer tracks it at all. + + Delivery provides one payload per credential and the protocol requires a + client to read only the first payload intent, so today the payload read is + the payload skills arrive on. These assert the behaviour that survives if + the first of those stops holding: another payload's ``xfer-full`` must not + publish an empty skill set, because with pruning on that deletes a + customer's materialized files. + """ + + def test_only_the_first_payload_intent_is_read(self) -> None: + """Reading only the first is what the protocol asks for, however many + arrive — the point of the rest of this class is to make that safe.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + events( + ( + "server-intent", + { + "payloads": [ + { + "id": "agent-skill", + "target": 1, + "intentCode": "xfer-full", + }, + {"id": "env-flags", "target": 2, "intentCode": "none"}, + ] + }, + ), + ("put-object", put_skill()), + ("payload-transferred", transferred()), + ), + ) + assert len(held) == 1 + + def test_more_than_one_payload_intent_warns_once(self, caplog: Any) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + intent = { + "payloads": [ + {"id": "env-flags", "target": 1, "intentCode": "xfer-changes"}, + {"id": "agent-skill", "target": 2, "intentCode": "xfer-changes"}, + ] + } + with caplog.at_level("WARNING"): + reader.handle("server-intent", intent) + reader.handle("server-intent", intent) + assert len(_payload_warnings(caplog, "described 2 payloads")) == 1 + + def test_one_payload_intent_warns_about_nothing(self, caplog: Any) -> None: + with caplog.at_level("WARNING"): + drive( + _ProtocolReader(_SkillObjectSet()), + skill_payload(("put-object", put_skill())), + ) + assert _payload_warnings(caplog, "payload") == [] + + def test_another_payloads_full_transfer_does_not_empty_the_skills_held( + self, caplog: Any + ) -> None: + """ + The case this guard exists for. A flag payload's ``xfer-full`` starts an + empty pending set; applying it at ``payload-transferred`` would publish + every skill as revoked, which a reconcile with pruning on reads as + "delete these files". + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, skill_payload(("put-object", put_skill()))) + with caplog.at_level("WARNING"): + outcomes = drive( + reader, + skill_payload( + ("put-object", put_flag()), payload_id="env-flags", state="basis-2" + ), + ) + assert held.get("pdf-extraction", None) is not None + assert reader.diagnostics.payloads_ignored == 1 + assert len(_payload_warnings(caplog, "was not applied")) == 1 + # Nothing changed, so no listener is woken to reconcile against it. + assert outcomes[-1].changes == [] + + def test_a_declined_transfer_warns_once_however_often_it_repeats( + self, caplog: Any + ) -> None: + """A polling connection sees the other payload on every poll.""" + reader = _ProtocolReader(_SkillObjectSet()) + drive(reader, skill_payload(("put-object", put_skill()))) + foreign = skill_payload(("put-object", put_flag()), payload_id="env-flags") + with caplog.at_level("WARNING"): + drive(reader, foreign) + drive(reader, foreign) + assert len(_payload_warnings(caplog, "was not applied")) == 1 + assert reader.diagnostics.payloads_ignored == 2 + + def test_a_full_transfer_of_the_skill_payload_still_empties_it(self) -> None: + """Every skill deleted is a real state, and the guard must not mask it.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, skill_payload(("put-object", put_skill()))) + drive(reader, skill_payload(state="basis-2")) + assert len(held) == 0 + assert reader.diagnostics.payloads_ignored == 0 + + def test_a_revocation_identifies_the_payload_as_the_skill_payload(self) -> None: + """A payload that only revokes is still a payload skills arrive on.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + skill_payload(("delete-object", delete_skill()), code="xfer-changes"), + ) + drive( + reader, skill_payload(("put-object", put_skill()), payload_id="env-flags") + ) + assert reader.diagnostics.payloads_ignored == 1 + + def test_the_payload_is_identified_from_the_selector_when_no_id_is_named( + self, + ) -> None: + """``payload-transferred``'s selector is the only other place a completed + transfer names its payload.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + unnamed = {"payloads": [{"target": 1, "intentCode": "xfer-full"}]} + drive( + reader, + events( + ("server-intent", unnamed), + ("put-object", put_skill()), + ("payload-transferred", transferred("(p:agent-skill:53)")), + ), + ) + drive( + reader, + events( + ("server-intent", unnamed), + ("put-object", put_flag()), + ("payload-transferred", transferred("(p:env-flags:12)")), + ), + ) + assert held.get("pdf-extraction", None) is not None + assert reader.diagnostics.payloads_ignored == 1 + + def test_an_unidentifiable_payload_is_applied_rather_than_withheld(self) -> None: + """ + A transfer naming no payload at all is the store's own, since delivery + sends it one payload. Withholding it would break the common case to + defend against a hypothetical one. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, skill_payload(("put-object", put_skill()))) + drive( + reader, + events( + ("server-intent", {"payloads": [{"intentCode": "xfer-full"}]}), + ("put-object", put_skill(object_version=4)), + ("payload-transferred", {"version": 44}), + ), + ) + assert held.get("pdf-extraction", None)["version"] == 4 + assert reader.diagnostics.payloads_ignored == 0 + + def test_the_first_transfer_of_a_connection_is_the_residual( + self, caplog: Any + ) -> None: + """ + Before a skill has arrived there is nothing to compare a payload + against, so another payload's ``xfer-full`` arriving first cannot be + told apart. The multiple-payload WARNING is the only signal there is, + which is why it exists. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + with caplog.at_level("WARNING"): + drive( + reader, + events( + ( + "server-intent", + { + "payloads": [ + {"id": "env-flags", "intentCode": "xfer-full"}, + {"id": "agent-skill", "intentCode": "xfer-full"}, + ] + }, + ), + ("put-object", put_flag()), + ("payload-transferred", transferred()), + ), + ) + assert len(held) == 0 + assert len(_payload_warnings(caplog, "described 2 payloads")) == 1 + + # --------------------------------------------------------------------------- # Interface parity with InMemorySkillStore # --------------------------------------------------------------------------- From efc4ca77daf50a878b5988a591613cd96b778fcb Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 15:24:45 -0400 Subject: [PATCH 3/7] =?UTF-8?q?fix(client):=20Agent=20Skills=20=E2=80=94?= =?UTF-8?q?=20read=20the=20skill's=20version=20off=20the=20wire=20key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SDK-facing FDv2 channel now delivers skills the way streamer #4681 and gonfalon #70638 spell them: object kinds are open strings, the agent-skill payload is classified `generic`, and every generic object carries only `key`, `kind`, `version` and `object`, exactly like a flag. A skill arrives under kind `skill` with its own version folded into the key as `:`. There is no `category` field and no `objectVersion` field; both came from an earlier streamer draft that never shipped. Identification is now the kind alone. The wire key is split in one place, `_split_wire_key`, and both the put and the delete translation go through it. A key that will not split cleanly is held rather than dropped — version-less, or with the offending text as its version — so verification withholds it with `invalid_version` under a key the caller recognises; only a key with nothing before the delimiter is dropped, since there is no identity to hold it under. `SDK_DATA_MODEL_VERSION` goes with it: the connection's `mv` parameter only accepts flag model versions, and generic payloads ignore it. The transport stops sending it in the following change. Co-Authored-By: Claude Fable 5.1 --- packages/client/agents.md | 46 ++--- .../src/launchdarkly_ai_server/skills_core.py | 6 +- .../src/launchdarkly_ai_server/skills_fdv2.py | 159 ++++++++++++------ packages/client/tests/test_skills_fdv2.py | 130 ++++++++++---- 4 files changed, 233 insertions(+), 108 deletions(-) diff --git a/packages/client/agents.md b/packages/client/agents.md index e73c96d4..978dc8ad 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,7 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | -| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the `objectVersion`/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | +| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the wire-key/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | | `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | @@ -226,29 +226,37 @@ shape `skills_core.SkillStore` documents. It lives below the store interface; ** that interface knows it exists**. If a transport change ever seems to require editing an accessor, verification, or `write_skills`, the adapter boundary is wrong. -**`objectVersion` is the skill's version. `version` is the payload's.** On the wire a skill -`put-object` carries both, and they are not interchangeable: +**The skill's version is in the object's `key`. `version` is the payload's.** Each version +of a skill is its own object on the wire, identified as `:`: ```json -{"key":"pdf-extraction","kind":"inline-resource","category":"skill", - "objectVersion":3,"version":42, +{"key":"pdf-extraction:3","kind":"skill","version":42, "object":{"contentType":"text/markdown","content":"…","contentHash":"…","name":"…"}} ``` -`objectVersion` (3) is what a `{key, version}` reference pins and what becomes the stored -`version`. `version` (42) is the version of the *payload* the object arrived in — it moves -when anything in the environment moves, including a flag with nothing to do with skills. -Reading it as the skill's version fails **silently**: the object verifies, the hash matches, -and the caller gets content under a version number that means nothing. Flags and segments -carry only `version` and omit both `category` and `objectVersion`, which is exactly why the -two fields look interchangeable. `_store_object_from_put` is the only place the translation -happens, and `TestVersionTranslation` asserts it in both directions. - -**Skills are identified by `kind == "inline-resource" && category == "skill"`; everything else -is ignored, not rejected.** An environment's payload assignment carries its flag payload -alongside its agent-skill payload, so flag and segment objects arrive as a matter of course. -Erroring on an unrecognised kind would turn a normal payload into a permanent reconnect -loop — a flag-delivery outage caused by a skills rollout. +The `3` after the delimiter is what a `{key, version}` reference pins and what becomes the +stored `version`, under the stored key `pdf-extraction`. `version` (42) is the version of the +*payload* the object arrived in — it moves when anything in the environment moves, +including a flag with nothing to do with skills. Reading it as the skill's version fails +**silently**: the object verifies, the hash matches, and the caller gets content under a +version number that means nothing. There is no separate field for the skill's version: the +agent-skill payload is a *generic* payload, and generic objects carry only `key`, `kind`, +`version` and `object`, exactly like a flag. `_split_wire_key` is the only place the wire key +is read, `_store_object_from_put` and `_tombstone_from_delete` both go through it, and +`TestVersionTranslation` asserts the translation in both directions. A wire key that will +not split cleanly is *held*, not dropped — version-less, or with the offending text as its +version — so verification withholds it with `invalid_version` under a key the caller +recognises; only a key with nothing before the delimiter is dropped, since there is no +identity to hold it under. + +**Skills are identified by `kind == "skill"`; everything else is ignored, not rejected.** +Object kinds on the SDK-facing channel are open strings, and the agent-skill payload is +classified `generic`, so a skill arrives under the kind its producer registered — the bare +category name — not under a broader wrapper kind with a narrowing field. An environment's +payload assignment carries its flag payload alongside its agent-skill payload, so flag and +segment objects arrive as a matter of course. Erroring on an unrecognised kind would turn a +normal payload into a permanent reconnect loop — a flag-delivery outage caused by a skills +rollout. **Changes commit at `payload-transferred`, not as objects arrive.** A payload version is the unit of consistency: a half-applied full transfer would publish a state the server never diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index 22166799..d2d870c4 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -56,9 +56,9 @@ An **internal seam value**, deliberately not exported from the package root. It is the string ``skills.py`` and ``skills_fs.py`` pass to ``SkillStore.get_object`` and ``SkillStore.all_objects``, and a store adapter is free to map it onto -whatever the transport underneath actually uses — a delivery payload may well -carry skills under a broader kind with a narrower category, in which case -translating that pair to this one value is the adapter's job. +whatever the transport underneath actually uses — the value happens to match +the kind LaunchDarkly's delivery channel uses today, but a transport that spelt +it differently would translate, and that translation is the adapter's job. Exporting it would publish an SDK-side seam string as though it were the wire contract, which is a claim this side cannot make and would be hard to walk back diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index f644a349..d20a0e2f 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -3,7 +3,7 @@ The half of the delivery transport that has no I/O: identifying skill objects on the wire, translating them into the raw object shape the ``SkillStore`` -interface defines, holding them by ``(key, objectVersion)``, and applying a +interface defines, holding them by ``(key, version)``, and applying a payload's events as one consistent commit. ``FDv2SkillStore``, the store that puts a network connection underneath this, follows in a separate change. @@ -42,9 +42,9 @@ the same payload. ``_ProtocolReader`` keeps the pair apart anyway, because the cost of conflating them is an emptied skill set. -The design rationale — why ``objectVersion`` is not ``version``, why changes -commit at ``payload-transferred`` — is in ``agents.md`` under *The delivery -transport*. +The design rationale — why the skill's version is read from the object's +``key`` and never from ``version``, why changes commit at +``payload-transferred`` — is in ``agents.md`` under *The delivery transport*. """ from __future__ import annotations @@ -63,21 +63,27 @@ # The wire contract # --------------------------------------------------------------------------- -FDV2_OBJECT_KIND = "inline-resource" +FDV2_OBJECT_KIND = "skill" """ -The FDv2 ``kind`` skills are delivered under. Together with -``FDV2_OBJECT_CATEGORY`` it maps onto the single interface value -``skills_core.SKILL_OBJECT_KIND``; that translation is this adapter's job. +The FDv2 ``kind`` skills are delivered under. + +Object kinds on the SDK-facing channel are open strings: the agent-skill payload +is classified ``generic`` and every object in it carries the kind its producer +registered, which for skills is the bare category name. Delivery lower-cases the +kind, so an exact comparison is the whole test. The kind happens to equal +``skills_core.SKILL_OBJECT_KIND`` today; they are still separate constants, +because one is a wire value LaunchDarkly owns and the other is an SDK seam. """ -FDV2_OBJECT_CATEGORY = "skill" -"""The ``category`` that narrows ``inline-resource`` to an agent skill.""" - -SDK_DATA_MODEL_VERSION = 1 +FDV2_KEY_DELIMITER = ":" """ -The ``mv`` request parameter. The one request parameter whose value could not be -confirmed against a live server, so treat the default as provisional and -override it through the store's ``data_model_version`` if needed. +What separates a skill's key from its version inside the object's wire ``key``. + +A generic object is identified on the wire as ``:`` — the skill's +own key, one delimiter, the skill's own version — because each version of a +skill is a distinct object in the payload. Delivery forbids the delimiter inside +a registered category and skill keys cannot contain it, so a well-formed wire key +has exactly one. """ _EVENT_SERVER_INTENT = "server-intent" @@ -204,7 +210,7 @@ class StoreDiagnostics: # --------------------------------------------------------------------------- -# Deserialisation — where objectVersion is not version +# Deserialisation — where the skill's version lives in the key, not in version # --------------------------------------------------------------------------- @@ -220,17 +226,58 @@ def _is_skill_event(data: Any) -> bool: """ Whether one ``put-object`` / ``delete-object`` payload is a skill. - Both halves are required: ``inline-resource`` may carry other categories, - and flags and segments omit ``category`` entirely. Every other kind is - ignored, not rejected, because flag and segment objects share the connection - and erroring on them would turn a normal payload into a reconnect loop. + The kind alone decides it. Every other kind is ignored, not rejected, + because flag and segment objects share the connection and erroring on them + would turn a normal payload into a reconnect loop. """ if not isinstance(data, dict): return False - return ( - data.get("kind") == FDV2_OBJECT_KIND - and data.get("category") == FDV2_OBJECT_CATEGORY - ) + return data.get("kind") == FDV2_OBJECT_KIND + + +@dataclass(frozen=True) +class _WireIdentity: + """A skill object's wire ``key``, split into the skill's key and version.""" + + key: str + version: Any + """``int`` when the wire carried one; the offending text when it did not; + absent (``_NO_VERSION``) when the wire key had no delimiter at all.""" + + +_NO_VERSION = object() + + +def _split_wire_key(wire_key: Any) -> _WireIdentity | None: + """ + Reads ``:`` off one object's wire ``key``. + + Lenient where leniency keeps the object diagnosable and strict only where + there is nothing to diagnose: + + - No delimiter: the whole wire key is the skill key and there is no version, + so the object is held version-less and verification reports + ``invalid_version`` under a key the caller can recognise. + - A version that is not a run of digits (``"pdf:latest"``, ``"pdf:"``, + ``"a:1:2"``): the text is carried through *as the version*, for the same + reason — the caller learns that ``pdf`` arrived broken, not that it is + absent. + - An empty key before the delimiter (``":3"``): there is no identity to hold + it under, so ``None``, and the caller drops it. + + Leading zeros are accepted (``"pdf:03"`` is version 3) since ``int`` is the + identity a reference pins, not the spelling. + """ + if not isinstance(wire_key, str) or not wire_key: + return None + key, delimiter, version_text = wire_key.partition(FDV2_KEY_DELIMITER) + if not key: + return None + if not delimiter: + return _WireIdentity(key=key, version=_NO_VERSION) + if version_text.isascii() and version_text.isdigit(): + return _WireIdentity(key=key, version=int(version_text)) + return _WireIdentity(key=key, version=version_text) def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: @@ -240,33 +287,36 @@ def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: **The one translation this adapter must get right:** - wire ``objectVersion`` → stored ``version`` (the skill's own version) - wire ``version`` → dropped (the *payload* version) + wire ``key`` → stored ``key`` and ``version`` (split on ``:``) + wire ``version`` → dropped (the *payload* version) - ``objectVersion`` is what a ``{key, version}`` reference pins; ``version`` - moves whenever anything in the environment moves. Confusing them fails - silently: the object verifies and the caller gets content under a version - number that means nothing. + Each version of a skill is its own object on the wire, identified as + ``:``; that version is what a ``{key, version}`` reference + pins. The event's ``version`` field is the version of the payload the object + arrived in and moves whenever anything in the environment moves. Confusing + them fails silently: the object verifies and the caller gets content under a + version number that means nothing. - Returns ``None`` only when ``key`` is not a string, since a keyless object - has no identity to store it under. Every other defect is carried through - verbatim so that verification withholds it with a reason code rather than - the transport dropping it into indistinguishable absence. + Returns ``None`` only when the wire ``key`` carries no skill key at all, + since such an object has no identity to store it under. Every other defect + is carried through so that verification withholds it with a reason code + rather than the transport dropping it into indistinguishable absence. """ - key = data.get("key") - if not isinstance(key, str) or not key: + identity = _split_wire_key(data.get("key")) + if identity is None: logger.warning( - "An FDv2 skill put-object carried no string 'key' and could not be " - "stored under any identity; it was dropped." + "An FDv2 skill put-object carried no usable 'key' (%r) and could not " + "be stored under any identity; it was dropped.", + data.get("key"), ) return None - raw: dict[str, Any] = {"key": key} + raw: dict[str, Any] = {"key": identity.key} - # A membership test rather than a `.get` default, so an explicitly-null - # objectVersion stays null and reaches verification as `invalid_version`. - if "objectVersion" in data: - raw["version"] = data["objectVersion"] + # Absent stays absent and malformed stays malformed, so verification sees + # what arrived (as `invalid_version`) rather than something invented here. + if identity.version is not _NO_VERSION: + raw["version"] = identity.version envelope = data.get("object") if isinstance(envelope, dict): @@ -294,25 +344,26 @@ def _payload_id_from_selector(state: Any) -> str | None: def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: """ - Narrows one FDv2 skill ``delete-object`` to the identity it revokes, with - the same ``objectVersion`` translation as a put. + Narrows one FDv2 skill ``delete-object`` to the identity it revokes, reading + the wire ``key`` the same way a put does. An ``object_version`` of ``None`` means the delete named no usable version and is read as "revoke every version of this key". That is the safe direction: the alternative is continuing to serve content LaunchDarkly has - withdrawn. + withdrawn. It also removes whatever a malformed put of the same wire key + left held, since that was stored version-less under the same skill key. """ - key = data.get("key") - if not isinstance(key, str) or not key: + identity = _split_wire_key(data.get("key")) + if identity is None: logger.warning( - "An FDv2 skill delete-object carried no string 'key'; it was ignored." + "An FDv2 skill delete-object carried no usable 'key' (%r); it was ignored.", + data.get("key"), ) return None - object_version = data.get("objectVersion") return _Tombstone( - key=key, - object_version=object_version - if is_valid_skill_version(object_version) + key=identity.key, + object_version=identity.version + if is_valid_skill_version(identity.version) else None, ) @@ -324,7 +375,7 @@ def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: class _SkillObjectSet: """ - Raw skill objects held in memory, keyed by ``(key, objectVersion)``. + Raw skill objects held in memory, keyed by ``(key, version)``. Lookup semantics are identical to ``InMemorySkillStore``'s, down to the fall-through to a version-less entry, so that the store a caller configures diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index 87596609..b88d14d9 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -1,8 +1,8 @@ """ Tests for the FDv2 skill delivery protocol. -Wire semantics — which objects are skills, ``objectVersion`` versus ``version``, -revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted +Wire semantics — which objects are skills, the skill's version in the wire ``key`` +versus the payload's in ``version``, revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted against ``_ProtocolReader``, which has no I/O, so each case reads as the contract it is rather than as a server script. """ @@ -17,7 +17,7 @@ from launchdarkly_ai_server import InMemorySkillStore from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND from launchdarkly_ai_server.skills_fdv2 import ( - FDV2_OBJECT_CATEGORY, + FDV2_KEY_DELIMITER, FDV2_OBJECT_KIND, _is_skill_event, _ProtocolReader, @@ -40,6 +40,18 @@ def _hash(content: str) -> str: # --------------------------------------------------------------------------- +def wire_key(key: str, object_version: Any) -> str: + """ + The wire ``key`` of one skill object: ``:``. + + ``None`` builds a key with no version at all, which is how the tests spell a + malformed object; anything else is spelled after the delimiter verbatim. + """ + if object_version is None: + return key + return f"{key}{FDV2_KEY_DELIMITER}{object_version}" + + def put_skill( key: str = "pdf-extraction", *, @@ -62,10 +74,8 @@ def put_skill( content_hash if content_hash is not None else _hash(content) ) return { - "key": key, + "key": wire_key(key, object_version), "kind": FDV2_OBJECT_KIND, - "category": FDV2_OBJECT_CATEGORY, - "objectVersion": object_version, "version": payload_version, "object": envelope, } @@ -75,16 +85,14 @@ def delete_skill( key: str = "pdf-extraction", *, object_version: Any = 3, payload_version: int = 43 ) -> dict[str, Any]: return { - "key": key, + "key": wire_key(key, object_version), "kind": FDV2_OBJECT_KIND, - "category": FDV2_OBJECT_CATEGORY, - "objectVersion": object_version, "version": payload_version, } def put_flag(key: str = "my-flag", version: int = 17) -> dict[str, Any]: - """A flag ``put-object``: no ``category``, no ``objectVersion``.""" + """A flag ``put-object``: the same envelope fields, a different ``kind``.""" return { "key": key, "kind": "flag", @@ -141,30 +149,37 @@ def full_payload( class TestObjectIdentification: - def test_kind_and_category_together_identify_a_skill(self) -> None: + def test_the_kind_alone_identifies_a_skill(self) -> None: assert _is_skill_event(put_skill()) is True + def test_the_kind_is_the_bare_category_name(self) -> None: + """ + Object kinds on the channel are open strings and the agent-skill payload + is ``generic``, so a skill arrives under the kind its producer + registered — ``skill`` — not under a broader wrapper kind. + """ + assert FDV2_OBJECT_KIND == "skill" + def test_a_flag_is_not_a_skill(self) -> None: assert _is_skill_event(put_flag()) is False def test_a_segment_is_not_a_skill(self) -> None: assert _is_skill_event(put_segment()) is False - def test_inline_resource_of_another_category_is_not_a_skill(self) -> None: - """``inline-resource`` is a broad kind, so the category is required too.""" + def test_another_generic_kind_is_not_a_skill(self) -> None: + """A generic payload may carry other registered kinds one day.""" other = put_skill() - other["category"] = "prompt-template" + other["kind"] = "prompt-template" assert _is_skill_event(other) is False - def test_skill_category_under_another_kind_is_not_a_skill(self) -> None: + def test_a_skill_shaped_envelope_under_another_kind_is_not_a_skill(self) -> None: other = put_skill() other["kind"] = "some-future-kind" assert _is_skill_event(other) is False - def test_a_flag_shaped_object_with_no_category_is_not_a_skill(self) -> None: - """Flags and segments omit ``category`` entirely — the documented shape.""" - assert "category" not in put_flag() - assert "objectVersion" not in put_flag() + def test_nothing_but_the_kind_is_consulted(self) -> None: + """No secondary field narrows the kind, and none may be required.""" + assert set(put_skill()) == {"key", "kind", "version", "object"} @pytest.mark.parametrize("value", [None, "skill", 3, [], ()]) def test_non_dict_events_are_not_skills(self, value: Any) -> None: @@ -172,15 +187,27 @@ def test_non_dict_events_are_not_skills(self, value: Any) -> None: # --------------------------------------------------------------------------- -# objectVersion is not version +# The skill's version is in the wire key; `version` is the payload's # --------------------------------------------------------------------------- class TestVersionTranslation: - def test_object_version_becomes_the_seam_version(self) -> None: + def test_the_wire_key_is_key_colon_version(self) -> None: + assert ( + put_skill("pdf-extraction", object_version=3)["key"] == "pdf-extraction:3" + ) + + def test_the_version_after_the_delimiter_becomes_the_seam_version(self) -> None: raw = _store_object_from_put(put_skill(object_version=3, payload_version=42)) assert raw is not None assert raw["version"] == 3 + assert isinstance(raw["version"], int) + + def test_the_key_before_the_delimiter_becomes_the_seam_key(self) -> None: + """A caller asks for ``pdf-extraction``, never for ``pdf-extraction:3``.""" + raw = _store_object_from_put(put_skill("pdf-extraction", object_version=3)) + assert raw is not None + assert raw["key"] == "pdf-extraction" def test_the_payload_version_never_reaches_the_seam(self) -> None: """ @@ -200,36 +227,75 @@ def test_the_two_are_distinguished_even_when_the_payload_version_is_lower( assert raw is not None assert raw["version"] == 99 - def test_a_missing_object_version_is_not_defaulted_from_the_payload(self) -> None: - wire = put_skill() - del wire["objectVersion"] - raw = _store_object_from_put(wire) + def test_a_key_with_no_delimiter_is_held_version_less(self) -> None: + """Not defaulted from the payload version, and not dropped: verification + reports ``invalid_version`` under a key the caller recognises.""" + raw = _store_object_from_put(put_skill(object_version=None)) assert raw is not None + assert raw["key"] == "pdf-extraction" assert "version" not in raw - def test_an_explicitly_null_object_version_is_carried_through_as_null(self) -> None: - """Carried, not invented: verification reports ``invalid_version``.""" - raw = _store_object_from_put(put_skill(object_version=None)) + @pytest.mark.parametrize("spelling", ["latest", "", "3.0", "-1", "1:2", "３"]) + def test_a_version_that_is_not_digits_is_carried_through_as_invalid( + self, spelling: str + ) -> None: + """Carried, not invented: verification reports ``invalid_version`` for + the object rather than the transport reporting it absent.""" + raw = _store_object_from_put(put_skill(object_version=spelling)) + assert raw is not None + assert raw["key"] == "pdf-extraction" + assert raw["version"] == spelling + + def test_leading_zeros_spell_the_same_version(self) -> None: + raw = _store_object_from_put(put_skill(object_version="03")) assert raw is not None - assert raw["version"] is None + assert raw["version"] == 3 - def test_a_delete_translates_object_version_too(self) -> None: + def test_a_delete_reads_the_wire_key_the_same_way(self) -> None: tombstone = _tombstone_from_delete( delete_skill(object_version=3, payload_version=43) ) assert tombstone is not None + assert tombstone.key == "pdf-extraction" assert tombstone.object_version == 3 - def test_a_delete_with_no_usable_object_version_revokes_every_version(self) -> None: - tombstone = _tombstone_from_delete(delete_skill(object_version=None)) + @pytest.mark.parametrize("spelling", [None, "latest", "0"]) + def test_a_delete_with_no_usable_version_revokes_every_version( + self, spelling: Any + ) -> None: + tombstone = _tombstone_from_delete(delete_skill(object_version=spelling)) assert tombstone is not None + assert tombstone.key == "pdf-extraction" assert tombstone.object_version is None + @pytest.mark.parametrize("bad_key", [":3", "", None, 3]) + def test_a_put_with_no_skill_key_is_dropped_because_it_has_no_identity( + self, bad_key: Any + ) -> None: + wire = put_skill() + wire["key"] = bad_key + assert _store_object_from_put(wire) is None + def test_a_keyless_put_is_dropped_because_it_has_no_identity(self) -> None: wire = put_skill() del wire["key"] assert _store_object_from_put(wire) is None + def test_a_delete_with_no_skill_key_is_ignored(self) -> None: + wire = delete_skill() + wire["key"] = ":3" + assert _tombstone_from_delete(wire) is None + + def test_the_stored_identity_round_trips_to_the_wire_key(self) -> None: + """``_SkillObjectSet.snapshot`` spells its opaque keys the way the wire + does, so a held object can be matched back to the event that carried it.""" + held = _SkillObjectSet() + wire = put_skill("pdf-extraction", object_version=3) + raw = _store_object_from_put(wire) + assert raw is not None + held.put(raw) + assert set(held.snapshot()) == {wire["key"]} + def test_the_envelope_is_copied_verbatim(self) -> None: raw = _store_object_from_put(put_skill()) assert raw is not None From b6a25f9f455258dabb8d13b13978123239327376 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Thu, 10 Sep 2026 16:05:04 -0400 Subject: [PATCH 4/7] =?UTF-8?q?feat(client):=20Agent=20Skills=20=E2=80=94?= =?UTF-8?q?=20the=20FDv2=20delivery=20transport?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FDv2SkillStore puts LaunchDarkly's SDK-facing FDv2 channel underneath the protocol layer: GET /sdk/poll and GET /sdk/stream, authenticated with the environment's server-side SDK key, streaming by default. It carries basis across requests, sends If-None-Match and treats 304 as a current answer, retries with capped jittered backoff, honours Retry-After only up to max_backoff, gives up after a bounded run of consecutive failures where a committed payload resets the count, and keeps serving last known good through every failure. A mobile key or client-side environment ID is refused in the constructor. Standard library only. close interrupts the socket rather than only setting a flag, because the delivery thread lives in a read no flag can reach; without that every shutdown of a healthy stream waited out the full join timeout. The no-store message now names FDv2SkillStore first, and watch_skills points at it as the store with a delivery transport. Co-Authored-By: Claude Fable 5.1 --- packages/client/README.md | 62 + packages/client/agents.md | 25 +- .../src/launchdarkly_ai_server/__init__.py | 5 +- .../src/launchdarkly_ai_server/skills_core.py | 13 +- .../src/launchdarkly_ai_server/skills_fdv2.py | 748 ++++++++- .../launchdarkly_ai_server/skills_watch.py | 2 +- packages/client/tests/test_skills.py | 12 + packages/client/tests/test_skills_fdv2.py | 1332 ++++++++++++++++- packages/client/tests/test_skills_watch.py | 5 +- 9 files changed, 2162 insertions(+), 42 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 41d5c6b3..42b1f40d 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -445,6 +445,66 @@ which OS ran the write. The keys stay valid everywhere else: an AI Config refere named `aux` parses, and its other fields are unaffected. If you have a skill named for a device, rename it. +#### Receiving skills from LaunchDarkly + +`InMemorySkillStore` is for tests and bring-your-own-content. In production, skill content +arrives through `FDv2SkillStore`, which speaks LaunchDarkly's SDK-facing FDv2 delivery +channel — the same `GET /sdk/poll` and `GET /sdk/stream` endpoints the base SDK's FDv2 data +source uses, authenticated with the environment's server-side SDK key. + +```python +import os + +from launchdarkly_ai_server import FDv2SkillStore, init_client, watch_skills + +store = FDv2SkillStore(os.environ["LD_SDK_KEY"]).start() +store.wait_for_skills(timeout=10) +await init_client(options={"skillStore": store}) + +# Materialize now, and re-materialize whenever delivery changes. +report, watcher = await watch_skills("*", ".claude/skills") +try: + ... +finally: + watcher.close() + store.close() +``` + +**Nothing above the store changes.** The accessors, verification, and `write_skills` see raw +objects through the `SkillStore` interface and cannot tell which store produced them. + +**Server-side only.** Skills are for server-side agent runtimes and skill content is +customer-confidential. A mobile key (`mob-…`) or a client-side environment ID raises from the +constructor. + +**Streaming is the default, and it is what makes revocation fast.** A `delete-object` reaches +a live stream in seconds; with `mode="poll"` it arrives within one `poll_interval`. Paired +with `watch_skills`, a revoked skill's `SKILL.md` leaves the disk without a restart. During an +outage the store keeps serving the last content it received and `write_skills`' default +`on_unavailable="keep"` leaves managed files alone — an outage must not read as "everything +was revoked". + +**One network timeout, and its default depends on the mode.** `read_timeout` bounds every +socket operation of a request, connecting included. In `mode="poll"` it bounds the whole +request and defaults to 10 seconds; in `mode="stream"` it bounds each wait for the next bytes +and defaults to 300 seconds, well beyond LaunchDarkly's heartbeat interval. + +**The connection also carries your flags.** A client cannot request only the skill payload, +so a skills-enabled environment delivers flag and segment objects on the same connection. +They are skipped, not evaluated — this store does no evaluation of any kind — and +`diagnostics.objects_ignored` counts them. + +> **Beta caveats, worth knowing before you deploy.** Payload signing does not exist on this +> channel yet, so delivery is TLS-only and the content hash establishes self-consistency, not +> origin authenticity. The FDv2 protocol is opt-in per account: without it the endpoints +> return HTTP 403, which the store reports as a fatal error explaining what to do. `ld-relay` +> does not speak the FDv2 endpoints, so relay-only deployments cannot receive skills. + +**If every skill comes back empty, check `diagnostics.hashless_objects`.** Verification +withholds any delivered object without a `contentHash`, so a nonzero count means skills are +being withheld rather than that the environment has none. The store also logs an error per +hashless object naming the reason. There is deliberately no fallback that skips verification. + **Total path length is yours to bound, not the SDK's.** The 255-byte bound above is per *component*; the root is your path, so `` + `` + `/SKILL.md` can still exceed Windows' 260-character `MAX_PATH` with a perfectly legal key. Choose a short managed root on @@ -460,7 +520,9 @@ Windows. | `write_skills(skills, root, *, prune=True, timeout=10.0, on_unavailable="keep")` | Materialize skills under `root`, returning a `ReconcileReport`. `prune` removes formerly-managed skills no longer requested. `on_unavailable="raise"` raises instead of reporting when content cannot be retrieved. Raises `ValueError` for an unusable root, a negative `timeout`, or an unrecognised `on_unavailable`. **Performs synchronous filesystem I/O — see the note below.** | | `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)` / `remove_listener(kind, fn)`. | | `InMemorySkillStore(objects=None)` | A dict-backed store with `put(raw)`, for local development and testing. Holds several versions of a key. | +| `FDv2SkillStore(sdk_key, *, base_uri=…, mode="stream", …)` | The delivery transport: a store fed by LaunchDarkly over the SDK-facing FDv2 channel. `start()`, `wait_for_skills(timeout)`, `close()`, `diagnostics`, `failed`; also a context manager. **Server-side only** — a mobile key or client-side environment ID raises. See *Receiving skills from LaunchDarkly* above. | | `watch_skills(skills, root, …)` | `write_skills` plus a re-reconcile on every delivery change. Returns `(initial report, SkillWatcher)`; close the watcher when done. Revocation then takes effect within `debounce` of arriving rather than at the next restart. | +| `StoreDiagnostics` | What the transport has seen: `payloads_transferred`, `skill_objects_received`, `objects_ignored`, `objects_revoked`, `hashless_objects`, `connection_failures`, `last_error`. | Configure the store with `init_client(options={"skillStore": store})`. With none configured, the accessors raise `RuntimeError` explaining what to do and `write_skills` reports the diff --git a/packages/client/agents.md b/packages/client/agents.md index 978dc8ad..2a19c273 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,7 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | -| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the wire-key/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | +| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery transport — the FDv2 protocol, the wire-key/`version` translation, the held object set, and `FDv2SkillStore`. Sits **below** the store interface; imports `skills_core` only, and nothing imports it | | `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | @@ -221,10 +221,12 @@ of skills, and the `"*"` reconcile, since `//SKILL.md` is a single pa ### The delivery transport, and the one field that will bite you -`skills_fdv2.py` translates LaunchDarkly's FDv2 delivery protocol into raw objects in the -shape `skills_core.SkillStore` documents. It lives below the store interface; **nothing above -that interface knows it exists**. If a transport change ever seems to require editing an -accessor, verification, or `write_skills`, the adapter boundary is wrong. +`FDv2SkillStore` speaks LaunchDarkly's SDK-facing FDv2 channel (`GET /sdk/poll`, +`GET /sdk/stream`, server-side SDK key in `Authorization`, `basis` + `mv` params, +`If-None-Match`/304). It lives below the store interface and produces raw objects in the +shape `skills_core.SkillStore` documents; **nothing above that interface knows it exists**. If a transport +change ever seems to require editing an accessor, verification, or `write_skills`, the adapter +boundary is wrong. **The skill's version is in the object's `key`. `version` is the payload's.** Each version of a skill is its own object on the wire, identified as `:`: @@ -285,6 +287,19 @@ such skill" — and would let a prune delete the last known-good copy on disk. N a hash from the delivered content: that certifies the content against itself and verifies nothing. +**There is one network timeout, not two.** `urllib`'s `timeout` is the socket timeout for the +whole operation, so connect, headers and each read share it, and the module cannot bound the +connect separately without a custom connection class it should not carry. `read_timeout` is +therefore the only knob, and its default is per mode (`DEFAULT_POLL_TIMEOUT` for a whole poll +request, `DEFAULT_STREAM_READ_TIMEOUT` for the gap between reads on a stream). Do not add a +parameter that the standard library cannot honour; `TestTimeouts` measures the bound against a +socket that accepts and never answers. + +**`close` interrupts the socket, it does not just set a flag.** The delivery thread spends its +life blocked in a read that no flag can reach, and closing a response from another thread does +not unblock CPython's buffered reader. `_interrupt_read` shuts the socket down underneath it. +Without that, every shutdown of a *healthy* stream blocks for the full join timeout. + ### The reported outcome vocabulary, and the `Resolution` mapping `get_skill` returns `Skill | None`; `get_skill_result` returns a frozen `SkillOutcome` diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index f863d847..0f987117 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -46,6 +46,7 @@ skill_refs, ) from .skills_core import SkillStore +from .skills_fdv2 import FDv2SkillStore, StoreDiagnostics from .skills_fs import ( MANIFEST_FILENAME, MANIFEST_VERSION, @@ -238,7 +239,9 @@ "write_skills", "SkillStore", "InMemorySkillStore", - # skills — the eager re-reconcile + # skills — the FDv2 delivery transport, and the eager re-reconcile it enables + "FDv2SkillStore", + "StoreDiagnostics", "watch_skills", "SkillWatcher", # skills — the three closed-set unions a typed consumer needs to name diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index d2d870c4..b8043012 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -134,9 +134,18 @@ NO_STORE_MESSAGE = ( "No skill store is configured, so skill content cannot be retrieved. Configure " - 'one with init_client(options={"skillStore": store}) — InMemorySkillStore is ' - "available for local development and testing." + 'one with init_client(options={"skillStore": store}) — FDv2SkillStore receives ' + "content from LaunchDarkly, and InMemorySkillStore is available for local " + "development and testing." ) +""" +The first thing a user sees when no store is configured, so it names both stores. + +``FDv2SkillStore`` comes first because it is the answer in production, and a +message that offered only ``InMemorySkillStore`` would point a deployment at the +development store. Callers match on "skill store"; keep that phrase if the +wording changes. +""" # --------------------------------------------------------------------------- diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index d20a0e2f..da84a3b4 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -1,15 +1,10 @@ """ -Agent Skills — the FDv2 delivery protocol. +Agent Skills — the FDv2 delivery transport. -The half of the delivery transport that has no I/O: identifying skill objects -on the wire, translating them into the raw object shape the ``SkillStore`` -interface defines, holding them by ``(key, version)``, and applying a -payload's events as one consistent commit. ``FDv2SkillStore``, the store that -puts a network connection underneath this, follows in a separate change. - -It sits *below* the ``SkillStore`` interface, and everything above — the -accessors, integrity verification, the ``Skill`` dataclass, materialization — -is unaware of it. +The store implementation that talks to LaunchDarkly. It sits *below* the +``SkillStore`` interface: it produces raw wire objects in the shape +``skills_core`` documents, and everything above — the accessors, integrity +verification, the ``Skill`` dataclass, materialization — is unaware of it. Layering:: @@ -20,12 +15,12 @@ GET /sdk/poll, GET /sdk/stream, authenticated with the environment's server-side SDK key -Dependencies run one way: this module imports nothing from the feature beyond -the version validator in ``types_validation``, and nothing in the feature -imports it. It uses only the standard library, so it adds no dependency +Dependencies run one way: this module imports ``skills_core`` for the +interface's kind constant and nothing else from the feature, and nothing in the +feature imports it. It uses only the standard library, so it adds no dependency to a package whose sole runtime dependency is ``opentelemetry-api``. -Three things this layer does *not* do, on purpose: +Three things this module does *not* do, on purpose: - **It does not verify content.** Verification lives at the accessor boundary in ``skills_core`` so that it applies to every store equally, including a @@ -36,24 +31,29 @@ - **It does not evaluate anything.** Flag and segment objects that share the connection are skipped and counted, nothing more. -One assumption it *does* make, and states: **the payload intent it reads is the -payload skills arrive on.** Delivery sends one payload per credential and the -protocol tells a client to read only the first payload intent, so today those are -the same payload. ``_ProtocolReader`` keeps the pair apart anyway, because the -cost of conflating them is an emptied skill set. - The design rationale — why the skill's version is read from the object's ``key`` and never from ``version``, why changes commit at -``payload-transferred`` — is in ``agents.md`` under *The delivery transport*. +``payload-transferred``, why there is one network timeout — is in +``agents.md`` under *The delivery transport*. """ from __future__ import annotations +import json import logging +import math +import random import re +import socket +import threading +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable from dataclasses import dataclass, field -from typing import Any +from typing import Any, Literal +from .skills_core import SKILL_OBJECT_KIND from .types_validation import is_valid_skill_version logger = logging.getLogger(__name__) @@ -86,6 +86,20 @@ has exactly one. """ +DEFAULT_BASE_URI = "https://sdk.launchdarkly.com" +"""Where the SDK-facing FDv2 endpoints live. Overridable for Federal and private +instances.""" + +POLL_PATH = "/sdk/poll" +STREAM_PATH = "/sdk/stream" + +DEFAULT_POLL_TIMEOUT = 10.0 +"""Default ``read_timeout`` in ``"poll"`` mode: the bound on one whole request.""" + +DEFAULT_STREAM_READ_TIMEOUT = 300.0 +"""Default ``read_timeout`` in ``"stream"`` mode: the longest gap tolerated +between two reads. LaunchDarkly's heartbeats arrive well inside this.""" + _EVENT_SERVER_INTENT = "server-intent" _EVENT_PUT_OBJECT = "put-object" _EVENT_DELETE_OBJECT = "delete-object" @@ -115,6 +129,8 @@ no ``id``. """ +Mode = Literal["stream", "poll"] + _MOBILE_KEY_PREFIX = "mob-" _SERVER_KEY_PREFIX = "sdk-" _CLIENT_SIDE_ID = re.compile(r"\A[0-9a-f]{20,}\Z") @@ -504,7 +520,6 @@ def __init__(self, committed: _SkillObjectSet) -> None: self.diagnostics = StoreDiagnostics() # Identities already reported by ``_warn_hashless``. Per reader, so a # recreated store reports again and two stores never quieten each other. - # No lock: ``handle`` runs only on its owner's single delivery thread. self._warned_hashless: set[tuple[str, Any]] = set() # The payload the current intent describes, and the payload skills have # actually arrived on. One payload per connection makes these the same @@ -609,8 +624,8 @@ def _delete_object(self, data: Any) -> _TransferOutcome: self.diagnostics.objects_revoked += 1 # A revocation identifies the payload as ours just as a put does. self._skills_in_payload += 1 - # A tombstone carries identity and no content, so a listener that reads - # content must check for ``content`` rather than assume it. + # A tombstone carries identity and no content; see + # ``FDv2SkillStore.add_listener`` for what listeners should expect. self._changes.append( {"key": tombstone.key, "version": tombstone.object_version} ) @@ -780,3 +795,686 @@ def _warn_if_nothing_can_verify(committed: _SkillObjectSet) -> None: len(held), _HASHLESS_ADVICE, ) + + +# --------------------------------------------------------------------------- +# HTTP +# --------------------------------------------------------------------------- + + +class _FatalTransportError(Exception): + """A failure retrying cannot fix: bad credential, forbidden, wrong URI.""" + + +class _RecoverableTransportError(Exception): + """A failure worth retrying. Carries a server-requested delay when given one.""" + + def __init__(self, message: str, retry_after: float | None = None) -> None: + super().__init__(message) + self.retry_after = retry_after + + +_FORBIDDEN_ADVICE = ( + "The FDv2 protocol is opt-in per LaunchDarkly account and is served as HTTP " + "403 while it is off. Skill delivery needs it enabled; contact LaunchDarkly " + "support to enable it for your account." +) + + +def _retry_after_seconds(headers: Any) -> float | None: + """ + ``Retry-After`` in seconds, when the server sent a usable one. + + The HTTP-date form, and non-finite values such as ``inf`` or ``1e309`` that + ``float`` accepts, fall back to our own backoff: none of them is a delay, + and an infinite one would overflow the wait that honours it. + """ + if headers is None: + return None + try: + raw = headers.get("Retry-After") + except AttributeError: + return None + if raw is None: + return None + try: + seconds: float = float(str(raw).strip()) + except ValueError: + return None + if not math.isfinite(seconds): + return None + return max(0.0, seconds) + + +def _classify_status(status: int, headers: Any) -> Exception: + """Turns an HTTP error status into the right exception type.""" + if status == 401: + return _FatalTransportError( + "LaunchDarkly rejected the SDK key (HTTP 401). Skill delivery cannot " + "start. Check that the key is the environment's server-side SDK key." + ) + if status == 403: + return _FatalTransportError( + f"LaunchDarkly returned HTTP 403. {_FORBIDDEN_ADVICE}" + ) + if status in (400, 405, 406, 414, 501): + return _FatalTransportError( + f"LaunchDarkly returned HTTP {status}, which retrying will not fix. " + "The request this adapter sent was not understood. It carries only " + "the SDK key and, after the first payload, a 'basis' selector, so " + "check the base URI and that the endpoint speaks FDv2." + ) + return _RecoverableTransportError( + f"LaunchDarkly returned HTTP {status}", _retry_after_seconds(headers) + ) + + +def _interrupt_read(response: Any) -> None: + """ + Best-effort interruption of a read blocked on *response*, from another thread. + + Closing the response is not enough: CPython's buffered reader stays parked in + ``readline`` until bytes arrive. Shutting the *socket* down underneath it + unblocks it immediately. Reaching the socket means walking urllib's private + attribute chain, so every step is guarded and failure is silent: the + delivery thread is a daemon and ``close``'s join timeout is the backstop. + """ + for path in (("fp", "raw", "_sock"), ("fp", "_sock"), ("_sock",)): + found: Any = response + for name in path: + found = getattr(found, name, None) + if found is None: + break + if found is not None and hasattr(found, "shutdown"): + try: + found.shutdown(socket.SHUT_RDWR) + except OSError: + pass + return + + +class _StreamConnection: + """ + One open streaming connection: an event iterator plus a way to interrupt it + from another thread, which is what ``FDv2SkillStore.close`` needs. + """ + + def __init__(self, response: Any) -> None: + self._response = response + self.events = _iter_sse(response) + + def close(self) -> None: + """Interrupts the read. Safe to call from any thread, and twice.""" + _interrupt_read(self._response) + try: + self._response.close() + except Exception: + pass + + +@dataclass(frozen=True) +class _PollResult: + not_modified: bool + events: list[tuple[str, Any]] + etag: str | None + + +class _Requester: + """ + The only place this module opens a socket. Standard library only, on purpose. + + *read_timeout* is applied to every socket operation of a request. ``urllib`` + has no separate connect timeout: its ``timeout`` becomes the socket timeout + for the whole operation, so connecting, waiting for headers and each body + read are all bounded by the same value. + """ + + def __init__( + self, + sdk_key: str, + base_uri: str, + *, + read_timeout: float, + opener: Any = None, + ) -> None: + self._sdk_key = sdk_key + self._base_uri = base_uri.rstrip("/") + self._read_timeout = read_timeout + # Injectable so tests can drive a fake endpoint without a socket. + self._opener = opener or urllib.request.build_opener() + + def _url(self, path: str, basis: str | None) -> str: + """ + The request URL: the path, plus ``basis`` once a payload has committed. + + Deliberately no ``mv`` (data model version). That parameter selects the + *flag* data model and the connection rejects any value but the flag + default; the agent-skill payload is generic, is served regardless of it, + and has no model version of its own to ask for. + """ + if not basis: + return f"{self._base_uri}{path}" + return f"{self._base_uri}{path}?{urllib.parse.urlencode({'basis': basis})}" + + def _request( + self, path: str, basis: str | None, headers: dict[str, str] + ) -> urllib.request.Request: + all_headers = {"Authorization": self._sdk_key, **headers} + return urllib.request.Request( + self._url(path, basis), headers=all_headers, method="GET" + ) + + def poll(self, basis: str | None, etag: str | None) -> _PollResult: + """One ``GET /sdk/poll``. A 304 is a first-class outcome, not an error.""" + headers = {"Accept": "application/json"} + if etag: + headers["If-None-Match"] = etag + request = self._request(POLL_PATH, basis, headers) + try: + with self._opener.open(request, timeout=self._read_timeout) as response: + status = getattr(response, "status", None) or response.getcode() + if status == 304: + return _PollResult(not_modified=True, events=[], etag=etag) + body = response.read() + new_etag = response.headers.get("ETag") or etag + except urllib.error.HTTPError as exc: + if exc.code == 304: + # urllib raises on 304 when no redirect handler swallows it. + return _PollResult(not_modified=True, events=[], etag=etag) + raise _classify_status(exc.code, exc.headers) from exc + except Exception as exc: + raise _RecoverableTransportError( + f"polling request failed: {type(exc).__name__}: {exc}" + ) from exc + + return _PollResult( + not_modified=False, events=_decode_poll_body(body), etag=new_etag + ) + + def stream(self, basis: str | None) -> _StreamConnection: + """Opens ``GET /sdk/stream``.""" + request = self._request( + STREAM_PATH, + basis, + {"Accept": "text/event-stream", "Cache-Control": "no-cache"}, + ) + try: + response = self._opener.open(request, timeout=self._read_timeout) + except urllib.error.HTTPError as exc: + raise _classify_status(exc.code, exc.headers) from exc + except Exception as exc: + raise _RecoverableTransportError( + f"streaming request failed: {type(exc).__name__}: {exc}" + ) from exc + return _StreamConnection(response) + + +def _decode_poll_body(body: bytes) -> list[tuple[str, Any]]: + """ + Unwraps ``{"events": [...]}``. Polling and streaming carry identical event + objects, which is why the protocol reader is shared between the two modes. + """ + try: + parsed = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise _RecoverableTransportError( + f"polling response was not valid JSON: {exc}" + ) from exc + if not isinstance(parsed, dict) or not isinstance(parsed.get("events"), list): + raise _RecoverableTransportError("polling response had no 'events' array") + events: list[tuple[str, Any]] = [] + for entry in parsed["events"]: + if not isinstance(entry, dict): + continue + name = entry.get("event") + if isinstance(name, str): + events.append((name, entry.get("data"))) + return events + + +def _iter_sse(response: Any) -> Any: + """ + Decodes an SSE body into ``(event name, data)`` pairs. + + Minimal on purpose: ``event:``/``data:`` fields, multi-line ``data`` joined + with newlines, a blank line dispatching, and ``:`` comments skipped. + """ + try: + name: str | None = None + data_lines: list[str] = [] + for raw_line in response: + line = raw_line.decode("utf-8", errors="replace").rstrip("\r\n") + if line == "": + if name is not None: + payload = "\n".join(data_lines) + try: + parsed = json.loads(payload) if payload else None + except json.JSONDecodeError: + logger.warning( + "Discarding FDv2 '%s' event whose data was not JSON", name + ) + parsed = None + else: + yield name, parsed + name = None + data_lines = [] + continue + if line.startswith(":"): + continue + field_name, _, value = line.partition(":") + value = value[1:] if value.startswith(" ") else value + if field_name == "event": + name = value + elif field_name == "data": + data_lines.append(value) + finally: + try: + response.close() + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Backoff +# --------------------------------------------------------------------------- + + +def _backoff_delay( + attempt: int, *, base: float, maximum: float, jitter: float = 0.5 +) -> float: + """ + Exponential backoff with jitter, capped at *maximum*. + + Jitter is subtractive over the whole range rather than added on top, so the + cap is a real ceiling: a fleet restarted together must not reconnect in + lockstep, and must not exceed the interval the cap promises. + """ + # float(2 ** n): the integer power is untyped to mypy. + ceiling: float = min(maximum, base * float(2 ** max(0, attempt - 1))) + return ceiling * (1.0 - jitter * random.random()) + + +# --------------------------------------------------------------------------- +# The store +# --------------------------------------------------------------------------- + + +class FDv2SkillStore: + """ + A ``SkillStore`` fed by LaunchDarkly's SDK-facing FDv2 delivery channel. + + Constructed with the environment's server-side SDK key, started explicitly, + and passed to ``init_client``:: + + store = FDv2SkillStore(sdk_key=os.environ["LD_SDK_KEY"]) + store.start() + store.wait_for_skills(timeout=10) + await init_client(options={"skillStore": store}) + + skill = await get_skill("pdf-extraction") + ... + store.close() + + It also works as a context manager. + + **Server-side only.** A mobile key or a client-side environment ID is + refused in the constructor. + + **Delivery is in the background; retrieval is not.** A daemon thread owns + the connection and fills memory, and ``get_object`` only ever reads what has + already arrived. A process that calls ``get_skill`` immediately after + ``start()`` may see an empty store; ``wait_for_skills`` orders boot against + the first payload. + + **Last known good survives an outage.** A transport failure never empties + the store and never makes ``get_object`` raise, which is what makes + ``write_skills(on_unavailable="keep")`` correct. ``diagnostics`` and + ``failed`` report the degradation. + + **What arrives is untrusted.** Raw wire objects are held verbatim and + verified at the accessor boundary, not here. In particular an object with no + ``contentHash`` is held and then *withheld*; see + ``StoreDiagnostics.hashless_objects``. + """ + + def __init__( + self, + sdk_key: str, + *, + base_uri: str = DEFAULT_BASE_URI, + mode: Mode = "stream", + poll_interval: float = 30.0, + read_timeout: float | None = None, + initial_backoff: float = 1.0, + max_backoff: float = 30.0, + max_consecutive_failures: int = 10, + _requester: Any = None, + ) -> None: + """ + *mode* is ``"stream"`` by default. Prefer it: a ``delete-object`` reaches + a live stream in seconds. ``"poll"`` exists for environments that cannot + hold a long-lived connection, and revocation there is one + ``poll_interval`` late. + + *read_timeout* is the only network timeout and bounds every socket + operation of a request, so its meaning and default follow the mode: in + ``"poll"`` it bounds the whole request (``DEFAULT_POLL_TIMEOUT``); in + ``"stream"`` it bounds each wait for the next bytes + (``DEFAULT_STREAM_READ_TIMEOUT``). Must be positive when given. + + *max_backoff* caps every delay between retries, including one the server + asks for with ``Retry-After``. + + *max_consecutive_failures* bounds the retry loop. On exceeding it the + transport stops, logs an error, and the store keeps serving last known + good; ``failed`` reports it. Only failures in a row count: a committed + payload resets the count. + """ + _require_server_side_credential(sdk_key) + if mode not in ("stream", "poll"): + raise ValueError(f'mode must be "stream" or "poll", got {mode!r}') + if poll_interval <= 0: + raise ValueError(f"poll_interval must be positive, got {poll_interval!r}") + if read_timeout is None: + read_timeout = ( + DEFAULT_STREAM_READ_TIMEOUT + if mode == "stream" + else DEFAULT_POLL_TIMEOUT + ) + elif not (math.isfinite(read_timeout) and read_timeout > 0): + raise ValueError(f"read_timeout must be positive, got {read_timeout!r}") + + self._mode: Mode = mode + self._poll_interval = poll_interval + self._initial_backoff = initial_backoff + self._max_backoff = max_backoff + self._max_consecutive_failures = max_consecutive_failures + + self._objects = _SkillObjectSet() + self._reader = _ProtocolReader(self._objects) + self._lock = threading.RLock() + self._listeners: dict[str, list[Callable[[dict[str, Any]], Any]]] = {} + + self._basis: str | None = None + self._etag: str | None = None + + self._requester = _requester or _Requester( + sdk_key.strip(), + base_uri, + read_timeout=read_timeout, + ) + + self._stop = threading.Event() + self._first_payload = threading.Event() + self._thread: threading.Thread | None = None + self._failed_reason: str | None = None + # The open streaming connection, so ``close`` can interrupt its read. + self._connection: Any = None + # Recoverable failures since the last committed payload. Reset at the + # commit rather than when a connection returns: a stream only ever ends + # by being dropped, so resetting on return would count every healthy, + # server-recycled connection as a failure. + self._failures = 0 + + # -- lifecycle --------------------------------------------------------- + + def start(self) -> FDv2SkillStore: + """ + Starts the delivery thread. Idempotent; returns ``self`` so it chains. + + Does not block: use ``wait_for_skills`` when boot ordering matters. + """ + with self._lock: + if self._thread is not None and self._thread.is_alive(): + return self + self._stop.clear() + self._thread = threading.Thread( + target=self._run, name="ld-ai-skills-fdv2", daemon=True + ) + self._thread.start() + return self + + def close(self, timeout: float = 5.0) -> None: + """ + Stops delivery. Idempotent, and safe to call from any thread. + + Held content is *not* dropped: a closed store still answers from what it + received. Detaching the store from the accessors is the job of the + package-level ``launchdarkly_ai_server.shutdown()`` coroutine. + """ + self._stop.set() + # The delivery thread is normally blocked in a socket read that no flag + # can reach; without this the join waits out its full timeout. + with self._lock: + connection = self._connection + if connection is not None: + connection.close() + thread = self._thread + if ( + thread is not None + and thread.is_alive() + and thread is not threading.current_thread() + ): + thread.join(timeout=timeout) + + def __enter__(self) -> FDv2SkillStore: + return self.start() + + def __exit__(self, *_exc: Any) -> None: + self.close() + + def wait_for_skills(self, timeout: float = 10.0) -> bool: + """ + Blocks until the first payload has been committed, or *timeout* elapses. + + ``True`` means a payload arrived — not that any skill in it verified, and + not that the environment has any skills. ``diagnostics`` answers the rest. + """ + return self._first_payload.wait(timeout=timeout) + + @property + def failed(self) -> str | None: + """Why delivery stopped for good, or ``None`` while it is running.""" + with self._lock: + return self._failed_reason + + @property + def diagnostics(self) -> StoreDiagnostics: + """A snapshot of what the transport has seen. See ``StoreDiagnostics``.""" + with self._lock: + return StoreDiagnostics(**vars(self._reader.diagnostics)) + + # -- the SkillStore interface ----------------------------------------- + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + if kind != SKILL_OBJECT_KIND: + return None + with self._lock: + return self._objects.get(key, version) + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + if kind != SKILL_OBJECT_KIND: + return {} + with self._lock: + return self._objects.snapshot() + + def add_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Registers *fn* to be called once per changed object, at + ``payload-transferred`` rather than as objects stream in. + + A put notifies with the raw skill object. A revocation notifies with a + ``{"key", "version"}`` tombstone carrying no content, so a listener that + reads content must check for ``content`` rather than assume it. + + *fn* runs on the delivery thread. Keep it cheap and non-blocking. An + exception it raises is logged and swallowed, because a broken listener + must not be able to kill delivery. + """ + with self._lock: + self._listeners.setdefault(kind, []).append(fn) + + def remove_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Unregisters *fn* from *kind*. Safe to call from any thread, including + from inside a listener: a removal during one commit takes effect from + the next. + + Removes one occurrence; removing a callable that is not registered is a + no-op, so ``SkillWatcher.close`` can detach unconditionally. + """ + with self._lock: + listeners = self._listeners.get(kind) + if listeners is None: + return + try: + listeners.remove(fn) + except ValueError: + return + + def _notify(self, changes: list[dict[str, Any]]) -> None: + with self._lock: + listeners = list(self._listeners.get(SKILL_OBJECT_KIND, [])) + for raw in changes: + for listener in listeners: + try: + listener(raw) + except Exception: + logger.error( + "A skill store change listener raised; delivery continues", + exc_info=True, + ) + + # -- the delivery loop ------------------------------------------------- + + def _run(self) -> None: + while not self._stop.is_set(): + try: + if self._mode == "stream": + self._stream_once() + else: + self._poll_once() + # A poll that returned is a current answer even when it committed + # nothing (HTTP 304). A stream never returns normally; its + # successes are counted at each commit in ``_apply``. + self._record_success() + except _FatalTransportError as exc: + self._give_up(str(exc)) + return + except _RecoverableTransportError as exc: + with self._lock: + self._failures += 1 + failures = self._failures + self._reader.diagnostics.connection_failures = failures + self._reader.diagnostics.last_error = str(exc) + if failures > self._max_consecutive_failures: + self._give_up( + f"gave up after {failures} consecutive failures; " + f"last error: {exc}" + ) + return + delay = exc.retry_after + if delay is None or not math.isfinite(delay): + delay = _backoff_delay( + failures, base=self._initial_backoff, maximum=self._max_backoff + ) + # ``Retry-After`` is a request and ``max_backoff`` is a promise. + # The header may come from a proxy rather than LaunchDarkly, and + # a value in the hours would park revocation for that long. + delay = min(delay, self._max_backoff) + logger.warning( + "Skill delivery failed (%s); retrying in %.1fs", exc, delay + ) + if self._stop.wait(delay): + return + continue + except Exception as exc: # pragma: no cover - defensive + self._give_up(f"unexpected error in skill delivery: {exc!r}") + logger.error("Unexpected error in skill delivery", exc_info=True) + return + + if self._mode == "poll" and self._stop.wait(self._poll_interval): + return + + def _record_success(self) -> None: + with self._lock: + self._failures = 0 + self._reader.diagnostics.connection_failures = 0 + + def _give_up(self, reason: str) -> None: + with self._lock: + self._failed_reason = reason + self._reader.diagnostics.last_error = reason + logger.error( + "Skill delivery has stopped and will not retry: %s. The store keeps " + "serving the last content it received; skills will not update until " + "the process restarts with a working connection.", + reason, + ) + # Unblock anyone waiting on a first payload that is never coming. + self._first_payload.set() + + def _apply(self, name: str, data: Any) -> None: + """ + Feeds one event to the reader, publishes a commit, and raises the + transport error the event calls for, if any. + """ + with self._lock: + outcome = self._reader.handle(name, data) + if outcome.committed and outcome.basis is not None: + self._basis = outcome.basis + if outcome.committed: + # A commit breaks the row of consecutive failures. + self._record_success() + self._first_payload.set() + if outcome.changes: + self._notify(outcome.changes) + if outcome.fatal: + raise _FatalTransportError(outcome.fatal) + if outcome.disconnect: + raise _RecoverableTransportError(outcome.disconnect) + + def _poll_once(self) -> None: + with self._lock: + basis, etag = self._basis, self._etag + result = self._requester.poll(basis, etag) + with self._lock: + self._etag = result.etag + if result.not_modified: + logger.debug("Skill payload unchanged (HTTP 304)") + # A 304 counts as a first payload, so a boot that reconnects with a + # cached basis is not blocked on a transfer the server will not send. + self._first_payload.set() + return + for name, data in result.events: + self._apply(name, data) + + def _stream_once(self) -> None: + with self._lock: + basis = self._basis + connection = self._requester.stream(basis) + with self._lock: + self._connection = connection + try: + # ``close`` may have run while the connect was in flight and found + # no connection to interrupt; this is the last chance to notice + # before the read below blocks. + if self._stop.is_set(): + return + for name, data in connection.events: + if self._stop.is_set(): + return + self._apply(name, data) + except Exception: + if self._stop.is_set(): + # ``close`` interrupted the read on purpose. + return + raise + finally: + connection.close() + with self._lock: + self._connection = None + # A stream that ends without a goodbye is a dropped connection. + raise _RecoverableTransportError("the FDv2 stream closed unexpectedly") diff --git a/packages/client/src/launchdarkly_ai_server/skills_watch.py b/packages/client/src/launchdarkly_ai_server/skills_watch.py index 05263057..e421df6a 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_watch.py +++ b/packages/client/src/launchdarkly_ai_server/skills_watch.py @@ -267,7 +267,7 @@ async def watch_skills( "watch_skills needs a skill store that implements add_listener(kind, " "fn); the configured store does not, so delivery changes cannot be " "observed. Use write_skills for a one-shot reconcile, or configure a " - "store with a delivery transport." + "store with a delivery transport (FDv2SkillStore)." ) if debounce < 0: raise ValueError(f"debounce must not be negative, got {debounce!r}") diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index d3f83095..a07ea6ae 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -603,6 +603,18 @@ async def test_all_skills_raises_actionably_when_no_store(self) -> None: with pytest.raises(RuntimeError, match="skill store"): await all_skills() + async def test_the_no_store_message_names_the_delivery_store_first(self) -> None: + """ + A deployment that hits this message must be pointed at the store that + receives content from LaunchDarkly, not only at the development one. + """ + with pytest.raises(RuntimeError) as reported: + await get_skill("a") + message = str(reported.value) + assert "FDv2SkillStore" in message + assert "InMemorySkillStore" in message + assert message.index("FDv2SkillStore") < message.index("InMemorySkillStore") + async def test_shutdown_clears_the_store( self, make_raw_skill: Any, mock_ld_client: Any ) -> None: diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index b88d14d9..0cce2f0c 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -1,26 +1,55 @@ """ -Tests for the FDv2 skill delivery protocol. - -Wire semantics — which objects are skills, the skill's version in the wire ``key`` -versus the payload's in ``version``, revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted -against ``_ProtocolReader``, which has no I/O, so each case reads as the contract -it is rather than as a server script. +Tests for the FDv2 skill delivery transport. + +Two layers, deliberately: + +- **A real fake endpoint.** ``_FakeFDv2Endpoint`` is an in-process + ``ThreadingHTTPServer`` that implements the wire contract — the ``basis`` + query parameter, ``Authorization``, ``If-None-Match``/304, the + ``{"events": [...]}`` polling envelope, and SSE for streaming. The store under + test opens real sockets against it, so request construction and header + handling are exercised rather than mocked. +- **The protocol reader driven directly.** Wire semantics — which objects are + skills, the skill's version in the wire ``key`` versus the payload's in + ``version``, revocation, mixed payloads — are + asserted against ``_ProtocolReader``, which has no I/O, so those cases read as + the contract they are instead of as a server script. """ from __future__ import annotations import hashlib +import json +import socket +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, ClassVar +from urllib.parse import parse_qs, urlparse import pytest -from launchdarkly_ai_server import InMemorySkillStore +from launchdarkly_ai_server import ( + FDv2SkillStore, + InMemorySkillStore, + all_skills, + get_skill, + get_skill_result, + init_client, + watch_skills, +) from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND from launchdarkly_ai_server.skills_fdv2 import ( + DEFAULT_POLL_TIMEOUT, + DEFAULT_STREAM_READ_TIMEOUT, FDV2_KEY_DELIMITER, FDV2_OBJECT_KIND, + _backoff_delay, _is_skill_event, _ProtocolReader, + _RecoverableTransportError, + _Requester, + _retry_after_seconds, _SkillObjectSet, _store_object_from_put, _tombstone_from_delete, @@ -28,6 +57,7 @@ pytestmark = pytest.mark.usefixtures("reset_skill_state") +SDK_KEY = "sdk-00000000-0000-4000-8000-000000000000" SKILL_BODY = "---\nname: PDF Extraction\n---\nExtract text from PDFs.\n" @@ -143,6 +173,182 @@ def full_payload( ) +# --------------------------------------------------------------------------- +# The fake endpoint +# --------------------------------------------------------------------------- + + +class _FakeFDv2Endpoint: + """ + An in-process server implementing the SDK-facing FDv2 contract. + + Scripted per request: ``queue_poll`` appends a response for the next + ``/sdk/poll``, ``queue_stream`` appends a sequence of SSE events for the next + ``/sdk/stream``. Every request's method, path, query and headers are recorded + in ``requests`` so the tests can assert on what the store actually sent — + which is the only way ``basis`` round-tripping and ``If-None-Match`` can be + checked at all. + """ + + def __init__(self) -> None: + self.requests: list[dict[str, Any]] = [] + self._polls: list[dict[str, Any]] = [] + self._streams: list[list[dict[str, Any]]] = [] + self._lock = threading.Lock() + self.hold_stream_open = False + self._release = threading.Event() + + endpoint = self + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args: Any) -> None: + return + + def do_GET(self) -> None: + parsed = urlparse(self.path) + query = {k: v[0] for k, v in parse_qs(parsed.query).items()} + with endpoint._lock: + endpoint.requests.append( + { + "path": parsed.path, + "query": query, + "authorization": self.headers.get("Authorization"), + "if_none_match": self.headers.get("If-None-Match"), + "accept": self.headers.get("Accept"), + } + ) + if parsed.path == "/sdk/poll": + endpoint._serve_poll(self) + elif parsed.path == "/sdk/stream": + endpoint._serve_stream(self) + else: + self.send_response(404) + self.send_header("Content-Length", "0") + self.end_headers() + + class Server(ThreadingHTTPServer): + # Handler threads are not joined on shutdown: a test that ends while + # a stream is deliberately held open should not pay for the hold. + daemon_threads = True + + self._server = Server(("127.0.0.1", 0), Handler) + # A short poll interval so `shutdown` is prompt: the default 0.5s is + # paid at the teardown of every test that touches the endpoint. + self._thread = threading.Thread( + target=lambda: self._server.serve_forever(poll_interval=0.01), daemon=True + ) + self._thread.start() + + @property + def base_uri(self) -> str: + host, port = self._server.server_address[:2] + return f"http://{host}:{port}" + + # -- scripting --------------------------------------------------------- + + def queue_poll( + self, + payload_events: list[dict[str, Any]] | None = None, + *, + status: int = 200, + etag: str | None = None, + retry_after: str | None = None, + ) -> None: + with self._lock: + self._polls.append( + { + "status": status, + "events": payload_events or [], + "etag": etag, + "retry_after": retry_after, + } + ) + + def queue_stream(self, payload_events: list[dict[str, Any]]) -> None: + with self._lock: + self._streams.append(payload_events) + + # -- serving ----------------------------------------------------------- + + def _serve_poll(self, handler: BaseHTTPRequestHandler) -> None: + with self._lock: + response = ( + self._polls.pop(0) if self._polls else {"status": 304, "events": []} + ) + status = response["status"] + handler.send_response(status) + if response.get("etag"): + handler.send_header("ETag", response["etag"]) + if response.get("retry_after"): + handler.send_header("Retry-After", response["retry_after"]) + if status in (200,): + body = json.dumps({"events": response["events"]}).encode("utf-8") + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + return + handler.send_header("Content-Length", "0") + handler.end_headers() + + def _serve_stream(self, handler: BaseHTTPRequestHandler) -> None: + with self._lock: + payload_events = self._streams.pop(0) if self._streams else [] + handler.send_response(200) + handler.send_header("Content-Type", "text/event-stream") + handler.send_header("Cache-Control", "no-cache") + handler.send_header("Transfer-Encoding", "chunked") + handler.end_headers() + for event in payload_events: + chunk = ( + f"event: {event['event']}\ndata: {json.dumps(event.get('data'))}\n\n" + ).encode() + handler.wfile.write(f"{len(chunk):X}\r\n".encode() + chunk + b"\r\n") + handler.wfile.flush() + if self.hold_stream_open: + # Keeps the connection up so a test can assert on the store's state + # without racing the reconnect path. Released on ``close`` so the + # hold costs the suite nothing once the test is done with it. + self._release.wait(timeout=10) + handler.wfile.write(b"0\r\n\r\n") + + def close(self) -> None: + self._release.set() + self._server.shutdown() + self._server.server_close() + + +@pytest.fixture +def endpoint() -> Any: + server = _FakeFDv2Endpoint() + yield server + server.close() + + +def poll_store(endpoint: Any, **kwargs: Any) -> FDv2SkillStore: + return FDv2SkillStore( + SDK_KEY, + base_uri=endpoint.base_uri, + mode="poll", + poll_interval=kwargs.pop("poll_interval", 0.05), + initial_backoff=kwargs.pop("initial_backoff", 0.01), + max_backoff=kwargs.pop("max_backoff", 0.05), + read_timeout=kwargs.pop("read_timeout", 5.0), + **kwargs, + ) + + +def wait_until(predicate: Any, timeout: float = 5.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.01) + return predicate() + + # --------------------------------------------------------------------------- # Identifying skill objects, and ignoring everything else # --------------------------------------------------------------------------- @@ -827,6 +1033,608 @@ def test_snapshot_agrees(self) -> None: assert memory.all_objects(SKILL_OBJECT_KIND) == objects.snapshot() +# --------------------------------------------------------------------------- +# The store against the fake endpoint +# --------------------------------------------------------------------------- + + +class TestPollingAgainstTheEndpoint: + def test_a_polled_skill_becomes_retrievable_through_the_accessors( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + raw = store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") + assert raw is not None + assert raw["version"] == 3 + + def test_the_request_carries_the_sdk_key_and_no_data_model_version( + self, endpoint: Any + ) -> None: + """ + No ``mv``: that parameter selects the *flag* data model, the connection + rejects any value but the flag default, and the generic agent-skill + payload is served regardless of it. Sending ``mv=1`` — the skill + payload's own model version — gets the whole connection refused. + """ + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + first = endpoint.requests[0] + assert first["path"] == "/sdk/poll" + assert first["authorization"] == SDK_KEY + assert "mv" not in first["query"] + + def test_the_first_request_sends_no_basis(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert "basis" not in endpoint.requests[0]["query"] + + def test_the_basis_from_payload_transferred_is_echoed_on_the_next_request( + self, endpoint: Any + ) -> None: + endpoint.queue_poll( + full_payload(("put-object", put_skill()), state="selector-abc") + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert endpoint.requests[1]["query"]["basis"] == "selector-abc" + + def test_the_basis_advances_across_successive_payloads(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()), state="basis-1")) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("second")), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint): + assert wait_until(lambda: len(endpoint.requests) >= 3) + bases = [r["query"].get("basis") for r in endpoint.requests[:3]] + assert bases == [None, "basis-1", "basis-2"] + + def test_an_etag_is_returned_as_if_none_match(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill())), etag='W/"v1"') + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert endpoint.requests[1]["if_none_match"] == 'W/"v1"' + + def test_a_304_keeps_the_held_content(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill())), etag='W/"v1"') + endpoint.queue_poll(status=304) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert wait_until(lambda: len(endpoint.requests) >= 3) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + assert store.diagnostics.payloads_transferred == 1 + assert store.failed is None + + def test_a_304_before_any_payload_still_releases_wait_for_skills( + self, endpoint: Any + ) -> None: + """A reconnect with a cached basis has nothing to transfer; boot must not + block on a payload the server has no reason to send.""" + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + + def test_a_mixed_payload_over_the_wire_yields_only_the_skill( + self, endpoint: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_flag("flag-a")), + ("put-object", put_segment("beta")), + ("put-object", put_skill("pdf-extraction")), + ("put-object", put_flag("flag-b")), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + held = store.all_objects(SKILL_OBJECT_KIND) + assert len(held) == 1 + assert next(iter(held.values()))["key"] == "pdf-extraction" + assert store.diagnostics.objects_ignored == 3 + + def test_a_revocation_over_the_wire_removes_the_skill(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + assert wait_until( + lambda: ( + store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is None + and store.diagnostics.objects_revoked == 1 + ) + ) + + def test_the_store_asks_for_only_the_kind_it_serves(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert store.get_object("flag", "pdf-extraction") is None + assert store.all_objects("flag") == {} + + +class TestStreamingAgainstTheEndpoint: + def test_a_streamed_payload_lands(self, endpoint: Any) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore( + SDK_KEY, base_uri=endpoint.base_uri, mode="stream", initial_backoff=0.01 + ) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + finally: + store.close() + + def test_the_stream_request_advertises_event_stream(self, endpoint: Any) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + try: + store.start() + store.wait_for_skills(timeout=5) + finally: + store.close() + assert endpoint.requests[0]["path"] == "/sdk/stream" + assert endpoint.requests[0]["accept"] == "text/event-stream" + + def test_a_streamed_revocation_arrives_without_a_restart( + self, endpoint: Any + ) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream( + full_payload(("put-object", put_skill())) + + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ) + ) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + try: + store.start() + assert wait_until( + lambda: ( + store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is None + and store.diagnostics.objects_revoked == 1 + ) + ) + finally: + store.close() + + def test_a_dropped_stream_reconnects_with_the_basis_it_reached( + self, endpoint: Any + ) -> None: + endpoint.queue_stream( + full_payload(("put-object", put_skill()), state="basis-1") + ) + endpoint.queue_stream(events(("heart-beat", None))) + store = FDv2SkillStore( + SDK_KEY, + base_uri=endpoint.base_uri, + mode="stream", + initial_backoff=0.01, + max_backoff=0.05, + ) + try: + store.start() + assert wait_until(lambda: len(endpoint.requests) >= 2) + finally: + store.close() + assert endpoint.requests[1]["query"]["basis"] == "basis-1" + + def test_close_returns_promptly_while_a_stream_is_open(self, endpoint: Any) -> None: + """ + The delivery thread is blocked in a socket read that no stop flag can + reach, so ``close`` closes the connection under it. Without that, every + shutdown of a healthy stream waits out the join timeout. + """ + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + store.start() + assert store.wait_for_skills(timeout=5) is True + started = time.monotonic() + store.close(timeout=5.0) + assert time.monotonic() - started < 1.0 + + def test_an_interrupted_stream_is_not_reported_as_a_failure( + self, endpoint: Any + ) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + store.start() + store.wait_for_skills(timeout=5) + store.close() + assert store.failed is None + + def test_content_survives_a_reconnect(self, endpoint: Any) -> None: + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + endpoint.queue_stream(events(("heart-beat", None))) + store = FDv2SkillStore( + SDK_KEY, base_uri=endpoint.base_uri, mode="stream", initial_backoff=0.01 + ) + try: + store.start() + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + finally: + store.close() + + +# --------------------------------------------------------------------------- +# Failure handling +# --------------------------------------------------------------------------- + + +class _ScriptedConnection: + """Stands in for ``_StreamConnection``: an event iterator plus a close.""" + + def __init__(self, payload_events: Any) -> None: + self.events = iter(payload_events) + self.closed = False + + def close(self) -> None: + self.closed = True + + +class _ScriptedRequester: + """Raises a scripted sequence, so backoff is asserted without real sockets.""" + + def __init__(self, *outcomes: Any) -> None: + self.outcomes = list(outcomes) + self.calls: list[tuple[str | None, str | None]] = [] + + def poll(self, basis: str | None, etag: str | None) -> Any: + self.calls.append((basis, etag)) + outcome = ( + self.outcomes.pop(0) if self.outcomes else _RecoverableTransportError("x") + ) + if isinstance(outcome, Exception): + raise outcome + return outcome + + def stream(self, basis: str | None) -> Any: + self.calls.append((basis, None)) + outcome = ( + self.outcomes.pop(0) if self.outcomes else _RecoverableTransportError("x") + ) + if isinstance(outcome, Exception): + raise outcome + return _ScriptedConnection(outcome) + + +class _RecyclingRequester: + """ + A healthy server that recycles connections: every ``stream`` call succeeds, + transfers a full payload, and then ends the connection, as LaunchDarkly and + any proxy in between do to a long-lived stream. + """ + + def __init__(self) -> None: + self.connections = 0 + + def stream(self, basis: str | None) -> Any: + self.connections += 1 + return _ScriptedConnection( + [ + (e["event"], e["data"]) + for e in full_payload( + ("put-object", put_skill()), state=f"basis-{self.connections}" + ) + ] + ) + + +class _BlockingConnection: + """A stream that never produces an event until it is closed.""" + + def __init__(self) -> None: + self._closed = threading.Event() + + @property + def events(self) -> Any: + self._closed.wait() + return iter(()) + + def close(self) -> None: + self._closed.set() + + +class _SlowConnectRequester: + """ + A ``stream`` whose connect does not return until the test releases it, + standing in for a slow TLS handshake, followed by a read that never yields. + """ + + def __init__(self) -> None: + self.entered = threading.Event() + self.release = threading.Event() + + def stream(self, basis: str | None) -> Any: + self.entered.set() + self.release.wait(timeout=10) + return _BlockingConnection() + + +def stream_store(**kwargs: Any) -> FDv2SkillStore: + return FDv2SkillStore( + SDK_KEY, + mode="stream", + initial_backoff=kwargs.pop("initial_backoff", 0.001), + max_backoff=kwargs.pop("max_backoff", 0.002), + **kwargs, + ) + + +class TestFailureHandling: + def test_a_403_stops_delivery_and_explains_why( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll(status=403) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert "403" in store.failed + assert "opt-in" in store.failed + assert any("opt-in" in r.getMessage() for r in caplog.records) + + def test_a_401_stops_delivery(self, endpoint: Any) -> None: + endpoint.queue_poll(status=401) + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert "401" in store.failed + + def test_a_fatal_failure_releases_wait_for_skills_rather_than_hanging( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(status=401) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + assert store.failed is not None + + def test_a_fatal_failure_keeps_last_known_good_servable( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll(status=403) + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + + def test_a_500_is_retried(self, endpoint: Any) -> None: + endpoint.queue_poll(status=500) + endpoint.queue_poll(status=503) + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + assert store.failed is None + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + + def test_a_retry_resets_the_failure_count_on_success(self, endpoint: Any) -> None: + endpoint.queue_poll(status=500) + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) + assert wait_until(lambda: store.diagnostics.connection_failures == 0) + + def test_retries_are_bounded(self) -> None: + store = FDv2SkillStore( + SDK_KEY, + mode="poll", + poll_interval=0.01, + initial_backoff=0.001, + max_backoff=0.002, + max_consecutive_failures=3, + _requester=_ScriptedRequester(), + ) + try: + store.start() + assert wait_until(lambda: store.failed is not None) + # Four, not three: the bound is the number of failures *tolerated*, + # so the run that exceeds it is the one that gives up. + assert "gave up after 4 consecutive failures" in store.failed + finally: + store.close() + + def test_recycled_stream_connections_are_not_failures(self) -> None: + # A streaming connection only ever ends by being dropped, so a loop + # that counted every drop as a failure would give up on a healthy + # server after max_consecutive_failures + 1 recycles, and delivery + # (including revocation) would silently stop for the process lifetime. + requester = _RecyclingRequester() + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert wait_until(lambda: requester.connections >= 8) + assert store.failed is None + assert store.diagnostics.payloads_transferred >= 8 + # A drop is a failure until the next commit clears it, so the count + # may read 1 mid-reconnect. What it must never do is climb. + assert store.diagnostics.connection_failures <= 1 + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + finally: + store.close() + + def test_a_stream_commit_resets_the_failure_count(self) -> None: + payload = [ + (e["event"], e["data"]) for e in full_payload(("put-object", put_skill())) + ] + requester = _ScriptedRequester( + _RecoverableTransportError("x"), + _RecoverableTransportError("x"), + _RecoverableTransportError("x"), + payload, + ) + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=5) + # Three failures reach the bound, then a commit, then the exhausted + # requester fails on every reconnect. The count must start again at + # the commit: the stream's own drop is failure one, and three more + # connects are owed before giving up. Carrying the three over would + # give up on the drop itself, with no further connect at all. + assert wait_until(lambda: store.failed is not None) + assert "gave up after 4 consecutive failures" in store.failed + assert "last error: x" in store.failed + assert len(requester.calls) == 7 + finally: + store.close() + + def test_stream_retries_are_bounded(self) -> None: + store = stream_store( + max_consecutive_failures=3, _requester=_ScriptedRequester() + ) + try: + store.start() + assert wait_until(lambda: store.failed is not None) + assert "gave up after 4 consecutive failures" in store.failed + finally: + store.close() + + def test_a_retry_after_header_is_honoured(self) -> None: + requester = _ScriptedRequester( + _RecoverableTransportError("slow down", retry_after=0.25), + ) + store = FDv2SkillStore( + SDK_KEY, + mode="poll", + poll_interval=10.0, + initial_backoff=5.0, + _requester=requester, + ) + try: + started = time.monotonic() + store.start() + assert wait_until(lambda: len(requester.calls) >= 2, timeout=3) + elapsed = time.monotonic() - started + # The server asked for 0.25s; our own backoff would have been 5s. + assert 0.2 <= elapsed < 3.0 + finally: + store.close() + + def test_a_retry_after_header_is_parsed_off_the_wire(self, endpoint: Any) -> None: + endpoint.queue_poll(status=429, retry_after="0") + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint, initial_backoff=5.0) as store: + # If Retry-After were ignored the 5s backoff would blow the timeout. + assert store.wait_for_skills(timeout=3) is True + + @pytest.mark.parametrize("raw", ["inf", "Infinity", "-inf", "nan", "1e309"]) + def test_a_non_finite_retry_after_is_ignored(self, raw: str) -> None: + assert _retry_after_seconds({"Retry-After": raw}) is None + + def test_retry_after_parsing_keeps_its_edges(self) -> None: + assert _retry_after_seconds({"Retry-After": "0"}) == 0.0 + assert _retry_after_seconds({"Retry-After": "-5"}) == 0.0 + assert ( + _retry_after_seconds({"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}) + is None + ) + assert _retry_after_seconds({"Retry-After": "2.5"}) == 2.5 + + @pytest.mark.parametrize("retry_after", [float("inf"), float("nan"), 86400.0]) + def test_an_unreasonable_retry_after_neither_kills_delivery_nor_parks_it( + self, retry_after: float + ) -> None: + # An infinite wait would overflow inside the retry handler and kill the + # thread with `failed` still None; a day-long one would be honoured to + # the second. Both must fall back to the max_backoff cap and carry on. + requester = _ScriptedRequester( + _RecoverableTransportError("slow down", retry_after=retry_after), + [ + (e["event"], e["data"]) + for e in full_payload(("put-object", put_skill())) + ], + ) + store = stream_store(max_backoff=0.05, _requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=3) is True + assert store.failed is None + assert store._thread is not None and store._thread.is_alive() + finally: + store.close() + + def test_a_non_finite_retry_after_off_the_wire_falls_back_to_backoff( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(status=429, retry_after="inf") + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=3) is True + assert store.failed is None + + def test_backoff_is_exponential_and_capped(self) -> None: + assert _backoff_delay(1, base=1.0, maximum=30.0, jitter=0.0) == 1.0 + assert _backoff_delay(2, base=1.0, maximum=30.0, jitter=0.0) == 2.0 + assert _backoff_delay(3, base=1.0, maximum=30.0, jitter=0.0) == 4.0 + assert _backoff_delay(20, base=1.0, maximum=30.0, jitter=0.0) == 30.0 + + def test_jitter_never_exceeds_the_cap(self) -> None: + for attempt in range(1, 12): + for _ in range(50): + assert 0.0 <= _backoff_delay(attempt, base=1.0, maximum=5.0) <= 5.0 + + def test_a_malformed_polling_envelope_is_recoverable_not_fatal( + self, endpoint: Any + ) -> None: + store = FDv2SkillStore( + SDK_KEY, + mode="poll", + poll_interval=0.01, + initial_backoff=0.001, + _requester=_ScriptedRequester( + _RecoverableTransportError("polling response had no 'events' array") + ), + ) + try: + store.start() + assert wait_until(lambda: store.diagnostics.connection_failures >= 1) + assert store.failed is None + finally: + store.close() + + def test_a_listener_that_raises_does_not_kill_delivery(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill("first")))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("second")), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.add_listener(SKILL_OBJECT_KIND, lambda _raw: 1 / 0) + assert wait_until( + lambda: store.get_object(SKILL_OBJECT_KIND, "second") is not None + ) + assert store.failed is None + + # --------------------------------------------------------------------------- # The contentHash gap # --------------------------------------------------------------------------- @@ -854,6 +1662,63 @@ class TestMissingContentHash: itself and verify nothing. """ + async def test_a_hashless_skill_is_withheld_with_the_right_reason( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill(omit_hash=True)))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + outcome = await get_skill_result("pdf-extraction") + assert outcome.skill is None + assert outcome.reason == "integrity_failure" + assert await get_skill("pdf-extraction") is None + assert await all_skills() == [] + + async def test_the_object_is_still_held_so_the_outcome_is_not_absent( + self, endpoint: Any + ) -> None: + """ + Holding it is what makes the failure diagnosable. Dropping it at the + transport would report ``absent`` — indistinguishable from "no such + skill" — and would additionally let a prune delete the last known-good + copy already on disk. + """ + endpoint.queue_poll(full_payload(("put-object", put_skill(omit_hash=True)))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + raw = store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") + assert raw is not None + assert "contentHash" not in raw + await init_client(options={"skillStore": store}, client=object()) + assert (await get_skill_result("pdf-extraction")).reason != "absent" + + def test_the_store_counts_hashless_objects(self, endpoint: Any) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill("a", omit_hash=True)), + ("put-object", put_skill("b", omit_hash=True)), + ("put-object", put_skill("c")), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert store.diagnostics.hashless_objects == 2 + assert store.diagnostics.skill_objects_received == 3 + + def test_a_hashless_object_logs_an_error_naming_the_reason_code( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill(omit_hash=True)))) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + rendered = "\n".join(r.getMessage() for r in caplog.records) + assert "missing_content_hash" in rendered + assert "pdf-extraction" in rendered + assert "contentHash" in rendered + def test_a_redelivered_hashless_object_logs_once_per_store( self, caplog: Any ) -> None: @@ -895,3 +1760,456 @@ def test_two_live_stores_do_not_suppress_each_other(self, caplog: Any) -> None: drive(one, payload) drive(two, payload) assert len(_per_object_hashless_errors(caplog)) == 2 + + def test_a_wholly_hashless_payload_says_so_once( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill("a", omit_hash=True)), + ("put-object", put_skill("b", omit_hash=True)), + ) + ) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + summaries = [ + r + for r in caplog.records + if "No skill content will resolve" in r.getMessage() + ] + assert len(summaries) == 1 + assert "All 2 skill object(s)" in summaries[0].getMessage() + + def test_a_partly_hashed_payload_does_not_claim_total_failure( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill("a", omit_hash=True)), + ("put-object", put_skill("b")), + ) + ) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + rendered = "\n".join(r.getMessage() for r in caplog.records) + assert "No skill content will resolve" not in rendered + + async def test_a_hash_that_does_not_match_is_a_different_failure( + self, endpoint: Any + ) -> None: + """``missing_content_hash`` and ``hash_mismatch`` must not collapse: one + means the envelope carried no hash, the other means the content did not + match the hash it carried.""" + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(content_hash=_hash("something else"))) + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert store.diagnostics.hashless_objects == 0 + await init_client(options={"skillStore": store}, client=object()) + assert ( + await get_skill_result("pdf-extraction") + ).reason == "integrity_failure" + + async def test_a_hashed_skill_resolves_end_to_end(self, endpoint: Any) -> None: + """The positive control: a well-formed envelope resolves end to end.""" + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + skill = await get_skill("pdf-extraction") + assert skill is not None + assert skill.key == "pdf-extraction" + assert skill.version == 3 + assert skill.content == SKILL_BODY.encode("utf-8") + assert skill.content_hash == _hash(SKILL_BODY) + assert skill.name == "PDF Extraction" + + async def test_a_pinned_reference_resolves_to_the_pinned_object_version( + self, endpoint: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(object_version=2, content="v2 body")), + ("put-object", put_skill(object_version=5, content="v5 body")), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + pinned = await get_skill("pdf-extraction", version=2) + assert pinned is not None + assert pinned.content == b"v2 body" + newest = await get_skill("pdf-extraction") + assert newest is not None + assert newest.version == 5 + + async def test_the_payload_version_is_not_resolvable_as_a_skill_version( + self, endpoint: Any + ) -> None: + """ + The end-to-end form of the wire-key/``version`` assertion. + + Asking for the payload version resolves nothing — reported ``absent``, + because the store answers "I hold no such version" rather than answering + with the wrong one. The version that *does* resolve is the one after the + delimiter in the object's wire ``key``. + """ + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(object_version=3, payload_version=42)) + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + by_payload_version = await get_skill_result("pdf-extraction", version=42) + assert by_payload_version.skill is None + assert by_payload_version.reason == "absent" + assert await get_skill("pdf-extraction", version=3) is not None + + +# --------------------------------------------------------------------------- +# Server-side only +# --------------------------------------------------------------------------- + + +class TestServerSideOnly: + def test_a_mobile_key_is_refused(self) -> None: + with pytest.raises(ValueError, match="mobile key"): + FDv2SkillStore("mob-00000000-0000-4000-8000-000000000000") + + def test_a_client_side_environment_id_is_refused(self) -> None: + with pytest.raises(ValueError, match="client-side"): + FDv2SkillStore("0123456789abcdef01234567") + + def test_an_empty_credential_is_refused(self) -> None: + with pytest.raises(ValueError, match="server-side SDK key"): + FDv2SkillStore(" ") + + def test_a_server_side_key_is_accepted(self) -> None: + assert FDv2SkillStore(SDK_KEY) is not None + + def test_an_unrecognised_credential_shape_warns_but_is_allowed( + self, caplog: Any + ) -> None: + """Private instances and test doubles issue keys without the public prefix.""" + with caplog.at_level("WARNING"): + FDv2SkillStore("my-private-instance-credential") + assert any("server-side SDK key" in r.message for r in caplog.records) + + def test_an_unknown_mode_is_refused(self) -> None: + with pytest.raises(ValueError, match="stream"): + FDv2SkillStore(SDK_KEY, mode="mobile") # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# The eager re-reconcile, end to end over the transport +# --------------------------------------------------------------------------- + + +class TestWatchSkillsOverTheTransport: + """ + ``watch_skills`` against a live ``FDv2SkillStore``. The watcher's own + behaviour — debounce, refusal of a store without ``add_listener``, detaching + on close — is covered in ``test_skills_watch.py`` against the in-memory + store; these are the cases that only mean something with a transport + underneath: a wire-level revocation, a new skill version, and an outage. + """ + + async def test_a_revocation_prunes_without_a_restart( + self, endpoint: Any, tmp_path: Any + ) -> None: + """ + The store's change listener drives the reconcile, so the file goes away + seconds after the ``delete-object`` rather than at the next process start. + """ + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + + with poll_store(endpoint, poll_interval=0.2) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + report, watcher = await watch_skills( + "*", tmp_path / "skills", debounce=0.05 + ) + try: + written = tmp_path / "skills" / "pdf-extraction" / "SKILL.md" + assert written.exists() + assert any(a.action == "written" for a in report.actions) + assert wait_until(lambda: not written.exists(), timeout=10) + finally: + watcher.close() + + async def test_a_new_version_is_rewritten_without_a_restart( + self, endpoint: Any, tmp_path: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill(content="first")))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill(object_version=4, content="second")), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + + with poll_store(endpoint, poll_interval=0.2) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + assert written.read_text() == "first" + assert wait_until(lambda: written.read_text() == "second", timeout=10) + finally: + watcher.close() + + async def test_the_default_keeps_last_known_good_during_an_outage( + self, endpoint: Any, tmp_path: Any + ) -> None: + """``on_unavailable="keep"`` is the default: an outage must not read as + "everything was revoked".""" + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll(status=500) + with poll_store(endpoint, poll_interval=0.05) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + assert written.exists() + # ``last_error`` rather than ``connection_failures``: the counter + # resets on the next successful poll, so asserting on it races + # the retry that is supposed to happen. + assert wait_until( + lambda: store.diagnostics.last_error is not None, timeout=10 + ) + time.sleep(0.3) + assert written.exists() + finally: + watcher.close() + + +# --------------------------------------------------------------------------- +# Listener registration +# --------------------------------------------------------------------------- + + +class TestListenerRegistration: + @staticmethod + def _skill_listeners(store: Any) -> list[Any]: + return list(store._listeners.get(SKILL_OBJECT_KIND, [])) + + def test_fdv2_remove_listener_of_an_unregistered_callable_is_a_no_op( + self, endpoint: Any + ) -> None: + with poll_store(endpoint) as store: + store.remove_listener(SKILL_OBJECT_KIND, print) + store.add_listener(SKILL_OBJECT_KIND, print) + store.remove_listener("flag", print) + store.remove_listener(SKILL_OBJECT_KIND, print) + store.remove_listener(SKILL_OBJECT_KIND, print) + assert self._skill_listeners(store) == [] + + +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + + +class TestLifecycle: + def test_start_is_idempotent(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store = poll_store(endpoint) + try: + assert store.start() is store + assert store.start() is store + assert store.wait_for_skills(timeout=5) + finally: + store.close() + + def test_close_is_idempotent(self, endpoint: Any) -> None: + store = poll_store(endpoint) + store.start() + store.close() + store.close() + + def test_close_during_a_slow_connect_returns_promptly(self) -> None: + # Before the connect returns there is no connection for close() to + # interrupt. If the delivery thread then enters the read anyway, close() + # sits out its whole join timeout on a stream that will never speak. + requester = _SlowConnectRequester() + store = stream_store(_requester=requester) + store.start() + assert requester.entered.wait(timeout=5) + threading.Timer(0.1, requester.release.set).start() + started = time.monotonic() + store.close(timeout=5.0) + elapsed = time.monotonic() - started + assert elapsed < 2.0 + assert store._thread is not None + assert not store._thread.is_alive() + + def test_a_closed_store_still_answers_from_what_it_received( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store = poll_store(endpoint) + store.start() + store.wait_for_skills(timeout=5) + store.close() + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + + def test_wait_for_skills_times_out_rather_than_hanging(self) -> None: + store = FDv2SkillStore( + SDK_KEY, mode="poll", poll_interval=60, _requester=_ScriptedRequester() + ) + try: + assert store.wait_for_skills(timeout=0.05) is False + finally: + store.close() + + def test_the_store_satisfies_the_seam_before_it_starts(self) -> None: + store = FDv2SkillStore(SDK_KEY) + assert store.get_object(SKILL_OBJECT_KIND, "anything") is None + assert store.all_objects(SKILL_OBJECT_KIND) == {} + + +# --------------------------------------------------------------------------- +# Timeouts +# --------------------------------------------------------------------------- + + +class _BlackHole: + """ + A listening socket that accepts connections and never sends a byte. + + This is the host ``read_timeout`` exists for: the TCP handshake completes, so + nothing fails fast, and then no response ever comes. A request against it can + only end by timing out, which makes the elapsed time a direct measurement of + the timeout actually applied. + """ + + def __init__(self) -> None: + self._listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._listener.bind(("127.0.0.1", 0)) + self._listener.listen(8) + self._accepted: list[socket.socket] = [] + self._stop = threading.Event() + self._thread = threading.Thread(target=self._accept_forever, daemon=True) + self._thread.start() + host, port = self._listener.getsockname() + self.base_uri = f"http://{host}:{port}" + + def _accept_forever(self) -> None: + self._listener.settimeout(0.05) + while not self._stop.is_set(): + try: + conn, _ = self._listener.accept() + except OSError: + continue + self._accepted.append(conn) + + def close(self) -> None: + self._stop.set() + self._thread.join(timeout=2) + for conn in self._accepted: + conn.close() + self._listener.close() + + +@pytest.fixture +def black_hole() -> Any: + server = _BlackHole() + yield server + server.close() + + +class TestTimeouts: + """ + ``read_timeout`` is the only network timeout, and every request honours it. + + The bounds asserted here are loose on purpose: the point is that a request + against an unresponsive host fails in roughly ``read_timeout`` rather than in + minutes, and that a regression back to a much longer default fails this + suite quickly instead of hanging it. + """ + + def test_a_poll_against_an_unresponsive_host_fails_within_read_timeout( + self, black_hole: Any + ) -> None: + requester = _Requester(SDK_KEY, black_hole.base_uri, read_timeout=0.3) + started = time.monotonic() + with pytest.raises(_RecoverableTransportError) as excinfo: + requester.poll(None, None) + elapsed = time.monotonic() - started + assert 0.2 <= elapsed < 2.0 + assert "timed out" in str(excinfo.value) + + def test_a_stream_against_an_unresponsive_host_fails_within_read_timeout( + self, black_hole: Any + ) -> None: + requester = _Requester(SDK_KEY, black_hole.base_uri, read_timeout=0.3) + started = time.monotonic() + with pytest.raises(_RecoverableTransportError): + requester.stream(None) + assert time.monotonic() - started < 2.0 + + def test_the_store_reports_the_timeout_and_keeps_going( + self, black_hole: Any + ) -> None: + store = FDv2SkillStore( + SDK_KEY, + base_uri=black_hole.base_uri, + mode="poll", + poll_interval=0.05, + initial_backoff=0.01, + max_backoff=0.05, + read_timeout=0.3, + ) + try: + store.start() + assert wait_until(lambda: store.diagnostics.connection_failures >= 1) + assert store.failed is None + assert "timed out" in (store.diagnostics.last_error or "") + finally: + store.close() + + def test_the_default_bound_depends_on_the_mode(self) -> None: + assert DEFAULT_POLL_TIMEOUT == 10.0 + assert DEFAULT_STREAM_READ_TIMEOUT == 300.0 + polling = FDv2SkillStore(SDK_KEY, mode="poll") + streaming = FDv2SkillStore(SDK_KEY, mode="stream") + assert polling._requester._read_timeout == DEFAULT_POLL_TIMEOUT + assert streaming._requester._read_timeout == DEFAULT_STREAM_READ_TIMEOUT + + @pytest.mark.parametrize("mode", ["poll", "stream"]) + def test_an_explicit_read_timeout_overrides_the_default(self, mode: Any) -> None: + store = FDv2SkillStore(SDK_KEY, mode=mode, read_timeout=42.0) + assert store._requester._read_timeout == 42.0 + + @pytest.mark.parametrize("value", [0.0, -1.0, float("inf"), float("nan")]) + def test_a_non_positive_read_timeout_is_rejected(self, value: float) -> None: + with pytest.raises(ValueError, match="read_timeout"): + FDv2SkillStore(SDK_KEY, read_timeout=value) + + def test_there_is_no_separate_connect_timeout(self) -> None: + # ``urllib`` cannot bound the connect separately from the reads, so the + # constructor does not offer a parameter that would only pretend to. + with pytest.raises(TypeError): + FDv2SkillStore(SDK_KEY, connect_timeout=2.0) # type: ignore[call-arg] diff --git a/packages/client/tests/test_skills_watch.py b/packages/client/tests/test_skills_watch.py index 48a95f07..b44ffea0 100644 --- a/packages/client/tests/test_skills_watch.py +++ b/packages/client/tests/test_skills_watch.py @@ -4,7 +4,10 @@ The watcher is wired to the ``SkillStore`` interface, not to any one transport: it needs a store that implements ``add_listener``, and nothing more. These tests therefore drive it from ``InMemorySkillStore``, whose ``put`` notifies its -listeners synchronously, and from small hand-written store doubles. +listeners synchronously, and from small hand-written store doubles. The +end-to-end path — a ``delete-object`` arriving over a live FDv2 connection and +pruning a skill's files — is exercised in ``test_skills_fdv2.py``, where the fake +endpoint lives. Every test writes only inside pytest's ``tmp_path``. The watcher runs a real worker thread, so tests wait on observable outcomes rather than on fixed sleeps From c285ff533991b3e74a2d0b976a91de1008302596 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:04:26 -0400 Subject: [PATCH 5/7] fix(client): retry an FDv2 stream that dies mid-read _Requester.stream wrapped only the connect as recoverable, so a read timeout, reset or truncated chunk in the body reached the delivery loop as whatever the socket raised. The loop read that as a bug and gave up: delivery stopped for the process lifetime, taking updates and revocations with it, the first time a socket died. read_timeout exists to bound a stream that has gone quiet so the loop can reconnect, and tripping it did the opposite. The body now carries the same promise the connect already did. Wrapping the line source rather than the whole read keeps protocol reader errors out of it: those are raised from the consumer's loop body, where they still surface as the bugs they are. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fdv2.py | 21 ++++++- packages/client/tests/test_skills_fdv2.py | 61 +++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index da84a3b4..18b81842 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -1032,6 +1032,25 @@ def _decode_poll_body(body: bytes) -> list[tuple[str, Any]]: return events +def _iter_stream_lines(response: Any) -> Any: + """ + Yields a streaming body's raw lines, presenting a read failure as retryable. + + A live stream dies mid-body far more often than it refuses to open: a read + timeout on a stream that went quiet, a reset, a truncated chunk. Each of + those arrives as whatever the socket raised, and the delivery loop retries + only the transport errors this module defines — anything else it reads as a + bug and stops for the process lifetime. Connecting is already wrapped in + ``_Requester.stream``; this is the same promise for the body. + """ + try: + yield from response + except Exception as exc: + raise _RecoverableTransportError( + f"reading the FDv2 stream failed: {type(exc).__name__}: {exc}" + ) from exc + + def _iter_sse(response: Any) -> Any: """ Decodes an SSE body into ``(event name, data)`` pairs. @@ -1042,7 +1061,7 @@ def _iter_sse(response: Any) -> Any: try: name: str | None = None data_lines: list[str] = [] - for raw_line in response: + for raw_line in _iter_stream_lines(response): line = raw_line.decode("utf-8", errors="replace").rstrip("\r\n") if line == "": if name is not None: diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index 0cce2f0c..af5d1812 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -23,6 +23,7 @@ import socket import threading import time +from http.client import IncompleteRead from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, ClassVar from urllib.parse import parse_qs, urlparse @@ -52,6 +53,7 @@ _retry_after_seconds, _SkillObjectSet, _store_object_from_put, + _StreamConnection, _tombstone_from_delete, ) @@ -1287,6 +1289,41 @@ def test_content_survives_a_reconnect(self, endpoint: Any) -> None: # --------------------------------------------------------------------------- +class _DyingResponse: + """ + A streaming body that transfers a payload and then fails mid-read. + + This is how a live stream actually ends: not with a clean end of body but + with a read timeout on a stream that went quiet, or a reset from the server + or a proxy in between. + """ + + def __init__(self, exc: BaseException) -> None: + self._exc = exc + + def __iter__(self) -> Any: + for event in full_payload(("put-object", put_skill())): + yield f"event: {event['event']}\n".encode() + yield f"data: {json.dumps(event['data'])}\n".encode() + yield b"\n" + raise self._exc + + def close(self) -> None: + pass + + +class _DyingStreamRequester: + """Every connection transfers a payload, then dies with *exc* mid-read.""" + + def __init__(self, exc: BaseException) -> None: + self.connections = 0 + self._exc = exc + + def stream(self, basis: str | None) -> Any: + self.connections += 1 + return _StreamConnection(_DyingResponse(self._exc)) + + class _ScriptedConnection: """Stands in for ``_StreamConnection``: an event iterator plus a close.""" @@ -1477,6 +1514,30 @@ def test_recycled_stream_connections_are_not_failures(self) -> None: finally: store.close() + @pytest.mark.parametrize( + "exc", + [ + TimeoutError("timed out"), + ConnectionResetError(54, "Connection reset by peer"), + IncompleteRead(b"partial"), + ], + ids=["read timeout", "reset", "truncated body"], + ) + def test_a_stream_that_dies_mid_read_reconnects(self, exc: BaseException) -> None: + # A stream fails in its body far more often than at its connect, and + # ``read_timeout`` exists to bound one that has gone quiet. Treating + # such a failure as unexpected would stop delivery — including + # revocation — for the process lifetime the first time a socket died. + requester = _DyingStreamRequester(exc) + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + assert wait_until(lambda: requester.connections >= 5) + assert store.failed is None + finally: + store.close() + def test_a_stream_commit_resets_the_failure_count(self) -> None: payload = [ (e["event"], e["data"]) for e in full_payload(("put-object", put_skill())) From b7ee71a3477bad38d31123bed63e17a6203f3cbd Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Mon, 14 Sep 2026 14:36:58 -0400 Subject: [PATCH 6/7] fix(client): keep FDv2 delivery alive on an idle stream, and shut down promptly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three faults in the delivery loop, all of which left the store reporting itself healthy while doing less than it claimed. **An up-to-date stream tripped the failure cap.** The consecutive-failure count reset only at a commit, and an environment whose skills are not changing answers every reconnect with `intentCode: "none"` and transfers nothing. A stream only ever ends by being dropped, so each recycle of a perfectly healthy idle connection counted as a failure — announced with a `goodbye` or not — and `max_consecutive_failures + 1` of them stopped delivery for the process lifetime, revocations included. The reset on commit covered only the case where content had changed, which is the case that was easy to test and not the case that runs in production. `_TransferOutcome` now reports `up_to_date`, and a complete answer that transfers nothing breaks the row of failures exactly as a commit does. An intent this module does not recognise is still not an answer. **`close` could not interrupt a poll.** The interrupt reached the streaming connection only, so polling parked in its request with nothing to reach and `close` returned when its join timed out — on a 300s-class request, long after the process meant to exit. `_Requester` now tracks the response of a poll in flight and offers `interrupt`, which `close` calls alongside the stream's own. A request still inside its connect has no response to reach; that one is bounded by `read_timeout`, and `start` no longer leaves the store inert when a join times out around it. An interrupt we asked for is no longer recorded as a delivery failure. **`close` left a waiter parked.** `wait_for_skills` waited on the first payload alone, so a shutdown racing a waiter added the waiter's whole timeout to it. Delivery ending is now its own event: a waiter is released by a payload, a give-up or a close, and reports whether a payload actually arrived rather than merely that it was let go. That also settles what `_give_up` had been quietly asserting — it set the first-payload flag to unblock waiters, which made `wait_for_skills` answer `True` for a store holding nothing. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fdv2.py | 123 ++++++- packages/client/tests/test_skills_fdv2.py | 315 +++++++++++++++++- 2 files changed, 420 insertions(+), 18 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 18b81842..6a84092e 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -487,6 +487,14 @@ class _TransferOutcome: basis: str | None = None fatal: str | None = None disconnect: str | None = None + up_to_date: bool = False + """ + The server said what we hold is current and it has nothing to transfer. + + A complete answer that commits nothing, which is exactly what a 304 is to a + poll. The delivery loop counts it as a healthy connection; see + ``FDv2SkillStore._apply``. + """ class _ProtocolReader: @@ -576,6 +584,10 @@ def _server_intent(self, data: Any) -> _TransferOutcome: if intent != _INTENT_TRANSFER_NONE: logger.debug("Ignoring FDv2 server-intent with intentCode %r", intent) self._pending = None + # ``none`` is a complete answer that carries nothing. An intent this + # module does not recognise is not an answer at all, so only the + # former reports itself up to date. + return _TransferOutcome(up_to_date=intent == _INTENT_TRANSFER_NONE) return _TransferOutcome() def _target_for(self, data: Any) -> _SkillObjectSet | None: @@ -942,6 +954,25 @@ def __init__( self._read_timeout = read_timeout # Injectable so tests can drive a fake endpoint without a socket. self._opener = opener or urllib.request.build_opener() + self._lock = threading.Lock() + # The response of a poll in flight, so ``interrupt`` can reach its + # socket from another thread. Polling only: a stream's response is + # handed straight to the caller as a ``_StreamConnection``, which + # carries an interrupt of its own. + self._in_flight: Any = None + + def interrupt(self) -> None: + """ + Unblocks a poll parked in its body read, from another thread. + + Best effort, and safe to call when nothing is in flight. A request still + inside its connect has no response to reach yet and is bounded only by + ``read_timeout``; ``FDv2SkillStore.start`` covers what that leaves. + """ + with self._lock: + response = self._in_flight + if response is not None: + _interrupt_read(response) def _url(self, path: str, basis: str | None) -> str: """ @@ -972,11 +1003,17 @@ def poll(self, basis: str | None, etag: str | None) -> _PollResult: request = self._request(POLL_PATH, basis, headers) try: with self._opener.open(request, timeout=self._read_timeout) as response: - status = getattr(response, "status", None) or response.getcode() - if status == 304: - return _PollResult(not_modified=True, events=[], etag=etag) - body = response.read() - new_etag = response.headers.get("ETag") or etag + with self._lock: + self._in_flight = response + try: + status = getattr(response, "status", None) or response.getcode() + if status == 304: + return _PollResult(not_modified=True, events=[], etag=etag) + body = response.read() + new_etag = response.headers.get("ETag") or etag + finally: + with self._lock: + self._in_flight = None except urllib.error.HTTPError as exc: if exc.code == 304: # urllib raises on 304 when no redirect handler swallows it. @@ -1225,6 +1262,18 @@ def __init__( self._stop = threading.Event() self._first_payload = threading.Event() + """A payload has committed. The fact ``wait_for_skills`` reports.""" + self._delivery_ended = threading.Event() + """ + Delivery has stopped, by ``close`` or by ``_give_up``. Kept apart from + ``_first_payload`` because it is not one: a waiter has to be let go + either way, but only a payload makes ``wait_for_skills`` true. + """ + self._released = threading.Event() + """ + Either of the two above, and what a waiter actually parks on: an + ``Event`` cannot wait on two, so the setters funnel through here. + """ self._thread: threading.Thread | None = None self._failed_reason: str | None = None # The open streaming connection, so ``close`` can interrupt its read. @@ -1244,7 +1293,13 @@ def start(self) -> FDv2SkillStore: Does not block: use ``wait_for_skills`` when boot ordering matters. """ with self._lock: + self._rearm_waiters() if self._thread is not None and self._thread.is_alive(): + # A ``close`` whose join timed out leaves the previous thread + # running with the stop flag still set. Clearing it lets that + # thread carry on delivering, rather than leaving a store that + # reports itself started and never delivers again. + self._stop.clear() return self self._stop.clear() self._thread = threading.Thread( @@ -1253,6 +1308,16 @@ def start(self) -> FDv2SkillStore: self._thread.start() return self + def _rearm_waiters(self) -> None: + """ + Re-arms ``wait_for_skills`` for a store being started again after a + ``close``. A payload already held stays an answer; an ended delivery + does not, or the next waiter would be released before it began. + """ + self._delivery_ended.clear() + if not self._first_payload.is_set(): + self._released.clear() + def close(self, timeout: float = 5.0) -> None: """ Stops delivery. Idempotent, and safe to call from any thread. @@ -1262,12 +1327,17 @@ def close(self, timeout: float = 5.0) -> None: package-level ``launchdarkly_ai_server.shutdown()`` coroutine. """ self._stop.set() + # A waiter parked in ``wait_for_skills`` is owed an answer now rather + # than at the end of its timeout; delivery is over either way. + self._end_delivery() # The delivery thread is normally blocked in a socket read that no flag - # can reach; without this the join waits out its full timeout. + # can reach; without this the join waits out its full timeout. Streaming + # parks in the connection, polling in the request, so interrupt both. with self._lock: connection = self._connection if connection is not None: connection.close() + self._requester.interrupt() thread = self._thread if ( thread is not None @@ -1288,8 +1358,24 @@ def wait_for_skills(self, timeout: float = 10.0) -> bool: ``True`` means a payload arrived — not that any skill in it verified, and not that the environment has any skills. ``diagnostics`` answers the rest. + + Returns early, ``False``, when delivery ends before any payload does: + a ``close`` from another thread, or a failure delivery cannot retry. + Waiting out the full timeout for an answer that has already arrived + would delay every shutdown that raced a waiter. """ - return self._first_payload.wait(timeout=timeout) + self._released.wait(timeout=timeout) + return self._first_payload.is_set() + + def _publish_first_payload(self) -> None: + """Records the first committed payload and lets any waiter go.""" + self._first_payload.set() + self._released.set() + + def _end_delivery(self) -> None: + """Records that delivery has stopped and lets any waiter go.""" + self._delivery_ended.set() + self._released.set() @property def failed(self) -> str | None: @@ -1383,6 +1469,11 @@ def _run(self) -> None: self._give_up(str(exc)) return except _RecoverableTransportError as exc: + if self._stop.is_set(): + # ``close`` interrupted the request on purpose. Counting it + # would spend a retry from the bounded budget and leave a + # misleading ``last_error`` on a healthy store. + return with self._lock: self._failures += 1 failures = self._failures @@ -1432,8 +1523,8 @@ def _give_up(self, reason: str) -> None: "the process restarts with a working connection.", reason, ) - # Unblock anyone waiting on a first payload that is never coming. - self._first_payload.set() + # Let go of anyone waiting on a first payload that is never coming. + self._end_delivery() def _apply(self, name: str, data: Any) -> None: """ @@ -1444,10 +1535,16 @@ def _apply(self, name: str, data: Any) -> None: outcome = self._reader.handle(name, data) if outcome.committed and outcome.basis is not None: self._basis = outcome.basis - if outcome.committed: - # A commit breaks the row of consecutive failures. + if outcome.committed or outcome.up_to_date: + # Both break the row of consecutive failures: a commit is a payload + # delivered, and ``up_to_date`` is the server confirming we already + # hold it. Counting only the commit would give up on a healthy + # stream serving an environment whose skills are not changing: + # nothing to transfer means no commit, while every recycled + # connection still ends in a drop. self._record_success() - self._first_payload.set() + if outcome.committed: + self._publish_first_payload() if outcome.changes: self._notify(outcome.changes) if outcome.fatal: @@ -1465,7 +1562,7 @@ def _poll_once(self) -> None: logger.debug("Skill payload unchanged (HTTP 304)") # A 304 counts as a first payload, so a boot that reconnects with a # cached basis is not blocked on a transfer the server will not send. - self._first_payload.set() + self._publish_first_payload() return for name, data in result.events: self._apply(name, data) diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index af5d1812..d063e451 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -538,6 +538,17 @@ def test_a_full_transfer_commits_at_payload_transferred(self) -> None: assert outcomes[-1].committed is True assert outcomes[-1].basis == "basis-1" + def test_an_up_to_date_intent_is_reported_as_such(self) -> None: + """``intentCode: "none"`` is the stream's 304: current, nothing to send.""" + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle("server-intent", server_intent("none")) + assert outcome.up_to_date is True + assert outcome.committed is False + assert outcome.disconnect is None + # A transfer intent is a promise of content, not an up-to-date answer. + transfer = reader.handle("server-intent", server_intent("xfer-full")) + assert transfer.up_to_date is False + def test_nothing_is_visible_before_payload_transferred(self) -> None: """A payload version is the unit of consistency; half of one is not a state.""" held = _SkillObjectSet() @@ -1312,7 +1323,17 @@ def close(self) -> None: pass -class _DyingStreamRequester: +class _FakeRequester: + """ + Base for the requester fakes: supplies the ``interrupt`` the store calls on + ``close``, so each fake only scripts the part it is about. + """ + + def interrupt(self) -> None: + """No real socket to reach; these fakes end their own connections.""" + + +class _DyingStreamRequester(_FakeRequester): """Every connection transfers a payload, then dies with *exc* mid-read.""" def __init__(self, exc: BaseException) -> None: @@ -1335,7 +1356,7 @@ def close(self) -> None: self.closed = True -class _ScriptedRequester: +class _ScriptedRequester(_FakeRequester): """Raises a scripted sequence, so backoff is asserted without real sockets.""" def __init__(self, *outcomes: Any) -> None: @@ -1361,7 +1382,7 @@ def stream(self, basis: str | None) -> Any: return _ScriptedConnection(outcome) -class _RecyclingRequester: +class _RecyclingRequester(_FakeRequester): """ A healthy server that recycles connections: every ``stream`` call succeeds, transfers a full payload, and then ends the connection, as LaunchDarkly and @@ -1383,6 +1404,53 @@ def stream(self, basis: str | None) -> Any: ) +class _UpToDateRecyclingRequester(_FakeRequester): + """ + A healthy server with nothing new to say: every connection answers + ``intentCode: "none"`` — the stream's equivalent of a 304 — transfers + nothing, and is then recycled. This is the steady state of an environment + whose skills are not changing, which is most environments most of the time. + """ + + def __init__(self, farewell: bool = False) -> None: + self.connections = 0 + self._farewell = farewell + + def stream(self, basis: str | None) -> Any: + self.connections += 1 + script: list[tuple[str, Any]] = [ + ("server-intent", server_intent("none")), + ("heart-beat", {}), + ] + if self._farewell: + # A recycle is often announced rather than abrupt. + script.append(("goodbye", {"reason": "connection recycled"})) + return _ScriptedConnection(script) + + +class _SlowPollRequester(_FakeRequester): + """ + A poll whose request does not return until the test releases it, standing in + for one blocked where no interrupt can reach: inside its connect. + """ + + def __init__(self) -> None: + self.entered = threading.Event() + self.release = threading.Event() + + def poll(self, basis: str | None, etag: str | None) -> Any: + self.entered.set() + self.release.wait(timeout=10) + raise _RecoverableTransportError("released") + + +class _SilentStreamRequester(_FakeRequester): + """A stream that connects and then delivers nothing until it is closed.""" + + def stream(self, basis: str | None) -> Any: + return _BlockingConnection() + + class _BlockingConnection: """A stream that never produces an event until it is closed.""" @@ -1398,7 +1466,7 @@ def close(self) -> None: self._closed.set() -class _SlowConnectRequester: +class _SlowConnectRequester(_FakeRequester): """ A ``stream`` whose connect does not return until the test releases it, standing in for a slow TLS handshake, followed by a read that never yields. @@ -1447,7 +1515,11 @@ def test_a_fatal_failure_releases_wait_for_skills_rather_than_hanging( ) -> None: endpoint.queue_poll(status=401) with poll_store(endpoint) as store: - assert store.wait_for_skills(timeout=5) is True + started = time.monotonic() + # Released promptly, and ``False``: no payload arrived, and saying + # otherwise would send a caller on to read a store holding nothing. + assert store.wait_for_skills(timeout=5) is False + assert time.monotonic() - started < 2.0 assert store.failed is not None def test_a_fatal_failure_keeps_last_known_good_servable( @@ -1514,6 +1586,28 @@ def test_recycled_stream_connections_are_not_failures(self) -> None: finally: store.close() + @pytest.mark.parametrize("farewell", [False, True], ids=["dropped", "goodbye"]) + def test_an_up_to_date_recycled_stream_is_not_a_failure( + self, farewell: bool + ) -> None: + # Resetting at a commit covers only a connection that carried new + # content. An environment whose skills are not changing answers every + # reconnect with ``intentCode: "none"`` and transfers nothing, so a loop + # that counted those drops would give up on a *healthy* idle stream + # after max_consecutive_failures + 1 recycles — and revocation, the one + # thing streaming exists to deliver promptly, would never arrive again. + requester = _UpToDateRecyclingRequester(farewell=farewell) + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert wait_until(lambda: requester.connections >= 8) + assert store.failed is None + # As with a payload-carrying recycle, the count may read 1 + # mid-reconnect. What it must never do is climb. + assert store.diagnostics.connection_failures <= 1 + finally: + store.close() + @pytest.mark.parametrize( "exc", [ @@ -2194,6 +2288,64 @@ def close(self) -> None: self._listener.close() +class _StalledBody: + """ + A listening socket that answers with headers and then stalls the body. + + Distinct from ``_BlackHole``: here the request succeeds far enough to hand + urllib a response, and the caller then parks in ``read``. That is the state + ``close`` has to interrupt — and, unlike a request still inside its connect, + the state an interrupt can actually reach. + """ + + def __init__(self) -> None: + self._listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._listener.bind(("127.0.0.1", 0)) + self._listener.listen(8) + self._accepted: list[socket.socket] = [] + self.serving = threading.Event() + self._stop = threading.Event() + self._thread = threading.Thread(target=self._serve_forever, daemon=True) + self._thread.start() + host, port = self._listener.getsockname() + self.base_uri = f"http://{host}:{port}" + + def _serve_forever(self) -> None: + self._listener.settimeout(0.05) + while not self._stop.is_set(): + try: + conn, _ = self._listener.accept() + except OSError: + continue + self._accepted.append(conn) + try: + conn.recv(4096) + # A length far longer than the body that follows, so the read + # blocks rather than seeing the end of the message. + conn.sendall( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" + b"Content-Length: 4096\r\n\r\n" + ) + except OSError: + continue + self.serving.set() + + def close(self) -> None: + self._stop.set() + self._thread.join(timeout=2) + for conn in self._accepted: + conn.close() + self._listener.close() + + +@pytest.fixture +def stalled_body() -> Any: + server = _StalledBody() + yield server + server.close() + + @pytest.fixture def black_hole() -> Any: server = _BlackHole() @@ -2274,3 +2426,156 @@ def test_there_is_no_separate_connect_timeout(self) -> None: # constructor does not offer a parameter that would only pretend to. with pytest.raises(TypeError): FDv2SkillStore(SDK_KEY, connect_timeout=2.0) # type: ignore[call-arg] + + +class TestWaitingForSkills: + """ + ``wait_for_skills`` answers with what happened, and never outlives it. + + Its budget is a boot-ordering allowance, not a delay to spend: a store that + already knows no payload is coming owes the caller that answer immediately. + """ + + def test_close_releases_a_waiter_rather_than_leaving_it_parked(self) -> None: + # A shutdown racing a waiter is the ordinary case, not an exotic one: + # ``close`` on the main thread while a worker is still waiting for its + # first payload. Parking that worker for the rest of its timeout adds + # the whole budget to a process that has already decided to stop. + store = stream_store(_requester=_SilentStreamRequester()) + store.start() + answers: list[bool] = [] + waiter = threading.Thread( + target=lambda: answers.append(store.wait_for_skills(timeout=10)), + daemon=True, + ) + waiter.start() + time.sleep(0.2) + started = time.monotonic() + store.close() + waiter.join(timeout=5) + assert not waiter.is_alive() + assert time.monotonic() - started < 2.0 + assert answers == [False] + + def test_a_payload_already_held_still_answers_true_after_close(self) -> None: + # ``close`` does not drop content, so it must not turn the answer about + # that content into a lie either. + requester = _RecyclingRequester() + store = stream_store(_requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + finally: + store.close() + assert store.wait_for_skills(timeout=5) is True + + def test_a_restarted_store_waits_again(self) -> None: + # The released flag is sticky by design, so a store closed before any + # payload and then started again has to re-arm: otherwise the next + # waiter is let go before delivery has had a chance to begin. + store = stream_store(_requester=_SilentStreamRequester()) + store.start() + store.close() + assert store.wait_for_skills(timeout=0.1) is False + store.start() + try: + started = time.monotonic() + assert store.wait_for_skills(timeout=0.5) is False + # Waited, rather than being released by the previous close. + assert time.monotonic() - started >= 0.4 + finally: + store.close() + + +class TestPollShutdown: + """ + ``close`` has to interrupt a poll in flight, as it already does a stream. + + Without it the delivery thread stays parked in its request and ``close`` + returns only when the join times out — on a 300s-class request, long after + the process meant to exit. The bound is loose on purpose: the point is + promptly rather than a particular number of milliseconds. + """ + + def test_interrupt_unblocks_a_poll_stalled_in_its_body( + self, stalled_body: Any + ) -> None: + requester = _Requester(SDK_KEY, stalled_body.base_uri, read_timeout=30.0) + raised: list[BaseException] = [] + + def poll_until_interrupted() -> None: + try: + requester.poll(None, None) + except BaseException as exc: + raised.append(exc) + + thread = threading.Thread(target=poll_until_interrupted, daemon=True) + thread.start() + assert stalled_body.serving.wait(timeout=5) + # The response is in hand; give the read a moment to park in it. + time.sleep(0.2) + started = time.monotonic() + requester.interrupt() + thread.join(timeout=5) + assert not thread.is_alive() + assert time.monotonic() - started < 2.0 + assert raised and isinstance(raised[0], _RecoverableTransportError) + + def test_close_during_a_stalled_poll_returns_promptly( + self, stalled_body: Any + ) -> None: + store = FDv2SkillStore( + SDK_KEY, + base_uri=stalled_body.base_uri, + mode="poll", + poll_interval=0.05, + read_timeout=30.0, + ) + store.start() + assert stalled_body.serving.wait(timeout=5) + time.sleep(0.2) + started = time.monotonic() + store.close(timeout=5.0) + assert time.monotonic() - started < 2.0 + assert store._thread is not None and not store._thread.is_alive() + + def test_a_poll_we_interrupted_is_not_a_delivery_failure( + self, stalled_body: Any + ) -> None: + # Our own shutdown is not an outage: counting it would spend a retry + # from the bounded budget and leave a misleading ``last_error`` behind + # on a store whose content is still perfectly good. + store = FDv2SkillStore( + SDK_KEY, + base_uri=stalled_body.base_uri, + mode="poll", + poll_interval=0.05, + read_timeout=30.0, + ) + store.start() + assert stalled_body.serving.wait(timeout=5) + time.sleep(0.2) + store.close(timeout=5.0) + assert store.diagnostics.connection_failures == 0 + assert store.diagnostics.last_error is None + assert store.failed is None + + def test_a_close_that_timed_out_leaves_the_store_restartable(self) -> None: + # A request blocked inside its connect is beyond any interrupt, so + # ``close`` can still return with the thread alive. ``start`` must not + # then find that thread and return with the stop flag set: the store + # would report itself started and never deliver again. + requester = _SlowPollRequester() + store = FDv2SkillStore( + SDK_KEY, mode="poll", poll_interval=0.01, _requester=requester + ) + try: + store.start() + assert requester.entered.wait(timeout=5) + store.close(timeout=0.2) + assert store._thread is not None and store._thread.is_alive() + store.start() + assert store._stop.is_set() is False + finally: + requester.release.set() + store.close(timeout=2) From 88c225ebc296613713d32fcfa3f7fb474ffe6495 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Mon, 14 Sep 2026 14:47:44 -0400 Subject: [PATCH 7/7] fix(client): log a recycled FDv2 stream at debug rather than warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A stream only ever ends by being dropped, and LaunchDarkly — and any proxy in between — recycles a long-lived one. Every reconnect therefore logged "Skill delivery failed" at WARNING, for as long as the process ran. Until the previous commit that noise was bounded, because an idle stream gave up after eleven recycles and went quiet; now that delivery correctly survives them, it would run forever and describe a healthy store as failing. A connection that got a complete answer before it ended — a committed payload, or an up-to-date intent — delivered everything it was asked for, so its reconnect is now DEBUG and says so. A connection that ended without answering is the case the warning exists for and still gets it: a connect that never landed, or a transfer that died part-way through. Filling a customer's logs with a fault they do not have is not merely untidy; it teaches them that the level which means something can be ignored. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fdv2.py | 27 ++++++++++++++-- packages/client/tests/test_skills_fdv2.py | 32 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 6a84092e..71411c8a 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -1283,6 +1283,10 @@ def __init__( # by being dropped, so resetting on return would count every healthy, # server-recycled connection as a failure. self._failures = 0 + # Whether the current attempt got a complete answer before it ended. + # A stream only ever ends by being dropped, so this is what separates + # a recycled healthy connection from one that failed. + self._attempt_answered = False # -- lifecycle --------------------------------------------------------- @@ -1456,6 +1460,8 @@ def _notify(self, changes: list[dict[str, Any]]) -> None: def _run(self) -> None: while not self._stop.is_set(): + with self._lock: + self._attempt_answered = False try: if self._mode == "stream": self._stream_once() @@ -1477,6 +1483,7 @@ def _run(self) -> None: with self._lock: self._failures += 1 failures = self._failures + answered = self._attempt_answered self._reader.diagnostics.connection_failures = failures self._reader.diagnostics.last_error = str(exc) if failures > self._max_consecutive_failures: @@ -1494,9 +1501,22 @@ def _run(self) -> None: # The header may come from a proxy rather than LaunchDarkly, and # a value in the hours would park revocation for that long. delay = min(delay, self._max_backoff) - logger.warning( - "Skill delivery failed (%s); retrying in %.1fs", exc, delay - ) + if answered: + # LaunchDarkly, and any proxy in between, recycles a + # long-lived stream. A connection that answered before it + # ended delivered everything it was asked for, so the + # reconnect is routine rather than a fault worth warning + # about for as long as the process runs. + logger.debug( + "The FDv2 stream ended after a complete answer (%s); " + "reconnecting in %.1fs", + exc, + delay, + ) + else: + logger.warning( + "Skill delivery failed (%s); retrying in %.1fs", exc, delay + ) if self._stop.wait(delay): return continue @@ -1511,6 +1531,7 @@ def _run(self) -> None: def _record_success(self) -> None: with self._lock: self._failures = 0 + self._attempt_answered = True self._reader.diagnostics.connection_failures = 0 def _give_up(self, reason: str) -> None: diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index d063e451..b0c6aa86 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -1608,6 +1608,38 @@ def test_an_up_to_date_recycled_stream_is_not_a_failure( finally: store.close() + def test_a_recycled_connection_reconnects_quietly(self, caplog: Any) -> None: + # A healthy idle stream reconnects for as long as the process runs, so + # warning on each one would fill a customer's logs with a fault they do + # not have and teach them to ignore the level that means something. + requester = _UpToDateRecyclingRequester() + store = stream_store(_requester=requester) + with caplog.at_level("DEBUG", logger="launchdarkly_ai_server.skills_fdv2"): + try: + store.start() + assert wait_until(lambda: requester.connections >= 5) + finally: + store.close() + assert store.failed is None + assert not [r for r in caplog.records if r.levelname == "WARNING"] + assert [r for r in caplog.records if "reconnecting in" in r.getMessage()] + + def test_a_connection_that_never_answered_still_warns(self, caplog: Any) -> None: + # The quiet path is earned by answering. A connection that failed before + # it told us anything is the case the warning exists for. + store = stream_store( + max_consecutive_failures=10, _requester=_ScriptedRequester() + ) + with caplog.at_level("DEBUG", logger="launchdarkly_ai_server.skills_fdv2"): + try: + store.start() + assert wait_until(lambda: store.diagnostics.connection_failures >= 3) + finally: + store.close() + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert warnings + assert all("Skill delivery failed" in r.getMessage() for r in warnings) + @pytest.mark.parametrize( "exc", [