diff --git a/packages/client/README.md b/packages/client/README.md index 852f5eff..ca834fed 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -262,6 +262,443 @@ asyncio.run(main()) --- +### Agent Skills + +Skills are versioned `SKILL.md` documents managed in LaunchDarkly and attached to AI Config +variations by reference. The SDK surfaces which skills a config references, retrieves their +content, and materializes them onto disk where agent runtimes (Claude Agent SDK, and +anything else following the `//SKILL.md` convention) discover them. + +```python +import asyncio +import hashlib +from pathlib import Path + +from launchdarkly_ai_server import ( + init_client, inspect_config, skill_refs, get_skill, write_skills, + InMemorySkillStore, +) + +SKILL_MD = "---\nname: PDF Extraction\n---\nExtract text from PDFs.\n" + +async def main(): + # A store supplies skill content. InMemorySkillStore is the dict-backed + # store for local development, testing, and bring-your-own-content use. + store = InMemorySkillStore() + store.put({ + "key": "pdf-extraction", + "version": 2, + "content": SKILL_MD, + # sha256, lowercase hex, over the verbatim utf-8 bytes. Content whose hash + # does not match is withheld, so this is not optional. + "contentHash": hashlib.sha256(SKILL_MD.encode("utf-8")).hexdigest(), + }) + await init_client(options={"skillStore": store}) + + # 1. Which skills does this config reference? Pure projection — no I/O. + info = await inspect_config("doc-agent", {"kind": "user", "key": "user-123"}) + refs = skill_refs(info["config"]) # [SkillReference(key='pdf-extraction', version=2)] + + # 2. Fetch content. Returns None rather than raising when a skill is unavailable. + skill = await get_skill("pdf-extraction") + if skill is not None: + print(skill.content) + + # 3. Write them where the agent runtime will look. Only the leaf directory is + # created, so the parent must already exist. + Path(".claude").mkdir(exist_ok=True) + report = await write_skills(refs, ".claude/skills") + for action in report.errors: + print(f"skill {action.key or ''}: {action.error}") + +asyncio.run(main()) +``` + +Pass `"*"` instead of a reference list to materialize every skill the store holds — but know +what you are asking for. `"*"` materializes the **whole project library**, which puts every +skill's `description` into the agent's context, including skills no AI Config references and +skills belonging to other teams. `write_skills(skill_refs(...), root)` is the form used above +because it materializes only what the resolved variation actually asked for; reach for `"*"` +when you genuinely want the whole library on disk. + +**`skills` is now a validated field.** Config parsing fails closed on a `skills` value that +is not a list of `{key, version}` objects (key matching `^[a-z0-9][a-z0-9-]*$`, version an +integer ≥ 1): the whole variation is rejected, `inspect_config` returns `config: None`, and +`extract_variation` raises. A variation that previously carried its own custom `skills` +field of a different shape must rename it before upgrading. + +**Integrity is not optional.** Content is only returned after its sha256 (lowercase hex, +over the verbatim UTF-8 bytes) matches the delivered `contentHash`, its key and version +revalidate, and its size is within 10 MiB. Anything that fails is withheld and treated as +missing — no unverified content ever reaches your code. A retrieval that withheld anything +logs a count at WARN, so a run that resolved nothing is not silent. + +#### Detecting integrity failures + +Every withheld skill emits one structured **ERROR** log record on the SDK's own logger +(`launchdarkly_ai_server.skills_core`), designed to be ingested by a SIEM and alerted on. +It is emitted **regardless of how telemetry is configured** — it is not conditional on any +opt-in, and it is the detection path that works when nothing leaves your process. + +The message text is the stable event name followed by compact JSON, so it is greppable and +`jq`-able under any handler configuration, and the same mapping is attached as +`extra["ld_skills"]` for a structured handler: + +``` +ERROR ld.skills.integrity_failure {"action":"withheld","event":"ld.skills.integrity_failure","expected_hash":"0000…0000","language":"python","observed_hash":"5fc8…6ec0","reason":"content hash mismatch","reason_code":"hash_mismatch","skill_key":"pdf-extraction","version":2} +``` + +**`ld.skills.integrity_failure` is a stability commitment.** It is the string to match on, +it will not be renamed, and the JSON keys are sorted so the line is byte-identical across +LaunchDarkly's AI SDKs for the same input. + +| Field | Description | +|---|---| +| `event` | Always `ld.skills.integrity_failure`. | +| `action` | Always `withheld` — the content was not returned to your code. | +| `skill_key` | The skill key, or `` when the delivered key was itself malformed. | +| `version` | The delivered version. Omitted when it was not a valid version. | +| `expected_hash` | The delivered `contentHash`, or `` when it was not one. Omitted when none was delivered. | +| `observed_hash` | The sha256 the SDK computed. Omitted when the failure happened before anything was hashed. | +| `reason_code` | A stable token naming the failure mode — see below. | +| `reason` | Human-readable detail, including byte counts where relevant. | +| `language` | Always `python`. | + +Absent optional fields are **omitted entirely** rather than emitted as `null`, so a field +existence check is meaningful. The skill body, and any attacker-controllable string that +could carry it, never appears in the record; neither does any filesystem path. + +| `reason_code` | Meaning | +|---|---| +| `not_an_object` | The delivered object was not a JSON object. | +| `invalid_key` | The key did not match `^[a-z0-9][a-z0-9-]*$` or exceeded 256 characters. | +| `invalid_version` | The version was not an integer ≥ 1. | +| `missing_content` | `content` was absent or not a string. | +| `missing_content_hash` | `contentHash` was absent or not a string. | +| `not_utf8` | The content string had no UTF-8 encoding, so there are no bytes that could have been hashed. | +| `over_size_cap` | The content exceeded the SDK's local size cap. | +| `hash_mismatch` | The computed sha256 did not match the delivered `contentHash`. | + +**`hash_mismatch` is the one worth paging on.** The other seven describe a malformed or +truncated payload; a mismatch means content was delivered whose bytes are not the bytes +LaunchDarkly hashed, which is a possible **active-tampering** signal. Alert on it, and +treat `expected_hash` / `observed_hash` as the evidence pair. + +#### Failing closed on tampering + +The log record above is the operator's surface. `get_skill_result` is the application's: +same retrieval, same verification, same telemetry as `get_skill`, and it reports which of +five outcomes happened instead of collapsing all of them to `None`. + +```python +from launchdarkly_ai_server import get_skill_result + +outcome = await get_skill_result("pdf-extraction") + +if outcome.reason == "integrity_failure": + # Content was delivered and did not verify. Do not degrade quietly. + raise SystemExit(f"refusing to start: {outcome.detail}") + +if outcome.reason == "store_unavailable": + # The store could not answer at all. Retry, alert, or carry on with what + # you already have — but this is an outage, not a revocation. + print(f"skill retrieval unavailable: {outcome.detail}") +elif outcome.reason in ("absent", "wrong_version"): + # Nothing was tampered with — this skill is simply not available to you. + print(f"continuing without a skill: {outcome.detail}") +elif outcome.skill is not None: + print(outcome.skill.content) +``` + +| `reason` | Meaning | +|---|---| +| `ok` | A verified skill was returned; `.skill` is set and `.detail` is `None`. | +| `absent` | The store answered, and does not hold that key. | +| `integrity_failure` | Content was delivered and failed verification, so it was withheld. **The one to fail closed on.** | +| `store_unavailable` | The store itself could not answer — it raised. An outage, not a deletion. | +| `wrong_version` | The store answered with a version other than the one asked for, so the answer was withheld. | + +`.detail` is human-readable and safe to log or show an operator — it names the key and the +failure mode, and never carries skill content or a filesystem path. Branch on `.reason`, +not on `.detail`. `.skill` is populated only when `.reason == "ok"`. `SkillOutcome` is +frozen, like every other value type here. + +**`get_skill` is unchanged.** It still returns `None` for all four failures and still never +raises for one, so no existing caller has to move. The two accessors run the same code path +and differ only in what they report — `get_skill_result` adds no second log record and no +second signal for a failure that already emitted one, so a caller can switch to it without +double-counting anything. + +`get_skills` and `all_skills` have no reported form: they still omit entries that could not +be resolved, and a run that omitted anything logs a count at WARN. Retrieve individually +with `get_skill_result` when you need the reason per key. + +**Versions are selected, not filtered.** A store may hold several versions of one key at +once, because a delivery payload does: the newest version of every skill, plus every +version a variation currently pins. `get_skill("k", version=1)` asks the store for version +1 and gets it even when a newer one is also held. `all_skills()` and `write_skills("*")` +collapse to one skill per key at its newest version, since `//SKILL.md` is a +single path. + +**The root's parent must exist.** `write_skills` creates the root itself but never its +ancestors, so a typo cannot scatter a directory tree across your project. An absent parent, +a root that is an existing file, and a root that is a symlink each raise `ValueError` — +these are caller errors, distinct from the per-skill `error` actions in the report. + +**`write_skills` is deliberately conservative** about your filesystem. It writes only +`//SKILL.md`, tracks what it owns in a manifest at +`/.launchdarkly-skills.json`, and will overwrite or delete **only** paths that +manifest records. A file you placed yourself is reported as an error and left untouched; it +never writes through a symlink; writes are atomic (temp file, `fsync`, rename) at mode +`0644`; and if the manifest is unreadable it performs no destructive action at all. Removing +a skill from a variation is how revocation works — the next reconcile prunes it. + +**Platform bound: the descriptor-pinned guarantee is POSIX-only.** On POSIX the managed root +is opened once per reconcile — `O_RDONLY|O_DIRECTORY|O_NOFOLLOW`, checked with `fstat`, and +held until the call returns — and every step under it runs relative to that descriptor, the +reads that decide an action as much as the action itself: the per-skill directory is created +and opened relative to the root; the manifest read, the check for an existing `SKILL.md`, the +byte comparison against it and the listing for orphaned temp files are all answered from the +pinned directory; and the unlink and the `rmdir` run relative to the directory, the manifest +write relative to the root. So a directory swapped for a symlink *after* its checks cannot +redirect a write or a delete, and cannot change *which* of them happens either — a prune +cannot be talked into skipping its unlink, and a compare cannot be shown a file from outside +the root: the descriptor names the inode that was checked, which closes the swap window rather +than narrowing it. Because the root itself is pinned this holds for the root and its ancestors too, +not only for `/` — but only from the instant the root is opened, which is why the +checklist below denies the agent write access to every ancestor. Windows has no +`*at()` syscall family, so there `write_skills` falls back to a per-component `lstat` check +taken immediately before each step. That floor is a check-then-use race rather than a closed +window: an attacker who already holds **write permission on the managed root** can still win +it. Windows reparse-point checks (`GetFileAttributesW` / `FILE_FLAG_OPEN_REPARSE_POINT`) are +deliberately not implemented in this release, and Windows is not a supported or tested +platform for it. Treat write permission on the managed root **or on any directory above it** +as the security boundary on every platform, and on Windows as the *only* one. + +**One exception, and it is what makes a crashed reconcile recoverable.** A file at a managed +path whose bytes are *already byte-identical* to the content LaunchDarkly resolved is +adopted — recorded in the manifest and reported `skipped_current` — rather than refused. +Without that, a process killed after a skill file lands but before the manifest is rewritten +leaves that file managed-but-unrecorded, which is indistinguishable from a file you wrote +yourself, so every later reconcile would refuse it and the skill would stay wedged until +someone intervened. Adoption cannot weaken the guarantee above, because bytes that differ in +any way are still refused and left untouched. Note that an adopted file becomes prunable +like any other managed file — which is the same outcome the crash pre-empted. + +**A few keys are legal to an AI Config but not to a filesystem.** A key becomes a single +directory name, so `write_skills` applies bounds of its own on top of the key grammar: no +mainstream filesystem allows a 256-byte path component, and Windows reserves 22 MS-DOS +device names (`con`, `prn`, `aux`, `nul`, `com1`–`com9`, `lpt1`–`lpt9`) that cannot be +directory names there. Either one is a reported `error` action for that skill, and the +rejection is unconditional rather than platform-gated — a managed root written from a Linux +container is routinely read from a Windows host, so the on-disk result must not depend on +which OS ran the write. The keys stay valid everywhere else: an AI Config referencing a skill +named `aux` parses, and its other fields are unaffected. If you have a skill named for a +device, rename it. + +#### Receiving skills from LaunchDarkly + +`InMemorySkillStore` is for tests and bring-your-own-content. In production, skill content +arrives through `FDv2SkillStore`, which speaks LaunchDarkly's SDK-facing FDv2 delivery +channel — the same `GET /sdk/poll` and `GET /sdk/stream` endpoints the base SDK's FDv2 data +source uses, authenticated with the environment's server-side SDK key. + +```python +import os + +from launchdarkly_ai_server import FDv2SkillStore, init_client, watch_skills + +store = FDv2SkillStore(os.environ["LD_SDK_KEY"]).start() +if not store.wait_for_skills(timeout=10): + # No payload arrived. Reconciling now would find an empty store; see below. + print(f"skill delivery has not answered yet: {store.failed or 'still waiting'}") +await init_client(options={"skillStore": store}) + +# Materialize now, and re-materialize whenever delivery changes. +report, watcher = await watch_skills("*", ".claude/skills") +try: + ... +finally: + watcher.close() + store.close() +``` + +**A reconcile that runs before delivery answers does not prune.** Through the `SkillStore` +interface, a store still waiting for its first payload and an environment that holds no +skills give the same answer — an empty one — and `write_skills("*")` would otherwise read +that as every skill having been revoked and delete the files it wrote on a previous run. +`FDv2SkillStore` reports readiness through the optional `is_initialized()`, so a reconcile +against a store that has not received a payload reports the retrieval unavailable and leaves +everything on disk alone. `report.ok` is `False` in that case, and the error names it. + +**Nothing above the store changes.** The accessors, verification, and `write_skills` see raw +objects through the `SkillStore` interface and cannot tell which store produced them. + +**Server-side only.** Skills are for server-side agent runtimes and skill content is +customer-confidential. A mobile key (`mob-…`) or a client-side environment ID raises from the +constructor. + +**The SDK key goes only where you pointed it.** `base_uri` must be `https://` (plain `http://` +is refused, except to a loopback host for a local test double), and redirects are never +followed, so a 3xx from a proxy or a misconfigured private instance stops delivery rather than +forwarding the key to whatever host the `Location` header names. + +**Reads are memory-bounded.** No poll body or streamed event is held past `MAX_RESPONSE_BYTES` +(64 MiB, far above any real payload); one that crosses it is dropped without being applied, the +store keeps serving what it last held, and delivery retries on its normal backoff. + +**Streaming is the default, and it is what makes revocation fast.** A `delete-object` reaches +a live stream in seconds; with `mode="poll"` it arrives within one `poll_interval`. Paired +with `watch_skills`, a revoked skill's `SKILL.md` leaves the disk without a restart. During an +outage the store keeps serving the last content it received and `write_skills`' default +`on_unavailable="keep"` leaves managed files alone — an outage must not read as "everything +was revoked". + +**One network timeout, and its default depends on the mode.** `read_timeout` bounds every +socket operation of a request, connecting included. In `mode="poll"` it bounds the whole +request and defaults to 10 seconds; in `mode="stream"` it bounds each wait for the next bytes +and defaults to 300 seconds, well beyond LaunchDarkly's heartbeat interval. + +**The connection also carries your flags.** A client cannot request only the skill payload, +so a skills-enabled environment delivers flag and segment objects on the same connection. +They are skipped, not evaluated — this store does no evaluation of any kind — and +`diagnostics.objects_ignored` counts them. + +> **Beta caveats, worth knowing before you deploy.** Payload signing does not exist on this +> channel yet, so delivery is TLS-only and the content hash establishes self-consistency, not +> origin authenticity. The FDv2 protocol is opt-in per account: without it the endpoints +> return HTTP 403, which the store reports as a fatal error explaining what to do. `ld-relay` +> does not speak the FDv2 endpoints, so relay-only deployments cannot receive skills. + +**If every skill comes back empty, check `diagnostics.hashless_objects`.** Verification +withholds any delivered object without a `contentHash`, so a nonzero count means skills are +being withheld rather than that the environment has none. The store also logs an error per +hashless object naming the reason. There is deliberately no fallback that skips verification. + +**Total path length is yours to bound, not the SDK's.** The 255-byte bound above is per +*component*; the root is your path, so `` + `` + `/SKILL.md` can still exceed +Windows' 260-character `MAX_PATH` with a perfectly legal key. Choose a short managed root on +Windows. + +| Export | Description | +|---|---| +| `skill_refs(config)` | Project a config's `skills` array into `list[SkillReference]`. Pure — no client, store, or network needed. Returns `[]` when absent. | +| `get_skill(key, *, version=None)` | One verified skill, or `None`. `version=None` means newest available; a specific `version` matches exactly. Raises only when no store is configured. | +| `get_skill_result(key, *, version=None)` | The same retrieval, reporting **why**: a frozen `SkillOutcome` with `.skill`, `.reason` (`ok` / `absent` / `integrity_failure` / `store_unavailable` / `wrong_version`), and `.detail`. Use it to fail closed on tampering — see *Failing closed on tampering* above. Raises only when no store is configured. | +| `get_skills(refs)` | Batch form. Accepts `SkillReference` values and bare key strings (string = latest). Results follow input order; missing or unverifiable entries are omitted. | +| `all_skills()` | Every verified skill the store holds, one per key at its newest version. | +| `write_skills(skills, root, *, prune=True, timeout=10.0, on_unavailable="keep")` | Materialize skills under `root`, returning a `ReconcileReport`. `prune` removes formerly-managed skills no longer requested. `on_unavailable="raise"` raises instead of reporting when content cannot be retrieved. Raises `ValueError` for an unusable root, a negative `timeout`, or an unrecognised `on_unavailable`. **Performs synchronous filesystem I/O — see the note below.** | +| `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `is_initialized()`, `add_listener(kind, fn)` / `remove_listener(kind, fn)`. A store without `is_initialized()` is treated as initialized. | +| `InMemorySkillStore(objects=None)` | A dict-backed store with `put(raw)`, for local development and testing. Holds several versions of a key. | +| `FDv2SkillStore(sdk_key, *, base_uri=…, mode="stream", …)` | The delivery transport: a store fed by LaunchDarkly over the SDK-facing FDv2 channel. `start()`, `wait_for_skills(timeout)`, `is_initialized()`, `close()`, `diagnostics`, `failed`; also a context manager. **Server-side only** — a mobile key or client-side environment ID raises. See *Receiving skills from LaunchDarkly* above. | +| `watch_skills(skills, root, …)` | `write_skills` plus a re-reconcile on every delivery change. Returns `(initial report, SkillWatcher)`; close the watcher when done. Revocation then takes effect within `debounce` of arriving rather than at the next restart. | +| `StoreDiagnostics` | What the transport has seen: `payloads_transferred`, `skill_objects_received`, `objects_ignored`, `objects_revoked`, `hashless_objects`, `connection_failures`, `last_error`. | + +Configure the store with `init_client(options={"skillStore": store})`. With none configured, +the accessors raise `RuntimeError` explaining what to do and `write_skills` reports the +failure in its report (or raises, with `on_unavailable="raise"`). `shutdown()` clears it. + +`ReconcileReport.actions` holds one `ReconcileAction` per outcome — `written`, `updated`, +`skipped_current`, `removed`, or `error` — each carrying `key`, `version`, the resolved +`path`, and `error`. `report.ok` is `True` when no action is an `error`, and +`report.errors` is just the `error` actions, so you rarely need to filter `actions` +yourself. A failure that belongs to the whole run rather than to one skill — an unreadable +manifest, for instance — carries the empty string as its `key`. + +The fixed on-disk values are exported too, so you do not have to hardcode them: +`MANIFEST_FILENAME` (`.launchdarkly-skills.json`, handy for a `.gitignore`), +`SKILL_FILENAME`, and `MANIFEST_VERSION`. So are the three closed-set types, for annotating +your own helpers: `ReconcileActionKind` (`written` / `updated` / `skipped_current` / +`removed` / `error`), `OnUnavailable` (`keep` / `raise`), and `SkillOutcomeReason` +(`absent` / `integrity_failure` / `ok` / `store_unavailable` / `wrong_version`). + +**`write_skills` blocks.** It is `async` for parity with the other accessors, but it awaits +nothing: every read, write, `fsync` and rename runs inline, +so a large reconcile holds the event loop for its duration. Wrap it in +`asyncio.to_thread` if that matters. For the same reason `timeout` is checked between +steps rather than interrupting one already in progress. Reconcile one root at a time, +though: because nothing yields today, a run is atomic against the rest of your loop, and +wrapping it to run concurrently makes two runs against the same root race on the manifest. + +`all_objects` returns one entry per `(key, version)` under keys that are **opaque** to the +SDK — identity is read from each object's own `key` and `version` fields, so a store is free +to key its own map however the transport underneath does. + +> `Skill.content` is `bytes` — the verified verbatim bytes LaunchDarkly delivered, exactly +> what was hashed. The SDK never parses or interprets them; if you want the frontmatter, +> decode and parse the content on your side. + +#### Privilege separation: the agent must not be able to rewrite its own skills + +**The recommended deployment runs `write_skills` as a different identity than the agent.** +Reconcile as one user, run the agent as another. Everything the reconcile puts on disk is +owner-write-only, and set explicitly rather than inherited from your umask: skill files and +the manifest at `0644` (via `fchmod` on the descriptor, so it cannot be redirected), the +per-skill `//` directories at `0755`, and the execute bit never set on anything. +Those modes are only a defense if the two identities actually differ — under a single identity +they describe a directory the agent can freely rewrite. + +**What to verify, as the identity that will run the agent.** The SDK cannot check this for you +(see below), so make it a deployment step: confirm the agent's identity has no write access to + +- the managed root itself, +- the per-skill directories `//` and the files `//SKILL.md`, +- the manifest at `/.launchdarkly-skills.json`, +- **the root's parent, and every ancestor directory above it.** Write access to an ancestor is + write access to the root by another route: it permits renaming the root aside and leaving a + symlink in its place, which redirects the reconcile — and the agent's own skill lookups — at + a directory the agent controls. In the documented layout `/.claude/skills` that parent + is `.claude`, which an agent identity is otherwise likely to own outright. + +```bash +# Run as the agent's user. Every line should print DENIED. +root=.claude/skills +for target in "$root" "$root/.launchdarkly-skills.json" "$root"/*/ "$root"/*/SKILL.md; do + [ -e "$target" ] || continue + if [ -w "$target" ]; then echo "WRITABLE — fix this: $target"; else echo "DENIED: $target"; fi +done + +# The root's ancestors, up to /. A writable one is enough to rename the root +# aside and put a symlink where it was, so these matter as much as the root. +ancestor=$(cd "$(dirname "$root")" && pwd) +while :; do + if [ -w "$ancestor" ]; then echo "WRITABLE ANCESTOR — fix this: $ancestor"; else echo "DENIED: $ancestor"; fi + [ "$ancestor" = / ] && break + ancestor=$(dirname "$ancestor") +done +``` + +Note that the managed root's own mode is **yours, not the SDK's**: `write_skills` creates only +that one leaf directory and does so with your umask, precisely because the root is a path you +chose. Own it — `chown reconcile-user:agent-group` and `chmod 0755` on the root is the shape +that makes the rest of the tree's modes mean something. + +**Why this is the mitigation that matters.** A `SKILL.md` is agent *instructions*. An agent +that can write its own skills directory can rewrite its own instructions, and an agent +processing untrusted input is exactly the thing that might be induced to do so. Write access +to the manifest is worse than write access to a skill, because the manifest is what tells the +*next* reconcile which paths the SDK owns and may delete: an agent that can edit it can keep a +skill LaunchDarkly has revoked, or aim the SDK's own delete path at something it should not +touch. `write_skills` re-validates every manifest entry from scratch for exactly that reason — +it treats that file as untrusted input, never as authorization — but an agent that cannot edit +it at all is the stronger position, and only your deployment can provide that. + +The same reasoning is why the ancestors are on the list. An identity that can rename a +directory above the root does not need write access to anything inside it: it can substitute +the whole tree, and everything the agent then loads as a skill is a file it wrote itself. That +is a different capability from racing the reconcile — no timing is involved, and it persists +until someone notices — and no amount of descriptor pinning inside `write_skills` addresses +it, because the substituted tree is what the agent reads, not what the SDK wrote. + +**The SDK deliberately does not report whether the root is writable.** There is no such field +on `ReconcileReport`, and its absence is a decision rather than an oversight. The SDK knows +only its own identity, which trivially has write access — it just wrote there. It cannot know +which identity will later run the agent, so any check it could make would answer a different +question than the one that matters, and would read as reassurance exactly where caution is +wanted. You know both identities; the SDK knows one. + +--- + ### Utility Helpers ```python @@ -292,3 +729,8 @@ All types are exported from this package. Handler packages import them from here | `GraphNode` / `GraphEdge` | A dataclass node (`.key`, `.config`, `.meta`, `.edges`, `.is_terminal`) and a dataclass directed edge (`.key`, `.source_key`, `.target_key`, `.handoff`) | | `ProviderGraphResponse` | A dataclass returned by `graph(...).invoke()`: `.response`, `.usage`, `.judge_results` | | `GraphTopology` | The parsed graph flag shape (`root` + `edges`) | +| `Skill` | A frozen skill document: `.key`, `.version`, `.content` (verified verbatim `bytes`), `.content_hash`, `.name?`, `.description?` | +| `SkillReference` | A frozen version-pinned pointer to a skill: `.key`, `.version` | +| `SkillOutcome` | A frozen retrieval outcome: `.skill`, `.reason` (`SkillOutcomeReason`), `.detail` | +| `ReconcileAction` | One `write_skills` outcome: `.key`, `.action`, `.version?`, `.path?`, `.error?` | +| `ReconcileReport` | The `write_skills` result: `.actions`, `.ok`, and `.errors` | diff --git a/packages/client/agents.md b/packages/client/agents.md index 381adcbf..5533afb3 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -28,7 +28,13 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/tracking.py` | `execute_and_track`, `execute_and_stream`, `wrap_tool_handlers`, `parse_usage` | | `src/launchdarkly_ai_server/graph.py` | `graph()`, `resolve_graph()`, `GraphInstance` | | `src/launchdarkly_ai_server/types.py` | All shared Python types — `AiConfigRep`, `ProviderHandler`, `LDContext`, `NativeTool`, etc. | -| `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape | +| `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | +| `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | +| `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | +| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery transport — the FDv2 protocol, the wire-key/`version` translation, the held object set, and `FDv2SkillStore`. Sits **below** the store interface; imports `skills_core` only, and nothing imports it | +| `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | +| `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | +| `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, `DirectoryMissing`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | | `src/launchdarkly_ai_server/registry.py` | `Registry`, `global_registry`, `compose`, `resolve_handlers`, `resolve_tools` | | `src/launchdarkly_ai_server/judges.py` | `run_judges`, `build_judge_tasks`, `run_judge` | @@ -55,6 +61,7 @@ from launchdarkly_ai_server import ( TrackData, UsageDict, HandlerResult, HandlerStreamEvent, StreamEvent, StreamChunkEvent, StreamDoneEvent, ExecuteStreamEvent, ExecuteStreamDoneEvent, VariationMeta, InitClientOptions, JudgeResult, ParseResult, ParseSuccess, ParseFailure, + Skill, SkillReference, ReconcileAction, ReconcileReport, ) # Utilities @@ -71,8 +78,26 @@ from launchdarkly_ai_server import execute_and_track, execute_and_stream, wrap_t # Entry points from launchdarkly_ai_server import config, graph, resolve_graph, init_evaluations + +# Agent Skills +from launchdarkly_ai_server import ( + skill_refs, get_skill, get_skill_result, get_skills, all_skills, write_skills, + SkillStore, InMemorySkillStore, SkillOutcome, + SKILL_FILENAME, MANIFEST_FILENAME, MANIFEST_VERSION, + ReconcileActionKind, OnUnavailable, SkillOutcomeReason, # the three closed-set unions +) ``` +`MAX_SKILL_CONTENT_BYTES` is deliberately *not* among them: it is a local enforcement +bound on content the platform produces, not a value this SDK defines, so exporting it +would semver-lock a number this side does not own. Keep it internal to `skills_core`. + +`SKILL_OBJECT_KIND` is not exported either, for a different reason: it is the string this +SDK hands a store, and a store adapter maps whatever the transport underneath calls a skill +onto it. Publishing it would advertise an SDK-side seam value as the wire contract — a claim +this side cannot make, and hard to walk back once a caller depends on it. An adapter that +needs to agree with it reaches it through `launchdarkly_ai_server.skills_core`. + When adding a new export, add it to `__init__.py`'s imports and `__all__`. Handler packages must never import from sub-paths (e.g. `launchdarkly_ai_server.client`). --- @@ -165,6 +190,539 @@ This is an OTel context value, not W3C baggage, so the id does not leak onto out --- +## Agent Skills + +Versioned `SKILL.md` documents attached to AI Config variations by reference, retrieved +through an injectable store, and materialized onto disk for agent runtimes to discover. +Three layers, in increasing order of blast radius: + +1. **Reference discovery** — `skill_refs(config)` projects the config's `skills` array into + typed `SkillReference` values. Pure: no network, no client, no store, no telemetry. + Validation of the array itself lives in `parse_ai_config` and is **fail closed** — one + malformed reference fails the whole config parse. +2. **Content accessors** — `get_skill`, `get_skill_result`, `get_skills`, `all_skills` read + through the `SkillStore` seam. Configure a store with + `init_client(options={"skillStore": store})`; with none configured the accessors raise + an actionable `RuntimeError`. A delivery transport can be added behind the seam + without touching the public API. +3. **Materialization** — `write_skills(skills, root)` writes `//SKILL.md` and + reconciles against a manifest at `/.launchdarkly-skills.json`. + +### The store seam, and why version is part of the lookup + +`SkillStore` is `get_object(kind, key, version=None)`, `all_objects(kind)`, and the optional +`is_initialized()` plus the optional pair `add_listener(kind, fn)` / +`remove_listener(kind, fn)`. Version is part of the **lookup +identity**, not a filter applied to the answer, and that is load-bearing: a delivery payload +carries the newest version of every skill *plus* every version any variation currently pins, +so two versions of one key coexist routinely. A store keyed by key alone would answer a pinned reference with the newest +object, and the caller would then have to reject it — turning the primary use case, a +version-pinned attachment, into a missing skill. `version=None` asks for the newest held. + +The equality check in `resolve_from_store` stays, now as a **defense** rather than as the +selection mechanism: the store is untrusted, so an answer that is not the version asked for +is withheld. + +`all_objects` returns one entry per `(key, version)` under keys that are **opaque** to this +SDK. Do not parse them and do not assume one per skill key; identity is read off each +object's own `key` and `version`, which are revalidated anyway. `newest_by_key` is the +one place that collapses the result to one object per key, because both whole-store +consumers need it — `all_skills`, since a list holding two versions of one key is not a set +of skills, and the `"*"` reconcile, since `//SKILL.md` is a single path. It keeps +an object too malformed to carry a usable key and version, so verification is what withholds +it and the key stays in the requested set where prune cannot touch its on-disk copy — unless +another version resolved that key anyway, in which case keeping it would only report a +withholding for a key that resolved. + +### `is_initialized()` is what stands between a slow boot and deleting a customer's files + +Through the store interface, "this environment holds no skills" and "delivery has not +answered yet" are the same empty answer, and `write_skills("*")` reads the first as every +skill having been revoked. So `_available_store` — the single gate that sets `unavailable` +and therefore suppresses pruning — consults `store_is_initialized`, and a store that has not +received its initial data blocks retrieval instead of authorizing a prune. A store that does +not implement the probe is treated as initialized, which is right for one populated by hand; +a probe that *raises* counts as not initialized, because a store that cannot say whether it +is ready is not one to delete on. Keep this check in that one gate: maintained in two places, +a condition added to one and not the other does not merely produce a wrong message — it +deletes the user's files. + +### The delivery transport, and the one field that will bite you + +`FDv2SkillStore` speaks LaunchDarkly's SDK-facing FDv2 channel (`GET /sdk/poll`, +`GET /sdk/stream`, server-side SDK key in `Authorization`, `basis` + `mv` params, +`If-None-Match`/304). It lives below the store interface and produces raw objects in the +shape `skills_core.SkillStore` documents; **nothing above that interface knows it exists**. If a transport +change ever seems to require editing an accessor, verification, or `write_skills`, the adapter +boundary is wrong. + +**The key travels over TLS only, and only to the base URI.** `_require_https_base_uri` refuses a +plain `http://` base URI in the constructor — the SDK key would go out in cleartext — with a +loopback exemption (`localhost`, `127.0.0.1`, `::1`) because the test suite's fake endpoints +listen there; and the opener is built with `_RefuseRedirects`, because `urllib`'s standard +redirect handler copies `Authorization` onto the redirected request, so any 3xx (304 aside, +which is a poll's not-modified answer) surfaces as an `HTTPError` that `_classify_status` maps +to a fatal, non-retried failure instead of a request carrying the key to the `Location` host. + +**Reads are memory-bounded.** `_read_bounded` reads a poll body in chunks, and +`_iter_stream_lines`/`_iter_sse` read each line with a size argument and total each event, all +against `MAX_RESPONSE_BYTES` (64 MiB): crossing it raises `_RecoverableTransportError`, so +nothing from that body or event is applied, the delivery loop abandons the reader's in-flight +payload, records the failure in `connection_failures`/`last_error`, and retries on the usual +backoff while the committed set stays served. The bound is a memory backstop for the transport +and is independent of `skills_core.MAX_SKILL_CONTENT_BYTES`, which caps one skill's content at +verification; do not derive one from the other. + +**The skill's version is in the object's `key`. `version` is the payload's.** Each version +of a skill is its own object on the wire, identified as `:`: + +```json +{"key":"pdf-extraction:3","kind":"skill","version":42, + "object":{"contentType":"text/markdown","content":"…","contentHash":"…","name":"…"}} +``` + +The `3` after the delimiter is what a `{key, version}` reference pins and what becomes the +stored `version`, under the stored key `pdf-extraction`. `version` (42) is the version of the +*payload* the object arrived in — it moves when anything in the environment moves, +including a flag with nothing to do with skills. Reading it as the skill's version fails +**silently**: the object verifies, the hash matches, and the caller gets content under a +version number that means nothing. There is no separate field for the skill's version: the +agent-skill payload is a *generic* payload, and generic objects carry only `key`, `kind`, +`version` and `object`, exactly like a flag. `_split_wire_key` is the only place the wire key +is read, `_store_object_from_put` and `_tombstone_from_delete` both go through it, and +`TestVersionTranslation` asserts the translation in both directions. A wire key that will +not split cleanly is *held*, not dropped — version-less, or with the offending text as its +version — so verification withholds it with `invalid_version` under a key the caller +recognises; only a key with nothing before the delimiter is dropped, since there is no +identity to hold it under. + +**Skills are identified by `kind == "skill"`; everything else is ignored, not rejected.** +Object kinds on the SDK-facing channel are open strings, and the agent-skill payload is +classified `generic`, so a skill arrives under the kind its producer registered — the bare +category name — not under a broader wrapper kind with a narrowing field. An environment's +payload assignment carries its flag payload alongside its agent-skill payload, so flag and +segment objects arrive as a matter of course. Erroring on an unrecognised kind would turn a +normal payload into a permanent reconnect loop — a flag-delivery outage caused by a skills +rollout. + +**Changes commit at `payload-transferred`, not as objects arrive.** A payload version is the +unit of consistency: a half-applied full transfer would publish a state the server never +described, and would briefly empty the store — which, with pruning on, is the difference +between a reconcile and deleting a customer's skill files. An interrupted transfer therefore +leaves last known good intact, and listeners fire once per commit. + +**The first payload intent is read, and is assumed to be the skill payload.** Delivery +provides one payload per credential and the protocol requires a client to ignore all but the +first payload intent, so `payloads[0]` is both what arrives and what the protocol says to +read. The cost of that assumption is that an `xfer-full` for somebody *else's* payload would +start an empty pending set, and the next `payload-transferred` would publish it — every skill +reported revoked, and with pruning on, a customer's files deleted. `_ProtocolReader` +therefore learns which payload skills arrive on, from the intent's `id` or from the +`(p::)` selector, and declines to apply a transfer of any other: once at +WARNING, counted in `diagnostics.payloads_ignored`, holding last known good. A transfer that +names no payload is applied, since one-payload delivery is the common case. The residual is +the first transfer of a connection — before a skill has arrived there is nothing to compare +against — which is what the separate WARNING on a multi-payload intent is for. + +**A hashless object is held, not dropped.** Verification withholds it with +`missing_content_hash`; the transport's job is to make that loud (an error per object, a +summary per wholly-hashless payload, `diagnostics.hashless_objects`) rather than to work +around it. Dropping it at the transport would report `absent` — indistinguishable from "no +such skill" — and would let a prune delete the last known-good copy on disk. Never synthesize +a hash from the delivered content: that certifies the content against itself and verifies +nothing. + +**There is one network timeout, not two.** `urllib`'s `timeout` is the socket timeout for the +whole operation, so connect, headers and each read share it, and the module cannot bound the +connect separately without a custom connection class it should not carry. `read_timeout` is +therefore the only knob, and its default is per mode (`DEFAULT_POLL_TIMEOUT` for a whole poll +request, `DEFAULT_STREAM_READ_TIMEOUT` for the gap between reads on a stream). Do not add a +parameter that the standard library cannot honour; `TestTimeouts` measures the bound against a +socket that accepts and never answers. + +**`close` interrupts the socket, it does not just set a flag.** The delivery thread spends its +life blocked in a read that no flag can reach, and closing a response from another thread does +not unblock CPython's buffered reader. `_interrupt_read` shuts the socket down underneath it. +Without that, every shutdown of a *healthy* stream blocks for the full join timeout. + +### The reported outcome vocabulary, and the `Resolution` mapping + +`get_skill` returns `Skill | None`; `get_skill_result` returns a frozen `SkillOutcome` +(`skill`, `reason`, `detail`) naming *which* outcome happened. Both are +`resolve_from_store` — one retrieval, one verification, one telemetry pass — and they differ +only in what they report. `get_skill`'s contract is load-bearing and **frozen**: `None` for +every failure, never raises for one, documented in its docstring and in the README. Change +it and every caller that treats `None` as "no skill" breaks silently. + +`SkillOutcomeReason` is five tokens, listed alphabetically for the same reason +`IntegrityReasonCode` is — so the vocabulary reads identically in the Python and TypeScript +SDKs, where the type name, the accessor name, and the tokens are all deliberately the same. +Do not rename one on one side. + +Internal `Resolution.reason` maps 1:1 onto it, set explicitly at every construction site: + +| `resolve_from_store` outcome | `reason` | +|---|---| +| the store raised (`unavailable=True`) | `store_unavailable` | +| `raw` is not a dict | `absent` | +| `verify_raw_skill` returned `None` | `integrity_failure` | +| `skill.version != wanted_version` | `wrong_version` | +| success | `ok` | + +**Adding a sixth internal outcome means choosing which public token it maps to.** +`Resolution.reason` has no default, so the compiler asks the question; answer it rather than +defaulting to `absent`, which claims the store does not hold the skill. If the new outcome +is genuinely neither of the five, the token set grows — on both sides, in the same commit. + +Two things the reason is deliberately *not*: + +- **Not derived from `Resolution.error`.** That string is prose for a human; recovering a + decision a caller fails closed on by matching it is the fragility the typed token exists + to remove. `detail` *is* that string, passed straight through — safe to surface (key and + failure mode only, never content, never a path), and not for matching on. +- **Not `Resolution.unavailable`.** The flag answers "may prune run?" and the token answers + "what does the caller learn?". They agree by construction — `unavailable` is `True` in + exactly the `store_unavailable` case — and both exist because `store_unavailable` must + stay distinct from `absent`: only a raising store suppresses pruning, since deleting + managed files after a failed lookup turns an outage into data loss. + +`get_skill_result` emits nothing of its own. The integrity log record and signal already +fired inside verification before `resolve_from_store` returned; recording anything here +would double-count one failure in a SIEM and in the product counter. + +There is no `get_skills_result` or `all_skills_result`. The batch accessors keep omitting +unresolved entries and keep logging the run-level WARN count, and a second accessor per +batch form would double the surface for a case nobody has asked for. + +### Security posture — do not relax any of this + +Store data is **untrusted input**; the transport is not part of the trust boundary. + +- **Skill content is an opaque byte buffer.** `Skill.content` is `bytes` — the verified + verbatim bytes, exactly what was hashed. The wire object delivers content as a JSON + string; the UTF-8 encode happens once, during verification, and from then on the SDK + never parses, decodes, or interprets the bytes anywhere: not in the integrity path, not + in an accessor, not during materialization. Consumers who want frontmatter parse it + themselves. +- **Integrity is mandatory and doubled, through one implementation.** Every raw object is + verified at the accessor boundary (key pattern and length, integer version >= 1, content + at most 10 MiB, sha256 lowercase hex over the verbatim bytes against `contentHash`) + and the hash is re-verified immediately before a write, both through + `skills_core.verified_bytes`, so the integrity signal's property set cannot depend on + which layer caught the defect. A `Skill` is only ever constructed from content that + passed. Nothing unverified reaches user code. +- **`contentHash` is required.** An object without one is withheld, not accepted on trust. + A payload built before the field is populated therefore yields nothing, which is why a + withholding run logs a run-level count at WARN — an empty result would otherwise be + indistinguishable from "this project has no skills". +- **No unencodable string ever reaches an encode.** `json.loads` turns a `\ud800` escape + into an unpaired surrogate with no UTF-8 representation; every `.encode("utf-8")` site + treats that as a verification failure. Never reach for `errors="surrogatepass"` — + fabricating bytes could satisfy the hash comparison. +- **Attacker-controlled strings are never echoed into telemetry.** `contentHash` and `key` + come off the wire, so a store could put the skill body in either; both are shape-checked + and redacted before they reach a signal or a log line. +- **The key is re-validated inside `write_skills`**, regardless of upstream validation — a + key becomes a directory name. Rejection happens before any filesystem call. +- **Never write through a symlink**, in either the skill directory or the target file, on + the write path *and* the prune path. +- **Destructive operations only on manifest-listed paths whose `key` matches.** A file at a + managed path with no matching manifest entry is reported as `error` and left alone — + *unless its bytes already are the resolved content*, in which case it is adopted (manifest + entry recorded, reported `skipped_current`). That single exception is what makes a + reconcile killed between the content writes and the final manifest rewrite recoverable + instead of permanently wedged, and it cannot be widened: the comparison is over the + verbatim bytes against the resolved `contentHash`, a read that fails is a refusal and + never an overwrite, and the read is bounded at `len(content) + 1` bytes so a file that + merely *begins* with the resolved content is refused too. Do not relax it to a prefix, a + length, an mtime, or the manifest's own recorded `sha256` — that field is untrusted and is + never a decision input. `skipped_current` is reused deliberately rather than adding an + `adopted` action kind; `ReconcileActionKind` is a public closed set. +- **Temp files are swept, within the same bounds as everything else.** `atomic_write` unlinks + its own temp file on any exception, but a `SIGKILL` leaves one behind that no manifest + entry records, and a non-empty directory defeats `_prune_one`'s `rmdir` — so one orphan + pins a skill directory forever. The sweep is the only place this SDK removes a file the + manifest does not list, and it is bounded on every axis: inside `//` only, for a + key that passes `_key_rejection_reason`; only names `safe_fs.is_temp_name` recognizes, + anchored at both ends and asked of `safe_fs` rather than re-spelled (a copy would drift + from the writer); only regular files, with the type read off the descriptor; listed off + the pinned descriptor (`os.listdir(fd)`), so the names come from the directory that was + pinned; unlinked through that same descriptor. It never raises and never aborts a run. +- **A corrupt manifest fails closed**: unreadable, unparseable, not an object, malformed + `entries`, larger than `_MAX_MANIFEST_BYTES`, or a `manifestVersion` outside + `1 <= v <= MANIFEST_VERSION` means no overwrites and no prunes, brand-new paths may still + be written, an `error` action names the manifest, and the manifest file itself is not + rewritten. The version is bounded on *both* sides: 1 is the first version ever written, so + 0 or a negative is not a manifest this SDK produced, and accepting one would act + destructively on it and then silently rewrite it as version 1. The size bound exists + because the manifest is the one file here whose length no caller can predict, it lives in + a directory the SDK does not own exclusively, and a reconcile must not be the thing that + exhausts the process — every read in `skills_fs` is bounded, and `_read_regular_file` + takes a required `max_bytes` so a new call site cannot opt out by omission. +- **An incomplete retrieval suppresses pruning.** Otherwise a transport outage would read + as "everything was revoked" and delete the customer's managed files. +- **Writes are atomic**: temp file created exclusively in the target's *own* directory, + mode `0644` set explicitly (never inherited from the umask, never executable), write, + fsync, `os.replace`, fsync the directory. `os.replace` is the single rename call site + and must not be swapped for `os.rename`. +- **Every operation under the root goes through a pinned descriptor, not a path — the + reads that decide an action included.** See "Descriptor-pinned filesystem access" below. + Re-resolving `/` from its path at write or unlink time reopens a swap window + that the checks above cannot cover; re-resolving it for the existence probe, the compare + read or the orphan listing lets a swap choose the *branch* instead — most seriously, a + prune whose probe is answered "absent" from a swapped directory skips its unlink, drops + the manifest entry, and reports `removed` while the revoked skill stays on disk. +- **A key valid to the data model may still be unrepresentable on disk.** The model allows + 256 characters; `NAME_MAX` is 255 bytes. Windows additionally reserves 22 MS-DOS device + names, none of which can be a directory name there: `con`, `prn`, `aux`, `nul`, + `com1`–`com9`, `lpt1`–`lpt9` (`com0` and `lpt0` are *not* reserved; do not add them). + `write_skills` rejects both before any filesystem call, and every per-skill filesystem + failure is caught at the loop so it becomes an `error` action — aborting the loop would + skip the manifest rewrite and orphan files already written in that run. +- **Those two bounds live in `_key_rejection_reason`, not in the key grammar, and must not + move.** `is_valid_skill_key` / `skill_key_rejection_reason` keep admitting an over-long or + reserved key on purpose. `parse_ai_config` fails closed on a bad `skills` entry, so a + grammar-level rejection would invalidate the *entire* AI Config — model, provider, + instructions, tools — for a Linux customer over a Windows-only constraint; and it would + silently shrink `skill_refs`, which is what authorizes a prune, converting "this skill + fails to write on Windows" into "this skill gets deleted on Linux". `_key_rejection_reason` + is shared by the write and prune paths, so one edit covers both destructive paths. + The reserved-name check is unconditional rather than `os.name == "nt"`-gated: a root + written from a Linux container is routinely read from a Windows host, and neither + repository has a Windows CI runner (every matrix job is `ubuntu-latest`), so a gated branch + would be untestable — the exact condition that produced the gap. No suffix stripping and no + case folding are needed, because the grammar admits no `.` and no `$` (so `con.txt` and + `CONIN$` are unreachable) and is lowercase-only. The residual the SDK cannot check is total + path length: the 255-byte bound is per *component*, and the root belongs to the customer, + so `MAX_PATH` overflow is a README note rather than a check. +- **A key is untrusted input everywhere it appears.** `skill_key_rejection_reason` is the + single canonical explanation, so the config parser and the reference projection reject a + key for the same stated reason — and so does every layer added later. A silently + shortened projection is not acceptable: every dropped entry is logged. + +### Telemetry seam + +Skills telemetry goes through a private emitter with one method, +`record(signal, properties)`, whose default implementation is a **no-op** — nothing leaves +the process in this release. `client.track()` is deliberately *not* used: it needs an LD +context, spends the customer's event volume, lands in their data export, and is silenced by +offline mode. No LD context is involved anywhere in this feature. + +Exactly three signals exist, and the list is an **allowlist, not a floor**: + +| Signal | When | Properties | +|---|---|---| +| `AgentControl Skill Integrity Failure` | any hash/size/shape verification failure | `skill_key`, `version?`, `expected_hash?`, `observed_hash?`, `language` | +| `AgentControl Skill Materialized` | each `written` / `updated` / `skipped_current` | `skill_key`, `content_bytes`, `content_hash`, `reconcile_action`, `language` | +| `AgentControl Skill Revoked Received` | prune removes a formerly managed skill | `skill_key`, `version`, `removed_from_disk`, `language` | + +### The integrity-failure log record + +The signal above is product telemetry; the **log record** beside it is the customer-owned +detection path, and the more load-bearing of the two. It is the only integrity surface that +works when telemetry is off, and the only one that exists at all in an instance with no +telemetry destination, so it is a documented contract in the README rather than a debugging +aid. `record_integrity_failure` writes both, and is the only place either is constructed. + +One ERROR record per withheld skill, message text = `INTEGRITY_FAILURE_EVENT` + a space + +`json.dumps(record, sort_keys=True, separators=(",", ":"))`, plus the same mapping under +`extra={"ld_skills": record}`. Fields: `event`, `action` (always `withheld`), `skill_key`, +`version?`, `expected_hash?`, `observed_hash?`, `reason_code`, `reason`, `language`. + +Each of those choices is load-bearing; do not undo one as a simplification. + +- **The event name is in the message text**, not only in `extra`. Severity cannot + discriminate — `resolve_from_store` and `list_raw_objects` in the same module also log + ERROR for a raising store — and the stdlib's default formatter drops `extra` entirely, so + an `extra`-only record is invisible under a plain `logging.basicConfig()`. +- **`ld.skills.integrity_failure` is documented for customers to match on**, which makes it + a compatibility surface. It must never be renamed. +- **`sort_keys=True` is not cosmetic.** The other language implementations build the object + in alphabetical key order, so sorting makes the serialized line byte-identical across + SDKs for the same input, modulo `language`. +- **Optional fields are omitted, never nulled**, so a SIEM field-existence check means + something. +- **The record spreads the signal's properties** rather than rebuilding them, so the two + cannot drift on the fields they share — in particular on which are redacted. Anything + added later that comes off the wire needs the same shape-check-then-redact treatment. +- **`reason_code` is in the record only.** The signal's property set is the allowlist above + and does not grow; the local record is where the detection vocabulary lives. + +`reason_code` is a **closed vocabulary of exactly eight tokens** — `IntegrityReasonCode`, a +`Literal`, so a typo at a call site is a type error — one per `record_integrity_failure` +call site, and the same eight in every language implementation: + +| `reason_code` | Call site | +|---|---| +| `not_an_object` | `verify_raw_skill` — raw object is not a dict | +| `invalid_key` | `verify_raw_skill` — fails `is_valid_skill_key` | +| `invalid_version` | `verify_raw_skill` — fails `is_valid_skill_version` | +| `missing_content` | `verify_raw_skill` — `content` absent or not a string | +| `missing_content_hash` | `verify_raw_skill` — `contentHash` absent or not a string | +| `not_utf8` | `verified_bytes` — `UnicodeEncodeError` on encode (wire-`str` path only; a `Skill` already holds bytes) | +| `over_size_cap` | `verified_bytes` — over `MAX_SKILL_CONTENT_BYTES` | +| `hash_mismatch` | `verified_bytes` — observed sha256 != `contentHash` | + +Adding a ninth failure mode means widening `IntegrityReasonCode`, adding a case to +`REASON_CODE_CASES` in `test_skills.py` (whose exhaustiveness assertion fails otherwise), +documenting it in the README table, **and** doing the same in the other language SDKs. A +token added on one side only is a drift bug: a customer's detection rule stops matching +where they cannot see it. + +`AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` +were considered and **deliberately excluded from SDK emission** — both are observable +server-side. Do not add them. The skill body never appears in a signal, a log line, or +an error message, and no signal carries a filesystem path (paths belong in the returned +`ReconcileReport`, which is user-facing API). An emitter that raises is caught and logged; +it never fails the operation. + +Module state lives in `skills_core.py`, the module `skills.py` and `skills_fs.py` share, +so there is exactly one store and one emitter however the feature is entered. All three signals are emitted from the `record_*` functions +next to the seam there — nothing outside that module calls `emit`, so the allowlist is +enforced in one place. + +The injection path is deliberately narrower than the state's location: `skills.py` owns +`_set_store`, `_set_emitter_for_testing` and `_clear_state`, which delegate to +`skills_core`. `init_client` and `shutdown` use those, tests inject through those +(`skills._set_store(store)` is the same setter `init_client` uses), and neither should +reach into `skills_core` directly. + +### Descriptor-pinned filesystem access + +A path check is only as good as the last path resolution after it. Every `lstat`, `realpath` +and containment check validates an *inode*, but a following +`os.replace(tmp, root / key / "SKILL.md")` re-resolves `/` from its *name* — so +anything holding write permission on a managed directory can move the validated directory +aside, leave a symlink in its place, and redirect the write (or an unlink) somewhere else. +Narrowing that window is not a fix; the race is winnable at any width. + +So the checks hand off to a descriptor and nothing re-resolves a path afterwards. The +primitives live in `safe_fs.py`, which knows nothing about skills: + +- `open_directory_nofollow` opens the directory with `O_RDONLY | O_DIRECTORY | O_NOFOLLOW` + and confirms `S_ISDIR` on the `fstat` (the explicit check is what covers platforms with no + `O_DIRECTORY`). `open_or_create_directory` wraps it with `os.mkdir` plus an `lstat` on the + `FileExistsError` path — `Path.mkdir(exist_ok=True)` accepts a symlink-to-directory as + "already there", which would reopen the hole the caller's check just closed. + `pinned_directory` holds either for the duration of a block, so a caller states the + platform split once as `if dir_fd is not None` and cannot forget the `os.close`. +- `atomic_write` creates the temp file with `O_CREAT | O_EXCL | O_NOFOLLOW` **at** that + descriptor (`_mkstemp_at`, since `tempfile` has no `dir_fd` form), `fchmod`s the + descriptor rather than `chmod`ing a path — probed, because Windows has no `os.fchmod` + before 3.13 and 3.12 is supported — writes, fsyncs, and renames with + `os.replace(tmp, name, src_dir_fd=fd, dst_dir_fd=fd)`, then fsyncs the directory so the + rename survives a crash. `atomic_write_in` is the same against a directory the caller does + not already hold open. `os.replace` is the single rename call site, reached by attribute + lookup so tests can intercept it, and `os.rename` must not be substituted for it — it is + also the only one with defined overwrite semantics on Windows. +- `unlink_file` probes and unlinks descriptor-relative too. `unlink` never follows a + *trailing* symlink, but it does resolve the directory above it, so the same swap turns a + removal into a delete of an attacker-chosen file. A symlink found where this SDK expects + its own file raises `SymlinkRefused` rather than being tidied away: the state on disk is + not what the caller believes, and that is the caller's to report. `_prune_one` goes + through it; `rmdir` is issued relative to the root descriptor, and is safe at the key + since it fails `ENOTDIR` on a symlink and only ever succeeds on an empty directory. +- `open_directory_nofollow` raises `DirectoryMissing` — a `ValueError` subclass — when + nothing at the path is a directory (`ENOENT`, `ENOTDIR`). That is how the skills side + learns a skill directory is absent *from the pin itself*, rather than from a separate + `exists()` on the path that a swap could answer differently: a prune of a file that is + already gone and a sweep of a directory never created both take that branch. + +The reads that decide an action are pinned the same way, in `skills_fs`. Each of `_write_one` +and `_prune_one` pins `/` relative to the root descriptor *before* it decides +anything and holds the pin through the action: the existence probe is +`os.stat(SKILL_FILENAME, dir_fd=skill_fd, follow_symlinks=False)`, the compare read is +`_read_regular_file(SKILL_FILENAME, dir_fd=skill_fd)`, and the orphan sweep lists +`os.listdir(skill_fd)`. So a swap after the pin cannot choose the branch — it cannot have +a prune skip its unlink and still report `removed`, cannot have a compare read adopt or +refuse over a file outside the root, and cannot feed the sweep names from elsewhere. Where +`dir_fd` is `None` (the `lstat` floor) each of these stays path-based, which is the +documented Windows bound. The manifest read was already pinned; nothing under the root is +read by path any more. + +Every `lstat`, `realpath` and containment check on the skills side lives in one shared +`_unsafe_path_reason`, so the write and prune paths cannot drift apart on what counts as +unsafe. Those checks stay, ahead of the pin, as defense in depth — they are not the boundary +and must not be removed. Symlink probes on the descriptor side are spelled +`os.stat(..., follow_symlinks=False)` rather than `os.lstat`, matching the name the +capability probe advertises. + +`safe_fs.SUPPORTS_DIR_FD` gates all of it, and the probe is not the obvious one. +`os.supports_dir_fd` is populated per underlying syscall, and CPython registers `renameat` +under `os.rename` only and `fstatat` under `os.stat` only — even though `os.replace` is the +same `renameat`-backed function and `os.lstat` is `fstatat` with `AT_SYMLINK_NOFOLLOW`. +Probing the names this module actually calls reports "unsupported" on every POSIX platform +and silently turns the defense off, so the probe names the advertised twins +(`{os.rename, os.open, os.unlink, os.stat}`) and a caller's symlink check is spelled +`os.stat(..., follow_symlinks=False)` rather than `os.lstat`. Where the family is absent +(Windows) `open_directory_nofollow` returns `None` after an `lstat` check instead of +attempting the descriptor open — `os.open` cannot open a directory there — and every caller +falls back to the identical full-path sequence, the per-component `lstat` floor. The +residual window on those platforms is documented rather than closed; the TOCTOU tests skip +off this same flag, deliberately, so a probe that wrongly reports "unsupported" cannot also +silently skip the tests that would have caught it. + +Both call shapes are admitted by the test seam. `os.replace` remains the single +interceptable rename call site; under the descriptor-relative shape `dst` is the bare string +`"SKILL.md"`, so an `endswith("SKILL.md")` spy filter still matches, and the +same-directory requirement is proved by descriptor identity (`src_dir_fd == dst_dir_fd`, +resolving to the skill directory's `(st_dev, st_ino)`) instead of by comparing path strings. +A spy must `fstat` the descriptor **inside** the intercepted call — the implementation closes +it as soon as the write returns. + +**The platform bound is POSIX-only, and that is a decision — do not quietly "fix" it.** +Windows reparse-point checks (`GetFileAttributesW`, `FILE_FLAG_OPEN_REPARSE_POINT`) are not +implemented because Windows is not a supported or tested platform for this release: there is +no Windows CI runner in either repository, so the checks would ship unverified, and the +TypeScript SDK could not match them at all — Node exposes no `*at()` family on *any* +platform, so its racy floor is universal rather than Windows-only. Implementing them in +Python alone would break cross-language parity and trade a documented bound for an unverified +one. Two follow-on facts: on Windows write permission on the managed root is the only +boundary, which is why the privilege-separated deployment is documented as the mitigation +rather than as advice; and this bound retroactively lowers the priority of the reserved-device-name +work above — keep that code, but do not read it as evidence that Windows is hardened. If +Windows becomes a supported platform, revisit both together, and add the CI runner first. + +**Privilege separation is the deployment-side half of this, and `ReconcileReport` must not +grow a writability field.** The recommended deployment runs the reconcile as a different +identity than the agent, so the `0644`/`0755` modes above actually deny something: the agent +reads its instructions and cannot rewrite them or the manifest. That is the mitigation for a +prompt-injected agent editing its own skills. The security review asked for the report to +surface whether the managed root is writable; we declined, and the reasoning is load-bearing +rather than a preference. The SDK knows only its *own* identity, which trivially has write +access — it just wrote there — and cannot know which identity will later run the agent. Any +check it could perform would answer a different question than the one asked and would create +false confidence exactly where caution is wanted. The operator's verification steps live in +the README instead. Do not add the field. + +### Deferred: bounded retries + +`timeout` is implemented — a monotonic deadline, checked before each retrieval, before +each write, and before each prune; only the final manifest rewrite runs past it, so files +already written are never orphaned. Bounded retries inside that deadline are **not** +implemented, and belong to the delivery transport, not to this layer. Three structural +reasons, all of which the transport changes: + +1. **There is nothing transient to retry.** `SkillStore.get_object` is a synchronous + in-process read against already-delivered data, modelled on the LaunchDarkly + data-store API. `InMemorySkillStore` reads a dict. A retry re-invokes customer code and + returns the same answer. +2. **The seam cannot classify a failure.** All it surfaces is "this raised". Retrying a + `PermissionError` or a malformed payload spends the caller's `timeout` on a certainty. + The transient/permanent taxonomy a retry policy needs is the transport's to define. +3. **Backoff has nowhere to sleep.** The retrieval path (`_resolve_requests`, + `_resolve_reference`, `_resolve_all`) is synchronous, called from an async + `write_skills`. Backoff would mean either `time.sleep` — blocking the event loop of every + caller — or async-ifying the whole path for a store that cannot benefit. + +Picking a bound and a backoff now would fix numbers in a cross-language contract with no +transport to calibrate them against, so there is **no** retry test and no assumable attempt +count. When the transport lands it owns the policy; keep both languages retry-free until +then, since the number of times a throwing store is invoked is observable and the two would +otherwise diverge. + +--- + ## OTel Setup The core client owns all OTel initialization. `init_client()` configures a `TracerProvider` with `ConversationIdSpanProcessor` and a `BatchSpanProcessor` plus an OTLP HTTP exporter when the optional OTel packages are installed. @@ -240,6 +798,43 @@ When `enabled` is `False`, `config` is always `None`. When `enabled` is `True` b --- +## Dependencies + +Tier 0, so the runtime surface is deliberately tiny: **one** hard dependency, and everything else either an optional extra, resolved dynamically at runtime, or dev-only. Nothing here may grow without a reason recorded in this table. + +### Runtime (`[project] dependencies`) + +| Package | Why | +|---|---| +| `opentelemetry-api>=1.25` | The tracer/span API used on every instrumented path (`tracking.py`, `graph.py`, `content.py`, `conversation.py`, `utils.py`). API-only — the *SDK* half is an optional extra, so a consumer that never configures OTel gets no-op spans rather than an `ImportError`. `conversation.py` imports `opentelemetry.sdk.trace.SpanProcessor` under `TYPE_CHECKING` only, for exactly this reason. | + +There is deliberately **no** `python-dotenv` here: `lifecycle.py` reads `os.environ` directly, so loading a `.env` file is the application's job rather than the SDK's. `python-dotenv` is in the workspace dev group for the examples only. + +### Optional extra (`[project.optional-dependencies] otel`) + +| Package | Why | +|---|---| +| `opentelemetry-sdk>=1.25` | Tracer provider, resources, and the batch span processor, imported inside `_setup_telemetry()` in `lifecycle.py`. Optional so telemetry is opt-in; absent ⇒ a `logger.warning` and no spans, never a raise. | +| `opentelemetry-exporter-otlp-proto-http>=1.25` | OTLP/HTTP span export and its compression enum. Same optionality, same loader. | + +Install with `pip install "launchdarkly-ai-server[otel]"`; see [OTel Setup](#otel-setup) for the endpoint variables. + +### Resolved dynamically, declared nowhere + +| Package | Why | +|---|---| +| `launchdarkly-server-sdk` | The LaunchDarkly server SDK, reached by `importlib.import_module("ldclient")` (falling back to `launchdarkly_server_sdk`) inside `init_client()`'s options path. Undeclared on purpose: the BYOC path (`init_client(client=...)`) targets environments that supply their own client, and a hard dependency would force an unused SDK into every such install. So it is imported late and raises actionably when missing — absent ⇒ a `RuntimeError` naming the `pip install`, and only on the path that needs it. | + +### Dev-only (workspace root `[dependency-groups] dev`) — the ones with a contract attached + +| Package | Why | +|---|---| +| `launchdarkly-server-sdk>=9.0`, and the `otel` extra mirrored (`opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-http`) | Each dynamically-resolved or optional package is repeated in the dev group so the test suite can import it. Something that is *only* optional would not be installed in this workspace and the tests covering its present-and-working path could not run. | +| `pytest>=8`, `pytest-asyncio>=0.24` | Test runner and the async support the whole suite relies on. `asyncio_mode = "auto"` is set at the workspace root, which is why no test in this package carries an `@pytest.mark.asyncio`. | +| `mypy>=1.10` (`strict`), `ruff>=0.15` | Type checker and linter/formatter. `mypy` strict mode is the only thing enforcing the `Literal[...]` closed set on `ReconcileAction.action` — unlike `write_skills`'s `on_unavailable`, which is also checked at runtime because the value can arrive from untyped code. | + +--- + ## Common Pitfalls ### 1. Calling `get_client()` before `init_client()` resolves @@ -250,6 +845,30 @@ When `enabled` is `False`, `config` is always `None`. When `enabled` is `True` b `execute_and_track` expects the handler to return a plain `dict` with at least `output` and `usage` keys. Do not return a custom class — `parse_usage` and the telemetry pipeline both access dict keys. +### 3. Interpreting skill content anywhere + +`Skill.content` is opaque `bytes` by construction. Do not add a parser, a decoder, or a +convenience accessor that reads meaning into it — no YAML/frontmatter parsing, no +"decode as UTF-8 for display", nothing. The SDK's whole contract is that content is the +verified verbatim byte buffer and nothing more; a consumer who wants structure parses it +on their side of the boundary. + +### 4. Assuming `write_skills` prunes on every run + +Pruning is suppressed when the manifest is corrupt or any retrieval was incomplete — both +mean the SDK cannot tell what it owns or what is still current, and deleting under that +uncertainty is data loss. A run whose report contains a manifest `error` will not have +pruned anything, so do not read "no `removed` actions" as "nothing is stale". + +### 5. Treating "absent from the resolved set" as always meaning revoked + +Revocation is pruning, but only for a skill the store genuinely no longer serves. An object +that is *present and unverifiable* is a different thing, and `_resolve_all` must emit a +failed `_PendingWrite` for it rather than filtering it out: dropping it silently leaves its +key out of the requested set, so prune deletes the last known-good copy on disk and reports +a routine `removed` with `report.ok` still true. Tampered content must never be able to +trigger deletion. + --- ## Adding a New Export @@ -265,3 +884,9 @@ When `enabled` is `False`, `config` is always `None`. When `enabled` is `True` b - Handler packages must import `LDContext` from `launchdarkly-ai-server` — not directly from any LD SDK. - Do not weaken the `parse_ai_config` validation — handler packages rely on `config` being valid when they receive it. - `parse_usage` must continue to accept `input_tokens/output_tokens`, `inputTokens/outputTokens`, and `input/output` as all existing handlers return one of these variants. +- `Skill.content` is opaque `bytes`. Do not add anything that parses or interprets it — no YAML library in this package's dependencies at any tier, and no accessor that decodes content. +- Do not route skills telemetry through `client.track()`, and do not introduce an LD context anywhere in the skills path. Signals go through the `skills_core.py` emitter seam, whose default is a no-op, and only via its `record_*` functions. +- Do not add a signal name outside the three in the Agent Skills table above — the list is an allowlist. `AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` were considered and deliberately excluded from SDK emission. +- Do not rename `ld.skills.integrity_failure`, and do not add a ninth `reason_code` in one language only — both are documented compatibility surfaces. See "The integrity-failure log record" above. +- Do not relax any of the `write_skills` filesystem defenses (local key re-validation, symlink refusal, manifest-authorized destruction, corrupt-manifest fail-closed, atomic `0644` writes). Each is a deliberate security property with abuse-case tests attached. +- Do not make `SkillStore` lookups key-only. Version is part of the lookup identity because a payload holds several versions of one key; a key-only seam cannot express a version-pinned reference. diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index f67dcc2f..7b317fe3 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -49,6 +49,24 @@ resolve_tools, ) from .sdk_info import SDK_INFO_CONTEXT, SDK_INFO_EVENT, register_ai_sdk_package +from .skills import ( + InMemorySkillStore, + all_skills, + get_skill, + get_skill_result, + get_skills, + skill_refs, +) +from .skills_core import SkillStore +from .skills_fdv2 import FDv2SkillStore, StoreDiagnostics +from .skills_fs import ( + MANIFEST_FILENAME, + MANIFEST_VERSION, + SKILL_FILENAME, + OnUnavailable, + write_skills, +) +from .skills_watch import SkillWatcher, watch_skills from .tracking import execute_and_stream, execute_and_track, wrap_tool_handlers from .types import ( NATIVE_TOOL_KEY, @@ -78,6 +96,13 @@ ProviderGraphResponse, ProviderHandler, ProviderResponse, + ReconcileAction, + ReconcileActionKind, + ReconcileReport, + Skill, + SkillOutcome, + SkillOutcomeReason, + SkillReference, StreamChunkEvent, StreamDoneEvent, StreamEvent, @@ -138,6 +163,12 @@ "ProviderGraphResponse", "ProviderHandler", "ProviderResponse", + "ReconcileAction", + "ReconcileActionKind", + "ReconcileReport", + "Skill", + "SkillOutcome", + "SkillReference", "StreamChunkEvent", "StreamDoneEvent", "StreamEvent", @@ -221,6 +252,28 @@ "graph", "resolve_graph", "GraphInstance", + # skills + "skill_refs", + "get_skill", + "get_skill_result", + "get_skills", + "all_skills", + "write_skills", + "SkillStore", + "InMemorySkillStore", + # skills — the FDv2 delivery transport, and the eager re-reconcile it enables + "FDv2SkillStore", + "StoreDiagnostics", + "watch_skills", + "SkillWatcher", + # skills — the three closed-set unions a typed consumer needs to name + "ReconcileActionKind", + "OnUnavailable", + "SkillOutcomeReason", + # skills — on-disk constants, identical across languages + "SKILL_FILENAME", + "MANIFEST_FILENAME", + "MANIFEST_VERSION", ] register_ai_sdk_package("launchdarkly-ai-server", __version__) diff --git a/packages/client/src/launchdarkly_ai_server/lifecycle.py b/packages/client/src/launchdarkly_ai_server/lifecycle.py index 1a37b25a..0ce71a89 100644 --- a/packages/client/src/launchdarkly_ai_server/lifecycle.py +++ b/packages/client/src/launchdarkly_ai_server/lifecycle.py @@ -6,6 +6,7 @@ import os from typing import Any +from . import skills from .sdk_info import flush_ai_sdk_info, reset_ai_sdk_info from .types import InitClientOptions @@ -131,13 +132,43 @@ async def init_client( - Pass *client* directly (BYOC) to skip the LaunchDarkly Python SDK path. - Otherwise, reads ``LD_SDK_KEY`` from env or ``options['sdkKey']``. + - ``options['skillStore']`` configures the store the Agent Skills accessors + read from. Absent by default, in which case they raise an actionable error. + + This function is idempotent for the client singleton: a second call returns + the existing client without re-initializing, and every option is ignored — + **except** ``skillStore``, which is applied on every successful call. That + asymmetry is deliberate, and it is what lets a client that was lazily + auto-initialized, or initialized without a store, be given one afterwards. + A ``skillStore`` of ``None`` (or absent) never clears an already-configured + store; use ``shutdown()`` for that. The store is installed only once + initialization has succeeded: a call that raises leaves no global state + behind, so a failed init cannot leave the skill accessors working against a + store the application believes was never installed. Returns the initialized ``LDClientInterface`` instance. """ - global _client - opts = options or {} + ld_client = await _resolve_client(opts, client) + + # The single success point: every path that raises returns before here, so + # "installed only on success" is one statement rather than a copy per exit. + skill_store = opts.get("skillStore") + if skill_store is not None: + skills._set_store(skill_store) + return ld_client + + +async def _resolve_client(opts: InitClientOptions, client: Any) -> Any: + """ + Returns the singleton client, initializing it on first call. + + Split from ``init_client`` so that function has exactly one success point to + hang the ``skillStore`` carve-out on. + """ + global _client + # Idempotent — if already initialized, return the existing client if _client is not None: flush_ai_sdk_info(_client) @@ -202,12 +233,18 @@ async def shutdown() -> None: """ Shuts down the singleton client. Idempotent — safe to call multiple times even if the client was never initialized or already shut down. + + Also clears the configured skill store (and telemetry emitter): after a + shutdown, re-pass ``skillStore`` to the next ``init_client`` if the skill + accessors should keep working. """ global _client, _tracer_provider local_client = _client local_provider = _tracer_provider + skills._clear_state() + # Null the singleton before any awaits so a second call is a no-op _client = None _tracer_provider = None @@ -245,6 +282,7 @@ def _reset_for_testing() -> None: global _client, _tracer_provider _client = None _tracer_provider = None + skills._clear_state() async def inspect_config( diff --git a/packages/client/src/launchdarkly_ai_server/safe_fs.py b/packages/client/src/launchdarkly_ai_server/safe_fs.py new file mode 100644 index 00000000..b022aafb --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/safe_fs.py @@ -0,0 +1,418 @@ +""" +Descriptor-pinned filesystem primitives. + +Nothing here knows what a skill is: this is the "write a file under a directory +an attacker may be racing you for" problem, solved once. ``skills_fs.py`` is the +only caller today. + +**The invariant:** every operation runs relative to a descriptor pinned to a +directory the caller already verified, never against a re-resolved path. A path +check is only as good as the last resolution after it. + +**POSIX only.** Windows has no ``*at()`` family, so only the per-component +``lstat`` floor runs there — a check-then-use race rather than a closed window. +Write permission on the managed root is therefore the only boundary on Windows, +which is why the README documents a privilege-separated deployment as the +mitigation rather than as advice. + +The threat model, the platform decision behind it, and what must not be relaxed +are in ``agents.md`` under *Descriptor-pinned filesystem access*. +""" + +from __future__ import annotations + +import errno +import os +import re +import secrets +import stat +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +_FILE_MODE = 0o644 +"""Mode set explicitly on every written file — never inherited from the umask, +and never executable.""" + +_SUPPORTS_FCHMOD = hasattr(os, "fchmod") +""" +Whether the mode can be set on the descriptor rather than on a path. + +POSIX always has ``os.fchmod``; Windows only gained it in CPython 3.13, and this +package supports 3.12. Probing for it rather than assuming it is what keeps the +documented Windows fallback a fallback instead of an ``AttributeError`` raised +after the temp file is already open. +""" + +SUPPORTS_DIR_FD = os.supports_dir_fd.issuperset( + # renameat, openat, unlinkat, fstatat, mkdirat, and unlinkat's AT_REMOVEDIR + # form — the six this module and its caller need. + {os.rename, os.open, os.unlink, os.stat, os.mkdir, os.rmdir} +) +""" +Whether the ``*at()`` syscall family is available. Gates every descriptor-pinned +operation in this module; ``False`` falls back to the ``lstat`` floor. + +**Do not "correct" the names in this probe.** ``os.supports_dir_fd`` is +populated per underlying syscall, and CPython registers ``renameat`` under +``rename`` only and ``fstatat`` under ``stat`` only — so probing the +``os.replace`` and ``os.lstat`` this module actually calls reports +"unsupported" on every POSIX platform and silently disables the defense. +""" + + +class DirectoryMissing(ValueError): + """ + Raised by ``open_directory_nofollow`` when nothing at the path is a + directory: the name is absent (``ENOENT``) or names a non-directory + (``ENOTDIR``). + + A ``ValueError`` subclass so a caller that treats every unpinnable directory + alike keeps its single ``except``; a distinct type so one for which "not + there" is an ordinary outcome — a prune whose file is already gone, a sweep + of a directory that was never created — can tell it from a refusal without + matching on a message or an errno. A symlink is never this: it is refused + as a symlink. + """ + + +def _at(directory: Path, dir_fd: int | None) -> str | Path: + """ + What to name *directory* by, given a descriptor for its parent. + + With a *dir_fd*, the bare final component, so the kernel resolves it inside + the pinned parent; without one, the full path. Spelled once because a single + call site left on the full path would silently reopen the window the + descriptor closes. + """ + return directory.name if dir_fd is not None else directory + + +def open_directory_nofollow( + directory: Path, *, dir_fd: int | None = None +) -> int | None: + """ + Opens *directory* without following a final symlink, and pins it. + + *dir_fd* is a descriptor for the *parent*. Passing one extends the guarantee + past the final component: ``O_NOFOLLOW`` refuses a link at *directory* + itself, but without a parent descriptor every ancestor is re-resolved on + each open. + + Returns ``None`` where the ``*at()`` family is absent, after an ``lstat`` + check for a real non-symlink directory. It must not attempt the descriptor + open there: ``os.open`` cannot open a directory on Windows, so that path + would fail every operation rather than fall back. + + Raises ``ValueError`` when the path will not open, or inspect, as a real + directory — ``DirectoryMissing`` when that is because nothing is there or + what is there is not a directory, so a caller can treat absence as an + outcome rather than a failure. Both are decided by the open itself (or the + ``lstat`` on the floor), never by a separate existence probe on the path. + """ + if not SUPPORTS_DIR_FD: + try: + mode = os.lstat(directory).st_mode + except FileNotFoundError as exc: + raise DirectoryMissing(f"the directory does not exist: {exc}") from exc + except OSError as exc: + raise ValueError(f"the directory could not be inspected: {exc}") from exc + if stat.S_ISLNK(mode): + raise ValueError("the directory is a symlink") + if not stat.S_ISDIR(mode): + raise DirectoryMissing("the path is not a directory") + return None + + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(_at(directory, dir_fd), flags, dir_fd=dir_fd) + except (FileNotFoundError, NotADirectoryError) as exc: + raise DirectoryMissing(f"the directory does not exist: {exc}") from exc + except OSError as exc: + raise ValueError( + f"the directory could not be opened without following links: {exc}" + ) from exc + try: + # O_DIRECTORY already guarantees this wherever the platform defines it; + # the explicit check is what covers the platforms that do not. + if not stat.S_ISDIR(os.fstat(fd).st_mode): + raise ValueError("the path is not a directory") + except BaseException: + os.close(fd) + raise + return fd + + +def open_or_create_directory( + directory: Path, *, dir_fd: int | None = None +) -> int | None: + """ + Creates *directory* if absent and returns a descriptor pinned to it. + + ``os.mkdir`` plus an ``lstat`` on the ``FileExistsError`` path, never + ``Path.mkdir(exist_ok=True)``: that accepts an existing + symlink-to-directory as "already there", reopening the hole the caller's + check just closed. + + *dir_fd* is a descriptor for the parent, and the ``mkdir`` needs it as much + as the open does — ``mkdir`` follows a symlink at its parent, so a create + against the full path is how a directory gets made, and then written into, + outside the root. + """ + # A parent descriptor is only usable where the ``*at()`` family is, and the + # mkdir below would raise rather than take the floor without this. + if not SUPPORTS_DIR_FD: + dir_fd = None + try: + os.mkdir(_at(directory, dir_fd), 0o755, dir_fd=dir_fd) + except FileExistsError: + # os.stat(follow_symlinks=False) on the descriptor path, os.lstat off it: + # identical results, and the former is the spelling os.supports_dir_fd + # advertises. + if dir_fd is not None: + mode = os.stat(directory.name, dir_fd=dir_fd, follow_symlinks=False).st_mode + else: + mode = os.lstat(directory).st_mode + if stat.S_ISLNK(mode): + raise ValueError("the directory is a symlink") from None + if not stat.S_ISDIR(mode): + raise ValueError("the path is not a directory") from None + return open_directory_nofollow(directory, dir_fd=dir_fd) + + +@contextmanager +def pinned_directory( + directory: Path, *, create: bool = False, dir_fd: int | None = None +) -> Iterator[int | None]: + """ + Holds *directory* pinned for the duration of the block, then releases it. + + Yields what the two openers above return — a descriptor, or ``None`` on the + ``lstat`` floor — so a caller states the platform split once and cannot + forget the ``os.close``. Raises ``ValueError`` for a directory that will not + pin. + + Note which descriptor is which: *dir_fd* pins the parent, and the yielded + one pins *directory* itself. + """ + dir_fd = ( + open_or_create_directory(directory, dir_fd=dir_fd) + if create + else open_directory_nofollow(directory, dir_fd=dir_fd) + ) + try: + yield dir_fd + finally: + if dir_fd is not None: + os.close(dir_fd) + + +class SymlinkRefused(OSError): + """ + Raised instead of removing a symlink found where a real file was expected. + + An ``OSError`` subclass so a caller that only cares that the removal failed + keeps its single ``except``; a distinct type so one that must report *this* + refusal specifically does not have to match on a message. + """ + + +def unlink_file(directory: Path, name: str, *, dir_fd: int | None) -> None: + """ + Removes ``/``, refusing to follow a symlink at *name*. + + Descriptor-relative for the same reason as ``atomic_write``: ``unlink`` + never follows a *trailing* symlink, but it does resolve the directory above + it, so a swapped ```` would turn this into a delete of an + attacker-chosen file. + + Raises ``SymlinkRefused`` when *name* is a symlink — refusing rather than + removing, because a link where the SDK expects its own file means the state + on disk is not what the manifest describes, and that is the caller's to + report rather than to tidy away. + """ + if dir_fd is None: + # No ``*at()`` family: the trailing-symlink check and the unlink are both + # path-based, the per-component floor. + target = directory / name + if target.is_symlink(): + raise SymlinkRefused(f"{name} is a symlink") + target.unlink() + return + + # os.stat(follow_symlinks=False), not os.lstat: identical result, and it is + # the spelling os.supports_dir_fd actually advertises. + probe = os.stat(name, dir_fd=dir_fd, follow_symlinks=False) + if stat.S_ISLNK(probe.st_mode): + raise SymlinkRefused(f"{name} is a symlink") + os.unlink(name, dir_fd=dir_fd) + + +_TEMP_SUFFIX = ".tmp" +"""Suffix on every temp file this module creates.""" + +_TEMP_TOKEN_BYTES = 8 +"""Bytes of randomness in a temp name, as ``secrets.token_hex`` takes them.""" + +_TEMP_TOKEN_PATTERN = re.compile( + # Two producers, one recognizer: ``secrets.token_hex`` on the descriptor + # path, ``tempfile.mkstemp``'s eight ``[a-z0-9_]`` characters on the + # fallback. Used with ``fullmatch``, so both branches are anchored. + rf"[0-9a-f]{{{_TEMP_TOKEN_BYTES * 2}}}|[a-z0-9_]{{8}}" +) + + +def temp_name_prefix(name: str) -> str: + """ + The prefix every temp file for *name* is created under. + + Spelled once because two callers must agree: ``atomic_write`` creates the + name and the orphan sweep recognizes it, and a second copy of the format + would drift from the writer. + """ + return f".{name}." + + +def is_temp_name(candidate: str, name: str) -> bool: + """ + Whether *candidate* is a name this module could have created for *name*. + + Deliberately narrow — prefix, random token and suffix must all match, with + nothing before or after — because the only thing a caller does with a + ``True`` here is delete the file. + """ + prefix = temp_name_prefix(name) + if not candidate.startswith(prefix) or not candidate.endswith(_TEMP_SUFFIX): + return False + token = candidate[len(prefix) : -len(_TEMP_SUFFIX)] + return _TEMP_TOKEN_PATTERN.fullmatch(token) is not None + + +def _mkstemp_at(dir_fd: int, prefix: str) -> tuple[int, str]: + """ + ``tempfile.mkstemp`` for a directory descriptor. + + ``tempfile`` has no ``dir_fd`` form, so this reproduces the part that + matters: ``O_CREAT | O_EXCL`` against an unpredictable name, retried on + collision, so a planted temp path is never written through. + """ + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + for _ in range(tempfile.TMP_MAX): + name = f"{prefix}{secrets.token_hex(_TEMP_TOKEN_BYTES)}{_TEMP_SUFFIX}" + try: + return os.open(name, flags, 0o600, dir_fd=dir_fd), name + except FileExistsError: + continue + raise OSError(errno.EEXIST, "no usable temporary file name was found") + + +def atomic_write( + directory: Path, name: str, data: bytes, *, dir_fd: int | None = None +) -> None: + """ + Writes *data* to ``/`` so no partial file is ever + observable. + + The temp file is created exclusively in the target's *own* directory — one + anywhere else would make the rename cross-device, and so not atomic — + written, fsynced, renamed over the target, and the directory fsynced so the + rename survives a crash. Mode is set explicitly rather than left to the + umask, and the execute bit is never set. + + Given a *dir_fd*, every one of those steps runs relative to that descriptor + and both names are bare filenames; without one, the identical sequence runs + against full paths. + + ``os.replace`` is the one and only rename call site. ``os.rename`` must not + be substituted for it: only ``os.replace`` has defined overwrite semantics + on Windows. + """ + at_fd = dir_fd if dir_fd is not None and SUPPORTS_DIR_FD else None + prefix = temp_name_prefix(name) + target: str | Path + + if at_fd is not None: + fd, temp = _mkstemp_at(at_fd, prefix) + target = name + else: + # mkstemp opens with O_CREAT|O_EXCL, so an existing temp path is never + # reused. + fd, temp = tempfile.mkstemp(dir=directory, prefix=prefix, suffix=_TEMP_SUFFIX) + target = directory / name + + try: + try: + # fchmod, not chmod: operating on the descriptor cannot be redirected + # by a swap of the temp path, and it makes the mode independent of + # the process umask (both creation paths open 0600). + if _SUPPORTS_FCHMOD: + os.fchmod(fd, _FILE_MODE) + elif at_fd is None: + # Windows before 3.13 has no fchmod — and no ``*at()`` family + # either, so ``temp`` is a full path here and the mode goes on it. + # That concedes nothing this platform was getting: it is already + # on the path-based floor, the temp name is unguessable, and the + # only bit Windows takes from a POSIX mode is read-only. A + # platform with descriptors but no fchmod does not exist; were + # there one it would keep the 0600 the exclusive open already set, + # rather than have a bare name chmoded relative to the cwd. + os.chmod(temp, _FILE_MODE) + view = memoryview(data) + while view: + view = view[os.write(fd, view) :] + os.fsync(fd) + finally: + os.close(fd) + if at_fd is not None: + os.replace(temp, target, src_dir_fd=at_fd, dst_dir_fd=at_fd) + else: + os.replace(temp, target) + except BaseException: + try: + if at_fd is not None: + os.unlink(temp, dir_fd=at_fd) + else: + os.unlink(temp) + except OSError: + pass + raise + + if at_fd is not None: + _fsync_directory_fd(at_fd) + else: + _fsync_directory(directory) + + +def atomic_write_in(directory: Path, name: str, data: bytes) -> None: + """ + ``atomic_write`` against a directory this module does not already hold open. + + The descriptor is taken with ``O_NOFOLLOW``, so a directory swapped for a + symlink between the caller's checks and the write fails it rather than + redirecting it. That still leaves the directory's *ancestors* re-resolved on + the open, so prefer ``atomic_write`` with a descriptor the caller already + holds; this is for callers that hold nothing better. + """ + with pinned_directory(directory) as dir_fd: + atomic_write(directory, name, data, dir_fd=dir_fd) + + +def _fsync_directory_fd(fd: int) -> None: + """Best effort — not every platform allows fsync on a directory descriptor.""" + try: + os.fsync(fd) + except OSError: + pass + + +def _fsync_directory(directory: Path) -> None: + """Best effort — not every platform lets a directory be opened for fsync.""" + try: + fd = os.open(directory, os.O_RDONLY) + except OSError: + return + try: + _fsync_directory_fd(fd) + finally: + os.close(fd) diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py new file mode 100644 index 00000000..c29bafcf --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -0,0 +1,344 @@ +""" +Agent Skills — reference discovery and content accessors. + +The public retrieval surface: projecting the skill references a resolved AI +Config carries, and retrieving skill content through a configurable store. + +The feature is three modules, and the dependencies run one way only: + +- ``skills_core.py`` — the store interface, module state, integrity + verification, and store resolution. Shared, and imports neither of the others. +- ``skills.py`` (this file) — ``skill_refs``, the accessors, and + ``InMemorySkillStore``. +- ``skills_fs.py`` — materialization onto disk. It owns the manifest format and + the on-disk filenames; nothing here knows about the filesystem. +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable, Sequence +from typing import Any + +from . import skills_core +from .skills_core import ( + SKILL_OBJECT_KIND, + list_raw_objects, + log_withholding_summary, + newest_by_key, + reference_target, + require_store, + resolve_from_store, + verify_raw_skill, +) +from .types import AiConfigRep, Skill, SkillOutcome, SkillReference +from .types_validation import ( + is_valid_skill_key, + is_valid_skill_version, + skill_key_rejection_reason, +) + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Injection points +# --------------------------------------------------------------------------- +# +# ``init_client`` and ``shutdown`` reach the configured store through these +# names. They delegate to ``skills_core``, which owns the state, so there is +# exactly one store and one emitter no matter which layer reaches for it. +# +# Bound directly to the implementations rather than wrapped: a delegating +# one-liner per name would give every state mutation two definitions to keep in +# agreement. ``_set_emitter_for_testing`` keeps its distinct name because it has +# no production caller. +_set_store = skills_core.set_store +_set_emitter_for_testing = skills_core.set_emitter +_clear_state = skills_core.clear_state + + +class InMemorySkillStore: + """ + A skill store backed by plain dicts. + + For local development, tests, and bring-your-own-content. Holds raw wire + objects verbatim and performs no validation of its own: verification belongs + at the accessor boundary, where it applies to every store equally. + + Several versions of one key coexist here, because they coexist in a real + delivery payload: the newest version of every skill, plus every version a + variation currently pins. ``get_object`` therefore selects on + ``(key, version)``, and ``version=None`` means "the newest held". + + An object whose ``version`` is not an integer >= 1 is still accepted and + still served, under its key alone. Withholding it is verification's job, not + the store's: a store that quietly refused it would make a malformed object + indistinguishable from an absent one, and no integrity signal would be + recorded. + """ + + def __init__(self, objects: dict[str, dict[str, Any]] | None = None) -> None: + self._versions: dict[str, dict[int, dict[str, Any]]] = {} + self._loose: dict[str, dict[str, Any]] = {} + self._listeners: dict[str, list[Callable[[dict[str, Any]], Any]]] = {} + for object_key, raw in (objects or {}).items(): + self._place(object_key, raw) + + def _place(self, fallback_key: str, raw: dict[str, Any]) -> None: + """Files one raw object under its own identity, verbatim.""" + key = raw.get("key") if isinstance(raw, dict) else None + if not isinstance(key, str): + key = fallback_key + version = raw.get("version") if isinstance(raw, dict) else None + if is_valid_skill_version(version): + self._versions.setdefault(key, {})[version] = raw + else: + self._loose[key] = raw + + def put(self, raw: dict[str, Any]) -> None: + """ + Adds or replaces a raw skill object, keyed by its own ``key`` and + ``version`` fields. + + Putting a second version of a key keeps both; putting the same + ``(key, version)`` twice replaces it. + + Notifies every skill-kind listener with the raw object as a single + positional argument. No validation happens here, so a listener sees + exactly what was put, unverified. + """ + key = raw.get("key") + if not isinstance(key, str): + raise ValueError("a raw skill object must carry a string 'key'") + self._place(key, raw) + for listener in self._listeners.get(SKILL_OBJECT_KIND, []): + listener(raw) + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + if kind != SKILL_OBJECT_KIND: + return None + held = self._versions.get(key, {}) + if not held: + # Nothing well-formed is filed under this key, so the version-less + # entry is all there is: serve it, and let verification withhold it + # with a signal rather than have it read as simply absent. A pin that + # misses while well-formed versions do exist is a plain miss, and + # answering it with a leftover malformed object would record an + # integrity failure for a skill whose integrity is not in question. + return self._loose.get(key) + if version is not None: + return held.get(version) + return held[max(held)] + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + """ + Every object held, one entry per ``(key, version)``. + + The dict keys are opaque store-internal identifiers, as ``SkillStore`` + documents. Do not parse them and do not assume one entry per skill key. + """ + if kind != SKILL_OBJECT_KIND: + return {} + out: dict[str, dict[str, Any]] = { + f"{key}:{version}": raw + for key, versions in self._versions.items() + for version, raw in versions.items() + } + out.update(self._loose) + return out + + def add_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Registers *fn* to be called with each raw object ``put`` under *kind*. + + Only ``kind == SKILL_OBJECT_KIND`` is ever notified, because ``put`` only + accepts skill objects; a listener registered under any other kind is + recorded and never fires. + """ + self._listeners.setdefault(kind, []).append(fn) + + def remove_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Unregisters *fn* from *kind*, so a subsequent ``put`` no longer calls it. + + Removes one occurrence: a callable registered twice must be removed twice. + Removing a callable that is not registered is a no-op, not an error, so a + consumer that detaches on close can do so unconditionally. + """ + listeners = self._listeners.get(kind) + if listeners is None: + return + try: + listeners.remove(fn) + except ValueError: + return + + +# --------------------------------------------------------------------------- +# Reference discovery +# --------------------------------------------------------------------------- + + +def skill_refs(config: AiConfigRep | None) -> list[SkillReference]: + """ + Projects a resolved AI Config's ``skills`` array into typed references. + + A pure projection — no network, no client, no store, no telemetry. Returns + ``[]`` when the config carries no skills. Compose it with the accessors for + per-context resolution: ``await get_skills(skill_refs(config))``. + + A config that came through ``parse_ai_config`` never contains an invalid + entry, since parsing fails closed on one. A hand-built dict can, and a + silently shortened projection would let ``write_skills`` prune the dropped + skill's on-disk copy, so every dropped entry is logged. + """ + if not isinstance(config, dict): + return [] + + raw = config.get("skills") + if not isinstance(raw, list): + return [] + + refs: list[SkillReference] = [] + for index, entry in enumerate(raw): + if not isinstance(entry, dict): + logger.warning( + "skills[%d] is not a {key, version} object; it was dropped " + "from the projection", + index, + ) + continue + key = entry.get("key") + version = entry.get("version") + # Branch on the TypeGuard predicate (not the reason string) so the type + # checker narrows ``key`` to ``str`` for the reference below. + if not is_valid_skill_key(key): + logger.warning( + "skills[%d].key %s; it was dropped from the projection", + index, + skill_key_rejection_reason(key), + ) + elif not is_valid_skill_version(version): + logger.warning( + "skills[%d].version must be an integer >= 1; it was dropped " + "from the projection", + index, + ) + else: + refs.append(SkillReference(key=key, version=version)) + return refs + + +# --------------------------------------------------------------------------- +# Content accessors +# --------------------------------------------------------------------------- + + +async def get_skill(key: str, *, version: int | None = None) -> Skill | None: + """ + Retrieves one verified skill by key. + + ``version=None`` means the newest version the store holds; a specific + ``version`` asks the store for that version and returns it only when the + store answers with it. A payload holding several versions of one key + resolves a pin to the pinned version, not to the newest. + Returns ``None`` — never raises — when the skill is missing, the requested + version is not the one held, or verification fails. Raises ``RuntimeError`` + only when no skill store is configured. + + There is no context parameter: skills have no targeting, so the SDK + credentials fully determine availability. Compose per-context resolution + explicitly with ``get_skills(skill_refs(config))``. + """ + return resolve_from_store(require_store(), key, version).skill + + +async def get_skill_result(key: str, *, version: int | None = None) -> SkillOutcome: + """ + Retrieves one verified skill, reporting *why* when there is none. + + Same retrieval, same verification, same telemetry as ``get_skill``; the two + differ only in what they report. ``get_skill`` collapses "no such skill", + "the store raised", "that is not the version held", and "the content failed + integrity verification" to one ``None``. This returns a ``SkillOutcome`` + whose ``reason`` names which of them happened, so a caller can fail closed on + suspected tampering while tolerating a merely-absent skill: + + ```python + outcome = await get_skill_result("pdf-extraction") + if outcome.reason == "integrity_failure": + raise SystemExit(f"refusing to run: {outcome.detail}") + if outcome.skill is not None: + print(outcome.skill.content) + ``` + + ``detail`` is human-readable and safe to surface — it names the key and the + failure mode, never any skill content or filesystem path. Branch on + ``reason``, not on ``detail``. + + Emits nothing of its own: verification has already recorded the log record + and the signal, and a second here would double-count one failure. Raises + ``RuntimeError`` only when no skill store is configured, as ``get_skill`` + does. + """ + resolved = resolve_from_store(require_store(), key, version) + return SkillOutcome( + skill=resolved.skill, reason=resolved.reason, detail=resolved.error + ) + + +async def get_skills(refs: Sequence[SkillReference | str]) -> list[Skill]: + """ + Retrieves a batch of verified skills. + + Accepts a mixed sequence of ``SkillReference`` values and bare key strings, + where a string means "the latest version". Results follow input order for + the skills that were found; entries that are missing, are the wrong version, + or fail verification are omitted rather than returned as placeholders — and + a run that omitted anything logs a count at WARN, so a batch that resolved + nothing is not silent. + """ + if isinstance(refs, str): + # str satisfies Sequence[str], so this type-checks; iterating it would + # silently look up one skill per character. + raise TypeError( + "get_skills takes a sequence of references; pass [key] rather than a " + f"bare string. Got {refs!r}." + ) + + store = require_store() + + requests = list(refs) + skills: list[Skill] = [] + for ref in requests: + key, wanted = reference_target(ref) + skill = resolve_from_store(store, key, wanted).skill + if skill is not None: + skills.append(skill) + log_withholding_summary("requested skills", len(requests), len(skills)) + return skills + + +async def all_skills() -> list[Skill]: + """ + Retrieves every verified skill the store currently holds. + + Skills that fail verification are omitted. Raises ``RuntimeError`` only when + no skill store is configured. + """ + objects, error = list_raw_objects(require_store()) + if error is not None: + return [] + + # One entry per key at its newest version: ``all_objects`` may hold several + # versions of one key, and a list carrying two of them is not a set of skills. + candidates = newest_by_key(objects) + skills: list[Skill] = [] + for _object_key, raw in candidates: + skill = verify_raw_skill(raw) + if skill is not None: + skills.append(skill) + log_withholding_summary("skills held by the store", len(candidates), len(skills)) + return skills diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py new file mode 100644 index 00000000..8da0ba8f --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -0,0 +1,783 @@ +""" +Agent Skills — the internals ``skills`` and ``skills_fs`` both need. + +Package-internal: nothing here is exported from ``launchdarkly_ai_server`` +except the two constants that are public API, and the dependency runs one way — +this module imports neither ``skills`` nor ``skills_fs``. + +It holds the store interface and the configured store, the telemetry emitter, +integrity verification, and store resolution. Each lives here in one copy so the +accessors and the materialization path cannot disagree: about whether a store is +configured, about which signals exist, about what verification accepts, or about +how a raising store is handled. + +**Everything a store hands back is untrusted input**; the transport is not part +of the trust boundary. Key, version, size, and content hash are revalidated here +on every pass, and no value off the wire is echoed into a signal or a log line +without a shape check. The reasoning, and the signal and log-record contracts +these must satisfy, are in ``agents.md`` under *Security posture*, *Telemetry* +and *The integrity-failure log record*. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import re +from dataclasses import dataclass +from typing import Any, Literal, Protocol, get_args + +from .types import Skill, SkillOutcomeReason, SkillReference +from .types_validation import is_valid_skill_key, is_valid_skill_version + +logger = logging.getLogger(__name__) + +SKILL_OBJECT_KIND = "skill" +""" +The kind this SDK asks a store for. + +Internal, and deliberately not exported from the package root: it is what the +accessors pass to ``SkillStore.get_object`` and ``SkillStore.all_objects``, and +a store adapter is free to map it onto whatever its transport uses underneath. +A store that needs to agree on a kind agrees with whatever the SDK hands it, +reached through ``launchdarkly_ai_server.skills_core``. +""" + +MAX_SKILL_CONTENT_BYTES = 10 * 1024 * 1024 +""" +Hard cap on skill content. Legitimately delivered skills are well under this +bound, so anything larger is withheld regardless of whether its hash checks out. + +Set well above LaunchDarkly's own limit on purpose. This is a backstop against +absurd input, not a second enforcement of the real bound, so the headroom lets +that bound grow without this constant moving. + +Not exported from the package root, unlike the on-disk and on-the-wire constants +beside it: it is a local enforcement bound rather than a value a caller needs to +agree with, and a caller pre-flighting "will my skill fit?" against it would be +reading the client's guess rather than the real limit. ``verified_bytes`` +reports the bound in its reason string when it is what withheld content. +""" + +_LANGUAGE = "python" + +_SHA256_HEX = re.compile(r"\A[0-9a-f]{64}\Z") +"""What a legitimate content hash looks like. Anything else is redacted before +it reaches telemetry: ``contentHash`` is untrusted, and a store that put the +skill body there would otherwise leak it into a signal.""" + +_SIGNAL_INTEGRITY_FAILURE = "AgentControl Skill Integrity Failure" +_SIGNAL_MATERIALIZED = "AgentControl Skill Materialized" +_SIGNAL_REVOKED = "AgentControl Skill Revoked Received" + +INTEGRITY_FAILURE_EVENT = "ld.skills.integrity_failure" +""" +Stable event identity for the local integrity-failure log record. + +A compatibility surface, not an implementation detail: this is the string a SIEM +matches on, so it must never be renamed. It appears verbatim in the message +text, not only in ``extra``, because the stdlib's default formatter drops +``extra`` and severity alone cannot discriminate — a raising store logs ERROR +from this module too. +""" + +_ACTION_WITHHELD = "withheld" +"""The only action an integrity failure results in: content is never returned.""" + +IntegrityReasonCode = Literal[ + "not_an_object", + "invalid_key", + "invalid_version", + "missing_content", + "missing_content_hash", + "not_utf8", + "over_size_cap", + "hash_mismatch", +] +""" +The closed ``reason_code`` vocabulary — one token per +``record_integrity_failure`` call site. Stable: a detection rule written against +these tokens keeps working, so adding one is a deliberate edit here rather than +a new string invented at the call site that needed it. +""" + +INTEGRITY_REASON_CODES: frozenset[str] = frozenset(get_args(IntegrityReasonCode)) +"""``IntegrityReasonCode`` as a runtime set, derived rather than restated.""" + +NO_STORE_MESSAGE = ( + "No skill store is configured, so skill content cannot be retrieved. Configure " + 'one with init_client(options={"skillStore": store}) — FDv2SkillStore receives ' + "content from LaunchDarkly, and InMemorySkillStore is available for local " + "development and testing." +) +""" +The first thing a user sees when no store is configured, so it names both stores, +``FDv2SkillStore`` first because it is the answer in production. Callers match on +"skill store"; keep that phrase if the wording changes. +""" + + +# --------------------------------------------------------------------------- +# The store interface +# --------------------------------------------------------------------------- + + +class SkillStore(Protocol): + """ + Structural interface every source of skill content satisfies. + + Duck-typed on purpose, mirroring how the LaunchDarkly client interface works + in this package: pass any object carrying these methods. + + Three members are **optional**, and are deliberately not declared here: a + Protocol member is required for structural compatibility, so declaring them + would reject every store that does not implement them. Each is probed for + instead, and each has a defined behaviour when absent. + + ``is_initialized()`` reports whether the store has received its initial data + — for a delivery transport, whether a payload has arrived yet. Absent means + initialized, which is right for a store populated by hand. It matters + because "the store holds nothing" and "the store has not heard yet" are the + same answer through ``all_objects``, and ``write_skills("*")`` would read the + first as "every skill was revoked": see ``store_is_initialized``. + + ``add_listener(kind, fn)`` lets a delivery transport push updates, and + ``remove_listener(kind, fn)`` lets a consumer such as ``watch_skills`` stop + receiving them; it removes one occurrence of *fn* under *kind* and is a + no-op when *fn* is not registered. A store offering the first should offer + the second: consumers skip detaching when it is absent, so such a store + works at the cost of a listener that lives as long as it does. + + The raw objects a store serves are wire-shaped, with camelCase field names:: + + {"key": "pdf-extraction", "version": 2, "content": "---\\n...", + "contentHash": "9f3a...", "name": "PDF Extraction", "description": "..."} + + **Version is part of the lookup identity, not a filter applied afterwards.** + A delivery payload holds the newest version of every skill *and* every + version any variation currently pins, so two versions of one key coexist + routinely. An interface keyed by key alone cannot express "the one this + variation pinned": it would answer with the newest, and the caller rejecting + it turns a pinned reference into a missing skill. So ``get_object`` takes the + wanted version, and ``version=None`` means "the newest you hold". + + ``all_objects`` returns one entry per *(key, version)* the store holds. Its + dict keys are **opaque store-internal identifiers** — do not parse them, and + do not assume one entry per skill key. Identity is read off each object's own + ``key`` and ``version`` fields, which are revalidated here anyway because + everything a store serves is untrusted. + """ + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: ... + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: ... + + +# --------------------------------------------------------------------------- +# Telemetry +# --------------------------------------------------------------------------- + + +class _TelemetryEmitter(Protocol): + def record(self, signal: str, properties: dict[str, Any]) -> None: ... + + +class _NoOpEmitter: + """ + The default emitter. + + No skills telemetry leaves the process: ``client.track()`` is the wrong + channel for it — that needs an LD context, spends the application's event + volume, and lands in its data export. Signals are still constructed and + recorded, so a transport can be installed behind this interface without + touching a call site. + """ + + def record(self, signal: str, properties: dict[str, Any]) -> None: + return None + + +_NOOP_EMITTER: _TelemetryEmitter = _NoOpEmitter() + +# --------------------------------------------------------------------------- +# Module state +# --------------------------------------------------------------------------- + +_store: SkillStore | None = None +_emitter: _TelemetryEmitter = _NOOP_EMITTER +"""Never ``None``: "no emitter installed" is spelled as the no-op, so ``emit`` +has one code path instead of re-deciding on every signal.""" + + +def set_store(store: Any) -> None: + """ + Replaces the configured store. Reached through ``skills._set_store``, the + documented injection point. + """ + global _store + _store = store + + +def set_emitter(emitter: Any) -> None: + """Replaces the telemetry emitter.""" + global _emitter + _emitter = emitter + + +def clear_state() -> None: + """Drops both the store and the emitter.""" + global _store, _emitter + _store = None + _emitter = _NOOP_EMITTER + + +def get_store() -> SkillStore | None: + """The configured store, or ``None``. The only reader of the global.""" + return _store + + +def require_store() -> SkillStore: + store = get_store() + if store is None: + raise RuntimeError(NO_STORE_MESSAGE) + return store + + +def store_is_initialized(store: Any) -> bool: + """ + Whether *store* has received its initial data. + + ``True`` for a store that does not implement the optional + ``is_initialized()``, since a hand-populated store is never waiting for + anything. Probed rather than declared on the Protocol for the reason + ``SkillStore`` gives: a declared member would reject every store without it. + + This is what keeps ``write_skills("*")`` from reading a store that has not + yet received a payload as an environment whose every skill was revoked. + Retrieval through such a store is reported unavailable, which suppresses + pruning — the same treatment a raising store gets, and for the same reason: + deleting the application's files because content could not be retrieved would + turn a slow boot into data loss. + + A probe that raises counts as not initialized. A store that cannot answer + whether it is ready is not one to authorize deletions on. + """ + probe = getattr(store, "is_initialized", None) + if not callable(probe): + return True + try: + return bool(probe()) + except Exception: + logger.warning( + "The skill store's is_initialized() raised; treating the store as " + "not yet initialized", + exc_info=True, + ) + return False + + +def emit(signal: str, properties: dict[str, Any]) -> None: + """ + Records one signal. Never raises into the calling operation — a broken + emitter must not be able to fail a retrieval or a reconcile. + """ + try: + _emitter.record(signal, properties) + except Exception: + logger.warning("Skills telemetry emitter raised; ignoring", exc_info=True) + + +def record_integrity_failure( + skill_key: str, + reason: str, + *, + reason_code: IntegrityReasonCode, + version: Any = None, + expected_hash: Any = None, + observed_hash: str | None = None, +) -> None: + """ + Records an integrity failure on both surfaces: one local log record, one + product signal. + + Carries hashes and byte counts only — the skill body never appears in a + signal, a log line, or an error message. + + The signal is product telemetry: no-op by default, with a fixed property + set. The log record is the application's own detection path — the only one + that works when telemetry is off — so it also carries the stable event name, + the action taken, the reason, and the machine-parseable ``reason_code``, in + both the message text and ``extra["ld_skills"]``. Neither form alone + survives every handler configuration; ``agents.md`` states the contract. + """ + # Key and expected hash come off the wire, so neither may be echoed + # verbatim: a store could put the skill body in either. Shape-check, then + # redact. Every field added below is a literal or SDK-authored; anything + # added later needs this same treatment. + safe_key = skill_key if is_valid_skill_key(skill_key) else "" + properties: dict[str, Any] = {"skill_key": safe_key, "language": _LANGUAGE} + if is_valid_skill_version(version): + properties["version"] = version + if isinstance(expected_hash, str): + properties["expected_hash"] = ( + expected_hash + if _SHA256_HEX.match(expected_hash) + else "" + ) + if observed_hash is not None: + properties["observed_hash"] = observed_hash + + # Spread the signal's properties rather than rebuilding them, so the record + # cannot drift from the signal on which fields are redacted or omitted. + # Absent optional fields stay absent; the record never carries a null. + record: dict[str, Any] = { + "event": INTEGRITY_FAILURE_EVENT, + "action": _ACTION_WITHHELD, + "reason_code": reason_code, + "reason": reason, + **properties, + } + # ``sort_keys`` is part of the record's format, not cosmetic: it is what + # makes the line stable for a given input. Do not drop it, and do not + # reorder the keys above expecting the output to follow. + logger.error( + "%s %s", + INTEGRITY_FAILURE_EVENT, + json.dumps(record, sort_keys=True, separators=(",", ":")), + extra={"ld_skills": record}, + ) + emit(_SIGNAL_INTEGRITY_FAILURE, properties) + + +def record_materialized( + skill_key: str, content_bytes: int, content_hash: str, reconcile_action: str +) -> None: + """ + Records a materialization. Carries no filesystem path of any kind: the same + reasoning that keeps the skill body out of telemetry keeps the directory + layout out. Paths live in the returned ``ReconcileReport`` instead, which is + API rather than telemetry. + """ + emit( + _SIGNAL_MATERIALIZED, + { + "skill_key": skill_key, + "content_bytes": content_bytes, + "content_hash": content_hash, + "reconcile_action": reconcile_action, + "language": _LANGUAGE, + }, + ) + + +def record_revoked(skill_key: str, version: Any) -> None: + """ + Records a revocation — a prune that removed a formerly managed skill. + + Lives here with the other two recorders rather than at the prune site, so + every signal this SDK can emit is visible in one place and nothing outside + this module touches ``emit``. + """ + # Both fields come off the manifest, which is untrusted — same rule as + # ``record_integrity_failure``: shape-check, then redact, so a hand-edited + # manifest cannot plant an arbitrary string in a signal. + safe_key = skill_key if is_valid_skill_key(skill_key) else "" + properties: dict[str, Any] = { + "skill_key": safe_key, + "removed_from_disk": True, + "language": _LANGUAGE, + } + if is_valid_skill_version(version): + properties["version"] = version + emit(_SIGNAL_REVOKED, properties) + + +# --------------------------------------------------------------------------- +# Integrity verification +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class VerifiedContent: + """Content that passed integrity verification.""" + + encoded: bytes + """The verbatim bytes, exactly as hashed.""" + content_hash: str + """The locally computed sha256 — never the caller's expected value.""" + + +@dataclass(frozen=True) +class VerificationFailure: + """Why content did not pass. The reason is safe to show a caller.""" + + reason: str + + +def verified_bytes( + key: str, content: str | bytes, expected_hash: str, version: int +) -> VerifiedContent | VerificationFailure: + """ + The whole content half of integrity verification: encode, size, hash. + + Accepts either shape content arrives in. A wire-shaped ``str`` is UTF-8 + encoded here, once — the only place that encode happens. ``bytes`` is an + already-verified ``Skill.content`` being re-verified, and is hashed directly. + + Returns the verbatim bytes and their locally computed sha256, or a + human-readable reason, having already recorded the integrity signal so it + does not depend on which caller noticed. The hash handed back is always the + one computed here, never the caller's expected value, which keeps an + untrusted string out of ``Skill``. + + Runs twice per skill by design — at the accessor boundary, and again + immediately before a write, since a ``Skill`` can also be constructed + directly by a caller. Do not optimise the second pass away by carrying the + first one's verdict forward: that puts a "trust the value computed upstream" + branch inside the one function whose job is not to. + """ + if isinstance(content, bytes): + encoded = content + else: + try: + encoded = content.encode("utf-8") + except UnicodeEncodeError: + # json.loads turns a "\ud800" escape into an unpaired surrogate, + # which has no UTF-8 encoding — so there are no bytes the server + # could have hashed. Never reach for errors="surrogatepass": it + # would fabricate bytes that could satisfy the hash comparison. + reason = "content is not encodable as UTF-8" + record_integrity_failure( + key, + reason, + reason_code="not_utf8", + version=version, + expected_hash=expected_hash, + ) + return VerificationFailure(reason) + + if len(encoded) > MAX_SKILL_CONTENT_BYTES: + reason = ( + f"content is {len(encoded)} bytes, over the " + f"{MAX_SKILL_CONTENT_BYTES} byte cap" + ) + record_integrity_failure( + key, + reason, + reason_code="over_size_cap", + version=version, + expected_hash=expected_hash, + ) + return VerificationFailure(reason) + + # sha256, lowercase hex, over the verbatim bytes — no canonicalization and + # no content parsing of any kind anywhere in the integrity path. + observed_hash = hashlib.sha256(encoded).hexdigest() + if observed_hash != expected_hash: + record_integrity_failure( + key, + "content hash mismatch", + reason_code="hash_mismatch", + version=version, + expected_hash=expected_hash, + observed_hash=observed_hash, + ) + return VerificationFailure("content hash mismatch") + + return VerifiedContent(encoded=encoded, content_hash=observed_hash) + + +def verify_raw_skill(raw: Any) -> Skill | None: + """ + Turns one untrusted raw store object into a ``Skill``, or withholds it. + + On any failure the skill is treated as missing, the integrity signal is + recorded, and an error is logged. No unverified content is ever returned to + user code. + """ + if not isinstance(raw, dict): + record_integrity_failure( + "", + "raw skill object is not an object", + reason_code="not_an_object", + ) + return None + + key = raw.get("key") + if not is_valid_skill_key(key): + record_integrity_failure( + key if isinstance(key, str) else "", + "key is not a valid skill key", + reason_code="invalid_key", + ) + return None + + version = raw.get("version") + if not is_valid_skill_version(version): + record_integrity_failure( + key, "version is not an integer >= 1", reason_code="invalid_version" + ) + return None + + content = raw.get("content") + if not isinstance(content, str): + record_integrity_failure( + key, + "content is missing or not a string", + reason_code="missing_content", + version=version, + ) + return None + + expected_hash = raw.get("contentHash") + if not isinstance(expected_hash, str): + record_integrity_failure( + key, + "contentHash is missing or not a string", + reason_code="missing_content_hash", + version=version, + ) + return None + + verified = verified_bytes(key, content, expected_hash, version) + if isinstance(verified, VerificationFailure): + return None + + name = raw.get("name") + description = raw.get("description") + return Skill( + key=key, + version=version, + content=verified.encoded, + content_hash=verified.content_hash, + name=name if isinstance(name, str) else None, + description=description if isinstance(description, str) else None, + ) + + +def log_withholding_summary(subject: str, requested: int, resolved: int) -> None: + """ + One WARN per run when content was withheld, naming the counts. + + Every individual withholding already records an integrity signal and an error + line, but a caller reading logs at WARN sees neither — and the case that + matters most is a run where *nothing* verified, because the result is then an + empty list indistinguishable from "this project has no skills". + + Called once per batch, not once per skill, so a large withholding run does + not itself become the noise. + """ + withheld = requested - resolved + if withheld <= 0: + return + if resolved == 0: + logger.warning( + "All %d %s were withheld and no skill content is available. Every " + "object failed verification — check that the delivered objects carry " + "a contentHash matching the sha256 of their content.", + requested, + subject, + ) + return + logger.warning( + "%d of %d %s were withheld and are unavailable; see the preceding errors " + "for the per-skill reason.", + withheld, + requested, + subject, + ) + + +def store_raised(exc: Exception) -> str: + """The one wording for "the store could not answer", used by every path.""" + return f"the skill store raised {type(exc).__name__}: {exc}" + + +def list_raw_objects( + store: SkillStore, +) -> tuple[dict[str, dict[str, Any]], str | None]: + """ + Every raw object the store holds, or the reason it could not answer. + + One entry per *(key, version)*, under keys that are opaque to this SDK — see + ``SkillStore``. Callers that need one skill per key have to collapse the + result themselves; ``newest_by_key`` does it. + + Returns the reason rather than raising, because both callers need the + distinction between "no skills" and "the store is broken", worded + identically. + + An answer that is not a mapping is a broken store, on the same footing as + one that raised — **not** an empty one. Collapsing it to ``{}`` would make a + store that served nothing usable indistinguishable from a store that holds + no skills, which reads downstream as "every skill was revoked". + """ + try: + objects = store.all_objects(SKILL_OBJECT_KIND) + except Exception as exc: + logger.error("Skill store raised while listing skills", exc_info=True) + return {}, store_raised(exc) + if not isinstance(objects, dict): + logger.error( + "Skill store listed skills as %s rather than an object", + type(objects).__name__, + ) + return {}, ( + f"the skill store listed skills as {type(objects).__name__} " + "rather than an object" + ) + return objects, None + + +def newest_by_key(objects: dict[str, dict[str, Any]]) -> list[tuple[str, Any]]: + """ + One raw object per skill key — the highest version of each, paired with the + store key it was served under. + + ``all_objects`` may hold several versions of one key, and both whole-store + callers want one skill per key: ``all_skills``, because a list holding two + versions of one key is not a set of skills, and the ``"*"`` reconcile, + because ``//SKILL.md`` is a single path. The store key is carried + through because the reconcile attributes a failure to it when the object's + own key is unusable. + + An object too malformed to carry a usable key and version is **kept**, so + verification is what withholds it: dropped silently it would fall out of the + requested set, and prune would then delete the last known-good copy on disk. + The exception is an object whose key resolved from another version anyway — + that key is already in the requested set, so keeping the malformed sibling + would only report a withholding for a key that resolved. + """ + best: dict[str, tuple[str, Any]] = {} + unusable: list[tuple[str, Any]] = [] + for object_key, raw in objects.items(): + skill_key = raw.get("key") if isinstance(raw, dict) else None + version = raw.get("version") if isinstance(raw, dict) else None + if not is_valid_skill_key(skill_key) or not is_valid_skill_version(version): + unusable.append((object_key, raw)) + continue + held = best.get(skill_key) + if held is None or version > held[1]["version"]: + best[skill_key] = (object_key, raw) + withheld = [ + (object_key, raw) + for object_key, raw in unusable + # ``is_valid_skill_key`` first: an unhashable key cannot be looked up. + if not ( + is_valid_skill_key(raw.get("key") if isinstance(raw, dict) else None) + and raw["key"] in best + ) + ] + return list(best.values()) + withheld + + +# --------------------------------------------------------------------------- +# Resolution internals — shared with the materialization path +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Resolution: + """One key resolved against a store: the skill, or why there is none.""" + + reason: SkillOutcomeReason + """ + Which of the five public outcomes this resolution is. + + Declared first and **without a default**, so every construction site has to + state which public token it maps to rather than inheriting one. + + ``get_skill_result`` publishes this value directly. It is carried as a token + rather than recovered from ``error``, because pattern-matching prose for a + decision a caller fails closed on is the fragility the typed outcome removes. + + Distinct from ``unavailable``, which answers a different question (may prune + run?), but the two can only disagree by a bug: ``unavailable`` is ``True`` in + exactly the ``store_unavailable`` case. + """ + skill: Skill | None = None + error: str | None = None + unavailable: bool = False + """ + ``True`` when the *store* could not answer — it raised — rather than when it + answered "no". Only the former suppresses pruning: deleting managed files + because a lookup failed would turn an outage into data loss. + """ + + +def resolve_from_store( + store: SkillStore, key: str, wanted_version: int | None +) -> Resolution: + """ + Fetches one key and verifies it — the sequence the accessors and the + materialization path share, written once so the two cannot drift apart on + the policy for a raising store. + + ``wanted_version`` goes *into* the lookup, because a store may hold several + versions of one key and only it can pick between them; ``None`` asks for the + newest. The equality check afterwards is kept as a **defense**, not as the + selection mechanism: the store is untrusted, so an answer that is not the + version that was asked for is withheld rather than returned. The key is + checked the same way and for the same reason: identity is read off the + object itself, so an answer served under a different key would otherwise be + returned under the caller's key while carrying its own. + """ + try: + raw = store.get_object(SKILL_OBJECT_KIND, key, wanted_version) + except Exception as exc: + logger.error("Skill store raised while retrieving '%s'", key, exc_info=True) + return Resolution( + reason="store_unavailable", + error=store_raised(exc), + unavailable=True, + ) + + if not isinstance(raw, dict): + return Resolution( + reason="absent", + error=f"skill '{key}' is not available from the configured skill store", + ) + + skill = verify_raw_skill(raw) + if skill is None: + return Resolution( + reason="integrity_failure", + error=f"skill '{key}' failed integrity verification and was withheld", + ) + if skill.key != key: + # ``integrity_failure`` rather than ``absent``: content was delivered + # and its identity did not verify, which is the one token a caller is + # expected to fail closed on. Reporting ``absent`` would file a store + # that substitutes one skill for another in the bucket the same caller + # is invited to tolerate. It is not ``wrong_version`` either — that + # token names a version mismatch specifically, and there is deliberately + # no ``wrong_key`` to parallel it. + return Resolution( + reason="integrity_failure", + error=( + f"skill '{key}' is not available: the store answered under " + f"key '{skill.key}'" + ), + ) + if wanted_version is not None and skill.version != wanted_version: + return Resolution( + reason="wrong_version", + error=( + f"skill '{key}' version {wanted_version} is not available " + f"(the store holds version {skill.version})" + ), + ) + return Resolution(reason="ok", skill=skill) + + +def reference_target(item: SkillReference | str) -> tuple[str, int | None]: + """Normalises a reference-or-key into ``(key, wanted version)``. + + A bare string means "the latest version the store holds". + """ + if isinstance(item, str): + return item, None + return item.key, item.version diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py new file mode 100644 index 00000000..c0435d19 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -0,0 +1,1829 @@ +""" +Agent Skills — the FDv2 delivery transport. + +The store implementation that talks to LaunchDarkly. It sits *below* the +``SkillStore`` interface: it produces raw wire objects in the shape +``skills_core`` documents, and everything above — the accessors, integrity +verification, the ``Skill`` dataclass, materialization — is unaware of it. + +Layering:: + + launchdarkly_ai_server + └─ SkillStore protocol (skills_core) ── the interface accessors call + └─ FDv2SkillStore (this module) ── deserialise, hold, serve + └─ LaunchDarkly's SDK-facing FDv2 channel + GET /sdk/poll, GET /sdk/stream, authenticated with the + environment's server-side SDK key + +Dependencies run one way: this module imports ``skills_core`` for the +interface's kind constant and nothing else from the feature, and nothing in the +feature imports it. It uses only the standard library, so it adds no dependency +to a package whose sole runtime dependency is ``opentelemetry-api``. + +Three things this module does *not* do, on purpose: + +- **It does not verify content.** Verification lives at the accessor boundary in + ``skills_core`` so that it applies to every store equally, including a + a store the application supplies itself. +- **It does not work around a missing ``contentHash``.** A hashless object is + held verbatim and *withheld* by verification with ``missing_content_hash``. + This module's job is to make that outcome loud — see ``StoreDiagnostics``. +- **It does not evaluate anything.** Flag and segment objects that share the + connection are skipped and counted, nothing more. + +Why the skill's version is read from the object's ``key`` and never from +``version``, why changes commit at ``payload-transferred``, and why there is one +network timeout are each explained where the code does it. +""" + +from __future__ import annotations + +import json +import logging +import math +import random +import re +import socket +import threading +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any, Literal + +from .skills_core import SKILL_OBJECT_KIND +from .types_validation import is_valid_skill_version + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# The wire contract +# --------------------------------------------------------------------------- + +FDV2_OBJECT_KIND = "skill" +""" +The FDv2 ``kind`` skills are delivered under. + +Object kinds on the SDK-facing channel are open strings: the agent-skill payload +is classified ``generic`` and every object in it carries the kind its producer +registered, which for skills is the bare category name. Delivery lower-cases the +kind, so an exact comparison is the whole test. The value happens to equal +``skills_core.SKILL_OBJECT_KIND``; they remain separate constants, because one is +a wire value LaunchDarkly owns and the other is what this SDK asks a store for. +""" + +FDV2_KEY_DELIMITER = ":" +""" +What separates a skill's key from its version inside the object's wire ``key``. + +A generic object is identified on the wire as ``:`` — the skill's +own key, one delimiter, the skill's own version — because each version of a +skill is a distinct object in the payload. Delivery forbids the delimiter inside +a registered category and skill keys cannot contain it, so a well-formed wire key +has exactly one. +""" + +DEFAULT_BASE_URI = "https://sdk.launchdarkly.com" +"""Where the SDK-facing FDv2 endpoints live. Overridable for Federal and private +instances.""" + +POLL_PATH = "/sdk/poll" +STREAM_PATH = "/sdk/stream" + +DEFAULT_POLL_TIMEOUT = 10.0 +"""Default ``read_timeout`` in ``"poll"`` mode: the bound on one whole request.""" + +DEFAULT_STREAM_READ_TIMEOUT = 300.0 +"""Default ``read_timeout`` in ``"stream"`` mode: the longest gap tolerated +between two reads. LaunchDarkly's heartbeats arrive well inside this.""" + +MAX_RESPONSE_BYTES = 64 * 1024 * 1024 +"""The most the transport will hold in memory from one response. + +A memory backstop, not a content limit. Verification caps each skill's content +at 10 MiB in ``skills_core``; this bound is on what one poll body, or one +streamed event, may accumulate *before* verification can run against it, and +it is deliberately far above any payload LaunchDarkly legitimately serves. The +two bound different things and are set independently. A body or event that +crosses it is dropped unread as a recoverable transport failure: nothing from +it is committed, the store keeps serving what it last held, and the delivery +loop retries on its usual backoff.""" + +_READ_CHUNK_BYTES = 64 * 1024 +"""How much of a poll body is read per call while checking it against the bound.""" + +_EVENT_SERVER_INTENT = "server-intent" +_EVENT_PUT_OBJECT = "put-object" +_EVENT_DELETE_OBJECT = "delete-object" +_EVENT_PAYLOAD_TRANSFERRED = "payload-transferred" +_EVENT_HEARTBEAT = "heart-beat" +_EVENT_GOODBYE = "goodbye" +_EVENT_ERROR = "error" + +_INTENT_TRANSFER_FULL = "xfer-full" +_INTENT_TRANSFER_CHANGES = "xfer-changes" +_INTENT_TRANSFER_NONE = "none" + +_ENVELOPE_FIELDS = ("contentType", "content", "contentHash", "name", "description") +""" +The skill object envelope's fields, copied through verbatim. Nothing is coerced +or defaulted: a transport that filled in a missing field would be forging the +very thing verification exists to check. +""" + +_PAYLOAD_SELECTOR = re.compile(r"\(p:([^:()]+):\d+\)") +""" +The payload identity inside a transfer's selector, ``(p::)``. + +The selector is the only place a completed transfer names its own payload: +``put-object``, ``delete-object`` and ``payload-transferred`` carry no payload id +of their own. ``_ProtocolReader`` reads it as a fallback for an intent that named +no ``id``. +""" + +Mode = Literal["stream", "poll"] + +_MOBILE_KEY_PREFIX = "mob-" +_SERVER_KEY_PREFIX = "sdk-" +_CLIENT_SIDE_ID = re.compile(r"\A[0-9a-f]{20,}\Z") +"""A client-side environment ID: bare lowercase hex. Server-side and mobile keys +both carry a prefix, so this shape is unambiguous rather than heuristic.""" + + +# --------------------------------------------------------------------------- +# Server-side only +# --------------------------------------------------------------------------- + + +def _require_server_side_credential(sdk_key: str) -> None: + """ + Refuses a mobile key or a client-side environment ID. + + Skill content is customer-confidential, and payload assignment is shared + across credential types, so a client-side credential may well *succeed* + against these endpoints — which is why this refuses rather than relying on + the server to. Raises rather than logs: a store built on the wrong + credential should not exist. + """ + if not isinstance(sdk_key, str) or not sdk_key.strip(): + raise ValueError( + "FDv2SkillStore requires a LaunchDarkly server-side SDK key " + "(sdk-...); none was given." + ) + key = sdk_key.strip() + if key.startswith(_MOBILE_KEY_PREFIX): + raise ValueError( + "FDv2SkillStore was given a mobile key (mob-...). Agent Skills are a " + "server-side feature: skill content is customer-confidential and is " + "never delivered to a mobile or client-side process. Use the " + "environment's server-side SDK key (sdk-...)." + ) + if _CLIENT_SIDE_ID.match(key): + raise ValueError( + "FDv2SkillStore was given what looks like a client-side environment " + "ID. Agent Skills are a server-side feature: skill content is " + "customer-confidential and is never delivered to a client-side " + "process. Use the environment's server-side SDK key (sdk-...)." + ) + if not key.startswith(_SERVER_KEY_PREFIX): + # Not rejected: private instances and test doubles issue credentials + # without the public prefix. Only the two unambiguous shapes above are. + logger.warning( + "The credential given to FDv2SkillStore does not look like a " + "LaunchDarkly server-side SDK key (sdk-...). Skills are delivered " + "only to server-side credentials; if this is a client-side or mobile " + "credential the connection will be rejected or will deliver nothing." + ) + + +_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"}) +"""The only hosts a plain ``http://`` base URI may name: a local test double.""" + + +def _require_https_base_uri(base_uri: str) -> None: + """ + Refuses a base URI that would send the SDK key in cleartext. + + Every request carries the environment's server-side SDK key in + ``Authorization``, so the transport is ``https://`` only. The one exception + is ``http://`` to a loopback host (``localhost``, ``127.0.0.1``, ``::1``), + which never leaves the machine and is what a local test double listens on. + Raises rather than logs, for the same reason the credential check does: a + store that would leak its key should not exist. + """ + if not isinstance(base_uri, str) or not base_uri.strip(): + raise ValueError( + "FDv2SkillStore requires an https:// base URI; none was given." + ) + parts = urllib.parse.urlsplit(base_uri.strip()) + if parts.scheme == "https" and parts.hostname: + return + if parts.scheme == "http" and parts.hostname in _LOOPBACK_HOSTS: + return + if parts.scheme == "http": + raise ValueError( + f"FDv2SkillStore refuses base_uri {base_uri!r}: a plain http:// URI " + "would send the server-side SDK key in cleartext. Use https:// " + "(the default is https://sdk.launchdarkly.com). Plain http:// is " + "allowed only for a loopback host (localhost, 127.0.0.1, ::1) " + "serving a local test double." + ) + raise ValueError( + f"FDv2SkillStore refuses base_uri {base_uri!r}: expected an https:// URI " + "with a host, such as https://sdk.launchdarkly.com." + ) + + +# --------------------------------------------------------------------------- +# Diagnostics +# --------------------------------------------------------------------------- + + +@dataclass +class StoreDiagnostics: + """ + What the transport has seen. Read-only from a caller's perspective. + + Not part of the ``SkillStore`` interface. It exists because "this environment + has no skills" and "every skill was withheld" are easy to mistake for each + other, and a counter is easier to assert on than a log line. + """ + + payloads_transferred: int = 0 + """Completed ``payload-transferred`` commits since the store started.""" + skill_objects_received: int = 0 + """``put-object`` events identified as skills, across all payloads.""" + objects_ignored: int = 0 + """Objects skipped because they were not skills: flags, segments, and any + future kind. Skipping is the contract, not a failure.""" + objects_revoked: int = 0 + """``delete-object`` events applied to skills.""" + payloads_ignored: int = 0 + """ + Transfers not applied because they completed a payload other than the one + skills arrive on. Zero while delivery sends one payload per connection. + """ + hashless_objects: int = 0 + """ + Skill objects whose envelope carried no ``contentHash``. + + **Nonzero means skills are being withheld**: verification withholds every one + of these with ``missing_content_hash``. + """ + connection_failures: int = 0 + """Recoverable transport failures since the last successful transfer.""" + last_error: str | None = None + """The most recent transport error, if any. Human-readable; do not parse.""" + + +# --------------------------------------------------------------------------- +# Deserialisation — where the skill's version lives in the key, not in version +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _Tombstone: + """A ``delete-object`` narrowed to the identity it revokes.""" + + key: str + object_version: int | None + + +def _is_skill_event(data: Any) -> bool: + """ + Whether one ``put-object`` / ``delete-object`` payload is a skill. + + The kind alone decides it. Every other kind is ignored, not rejected, + because flag and segment objects share the connection and erroring on them + would turn a normal payload into a reconnect loop. + """ + if not isinstance(data, dict): + return False + return data.get("kind") == FDV2_OBJECT_KIND + + +@dataclass(frozen=True) +class _WireIdentity: + """A skill object's wire ``key``, split into the skill's key and version.""" + + key: str + version: Any + """``int`` when the wire carried one; the offending text when it did not; + absent (``_NO_VERSION``) when the wire key had no delimiter at all.""" + + +_NO_VERSION = object() + + +def _split_wire_key(wire_key: Any) -> _WireIdentity | None: + """ + Reads ``:`` off one object's wire ``key``. + + Lenient where leniency keeps the object diagnosable and strict only where + there is nothing to diagnose: + + - No delimiter: the whole wire key is the skill key and there is no version, + so the object is held version-less and verification reports + ``invalid_version`` under a key the caller can recognise. + - A version that is not a run of digits (``"pdf:latest"``, ``"pdf:"``, + ``"a:1:2"``): the text is carried through *as the version*, for the same + reason — the caller learns that ``pdf`` arrived broken, not that it is + absent. + - An empty key before the delimiter (``":3"``): there is no identity to hold + it under, so ``None``, and the caller drops it. + + Leading zeros are accepted (``"pdf:03"`` is version 3) since ``int`` is the + identity a reference pins, not the spelling. + """ + if not isinstance(wire_key, str) or not wire_key: + return None + key, delimiter, version_text = wire_key.partition(FDV2_KEY_DELIMITER) + if not key: + return None + if not delimiter: + return _WireIdentity(key=key, version=_NO_VERSION) + if version_text.isascii() and version_text.isdigit(): + return _WireIdentity(key=key, version=int(version_text)) + return _WireIdentity(key=key, version=version_text) + + +def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: + """ + Translates one FDv2 skill ``put-object`` into the raw object shape the + ``SkillStore`` interface defines. + + **The one translation this adapter must get right:** + + wire ``key`` → stored ``key`` and ``version`` (split on ``:``) + wire ``version`` → dropped (the *payload* version) + + Each version of a skill is its own object on the wire, identified as + ``:``; that version is what a ``{key, version}`` reference + pins. The event's ``version`` field is the version of the payload the object + arrived in and moves whenever anything in the environment moves. Confusing + them fails silently: the object verifies and the caller gets content under a + version number that means nothing. + + Returns ``None`` only when the wire ``key`` carries no skill key at all, + since such an object has no identity to store it under. Every other defect + is carried through so that verification withholds it with a reason code + rather than the transport dropping it into indistinguishable absence. + """ + identity = _split_wire_key(data.get("key")) + if identity is None: + logger.warning( + "An FDv2 skill put-object carried no usable 'key' (%r) and could not " + "be stored under any identity; it was dropped.", + data.get("key"), + ) + return None + + raw: dict[str, Any] = {"key": identity.key} + + # Absent stays absent and malformed stays malformed, so verification sees + # what arrived (as `invalid_version`) rather than something invented here. + if identity.version is not _NO_VERSION: + raw["version"] = identity.version + + envelope = data.get("object") + if isinstance(envelope, dict): + for wire_field in _ENVELOPE_FIELDS: + if wire_field in envelope: + raw[wire_field] = envelope[wire_field] + return raw + + +def _payload_id_of(intent: Any) -> str | None: + """The payload id one payload intent names, when it names a usable one.""" + if not isinstance(intent, dict): + return None + value = intent.get("id") + return value if isinstance(value, str) and value else None + + +def _payload_id_from_selector(state: Any) -> str | None: + """The payload id inside a transfer's selector, when it carries one.""" + if not isinstance(state, str): + return None + match = _PAYLOAD_SELECTOR.search(state) + return match.group(1) if match else None + + +def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: + """ + Narrows one FDv2 skill ``delete-object`` to the identity it revokes, reading + the wire ``key`` the same way a put does. + + An ``object_version`` of ``None`` means the delete named no usable version + and is read as "revoke every version of this key". That is the safe + direction: the alternative is continuing to serve content LaunchDarkly has + withdrawn. It also removes whatever a malformed put of the same wire key + left held, since that was stored version-less under the same skill key. + """ + identity = _split_wire_key(data.get("key")) + if identity is None: + logger.warning( + "An FDv2 skill delete-object carried no usable 'key' (%r); it was ignored.", + data.get("key"), + ) + return None + return _Tombstone( + key=identity.key, + object_version=identity.version + if is_valid_skill_version(identity.version) + else None, + ) + + +# --------------------------------------------------------------------------- +# The held object set +# --------------------------------------------------------------------------- + + +class _SkillObjectSet: + """ + Raw skill objects held in memory, keyed by ``(key, version)``. + + Lookup semantics are identical to ``InMemorySkillStore``'s, down to when a + version-less entry is reachable, so the store an application configures + cannot change how a pinned reference resolves. Reimplemented rather than + inherited because the transport needs ``delete`` and the atomic + ``replace_with`` a full transfer requires. + + An object too malformed to carry a usable version is still held, under its + key alone, so verification withholds it with a signal rather than the + transport dropping it. + """ + + def __init__(self) -> None: + self._versions: dict[str, dict[int, dict[str, Any]]] = {} + self._loose: dict[str, dict[str, Any]] = {} + + def put(self, raw: dict[str, Any]) -> None: + key = raw["key"] + version = raw.get("version") + if is_valid_skill_version(version): + self._versions.setdefault(key, {})[version] = raw + else: + self._loose[key] = raw + + def delete(self, tombstone: _Tombstone) -> list[dict[str, Any]]: + """Removes what *tombstone* revokes; returns the raw objects that went away.""" + removed: list[dict[str, Any]] = [] + if tombstone.object_version is None: + held = self._versions.pop(tombstone.key, {}) + removed.extend(held.values()) + loose = self._loose.pop(tombstone.key, None) + if loose is not None: + removed.append(loose) + return removed + + held = self._versions.get(tombstone.key, {}) + gone = held.pop(tombstone.object_version, None) + if gone is not None: + removed.append(gone) + if not held: + self._versions.pop(tombstone.key, None) + return removed + + def get(self, key: str, version: int | None) -> dict[str, Any] | None: + held = self._versions.get(key, {}) + if not held: + # Nothing well-formed is filed under this key, so the version-less + # entry is all there is: serve it, and let verification withhold it + # with a signal rather than have it read as simply absent. A pin + # that misses while well-formed versions do exist is a plain miss, + # and answering it with a leftover malformed object would record an + # integrity failure for a skill whose integrity is not in question. + return self._loose.get(key) + if version is not None: + return held.get(version) + return held[max(held)] + + def snapshot(self) -> dict[str, dict[str, Any]]: + """One entry per ``(key, version)``, under keys opaque to the SDK.""" + out: dict[str, dict[str, Any]] = { + f"{key}:{version}": raw + for key, versions in self._versions.items() + for version, raw in versions.items() + } + out.update(self._loose) + return out + + def all_raw(self) -> list[dict[str, Any]]: + return list(self.snapshot().values()) + + def replace_with(self, other: _SkillObjectSet) -> None: + """Adopts *other*'s contents wholesale — how a full transfer commits.""" + self._versions = other._versions + self._loose = other._loose + + def copy(self) -> _SkillObjectSet: + clone = _SkillObjectSet() + clone._versions = {key: dict(v) for key, v in self._versions.items()} + clone._loose = dict(self._loose) + return clone + + def __len__(self) -> int: + return sum(len(v) for v in self._versions.values()) + len(self._loose) + + +# --------------------------------------------------------------------------- +# The protocol state machine — pure, no I/O +# --------------------------------------------------------------------------- + + +@dataclass +class _TransferOutcome: + """What one event did. Aggregated by the caller; nothing here does I/O.""" + + committed: bool = False + changes: list[dict[str, Any]] = field(default_factory=list) + basis: str | None = None + fatal: str | None = None + disconnect: str | None = None + up_to_date: bool = False + """ + The server said the content held is current and it has nothing to transfer. + + A complete answer that commits nothing, which is what a 304 is to a poll. + The delivery loop counts it as a healthy connection; see + ``FDv2SkillStore._apply``. + """ + + +class _ProtocolReader: + """ + Applies FDv2 events to an object set. Pure — no sockets, no threads, no + clock — so every wire case is testable without a server. + + **Changes are buffered and committed at ``payload-transferred``.** A payload + version is the unit of consistency: applying half of one would publish a + state the server never described, and on a full transfer would briefly empty + the store. Listeners therefore fire once per commit, not once per object. + + **The first payload intent is read, and is taken to be the skill payload.** + Delivery provides one payload per credential and the protocol requires a + client to ignore all but the first intent. Should that ever widen, an + ``xfer-full`` for another payload would publish the skill set empty — with + pruning on, the difference between a reconcile and deleting the + application's files. So this layer learns which payload skills arrive on and + declines to apply a transfer of any other. The residual case is the first + transfer of a connection, where there is nothing to compare against yet. + """ + + def __init__(self, committed: _SkillObjectSet) -> None: + self._committed = committed + self._intent: str | None = None + self._pending: _SkillObjectSet | None = None + self._changes: list[dict[str, Any]] = [] + self.diagnostics = StoreDiagnostics() + # Identities already reported by ``_warn_hashless``. Per reader, so a + # recreated store reports again and two stores never quieten each other. + self._warned_hashless: set[tuple[str, Any]] = set() + # The payload the current intent describes, and the payload skills have + # actually arrived on. One payload per connection makes these the same + # payload; the class docstring says why they are kept apart regardless. + self._intent_payload_id: str | None = None + self._skill_payload_id: str | None = None + self._skills_in_payload = 0 + self._warned_multiple_payloads = False + self._warned_foreign_payload = False + + # -- events ------------------------------------------------------------ + + def handle(self, name: str, data: Any) -> _TransferOutcome: + """Routes one event. Unknown event names are ignored, by contract.""" + if name == _EVENT_SERVER_INTENT: + return self._server_intent(data) + if name == _EVENT_PUT_OBJECT: + return self._put_object(data) + if name == _EVENT_DELETE_OBJECT: + return self._delete_object(data) + if name == _EVENT_PAYLOAD_TRANSFERRED: + return self._payload_transferred(data) + if name == _EVENT_ERROR: + return self._error(data) + if name == _EVENT_GOODBYE: + return self._goodbye(data) + if name == _EVENT_HEARTBEAT: + return _TransferOutcome() + logger.debug("Ignoring unknown FDv2 event '%s'", name) + return _TransferOutcome() + + def _server_intent(self, data: Any) -> _TransferOutcome: + payloads = data.get("payloads") if isinstance(data, dict) else None + if not isinstance(payloads, list) or not payloads: + return _TransferOutcome( + disconnect="server-intent carried no payload description" + ) + if len(payloads) > 1: + self._warn_multiple_payloads(payloads) + # The first payload only, as the protocol requires. + first = payloads[0] + intent = first.get("intentCode") if isinstance(first, dict) else None + self._intent = intent + self._intent_payload_id = _payload_id_of(first) + self._changes = [] + self._skills_in_payload = 0 + if intent == _INTENT_TRANSFER_FULL: + # Built alongside the live set rather than in place, so an + # interrupted transfer leaves last known good intact. + self._pending = _SkillObjectSet() + elif intent == _INTENT_TRANSFER_CHANGES: + self._pending = self._committed.copy() + else: + if intent != _INTENT_TRANSFER_NONE: + logger.debug("Ignoring FDv2 server-intent with intentCode %r", intent) + self._pending = None + # ``none`` is a complete answer that carries nothing. An intent this + # module does not recognise is not an answer at all, so only the + # former reports itself up to date. + return _TransferOutcome(up_to_date=intent == _INTENT_TRANSFER_NONE) + return _TransferOutcome() + + def _target_for(self, data: Any) -> _SkillObjectSet | None: + """ + The pending set a skill object event applies to, or ``None`` when the + event is not a skill or the current intent carries no objects. + + An object arriving with no ``server-intent`` at all is treated as a + delta against what is held rather than dropped. + """ + if not _is_skill_event(data): + self.diagnostics.objects_ignored += 1 + return None + if self._pending is None: + if self._intent is None: + self._intent = _INTENT_TRANSFER_CHANGES + if self._intent not in (_INTENT_TRANSFER_FULL, _INTENT_TRANSFER_CHANGES): + return None + self._pending = self._committed.copy() + return self._pending + + def _put_object(self, data: Any) -> _TransferOutcome: + target = self._target_for(data) + if target is None: + return _TransferOutcome() + raw = _store_object_from_put(data) + if raw is None: + return _TransferOutcome() + target.put(raw) + self._changes.append(raw) + self.diagnostics.skill_objects_received += 1 + self._skills_in_payload += 1 + if not isinstance(raw.get("contentHash"), str): + self.diagnostics.hashless_objects += 1 + self._warn_hashless(raw) + return _TransferOutcome() + + def _delete_object(self, data: Any) -> _TransferOutcome: + target = self._target_for(data) + if target is None: + return _TransferOutcome() + tombstone = _tombstone_from_delete(data) + if tombstone is None: + return _TransferOutcome() + target.delete(tombstone) + self.diagnostics.objects_revoked += 1 + # A revocation identifies the skill payload just as a put does. + self._skills_in_payload += 1 + # A tombstone carries identity and no content; see + # ``FDv2SkillStore.add_listener`` for what listeners should expect. + self._changes.append( + {"key": tombstone.key, "version": tombstone.object_version} + ) + return _TransferOutcome() + + def _payload_transferred(self, data: Any) -> _TransferOutcome: + state = data.get("state") if isinstance(data, dict) else None + version = data.get("version") if isinstance(data, dict) else None + payload_id = self._intent_payload_id or _payload_id_from_selector(state) + if self._pending is not None and self._is_foreign_payload(payload_id): + self._warn_foreign_payload(payload_id) + self.diagnostics.payloads_ignored += 1 + self._changes = [] + elif self._pending is not None: + self._committed.replace_with(self._pending) + _warn_if_nothing_can_verify(self._committed) + if self._skills_in_payload and payload_id is not None: + # Learnt, not configured: nothing below the interface is told + # which payload is which, so the payload that carried a skill + # put or revocation is the payload skills arrive on. + self._skill_payload_id = payload_id + self._pending = None + self._intent = None + self._intent_payload_id = None + self._skills_in_payload = 0 + changes = self._changes + self._changes = [] + self.diagnostics.payloads_transferred += 1 + logger.debug( + "FDv2 payload transferred: payload version %s, %d skill object(s) held", + version, + len(self._committed), + ) + return _TransferOutcome( + committed=True, + changes=changes, + basis=state if isinstance(state, str) and state else None, + ) + + def _abandon_in_flight(self) -> None: + """Drops the in-flight payload and keeps what is committed.""" + self._pending = None + self._intent = None + self._intent_payload_id = None + self._skills_in_payload = 0 + self._changes = [] + + def _error(self, data: Any) -> _TransferOutcome: + reason = data.get("reason") if isinstance(data, dict) else None + self._abandon_in_flight() + return _TransferOutcome(disconnect=f"server sent error: {reason}") + + def _goodbye(self, data: Any) -> _TransferOutcome: + reason = data.get("reason") if isinstance(data, dict) else None + catastrophe = bool(data.get("catastrophe")) if isinstance(data, dict) else False + silent = bool(data.get("silent")) if isinstance(data, dict) else False + self._abandon_in_flight() + if not silent: + logger.info("FDv2 connection closing: %s", reason) + if catastrophe: + return _TransferOutcome( + fatal=f"server sent a catastrophic goodbye: {reason}" + ) + return _TransferOutcome(disconnect=f"server said goodbye: {reason}") + + # -- payload identity ---------------------------------------------------- + + def _is_foreign_payload(self, payload_id: str | None) -> bool: + """ + Whether a transfer completes a payload other than the one skills arrive on. + + ``False`` unless both payloads are known, so one-payload delivery and + the first transfer of a connection are unaffected. + """ + return ( + self._skill_payload_id is not None + and payload_id is not None + and payload_id != self._skill_payload_id + ) + + # -- diagnostics --------------------------------------------------------- + + def _warn_multiple_payloads(self, payloads: list[Any]) -> None: + """ + One WARNING per reader for an intent describing more than one payload. + + Not an error: reading only the first is what the protocol asks for. But + it means the first payload is no longer *guaranteed* to be the skill + payload, and an intent for another payload arriving before any skill has + been seen is the one case ``_is_foreign_payload`` cannot catch. + """ + if self._warned_multiple_payloads: + return + self._warned_multiple_payloads = True + logger.warning( + "An FDv2 server-intent described %d payloads (%s). Only the first is " + "read, as the protocol requires, and it is taken to be the payload " + "skills arrive on. If skills stop resolving from this point, that is " + "the assumption that broke; contact LaunchDarkly support.", + len(payloads), + ", ".join(str(_payload_id_of(p)) for p in payloads), + ) + + def _warn_foreign_payload(self, payload_id: str | None) -> None: + """One WARNING per reader for a transfer this layer declined to apply.""" + if self._warned_foreign_payload: + return + self._warned_foreign_payload = True + logger.warning( + "An FDv2 transfer of payload %s was not applied to the skills held, " + "which arrive on payload %s. Applying it would have replaced them " + "with whatever that payload carried — nothing, in the case of a flag " + "payload. The skills held are unchanged.", + payload_id, + self._skill_payload_id, + ) + + def _warn_hashless(self, raw: dict[str, Any]) -> None: + """ + One ERROR per ``(key, version)`` whose envelope had no ``contentHash``. + + ERROR rather than WARN because an empty accessor result is otherwise + indistinguishable from an environment that has no skills. + """ + identity = (raw["key"], raw.get("version")) + if identity in self._warned_hashless: + return + self._warned_hashless.add(identity) + logger.error( + "Skill '%s' version %s arrived without a contentHash and will be withheld. %s", + raw["key"], + raw.get("version"), + _HASHLESS_ADVICE, + extra={"ld_skill_key": raw["key"], "ld_skill_version": raw.get("version")}, + ) + + +_HASHLESS_ADVICE = ( + "The delivered skill object carries no 'contentHash', so integrity " + "verification withholds it with reason_code 'missing_content_hash' and its " + "content will not resolve. The SDK cannot work around this: verification " + "hashes the delivered bytes and compares them against the envelope's " + "'contentHash', and there is nothing to compare against. 'contentHash' is a " + "sha256 over the verbatim UTF-8 content. Contact LaunchDarkly support if " + "skills in your environment arrive without one." +) + + +def _warn_if_nothing_can_verify(committed: _SkillObjectSet) -> None: + """ + One ERROR per committed payload in which *nothing* held can possibly verify. + + ``log_withholding_summary`` reports the same condition at the accessor + boundary, but only once a caller asks. This fires at delivery time, so it is + visible in a process that boots, materializes nothing, and exits. + """ + held = committed.all_raw() + if not held: + return + hashless = [raw for raw in held if not isinstance(raw.get("contentHash"), str)] + if len(hashless) != len(held): + return + logger.error( + "All %d skill object(s) in the delivered payload arrived without a " + "contentHash. No skill content will resolve from this store. %s", + len(held), + _HASHLESS_ADVICE, + ) + + +# --------------------------------------------------------------------------- +# HTTP +# --------------------------------------------------------------------------- + + +class _FatalTransportError(Exception): + """A failure retrying cannot fix: bad credential, forbidden, wrong URI.""" + + +class _RecoverableTransportError(Exception): + """A failure worth retrying. Carries a server-requested delay when given one.""" + + def __init__(self, message: str, retry_after: float | None = None) -> None: + super().__init__(message) + self.retry_after = retry_after + + +_FORBIDDEN_ADVICE = ( + "The FDv2 protocol is opt-in per LaunchDarkly account and is served as HTTP " + "403 while it is off. Skill delivery needs it enabled; contact LaunchDarkly " + "support to enable it for your account." +) + + +def _retry_after_seconds(headers: Any) -> float | None: + """ + ``Retry-After`` in seconds, when the server sent a usable one. + + The HTTP-date form, and non-finite values such as ``inf`` or ``1e309`` that + ``float`` accepts, fall back to this module's backoff: none is a delay, + and an infinite one would overflow the wait that honours it. + """ + if headers is None: + return None + try: + raw = headers.get("Retry-After") + except AttributeError: + return None + if raw is None: + return None + try: + seconds: float = float(str(raw).strip()) + except ValueError: + return None + if not math.isfinite(seconds): + return None + return max(0.0, seconds) + + +def _classify_status(status: int, headers: Any) -> Exception: + """Turns an HTTP error status into the right exception type.""" + if status == 401: + return _FatalTransportError( + "LaunchDarkly rejected the SDK key (HTTP 401). Skill delivery cannot " + "start. Check that the key is the environment's server-side SDK key." + ) + if status == 403: + return _FatalTransportError( + f"LaunchDarkly returned HTTP 403. {_FORBIDDEN_ADVICE}" + ) + if 300 <= status < 400 and status != 304: + return _FatalTransportError( + f"LaunchDarkly returned HTTP {status}, a redirect. Redirects are not " + "followed, so the SDK key is never forwarded to a host other than the " + "base URI. The SDK-facing FDv2 endpoints do not redirect; check the " + "base URI, and any proxy in between, for the address being redirected " + "to." + ) + if status in (400, 405, 406, 414, 501): + return _FatalTransportError( + f"LaunchDarkly returned HTTP {status}, which retrying will not fix. " + "The request this adapter sent was not understood. It carries only " + "the SDK key and, after the first payload, a 'basis' selector, so " + "check the base URI and that the endpoint speaks FDv2." + ) + return _RecoverableTransportError( + f"LaunchDarkly returned HTTP {status}", _retry_after_seconds(headers) + ) + + +def _interrupt_read(response: Any) -> None: + """ + Best-effort interruption of a read blocked on *response*, from another thread. + + Closing the response is not enough: CPython's buffered reader stays parked in + ``readline`` until bytes arrive. Shutting the *socket* down underneath it + unblocks it immediately. Reaching the socket means walking urllib's private + attribute chain, so every step is guarded and failure is silent: the + delivery thread is a daemon and ``close``'s join timeout is the backstop. + """ + for path in (("fp", "raw", "_sock"), ("fp", "_sock"), ("_sock",)): + found: Any = response + for name in path: + found = getattr(found, name, None) + if found is None: + break + if found is not None and hasattr(found, "shutdown"): + try: + found.shutdown(socket.SHUT_RDWR) + except OSError: + pass + return + + +class _StreamConnection: + """ + One open streaming connection: an event iterator plus a way to interrupt it + from another thread, which is what ``FDv2SkillStore.close`` needs. + """ + + def __init__(self, response: Any) -> None: + self._response = response + self.events = _iter_sse(response) + + def close(self) -> None: + """Interrupts the read. Safe to call from any thread, and twice.""" + _interrupt_read(self._response) + try: + self._response.close() + except Exception: + pass + + +class _RefuseRedirects(urllib.request.HTTPRedirectHandler): + """ + A redirect handler that follows nothing. + + The standard handler copies every request header onto the redirected + request, ``Authorization`` included, so a 3xx from a proxy or a misconfigured + private instance would hand the SDK key to whatever host ``Location`` names. + Declining here makes ``urllib`` surface the 3xx as an ``HTTPError``, which + ``_classify_status`` turns into a fatal, non-retried failure. Same-host + redirects are refused too: the endpoints this module calls do not redirect, + and a 304 is not a redirect and never reaches this handler. + """ + + def redirect_request( + self, + req: Any, + fp: Any, + code: Any, + msg: Any, + headers: Any, + newurl: Any, + ) -> None: + return None + + +def _build_opener() -> urllib.request.OpenerDirector: + """The default opener with its redirect handler replaced by a refusing one.""" + return urllib.request.build_opener(_RefuseRedirects) + + +@dataclass(frozen=True) +class _PollResult: + not_modified: bool + events: list[tuple[str, Any]] + etag: str | None + + +class _Requester: + """ + The only place this module opens a socket. Standard library only, on purpose. + + *read_timeout* is applied to every socket operation of a request. ``urllib`` + has no separate connect timeout: its ``timeout`` becomes the socket timeout + for the whole operation, so connecting, waiting for headers and each body + read are all bounded by the same value. + """ + + def __init__( + self, + sdk_key: str, + base_uri: str, + *, + read_timeout: float, + opener: Any = None, + ) -> None: + self._sdk_key = sdk_key + self._base_uri = base_uri.rstrip("/") + self._read_timeout = read_timeout + # Injectable, so an alternative transport can be supplied. The default + # never follows a redirect; see ``_RefuseRedirects``. + self._opener = opener or _build_opener() + self._lock = threading.Lock() + # The response of a poll in flight, so ``interrupt`` can reach its + # socket from another thread. Polling only: a stream's response is + # handed straight to the caller as a ``_StreamConnection``, which + # carries an interrupt of its own. + self._in_flight: Any = None + + def interrupt(self) -> None: + """ + Unblocks a poll parked in its body read, from another thread. + + Best effort, and safe to call when nothing is in flight. A request still + inside its connect has no response to reach yet and is bounded only by + ``read_timeout``; ``FDv2SkillStore.start`` covers what that leaves. + """ + with self._lock: + response = self._in_flight + if response is not None: + _interrupt_read(response) + + def _url(self, path: str, basis: str | None) -> str: + """ + The request URL: the path, plus ``basis`` once a payload has committed. + + Deliberately no ``mv`` (data model version). That parameter selects the + *flag* data model and the connection rejects any value but the flag + default; the agent-skill payload is generic, is served regardless of it, + and has no model version of its own to ask for. + """ + if not basis: + return f"{self._base_uri}{path}" + return f"{self._base_uri}{path}?{urllib.parse.urlencode({'basis': basis})}" + + def _request( + self, path: str, basis: str | None, headers: dict[str, str] + ) -> urllib.request.Request: + all_headers = {"Authorization": self._sdk_key, **headers} + return urllib.request.Request( + self._url(path, basis), headers=all_headers, method="GET" + ) + + def poll(self, basis: str | None, etag: str | None) -> _PollResult: + """One ``GET /sdk/poll``. A 304 is a first-class outcome, not an error.""" + headers = {"Accept": "application/json"} + if etag: + headers["If-None-Match"] = etag + request = self._request(POLL_PATH, basis, headers) + try: + with self._opener.open(request, timeout=self._read_timeout) as response: + with self._lock: + self._in_flight = response + try: + status = getattr(response, "status", None) or response.getcode() + if status == 304: + return _PollResult(not_modified=True, events=[], etag=etag) + body = _read_bounded(response, MAX_RESPONSE_BYTES) + new_etag = response.headers.get("ETag") or etag + finally: + with self._lock: + self._in_flight = None + except urllib.error.HTTPError as exc: + if exc.code == 304: + # urllib raises on any non-2xx, 304 included; it is a current + # answer here, not a redirect, and is handled before classifying. + return _PollResult(not_modified=True, events=[], etag=etag) + raise _classify_status(exc.code, exc.headers) from exc + except _RecoverableTransportError: + raise + except Exception as exc: + raise _RecoverableTransportError( + f"polling request failed: {type(exc).__name__}: {exc}" + ) from exc + + return _PollResult( + not_modified=False, events=_decode_poll_body(body), etag=new_etag + ) + + def stream(self, basis: str | None) -> _StreamConnection: + """Opens ``GET /sdk/stream``.""" + request = self._request( + STREAM_PATH, + basis, + {"Accept": "text/event-stream", "Cache-Control": "no-cache"}, + ) + try: + response = self._opener.open(request, timeout=self._read_timeout) + except urllib.error.HTTPError as exc: + raise _classify_status(exc.code, exc.headers) from exc + except Exception as exc: + raise _RecoverableTransportError( + f"streaming request failed: {type(exc).__name__}: {exc}" + ) from exc + return _StreamConnection(response) + + +def _read_bounded(response: Any, limit: int) -> bytes: + """ + Reads a whole poll body, holding no more than *limit* bytes of it. + + Read in chunks rather than all at once so a body that is never going to be + accepted is abandoned as soon as it crosses the bound, with at most one + byte over it in memory, instead of being buffered whole and measured after. + """ + chunks: list[bytes] = [] + seen = 0 + while True: + chunk = response.read(min(_READ_CHUNK_BYTES, limit + 1 - seen)) + if not chunk: + return b"".join(chunks) + seen += len(chunk) + if seen > limit: + raise _RecoverableTransportError( + f"polling response exceeded the {limit}-byte transport bound " + f"(at least {seen} bytes received); nothing from it was applied" + ) + chunks.append(chunk) + + +def _decode_poll_body(body: bytes) -> list[tuple[str, Any]]: + """ + Unwraps ``{"events": [...]}``. Polling and streaming carry identical event + objects, which is why the protocol reader is shared between the two modes. + """ + try: + parsed = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise _RecoverableTransportError( + f"polling response was not valid JSON: {exc}" + ) from exc + if not isinstance(parsed, dict) or not isinstance(parsed.get("events"), list): + raise _RecoverableTransportError("polling response had no 'events' array") + events: list[tuple[str, Any]] = [] + for entry in parsed["events"]: + if not isinstance(entry, dict): + continue + name = entry.get("event") + if isinstance(name, str): + events.append((name, entry.get("data"))) + return events + + +def _iter_stream_lines(response: Any, limit: int) -> Any: + """ + Yields a streaming body's raw lines, presenting a read failure as retryable + and refusing any single line longer than *limit* bytes. + + A live stream dies mid-body far more often than it refuses to open: a read + timeout on a stream that went quiet, a reset, a truncated chunk. Each of + those arrives as whatever the socket raised, and the delivery loop retries + only the transport errors this module defines — anything else it reads as a + bug and stops for the process lifetime. Connecting is already wrapped in + ``_Requester.stream``; this is the same promise for the body. + + Lines are read with a size argument rather than by iterating the response, + because an unbounded ``readline`` buffers until it finds a newline, and a + line that never ends would be held whole before anything here saw it. + """ + try: + while True: + line = response.readline(limit + 1) + if not line: + return + if len(line) > limit: + raise _RecoverableTransportError( + f"an FDv2 stream line exceeded the {limit}-byte transport " + "bound; the connection was dropped and nothing from the " + "in-flight payload was applied" + ) + yield line + except _RecoverableTransportError: + raise + except Exception as exc: + raise _RecoverableTransportError( + f"reading the FDv2 stream failed: {type(exc).__name__}: {exc}" + ) from exc + + +def _iter_sse(response: Any) -> Any: + """ + Decodes an SSE body into ``(event name, data)`` pairs. + + Minimal on purpose: ``event:``/``data:`` fields, multi-line ``data`` joined + with newlines, a blank line dispatching, and ``:`` comments skipped. + + One event may accumulate at most ``MAX_RESPONSE_BYTES`` across its lines. + Past that it is a recoverable transport failure: the generator raises, the + delivery loop drops the connection and retries, and the payload in flight + is abandoned rather than committed. + """ + limit = MAX_RESPONSE_BYTES + try: + name: str | None = None + data_lines: list[str] = [] + event_bytes = 0 + for raw_line in _iter_stream_lines(response, limit): + line = raw_line.decode("utf-8", errors="replace").rstrip("\r\n") + if line == "": + if name is not None: + payload = "\n".join(data_lines) + try: + parsed = json.loads(payload) if payload else None + except json.JSONDecodeError: + logger.warning( + "Discarding FDv2 '%s' event whose data was not JSON", name + ) + parsed = None + else: + yield name, parsed + name = None + data_lines = [] + event_bytes = 0 + continue + if line.startswith(":"): + continue + event_bytes += len(raw_line) + if event_bytes > limit: + raise _RecoverableTransportError( + f"an FDv2 stream event exceeded the {limit}-byte transport " + f"bound (at least {event_bytes} bytes received); the " + "connection was dropped and nothing from the in-flight " + "payload was applied" + ) + field_name, _, value = line.partition(":") + value = value[1:] if value.startswith(" ") else value + if field_name == "event": + name = value + elif field_name == "data": + data_lines.append(value) + finally: + try: + response.close() + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Backoff +# --------------------------------------------------------------------------- + + +def _backoff_delay( + attempt: int, *, base: float, maximum: float, jitter: float = 0.5 +) -> float: + """ + Exponential backoff with jitter, capped at *maximum*. + + Jitter is subtractive over the whole range rather than added on top, so the + cap is a real ceiling: a fleet restarted together must not reconnect in + lockstep, and must not exceed the interval the cap promises. + """ + # float(2 ** n): the integer power is untyped to mypy. + ceiling: float = min(maximum, base * float(2 ** max(0, attempt - 1))) + return ceiling * (1.0 - jitter * random.random()) + + +# --------------------------------------------------------------------------- +# The store +# --------------------------------------------------------------------------- + + +class FDv2SkillStore: + """ + A ``SkillStore`` fed by LaunchDarkly's SDK-facing FDv2 delivery channel. + + Constructed with the environment's server-side SDK key, started explicitly, + and passed to ``init_client``:: + + store = FDv2SkillStore(sdk_key=os.environ["LD_SDK_KEY"]) + store.start() + if not store.wait_for_skills(timeout=10): + ... # no payload yet: see is_initialized + await init_client(options={"skillStore": store}) + + skill = await get_skill("pdf-extraction") + ... + store.close() + + It also works as a context manager. + + **Server-side only.** A mobile key or a client-side environment ID is + refused in the constructor. + + **The SDK key goes only where it was pointed.** *base_uri* must be + ``https://`` — plain ``http://`` is refused except to a loopback host, for + local test doubles — and redirects are never followed, so a 3xx from a proxy + or a private instance is a fatal failure rather than a request carrying the + key to whatever host ``Location`` named. + + **Delivery is in the background; retrieval is not.** A daemon thread owns + the connection and fills memory, and ``get_object`` only ever reads what has + already arrived. A process that calls ``get_skill`` immediately after + ``start()`` may see an empty store; ``wait_for_skills`` orders boot against + the first payload, and ``is_initialized`` reports the same fact without + waiting — which is what stops ``write_skills("*")`` from pruning against a + store that has not heard yet. + + **Last known good survives an outage.** A transport failure never empties + the store and never makes ``get_object`` raise, which is what makes + ``write_skills(on_unavailable="keep")`` correct. ``diagnostics`` and + ``failed`` report the degradation. + + **Reads are memory-bounded.** No poll body or streamed event is held past + ``MAX_RESPONSE_BYTES``; one that crosses it is dropped unapplied as a + recoverable failure, the store keeps serving what it last held, and + delivery retries. + + **What arrives is untrusted.** Raw wire objects are held verbatim and + verified at the accessor boundary, not here. In particular an object with no + ``contentHash`` is held and then *withheld*; see + ``StoreDiagnostics.hashless_objects``. + """ + + def __init__( + self, + sdk_key: str, + *, + base_uri: str = DEFAULT_BASE_URI, + mode: Mode = "stream", + poll_interval: float = 30.0, + read_timeout: float | None = None, + initial_backoff: float = 1.0, + max_backoff: float = 30.0, + max_consecutive_failures: int = 10, + _requester: Any = None, + ) -> None: + """ + *base_uri* must be ``https://``; ``http://`` is accepted only for + ``localhost``, ``127.0.0.1`` or ``::1``. Raises ``ValueError`` otherwise. + + *mode* is ``"stream"`` by default. Prefer it: a ``delete-object`` reaches + a live stream in seconds. ``"poll"`` exists for environments that cannot + hold a long-lived connection, and revocation there is one + ``poll_interval`` late. + + *read_timeout* is the only network timeout and bounds every socket + operation of a request, so its meaning and default follow the mode: in + ``"poll"`` it bounds the whole request (``DEFAULT_POLL_TIMEOUT``); in + ``"stream"`` it bounds each wait for the next bytes + (``DEFAULT_STREAM_READ_TIMEOUT``). Must be positive when given. + + *max_backoff* caps every delay between retries, including one the server + asks for with ``Retry-After``. + + *max_consecutive_failures* bounds the retry loop. On exceeding it the + transport stops, logs an error, and the store keeps serving last known + good; ``failed`` reports it. Only failures in a row count: a committed + payload resets the count. + """ + _require_server_side_credential(sdk_key) + _require_https_base_uri(base_uri) + if mode not in ("stream", "poll"): + raise ValueError(f'mode must be "stream" or "poll", got {mode!r}') + if poll_interval <= 0: + raise ValueError(f"poll_interval must be positive, got {poll_interval!r}") + if read_timeout is None: + read_timeout = ( + DEFAULT_STREAM_READ_TIMEOUT + if mode == "stream" + else DEFAULT_POLL_TIMEOUT + ) + elif not (math.isfinite(read_timeout) and read_timeout > 0): + raise ValueError(f"read_timeout must be positive, got {read_timeout!r}") + + self._mode: Mode = mode + self._poll_interval = poll_interval + self._initial_backoff = initial_backoff + self._max_backoff = max_backoff + self._max_consecutive_failures = max_consecutive_failures + + self._objects = _SkillObjectSet() + self._reader = _ProtocolReader(self._objects) + self._lock = threading.RLock() + self._listeners: dict[str, list[Callable[[dict[str, Any]], Any]]] = {} + + self._basis: str | None = None + self._etag: str | None = None + + self._requester = _requester or _Requester( + sdk_key.strip(), + base_uri, + read_timeout=read_timeout, + ) + + self._stop = threading.Event() + self._first_payload = threading.Event() + """A payload has committed. The fact ``wait_for_skills`` reports.""" + self._delivery_ended = threading.Event() + """ + Delivery has stopped, by ``close`` or by ``_give_up``. Kept apart from + ``_first_payload`` because it is not one: a waiter has to be let go + either way, but only a payload makes ``wait_for_skills`` true. + """ + self._released = threading.Event() + """ + Either of the two above, and what a waiter actually parks on: an + ``Event`` cannot wait on two, so the setters funnel through here. + """ + self._thread: threading.Thread | None = None + self._failed_reason: str | None = None + # The open streaming connection, so ``close`` can interrupt its read. + self._connection: Any = None + # Recoverable failures since the last committed payload. Reset at the + # commit rather than when a connection returns: a stream only ever ends + # by being dropped, so resetting on return would count every healthy, + # server-recycled connection as a failure. + self._failures = 0 + # Whether the current attempt got a complete answer before it ended. + # A stream only ever ends by being dropped, so this is what separates + # a recycled healthy connection from one that failed. + self._attempt_answered = False + + # -- lifecycle --------------------------------------------------------- + + def start(self) -> FDv2SkillStore: + """ + Starts the delivery thread. Idempotent; returns ``self`` so it chains. + + Does not block: use ``wait_for_skills`` when boot ordering matters. + """ + with self._lock: + # Read before the rearm clears it: a thread inside ``_give_up`` is + # still alive and no longer delivering, so ``is_alive`` on its own + # would have this call adopt a run that is about to return and + # leave a store reporting no failure and never delivering again. + delivering = ( + self._failed_reason is None + and self._thread is not None + and self._thread.is_alive() + ) + self._rearm_waiters() + if delivering: + # A ``close`` whose join timed out leaves the previous thread + # running with the stop flag still set. Clearing it lets that + # thread carry on delivering, rather than leaving a store that + # reports itself started and never delivers again. + self._stop.clear() + return self + self._stop.clear() + self._thread = threading.Thread( + target=self._run, name="ld-ai-skills-fdv2", daemon=True + ) + self._thread.start() + return self + + def _rearm_waiters(self) -> None: + """ + Re-arms ``wait_for_skills`` for a store being started again after a + ``close``. A payload already held stays an answer; an ended delivery + does not, or the next waiter would be released before it began. + + A terminal ``failed`` reason is dropped for the same reason: it says why + delivery stopped for good, and delivery is about to run again. Leaving + it would have a healthy store reporting a failure it has recovered from. + Called with the lock held, which is what ``failed`` reads under. + """ + self._delivery_ended.clear() + self._failed_reason = None + # The retry budget belongs to the run that spent it. Carrying it over + # would have a store that gave up after its failure limit give up again + # on the restarted run's first recoverable failure. + self._failures = 0 + if not self._first_payload.is_set(): + self._released.clear() + + def close(self, timeout: float = 5.0) -> None: + """ + Stops delivery. Idempotent, and safe to call from any thread. + + Held content is *not* dropped: a closed store still answers from what it + received. Detaching the store from the accessors is the job of the + package-level ``launchdarkly_ai_server.shutdown()`` coroutine. + """ + self._stop.set() + # A waiter parked in ``wait_for_skills`` is owed an answer now rather + # than at the end of its timeout; delivery is over either way. + self._end_delivery() + # The delivery thread is normally blocked in a socket read that no flag + # can reach; without this the join waits out its full timeout. Streaming + # parks in the connection, polling in the request, so interrupt both. + with self._lock: + connection = self._connection + if connection is not None: + connection.close() + self._requester.interrupt() + thread = self._thread + if ( + thread is not None + and thread.is_alive() + and thread is not threading.current_thread() + ): + thread.join(timeout=timeout) + + def __enter__(self) -> FDv2SkillStore: + return self.start() + + def __exit__(self, *_exc: Any) -> None: + self.close() + + def wait_for_skills(self, timeout: float = 10.0) -> bool: + """ + Blocks until the first payload has been committed, or *timeout* elapses. + + ``True`` means a payload arrived — not that any skill in it verified, and + not that the environment has any skills. ``diagnostics`` answers the rest. + + Returns early, ``False``, when delivery ends before any payload does: + a ``close`` from another thread, or a failure delivery cannot retry. + Waiting out the full timeout for an answer that has already arrived + would delay every shutdown that raced a waiter. + """ + self._released.wait(timeout=timeout) + return self._first_payload.is_set() + + def _publish_first_payload(self) -> None: + """Records the first committed payload and lets any waiter go.""" + self._first_payload.set() + self._released.set() + + def _end_delivery(self) -> None: + """Records that delivery has stopped and lets any waiter go.""" + self._delivery_ended.set() + self._released.set() + + def is_initialized(self) -> bool: + """ + Whether a payload has arrived, so reads reflect delivery rather than an + empty store still waiting for its first one. + + The optional half of the ``SkillStore`` interface, and the same fact + ``wait_for_skills`` returns — without the wait. ``write_skills("*")`` + consults it so a reconcile that runs before delivery reports the + retrieval unavailable rather than pruning every managed skill as though + the environment had revoked it. + + Stays ``True`` once a payload has arrived, including after ``close``: a + closed store still answers from what it received, and a later + reconcile against that content is a reconcile against real delivery. + """ + return self._first_payload.is_set() + + @property + def failed(self) -> str | None: + """Why delivery stopped for good, or ``None`` while it is running.""" + with self._lock: + return self._failed_reason + + @property + def diagnostics(self) -> StoreDiagnostics: + """A snapshot of what the transport has seen. See ``StoreDiagnostics``.""" + with self._lock: + return StoreDiagnostics(**vars(self._reader.diagnostics)) + + # -- the SkillStore interface ----------------------------------------- + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + if kind != SKILL_OBJECT_KIND: + return None + with self._lock: + return self._objects.get(key, version) + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + if kind != SKILL_OBJECT_KIND: + return {} + with self._lock: + return self._objects.snapshot() + + def add_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Registers *fn* to be called once per changed object, at + ``payload-transferred`` rather than as objects stream in. + + A put notifies with the raw skill object. A revocation notifies with a + ``{"key", "version"}`` tombstone carrying no content, so a listener that + reads content must check for ``content`` rather than assume it. + + *fn* runs on the delivery thread. Keep it cheap and non-blocking. An + exception it raises is logged and swallowed, because a broken listener + must not be able to kill delivery. + """ + with self._lock: + self._listeners.setdefault(kind, []).append(fn) + + def remove_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Unregisters *fn* from *kind*. Safe to call from any thread, including + from inside a listener: a removal during one commit takes effect from + the next. + + Removes one occurrence; removing a callable that is not registered is a + no-op, so ``SkillWatcher.close`` can detach unconditionally. + """ + with self._lock: + listeners = self._listeners.get(kind) + if listeners is None: + return + try: + listeners.remove(fn) + except ValueError: + return + + def _notify(self, changes: list[dict[str, Any]]) -> None: + with self._lock: + listeners = list(self._listeners.get(SKILL_OBJECT_KIND, [])) + for raw in changes: + for listener in listeners: + try: + listener(raw) + except Exception: + logger.error( + "A skill store change listener raised; delivery continues", + exc_info=True, + ) + + # -- the delivery loop ------------------------------------------------- + + def _run(self) -> None: + while not self._stop.is_set(): + with self._lock: + self._attempt_answered = False + try: + if self._mode == "stream": + self._stream_once() + else: + self._poll_once() + # A poll that returned is a current answer even when it committed + # nothing (HTTP 304). A stream never returns normally; its + # successes are counted at each commit in ``_apply``. + self._record_success() + except _FatalTransportError as exc: + self._give_up(str(exc)) + return + except _RecoverableTransportError as exc: + if self._stop.is_set(): + # ``close`` interrupted the request on purpose. Counting it + # would spend a retry from the bounded budget and leave a + # misleading ``last_error`` on a healthy store. + return + with self._lock: + # Whatever the dropped connection had transferred so far is + # not a payload; the next connection starts one afresh. + self._reader._abandon_in_flight() + self._failures += 1 + failures = self._failures + answered = self._attempt_answered + self._reader.diagnostics.connection_failures = failures + self._reader.diagnostics.last_error = str(exc) + if failures > self._max_consecutive_failures: + self._give_up( + f"gave up after {failures} consecutive failures; " + f"last error: {exc}" + ) + return + delay = exc.retry_after + if delay is None or not math.isfinite(delay): + delay = _backoff_delay( + failures, base=self._initial_backoff, maximum=self._max_backoff + ) + # ``Retry-After`` is a request and ``max_backoff`` is a promise. + # The header may come from a proxy rather than LaunchDarkly, and + # a value in the hours would park revocation for that long. + delay = min(delay, self._max_backoff) + if answered: + # LaunchDarkly, and any proxy in between, recycles a + # long-lived stream. A connection that answered before it + # ended delivered everything it was asked for, so the + # reconnect is routine rather than a fault worth warning + # about for as long as the process runs. + logger.debug( + "The FDv2 stream ended after a complete answer (%s); " + "reconnecting in %.1fs", + exc, + delay, + ) + else: + logger.warning( + "Skill delivery failed (%s); retrying in %.1fs", exc, delay + ) + if self._stop.wait(delay): + return + continue + except Exception as exc: # pragma: no cover - defensive + self._give_up(f"unexpected error in skill delivery: {exc!r}") + logger.error("Unexpected error in skill delivery", exc_info=True) + return + + if self._mode == "poll" and self._stop.wait(self._poll_interval): + return + + def _record_success(self) -> None: + with self._lock: + self._failures = 0 + self._attempt_answered = True + self._reader.diagnostics.connection_failures = 0 + + def _give_up(self, reason: str) -> None: + with self._lock: + self._failed_reason = reason + self._reader.diagnostics.last_error = reason + # Let go of anyone waiting on a first payload that is never coming. + # Recorded beside the reason and under the lock so the two are + # published together: a ``start`` that landed between them would + # re-arm the waiters and then have this dying thread end delivery + # on the fresh run, releasing its waiters before it had answered. + self._end_delivery() + logger.error( + "Skill delivery has stopped and will not retry: %s. The store keeps " + "serving the last content it received; skills will not update until " + "the process restarts with a working connection.", + reason, + ) + + def _apply(self, name: str, data: Any) -> None: + """ + Feeds one event to the reader, publishes a commit, and raises the + transport error the event calls for, if any. + """ + with self._lock: + outcome = self._reader.handle(name, data) + if outcome.committed and outcome.basis is not None: + self._basis = outcome.basis + if outcome.committed or outcome.up_to_date: + # Both break the row of consecutive failures: a commit is a payload + # delivered, and ``up_to_date`` is the server confirming the store + # already holds it. Counting only the commit would give up on a healthy + # stream serving an environment whose skills are not changing: + # nothing to transfer means no commit, while every recycled + # connection still ends in a drop. + self._record_success() + if outcome.committed: + self._publish_first_payload() + if outcome.changes: + self._notify(outcome.changes) + if outcome.fatal: + raise _FatalTransportError(outcome.fatal) + if outcome.disconnect: + raise _RecoverableTransportError(outcome.disconnect) + + def _poll_once(self) -> None: + with self._lock: + basis, etag = self._basis, self._etag + result = self._requester.poll(basis, etag) + with self._lock: + self._etag = result.etag + if result.not_modified: + logger.debug("Skill payload unchanged (HTTP 304)") + # A 304 counts as a first payload, so a boot that reconnects with a + # cached basis is not blocked on a transfer the server will not send. + self._publish_first_payload() + return + for name, data in result.events: + self._apply(name, data) + + def _stream_once(self) -> None: + with self._lock: + basis = self._basis + connection = self._requester.stream(basis) + with self._lock: + self._connection = connection + try: + # ``close`` may have run while the connect was in flight and found + # no connection to interrupt; this is the last chance to notice + # before the read below blocks. + if self._stop.is_set(): + return + for name, data in connection.events: + if self._stop.is_set(): + return + self._apply(name, data) + except Exception: + if self._stop.is_set(): + # ``close`` interrupted the read on purpose. + return + raise + finally: + connection.close() + with self._lock: + self._connection = None + # A stream that ends without a goodbye is a dropped connection. + raise _RecoverableTransportError("the FDv2 stream closed unexpectedly") diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py new file mode 100644 index 00000000..03aaeeb7 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -0,0 +1,1269 @@ +""" +Agent Skills — filesystem materialization. + +This is the part that writes to disk. Everything here takes already-verified +content and reconciles it against a managed root; ``skills.py`` owns retrieval +and verification and knows nothing about the filesystem. The descriptor-pinned +primitives every destructive step goes through live in ``safe_fs.py``. + +**The invariants.** The managed root is pinned to a descriptor once per +reconcile, and every operation under it — the destructive ones and the reads +that decide them — runs relative to that descriptor or to a skill directory +pinned relative to it. The existence probe, the compare read and the orphan +listing are pinned before they are consulted, so a directory swapped after the +pin cannot change which branch runs, only what a path check reports. +Destructive operations only ever touch paths ``/.launchdarkly-skills.json`` +records under a matching key. A corrupt manifest suppresses every destructive +action, and an incomplete retrieval suppresses pruning. Content is re-verified +immediately before the write. The path checks still run, but as defense in +depth rather than as the boundary. + +None of these may be relaxed. The threat model behind each — and what breaks if +one moves — is in ``agents.md`` under *Security posture* and *Descriptor-pinned +filesystem access*. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import stat +import time +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Literal + +from .safe_fs import ( + DirectoryMissing, + SymlinkRefused, + atomic_write, + is_temp_name, + open_directory_nofollow, + pinned_directory, + unlink_file, +) +from .skills_core import ( + NO_STORE_MESSAGE, + Resolution, + SkillStore, + VerificationFailure, + get_store, + list_raw_objects, + log_withholding_summary, + newest_by_key, + record_materialized, + record_revoked, + reference_target, + resolve_from_store, + store_is_initialized, + verified_bytes, + verify_raw_skill, +) +from .types import ( + ReconcileAction, + ReconcileActionKind, + ReconcileReport, + Skill, + SkillReference, +) +from .types_validation import ( + is_valid_skill_key, + is_valid_skill_version, + skill_key_rejection_reason, +) + +logger = logging.getLogger(__name__) + +MANIFEST_FILENAME = ".launchdarkly-skills.json" +"""The SDK's record of what it has written under a managed root.""" + +MANIFEST_VERSION = 1 +"""Manifest schema version this release writes, and the highest it can read.""" + +_MAX_MANIFEST_BYTES = 8 * 1024 * 1024 +""" +Hard cap on the manifest, so a reconcile cannot be made to read an arbitrary +amount into memory. + +Set far above any real manifest: an entry is a path, a key, a version, a digest +and a timestamp, so even tens of thousands of skills land a couple of orders of +magnitude below this. Anything larger is treated as corruption, which is what +every other unreadable manifest is — the file lives in a directory the SDK does +not own exclusively, and a reconcile must not be the thing that exhausts the +process. +""" + +SKILL_FILENAME = "SKILL.md" +"""The single file each skill materializes to, under ``//``.""" + +OnUnavailable = Literal["keep", "raise"] +"""How ``write_skills`` reacts to content it could not retrieve.""" + +_UNAVAILABLE_PREFIX = "skill retrieval unavailable: " +""" +Prefix on every error describing content that could not be retrieved. Callers +assert on it, so it lives in one place. +""" + +_MAX_PATH_COMPONENT_BYTES = 255 +""" +NAME_MAX on Linux and macOS, and the component limit on Windows. The data model +permits keys one byte longer than any of those can represent, so an over-long +key is rejected before any filesystem call — a reported action rather than an +ENAMETOOLONG from a stat deep inside the reconcile. +""" + + +_WINDOWS_RESERVED_NAMES = frozenset( + {"con", "prn", "aux", "nul"} + | {f"com{digit}" for digit in range(1, 10)} + | {f"lpt{digit}" for digit in range(1, 10)} +) +""" +The 22 MS-DOS device names Windows reserves, which cannot be directory names +there. Rejected on every platform, so the on-disk result never depends on which +OS ran the write. + +The bare names are the whole set: the key grammar admits no ``.`` or ``$`` and +is lowercase-only, so ``con.txt`` and ``CONIN$`` are unreachable and no suffix +stripping or case folding is needed. ``com0`` and ``lpt0`` are deliberately +absent — those are not reserved. +""" + + +# ------------------------------------------------------------------------- +# The reconcile entry point +# ------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _PendingWrite: + """One skill queued for the reconcile: resolved content, or why there is none.""" + + key: str + skill: Skill | None = None + error: str | None = None + + +async def write_skills( + skills: Sequence[Skill | SkillReference | str] | str, + root: str | os.PathLike[str], + *, + prune: bool = True, + timeout: float = 10.0, + on_unavailable: OnUnavailable = "keep", +) -> ReconcileReport: + """ + Materializes skills under a managed root at ``//SKILL.md``. + + *skills* is a sequence of ``Skill`` / ``SkillReference`` / key strings, or + the literal ``"*"`` meaning everything ``all_skills()`` returns. ``Skill`` + values are used as-is; references and strings resolve through the accessors, + so they need a configured store. + + The reconcile is manifest-driven (``/.launchdarkly-skills.json``): + destructive operations only ever touch paths the manifest records under a + matching key, so a file the SDK did not write is never overwritten or + deleted. ``prune`` removes formerly-managed skills that are no longer in the + requested set — which is also how revocation takes effect. ``timeout`` + bounds retrieval, the writes, and pruning; the final manifest rewrite + always runs, so files already written are never orphaned. ``on_unavailable`` + chooses between reporting a failed retrieval (``"keep"``, leaving existing + managed files alone) and raising (``"raise"``). + + Returns a ``ReconcileReport`` in which every outcome is visible; raises + ``ValueError`` for a caller error such as an unusable root. + + The root is opened once, up front, and that descriptor is held until the + call returns — including for the manifest read, so the record that + authorizes a removal comes from inside the pinned root. A root replaced + after validation is refused rather than followed. POSIX only; see + ``safe_fs``. + + **This call performs synchronous filesystem I/O and does not yield.** It is + ``async`` for signature parity with the other accessors, not because it + awaits anything: every read, write, ``fsync`` and rename runs inline, so a + large reconcile blocks the event loop for its duration. Wrap it in + ``asyncio.to_thread`` if that matters on your loop. ``timeout`` is checked + between steps rather than interrupting one in progress, for the same + reason. + + **One root, one reconcile at a time.** Because nothing here yields, a + reconcile is atomic against every other task on the loop. Wrapping it to run + concurrently makes that the caller's problem: two runs against the same root + interleave on the manifest, the loser's entries are lost, and a later + reconcile then refuses the files it wrote as files the SDK did not write. + """ + # Both of these are annotated as closed sets, but the values can still arrive + # from untyped code, so they are checked rather than assumed. + if on_unavailable not in ("keep", "raise"): + raise ValueError( + f'on_unavailable must be "keep" or "raise", got {on_unavailable!r}' + ) + if timeout < 0: + raise ValueError(f"timeout must not be negative, got {timeout!r}") + + deadline = time.monotonic() + timeout + root_path = _resolve_root(root) + + # Pinned once and held for the whole reconcile, which is what makes + # _resolve_root's validation mean anything afterwards. None where the *at() + # family is absent, and the lstat floor applies instead. + try: + root_fd = open_directory_nofollow(root_path) + except ValueError as exc: + # Not a caller error: the root passed _resolve_root a moment ago, so + # this is the swap itself being refused. It belongs to the run, and + # nothing has been touched. + return ReconcileReport( + actions=[ + _run_error( + f"the skills root {root_path} could not be pinned for the " + f"reconcile: {exc}; no action was taken" + ) + ] + ) + + try: + manifest, manifest_error = _load_manifest(root_path, root_fd) + entries: dict[str, Any] = manifest.get("entries", {}) + + actions: list[ReconcileAction] = [] + if manifest_error is not None: + # Run-level failure: there is no single skill key to hang it off. + actions.append(_run_error(manifest_error)) + + requests, incomplete = _resolve_requests(skills, deadline, on_unavailable) + + written, write_timed_out = _write_all( + root_path, root_fd, requests, entries, deadline + ) + actions.extend(written) + incomplete = incomplete or write_timed_out + + # Pruning is destructive, so it needs a trustworthy picture of both + # sides: a corrupt manifest leaves the SDK unsure what it owns, and an + # incomplete run — a retrieval that failed, or a deadline that expired + # mid-write — leaves it unsure what is still current. Either way, + # deleting would be a guess. + if prune and manifest_error is None and not incomplete: + actions.extend( + _prune( + root_path, + root_fd, + entries, + {request.key for request in requests}, + deadline, + ) + ) + + if manifest_error is None: + actions.extend(_rewrite_manifest(root_path, root_fd, manifest, entries)) + + return ReconcileReport(actions=actions) + finally: + if root_fd is not None: + os.close(root_fd) + + +_RUN_LEVEL_KEY = "" +""" +The documented sentinel for a failure that belongs to no single skill (see +``ReconcileAction``). Spelled once so every path that cannot attribute a +failure to a key agrees with the others. +""" + + +def _run_error(message: str) -> ReconcileAction: + """ + A failure belonging to the run rather than to one skill. + + Uses the run-level sentinel key; it is constructed here so every run-level + error agrees. + """ + return ReconcileAction(key=_RUN_LEVEL_KEY, action="error", error=message) + + +def _write_all( + root: Path, + root_fd: int | None, + requests: list[_PendingWrite], + entries: dict[str, Any], + deadline: float, +) -> tuple[list[ReconcileAction], bool]: + """ + Reconciles every pending write. Returns ``(actions, timed out mid-run)``. + + The loop never aborts: a per-skill failure becomes an ``error`` action and the + next skill is attempted, because returning early would skip the caller's + manifest rewrite and orphan every file already written in this run. + """ + actions: list[ReconcileAction] = [] + timed_out = False + + for request in requests: + if request.skill is None: + actions.append( + ReconcileAction( + key=request.key, + action="error", + error=request.error + or f"skill '{request.key}' could not be resolved", + ) + ) + continue + if time.monotonic() >= deadline: + timed_out = True + actions.append( + ReconcileAction( + key=request.key, + action="error", + error=( + "the timeout was exhausted before skill " + f"'{request.key}' could be written" + ), + ) + ) + continue + try: + actions.append(_write_one(root, root_fd, request.skill, entries)) + except OSError as exc: + # A safety net, not the primary defense. pathlib's stat probes swallow + # only ENOENT/ENOTDIR/EBADF/ELOOP and re-raise every other errno, so an + # unexpected filesystem condition must not abort the loop. + actions.append( + ReconcileAction( + key=request.skill.key, + action="error", + version=request.skill.version, + error=f"skill '{request.skill.key}' could not be reconciled: {exc}", + ) + ) + + return actions, timed_out + + +def _rewrite_manifest( + root: Path, + root_fd: int | None, + manifest: dict[str, Any], + entries: dict[str, Any], +) -> list[ReconcileAction]: + """ + Writes the updated manifest. Returns an error action, or nothing. + + Written relative to the root descriptor the caller holds, rather than + re-opening the root by path — the same reason every other step here is. + """ + manifest["manifestVersion"] = MANIFEST_VERSION + manifest["entries"] = entries + try: + # json.dumps is inside the guard: indent= selects the pure-Python encoder, + # and unknown fields must be round-tripped, so a deeply nested + # planted field can raise RecursionError here — after every skill file is + # already on disk. + serialized = json.dumps(manifest, indent=2, sort_keys=True).encode("utf-8") + atomic_write(root, MANIFEST_FILENAME, serialized, dir_fd=root_fd) + except Exception as exc: + return [_run_error(f"the skills manifest could not be written: {exc}")] + return [] + + +# ------------------------------------------------------------------------- +# Request resolution — content in, or a reason there is none +# ------------------------------------------------------------------------- + + +def _unavailable(reason: str) -> str: + """Wraps *reason* as a retrieval-unavailable message.""" + return f"{_UNAVAILABLE_PREFIX}{reason}" + + +@dataclass(frozen=True) +class _RetrievalBlocked: + """Why retrieval must not be attempted. The reason is caller-facing.""" + + reason: str + + +def _available_store(deadline: float, subject: str) -> SkillStore | _RetrievalBlocked: + """ + The configured store, or why retrieval must not be attempted. + + Written once because this gate is what sets ``unavailable`` and therefore + suppresses pruning. Maintained in two places, a condition added to one and + not the other would not merely produce a wrong message — it would delete + the application's files. + """ + if time.monotonic() >= deadline: + return _RetrievalBlocked( + _unavailable( + f"the timeout was exhausted before {subject} could be retrieved" + ) + ) + store = get_store() + if store is None: + return _RetrievalBlocked(_unavailable(NO_STORE_MESSAGE)) + if not store_is_initialized(store): + # A store still waiting for its first delivery answers every read with + # "nothing", which is indistinguishable from an environment that holds + # no skills — and the "*" form reads that as every skill having been + # revoked. Blocking here reports the run incomplete, which is what + # suppresses the prune. + return _RetrievalBlocked( + _unavailable( + "the skill store has not received its initial data, so " + f"{subject} could not be retrieved and nothing on disk was " + "changed. Wait for delivery before reconciling: " + "FDv2SkillStore.wait_for_skills(timeout) returns True once the " + "first payload has arrived." + ) + ) + return store + + +def _resolve_requests( + skills: Sequence[Skill | SkillReference | str] | str, + deadline: float, + on_unavailable: OnUnavailable, +) -> tuple[list[_PendingWrite], bool]: + """ + Turns the caller's input into one request per skill. + + Returns the requests plus whether any retrieval was left incomplete — an + absent store, a raising store, or an exhausted timeout. That flag suppresses + pruning: deleting managed files because retrieval failed would turn a + transport outage into data loss. + """ + if isinstance(skills, str): + if skills != "*": + raise ValueError( + 'write_skills takes a sequence of skills or the literal "*"; ' + f"got {skills!r}" + ) + return _resolve_all(deadline, on_unavailable) + + requests: list[_PendingWrite] = [] + incomplete = False + for item in skills: + if isinstance(item, Skill): + requests.append(_PendingWrite(key=item.key, skill=item)) + continue + + key, wanted = reference_target(item) + resolved = _resolve_reference(key, wanted, deadline) + if resolved.unavailable: + incomplete = True + if on_unavailable == "raise": + raise RuntimeError(resolved.error) + requests.append( + _PendingWrite(key=key, skill=resolved.skill, error=resolved.error) + ) + + return requests, incomplete + + +def _resolve_reference( + key: str, wanted_version: int | None, deadline: float +) -> Resolution: + """ + Resolves one reference for the materialization path. + + Same core as the accessors, plus the two conditions only this path treats as + data rather than as an exception: an exhausted deadline and an absent store. + """ + store = _available_store(deadline, f"'{key}'") + if isinstance(store, _RetrievalBlocked): + return Resolution( + reason="store_unavailable", error=store.reason, unavailable=True + ) + + resolved = resolve_from_store(store, key, wanted_version) + if resolved.unavailable and resolved.error is not None: + return Resolution( + reason="store_unavailable", + error=_unavailable(resolved.error), + unavailable=True, + ) + return resolved + + +def _unavailable_run( + error: str, on_unavailable: OnUnavailable +) -> tuple[list[_PendingWrite], bool]: + """ + One run-level retrieval failure — raised, or reported against the empty key. + + Always reports the run incomplete, which is what suppresses pruning: nothing + was retrieved, so every managed file on disk has to be assumed current. + """ + if on_unavailable == "raise": + raise RuntimeError(error) + return [_PendingWrite(key="", error=error)], True + + +def _pending_for_raw(object_key: str, raw: Any) -> _PendingWrite: + """ + One raw store object as a pending write — verified, or reported as failed. + + Present but unverifiable is NOT the same as revoked. Dropping it silently + would leave the key out of the requested set, so prune would delete the last + known-good copy already on disk and report a routine "removed" with + report.ok still true. A failed request instead gets the same treatment the + reference path already gives (see ``_resolve_reference``): the outcome is + surfaced, and the key stays in the requested set so nothing is pruned. + """ + skill = verify_raw_skill(raw) + if skill is not None: + return _PendingWrite(key=skill.key, skill=skill) + # The on-disk copy lives under the object's *own* key, which a custom store + # may key differently in ``all_objects``. The failure must be recorded under + # the object's key, or the copy written under it on an earlier run would + # fall out of the requested set and be pruned — the very deletion this + # function exists to prevent. + raw_key = raw.get("key") if isinstance(raw, dict) else None + key = raw_key if is_valid_skill_key(raw_key) else object_key + if not is_valid_skill_key(key): + # Neither key is usable, so this failure cannot be attributed to a skill + # — the run-level sentinel is the honest report. ``_resolve_all`` reads + # that sentinel back as an incomplete run, because a failure with no key + # cannot protect its copy on disk the way the branch below does. + return _PendingWrite( + key=_RUN_LEVEL_KEY, + error="the skill store served an object under an invalid key; " + "it was withheld", + ) + return _PendingWrite( + key=key, + error=f"skill '{key}' failed integrity verification and was " + "withheld; the copy already on disk was left alone", + ) + + +def _resolve_all( + deadline: float, on_unavailable: OnUnavailable +) -> tuple[list[_PendingWrite], bool]: + """Resolves the ``"*"`` form — everything the store currently holds.""" + store = _available_store(deadline, "the skill set") + if isinstance(store, _RetrievalBlocked): + return _unavailable_run(store.reason, on_unavailable) + + # Deliberately not via all_skills(), which reports a raising store as an + # empty result — that would look like "every skill was revoked" and let + # prune delete the lot. + objects, error = list_raw_objects(store) + if error is not None: + return _unavailable_run(_unavailable(error), on_unavailable) + + # One object per key, at its newest version. ``all_objects`` may hold several + # versions of one key, and //SKILL.md is a single path — writing it + # twice in one run is a bug rather than a policy. + candidates = newest_by_key(objects) + requests = [_pending_for_raw(key, raw) for key, raw in candidates] + log_withholding_summary( + "skills held by the store", + len(requests), + sum(1 for request in requests if request.skill is not None), + ) + # A withholding that could not be attributed to a key leaves the run + # incomplete. Every other failure keeps its key in the requested set, which + # is what holds prune off the copy on disk; a run-level failure has no key to + # do that with, so suppressing prune wholesale is the only thing left that + # stops an unreadable object reading as a revocation. + unattributed = any( + request.skill is None and request.key == _RUN_LEVEL_KEY for request in requests + ) + return requests, unattributed + + +# ------------------------------------------------------------------------- +# The managed root and its manifest +# ------------------------------------------------------------------------- + + +def _resolve_root(root: str | os.PathLike[str]) -> Path: + """ + Resolves the managed root once, up front. + + An unusable root is a caller error rather than a per-skill outcome, so this + raises. Only the leaf directory is ever created: recursively creating + missing ancestors would let a typo scatter a directory tree. + + This is a caller-error check, not a security boundary — it establishes only + that the root was usable at this instant. ``write_skills`` pins the returned + path immediately afterwards, and that is what carries the guarantee. + """ + path = Path(os.fspath(root)) + + # pathlib re-raises any errno outside ENOENT/ENOTDIR/EBADF/ELOOP, so an + # unreadable parent would surface as PermissionError where the docs + # promise ValueError. + try: + is_symlink = path.is_symlink() + exists = path.exists() + is_dir = path.is_dir() + except OSError as exc: + raise ValueError(f"the skills root could not be inspected: {exc}") from exc + + if is_symlink: + raise ValueError( + f"the skills root must be a real directory, not a symlink: {path}" + ) + + if exists: + if not is_dir: + raise ValueError(f"the skills root is not a directory: {path}") + else: + parent = path.parent + try: + parent_is_dir = parent.is_dir() + except OSError as exc: + raise ValueError( + f"the parent of the skills root could not be inspected: {exc}" + ) from exc + if not parent_is_dir: + raise ValueError( + f"the parent of the skills root does not exist: {parent}. " + "write_skills creates only the leaf directory." + ) + try: + path.mkdir() + except OSError as exc: + raise ValueError(f"the skills root could not be created: {exc}") from exc + + return Path(os.path.realpath(path)) + + +def _load_manifest( + root: Path, root_fd: int | None +) -> tuple[dict[str, Any], str | None]: + """ + Loads the manifest. Returns ``(manifest, error)``. + + A manifest that cannot be read, cannot be parsed, is not an object, carries a + ``manifestVersion`` this release does not understand, is larger than the + read cap, or has a malformed ``entries`` map is **corrupt**. The caller then + performs no destructive action and leaves the file itself alone: rewriting it + would destroy the only record of what the SDK owns, and acting on a manifest + it cannot read would mean guessing at which files those are. + + An absent manifest is not corrupt — that is simply a fresh root. + + Read relative to the root descriptor, because this file decides which files + the SDK may overwrite and delete: the pin covers the decision as well as the + actions, here as for every other read under the root. That single open also + makes absence ``ENOENT`` on the read itself, and refuses a symlink or FIFO + wearing the manifest's name as corruption rather than following or waiting + on it. + """ + fresh: dict[str, Any] = {"manifestVersion": MANIFEST_VERSION, "entries": {}} + + try: + raw = _read_regular_file( + MANIFEST_FILENAME if root_fd is not None else root / MANIFEST_FILENAME, + max_bytes=_MAX_MANIFEST_BYTES, + dir_fd=root_fd, + ) + except FileNotFoundError: + return fresh, None + except OSError as exc: + return {}, f"the skills manifest {MANIFEST_FILENAME} could not be read: {exc}" + + # ``_read_regular_file`` stops one byte past the cap, which is the byte that + # proves the overage without reading the rest of the file. + if len(raw) > _MAX_MANIFEST_BYTES: + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} is larger than the " + f"{_MAX_MANIFEST_BYTES} byte cap; refusing every destructive action" + ) + + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + # UnicodeDecodeError is a ValueError, not an OSError: non-UTF-8 bytes in + # the manifest are corruption, and must fail closed like any other. + return {}, f"the skills manifest {MANIFEST_FILENAME} could not be read: {exc}" + + try: + data = json.loads(text) + except (ValueError, RecursionError) as exc: + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} is not valid JSON ({exc}); " + "refusing every destructive action" + ) + + if not isinstance(data, dict): + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} is not a JSON object; " + "refusing every destructive action" + ) + + version = data.get("manifestVersion") + # Bounded at both ends. Above, because a manifest from a future release may + # record fields whose meaning this one would guess at. Below, because 1 is + # the first version ever written, so 0 or a negative is not an older schema + # this release could still read — it is a schema that never existed, and + # acting on its entries would be a guess. + if ( + not isinstance(version, int) + or isinstance(version, bool) + or not 1 <= version <= MANIFEST_VERSION + ): + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} declares manifestVersion " + f"{version!r}, which this SDK cannot read; refusing every destructive " + "action" + ) + + if not isinstance(data.get("entries"), dict): + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} has a malformed 'entries' " + "map; refusing every destructive action" + ) + + return data, None + + +# ------------------------------------------------------------------------- +# Per-skill reconcile +# ------------------------------------------------------------------------- + + +def _unsafe_path_reason( + root: Path, skill_dir: Path, target: Path, key: str, *, require_directory: bool +) -> str | None: + """ + The path defenses, in one place. + + Returns why ``//SKILL.md`` must not be touched, or ``None``. + Shared by the write and prune paths so the two cannot drift, and not to be + relaxed in either. + + Defense in depth rather than the boundary: each of these inspects a path, so + each is a check-then-use. They stay because they turn a hostile layout into + a reported refusal rather than a failed syscall, and because they are the + whole defense where the ``*at()`` family is absent. + + *require_directory* is the only difference between the two callers: a write + needs a real directory to write into, while a prune only needs to not follow + a link. The containment check is unconditional even when ``skill_dir`` does + not exist yet — ``realpath`` resolves the existing prefix and appends the + rest, so a fresh key under a valid root passes. + """ + if skill_dir.is_symlink(): + return f"{key} is a symlink" + if require_directory and skill_dir.exists() and not skill_dir.is_dir(): + return f"{key} exists and is not a directory" + if target.is_symlink(): + return "the target file is a symlink" + if Path(os.path.realpath(skill_dir)).parent != root: + return f"it resolves outside the managed root {root}" + return None + + +def _key_rejection_reason(key: Any) -> str | None: + """ + Why *key* must not become a directory name under the managed root, or ``None``. + + Re-validated locally whatever any upstream layer already did, and before any + filesystem call, because a key becomes a path component. Shared by the write + and prune paths, and not to be relaxed in either. + + ``key.encode`` below is safe only because it runs *after* the pattern check, + which admits no surrogate. Do not reorder the two. + """ + if not is_valid_skill_key(key): + return f"{key!r} is not a valid skill key: it {skill_key_rejection_reason(key)}" + # The data model allows 256 characters; no mainstream filesystem allows a + # 256-byte path component. Catch it here so it is a reported action rather + # than an ENAMETOOLONG raised from the first stat in the caller. + key_bytes = len(key.encode("utf-8")) + if key_bytes > _MAX_PATH_COMPONENT_BYTES: + return ( + f"skill key '{key[:32]}...' is {key_bytes} bytes, over the " + f"{_MAX_PATH_COMPONENT_BYTES}-byte limit for a single directory name" + ) + # Checked here rather than in the grammar, for the same reason as the byte + # bound above. See agents.md: a grammar-level rejection would fail a whole + # AI Config over one skill, and would shrink skill_refs, which is what + # authorizes a prune. + if key in _WINDOWS_RESERVED_NAMES: + return ( + f"skill key '{key}' is a name Windows reserves for a device and " + "cannot be a directory name there; it is rejected on every platform " + "so a managed root written on one OS is usable on the other" + ) + return None + + +def _write_one( + root: Path, root_fd: int | None, skill: Skill, entries: dict[str, Any] +) -> ReconcileAction: + """Reconciles one verified skill against the managed root.""" + key = skill.key + + def failed(message: str) -> ReconcileAction: + return ReconcileAction( + key=key, action="error", version=skill.version, error=message + ) + + rejection = _key_rejection_reason(key) + if rejection is not None: + return failed(f"{rejection}; nothing was written") + if not is_valid_skill_version(skill.version): + return failed( + f"skill '{key}' has version {skill.version!r}, which is not an " + "integer >= 1; nothing was written" + ) + + skill_dir = root / key + target = skill_dir / SKILL_FILENAME + relative = f"{key}/{SKILL_FILENAME}" + + unsafe = _unsafe_path_reason(root, skill_dir, target, key, require_directory=True) + if unsafe is not None: + return failed(f"'{relative}' was refused: {unsafe}; nothing was written") + + # Re-verify immediately before writing, through the same core the accessors + # use: a Skill can also be constructed directly by a caller. + verified = verified_bytes(key, skill.content, skill.content_hash, skill.version) + if isinstance(verified, VerificationFailure): + return failed( + f"skill '{key}' failed verification immediately before writing: " + f"{verified.reason}; nothing was written" + ) + encoded, content_hash = verified.encoded, verified.content_hash + + # Sweep before writing rather than after, so a temp file this run is about + # to create can never be a candidate. + _sweep_orphan_temp_files(root, root_fd, key) + + # Overwrite only what the manifest records as the SDK's under this key. + entry = entries.get(relative) + managed = isinstance(entry, dict) and entry.get("key") == key + + # The directory is pinned here, before anything is decided, and the pin is + # held through the write: the existence probe, the compare read and the + # rename all resolve against the same descriptor, so nothing swapped in + # between can change which branch runs or where the write lands. Created + # relative to *root_fd* for the same reason — ``mkdir`` follows a symlink at + # its parent. A directory that does not exist yet is created now rather than + # after the decision; that only ever happens when there is no file to + # compare against, so the write that follows is the one that fills it. + try: + with pinned_directory(skill_dir, create=True, dir_fd=root_fd) as skill_fd: + try: + on_disk = _read_skill_file(skill_dir, skill_fd, max_bytes=len(encoded)) + except OSError as exc: + if not managed: + # A read that failed proves nothing, and must never become + # an overwrite: it is the comparison below that would + # authorize one. + return failed( + f"'{relative}' exists, the manifest does not record it as " + f"managed under key '{key}', and it could not be read to " + f"compare against the resolved content: {exc}; refusing to " + "overwrite a file this SDK may not have written" + ) + return failed(f"'{relative}' could not be read: {exc}") + + if on_disk is None: + action: ReconcileActionKind = "written" + elif hashlib.sha256(on_disk).hexdigest() == content_hash: + # Hash first, and decide from the bytes. The manifest check + # below is what protects a file the SDK did not write, but on + # its own it also refuses one the SDK wrote and was killed + # before recording it, wedging every later reconcile. Comparing + # the bytes separates those two cases, and only content + # byte-identical to what LaunchDarkly resolved is adopted. This + # exception must not be widened — see agents.md. + # + # ``skipped_current`` rather than a new action kind: the bytes + # on disk already are the resolved content, as true for an + # adopted file as for one the SDK wrote. Adoption records a + # manifest entry, so the file becomes prunable later. + _update_entry(entries, relative, skill, content_hash) + record_materialized(key, len(encoded), content_hash, "skipped_current") + return ReconcileAction( + key=key, + action="skipped_current", + version=skill.version, + path=str(target), + ) + elif not managed: + return failed( + f"'{relative}' exists but the manifest does not record it as " + f"managed under key '{key}'; refusing to overwrite a file this " + "SDK did not write" + ) + else: + # Stale version or local tampering — LD-resolved content wins. + action = "updated" + + try: + atomic_write(skill_dir, SKILL_FILENAME, encoded, dir_fd=skill_fd) + except OSError as exc: + return failed(f"'{relative}' could not be written: {exc}") + except OSError as exc: + return failed(f"the directory for skill '{key}' could not be created: {exc}") + except ValueError as exc: + return failed(f"'{relative}' was refused: {exc}") + + _update_entry(entries, relative, skill, content_hash) + record_materialized(key, len(encoded), content_hash, action) + return ReconcileAction( + key=key, action=action, version=skill.version, path=str(target) + ) + + +def _skill_file_present(skill_dir: Path, skill_fd: int | None) -> bool: + """ + Whether ``SKILL.md`` is present in the pinned *skill_dir*. + + Probed relative to the descriptor, ``follow_symlinks=False``, so the answer + is about the directory that was pinned and not about wherever its path leads + now. Only ``ENOENT`` means absent: any other failure reports present, so the + step that follows — the read or the unlink — is the one that fails and says + why, rather than a probe deciding silently that there was nothing to do. + Path-based on the ``lstat`` floor, where there is no descriptor. + """ + if skill_fd is None: + return (skill_dir / SKILL_FILENAME).exists() + try: + os.stat(SKILL_FILENAME, dir_fd=skill_fd, follow_symlinks=False) + except FileNotFoundError: + return False + except OSError: + return True + return True + + +def _read_skill_file( + skill_dir: Path, skill_fd: int | None, *, max_bytes: int +) -> bytes | None: + """ + The compare read: the bytes at ``SKILL.md`` in the pinned *skill_dir*, or + ``None`` when there is no file there. + + Both the probe and the read resolve against *skill_fd*, so the bytes that + decide adoption, update or refusal are the ones in the directory that was + pinned. Any other failure propagates as the ``OSError`` it was, for the + caller to report. + """ + if not _skill_file_present(skill_dir, skill_fd): + return None + if skill_fd is None: + return _read_regular_file(skill_dir / SKILL_FILENAME, max_bytes=max_bytes) + return _read_regular_file(SKILL_FILENAME, max_bytes=max_bytes, dir_fd=skill_fd) + + +def _read_regular_file( + target: Path | str, *, max_bytes: int, dir_fd: int | None = None +) -> bytes: + """ + Reads *target*, refusing anything that is not a regular file. + + Each flag earns its place. ``O_NONBLOCK``: opening a FIFO with no writer + blocks forever, so a managed file swapped for one would hang the reconcile + and the event loop with it (a no-op for regular files). ``O_NOFOLLOW``: no + trailing symlink. ``O_BINARY``: 0 on POSIX, but without it a Windows + descriptor translates CRLF and the bytes stop being verbatim. The type check + reads ``fstat`` on the descriptor, never the path. + + ``max_bytes`` is required, not optional, so a new call site cannot pull an + arbitrary file into memory by omission. The read stops at ``max_bytes + 1``, + the extra byte distinguishing "at the cap" from "over it". + + Given a *dir_fd*, *target* is a bare filename resolved inside that + descriptor. Every read under the managed root — the manifest and each + compare read — passes one wherever the platform has descriptors, so the + bytes a decision is made from come from the directory that was pinned. + """ + flags = ( + os.O_RDONLY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0) + | getattr(os, "O_BINARY", 0) + ) + fd = os.open(target, flags, dir_fd=dir_fd) + try: + if not stat.S_ISREG(os.fstat(fd).st_mode): + raise OSError("the target file is not a regular file") + chunks: list[bytes] = [] + remaining = max_bytes + 1 + while remaining > 0: + chunk = os.read(fd, min(remaining, 65536)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + finally: + os.close(fd) + + +def _sweep_orphan_temp_files(root: Path, root_fd: int | None, key: str) -> None: + """ + Removes temp files a killed reconcile left behind under ``//``. + + ``atomic_write`` unlinks its own temp file on any exception, but a + ``SIGKILL`` between the create and the rename leaves one behind that no + manifest entry records — and ``_prune_one``'s ``rmdir`` only succeeds on an + empty directory, so one orphan pins a skill's directory permanently. + + This is the one place the SDK removes a file the manifest does not list, so + it is bounded on every axis: inside ``//`` only, for a key that + passes ``_key_rejection_reason``; only names ``safe_fs`` recognizes, asked + of ``safe_fs`` so the recognizer cannot drift from the writer; only regular + files; the listing read off the pinned descriptor, and every removal + relative to it. Never widen these — see agents.md. + + Never raises and never aborts the run: the reconcile has succeeded either + way, so a sweep that cannot happen is a warning. A directory that does not + exist is not a failure — there is nothing to sweep, and it is the pin that + says so rather than a separate probe of the path. + """ + if _key_rejection_reason(key) is not None: + return + skill_dir = root / key + + try: + with pinned_directory(skill_dir, dir_fd=root_fd) as dir_fd: + # ``os.listdir`` accepts the descriptor itself on POSIX, so the + # names come from the directory that was pinned; on the lstat floor + # there is no descriptor and the path is all there is. + listed = os.listdir(skill_dir if dir_fd is None else dir_fd) + for name in sorted(listed): + if is_temp_name(name, SKILL_FILENAME): + _remove_orphan_temp_file(skill_dir, name, dir_fd) + except DirectoryMissing: + return + except (OSError, ValueError) as exc: + logger.warning( + "orphaned temp files under skill '%s' could not be swept: %s", key, exc + ) + + +def _remove_orphan_temp_file(skill_dir: Path, name: str, dir_fd: int | None) -> None: + """ + Removes one recognized orphan. A per-file failure warns and moves on. + + The type check is what keeps the temp naming from being a way to have this + SDK delete something it did not write: a symlink or a FIFO wearing that name + is not a file ``atomic_write`` left behind, so it is not this function's to + remove. It is read off the descriptor, not the path, wherever there is one. + """ + try: + if dir_fd is not None: + mode = os.stat(name, dir_fd=dir_fd, follow_symlinks=False).st_mode + else: + mode = os.lstat(skill_dir / name).st_mode + if not stat.S_ISREG(mode): + return + unlink_file(skill_dir, name, dir_fd=dir_fd) + except (OSError, ValueError) as exc: + logger.warning("an orphaned temp file could not be removed: %s", exc) + + +def _update_entry( + entries: dict[str, Any], relative: str, skill: Skill, content_hash: str +) -> None: + """ + Records a managed path in the manifest. + + Merges into any existing entry rather than replacing it, so fields written by + a future SDK release survive this one's rewrite. + + ``sha256`` and ``writtenAt`` are recorded for forensics only. The reconcile + decides currency by hashing the bytes on disk, precisely because the manifest + is untrusted, so neither field is ever read back as a decision input. + """ + existing = entries.get(relative) + entry = dict(existing) if isinstance(existing, dict) else {} + entry["key"] = skill.key + entry["version"] = skill.version + entry["sha256"] = content_hash + entry["writtenAt"] = _utc_timestamp() + entries[relative] = entry + + +# ------------------------------------------------------------------------- +# Pruning — how revocation takes effect +# ------------------------------------------------------------------------- + + +def _prune_error(key: str, message: str, version: Any = None) -> ReconcileAction: + """ + A prune refusal. Mirrors ``_write_one``'s local ``failed`` helper. + + *version* comes off the manifest, which is untrusted, so it is validated + here rather than at each call site — the same guard the ``removed`` action + applies, so a refusal and a removal report the field identically. A caller + that does not know a version passes nothing rather than inventing one. + """ + return ReconcileAction( + key=key, + action="error", + version=version if is_valid_skill_version(version) else None, + error=message, + ) + + +def _prune( + root: Path, + root_fd: int | None, + entries: dict[str, Any], + requested: set[str], + deadline: float, +) -> list[ReconcileAction]: + """ + Removes managed skills that are no longer requested. + + This is also how revocation takes effect: a revoked skill is simply absent + from the resolved set, so the next reconcile removes it. There is + deliberately no opt-out. + + The deadline applies here just as it does to the writes: a skill left + unpruned is reported as an error and stays in the manifest, so the next + reconcile picks it up. + """ + actions: list[ReconcileAction] = [] + + for relative, entry in list(entries.items()): + if not isinstance(entry, dict): + continue + key = entry.get("key") + if not isinstance(key, str) or key in requested: + continue + + if time.monotonic() >= deadline: + actions.append( + _prune_error( + key, + f"the timeout was exhausted before '{relative}' could be " + "pruned; it was left in place", + entry.get("version"), + ) + ) + continue + + # Only a manifest path this SDK could have written is removable. + if ( + _key_rejection_reason(key) is not None + or relative != f"{key}/{SKILL_FILENAME}" + ): + actions.append( + _prune_error( + key, + f"manifest entry '{relative}' does not name a path this SDK " + f"could own under key '{key}'; it was left in place", + entry.get("version"), + ) + ) + continue + + try: + actions.append(_prune_one(root, root_fd, relative, key, entries)) + except OSError as exc: + actions.append( + _prune_error( + key, + f"'{relative}' could not be removed: {exc}", + entry.get("version"), + ) + ) + + return actions + + +def _unlink_skill_file( + skill_dir: Path, skill_fd: int | None, relative: str +) -> str | None: + """ + Performs the removal itself. Returns a failure reason, or ``None`` on success. + + *skill_fd* is the descriptor ``_prune_one`` already holds for the directory, + the same one the existence probe was answered from — so the file the probe + found is the file this removes. ``unlink`` never follows a trailing symlink + but does resolve the directory above it, which is why it must not be given + a path here. + """ + try: + unlink_file(skill_dir, SKILL_FILENAME, dir_fd=skill_fd) + except SymlinkRefused: + return f"'{relative}' was not removed: the target file is a symlink" + except OSError as exc: + return f"'{relative}' could not be removed: {exc}" + return None + + +def _prune_one( + root: Path, + root_fd: int | None, + relative: str, + key: str, + entries: dict[str, Any], +) -> ReconcileAction: + """Removes one managed skill file, and its directory when that empties it.""" + skill_dir = root / key + target = skill_dir / SKILL_FILENAME + version = entries[relative].get("version") + + unsafe = _unsafe_path_reason(root, skill_dir, target, key, require_directory=False) + if unsafe is not None: + return _prune_error(key, f"'{relative}' was not removed: {unsafe}", version) + + # Before the removal, so the ``rmdir`` below is not defeated by an orphaned + # temp file that nothing else on disk records. + _sweep_orphan_temp_files(root, root_fd, key) + + # Pinned relative to the root before the existence probe, and held through + # the unlink: a directory swapped after the pin cannot make the probe report + # a file that is not there — or, worse, report nothing where the SDK's file + # still is, which would drop the manifest entry and leave a revoked skill on + # disk with a report that says it was removed. A directory that is not there + # at all is the ordinary case for a file that is already gone. + removed_from_disk = False + try: + with pinned_directory(skill_dir, dir_fd=root_fd) as skill_fd: + if _skill_file_present(skill_dir, skill_fd): + failure = _unlink_skill_file(skill_dir, skill_fd, relative) + if failure is not None: + return _prune_error(key, failure, version) + removed_from_disk = True + except DirectoryMissing: + pass + except ValueError as exc: + return _prune_error(key, f"'{relative}' was not removed: {exc}", version) + + if removed_from_disk: + try: + # Relative to the root descriptor: rmdir is safe at the key (it + # fails ENOTDIR on a symlink and needs an empty directory), but a + # path-based call re-resolves the root above it. + if root_fd is not None: + os.rmdir(key, dir_fd=root_fd) + else: + skill_dir.rmdir() + except OSError: + pass # the directory is not empty: other files live here too + + entries.pop(relative, None) + + if removed_from_disk: + record_revoked(key, version) + + return ReconcileAction( + key=key, + action="removed", + version=version if is_valid_skill_version(version) else None, + path=str(target), + ) + + +def _utc_timestamp() -> str: + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/packages/client/src/launchdarkly_ai_server/skills_watch.py b/packages/client/src/launchdarkly_ai_server/skills_watch.py new file mode 100644 index 00000000..72cad733 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_watch.py @@ -0,0 +1,323 @@ +""" +Agent Skills — re-reconcile on delivery, so revocation does not wait for a restart. + +``write_skills`` is a one-shot reconcile: it materializes what the store holds +now. With a hand-populated store that is sufficient, and a revocation takes +effect at the next process restart. + +A streaming FDv2 connection changes the premise. A ``delete-object`` reaches a +live connection in **seconds**, and the store publishes a change listener, so +wiring the two together collapses the gap between "LaunchDarkly revoked this +skill" and "its ``SKILL.md`` is off the agent's disk" from a process lifetime to +a debounce interval. + +``on_unavailable="keep"`` stays the default: an outage must not read as +"everything was revoked". A watcher that pruned on a failed retrieval would +convert every transport failure into deletion of the application's skill files. + +Layering: this module sits *above* ``skills_fs`` and calls ``write_skills`` +without modifying it. Nothing in the reconcile, the accessors, or verification +knows this file exists. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import threading +from collections.abc import Callable, Sequence +from typing import Any + +from .skills_core import SKILL_OBJECT_KIND, get_store +from .skills_fs import OnUnavailable, write_skills +from .types import ReconcileReport, Skill, SkillReference + +logger = logging.getLogger(__name__) + +DEFAULT_DEBOUNCE_SECONDS = 0.5 +""" +How long a change waits for its neighbours before a reconcile runs. + +A full payload transfer commits many objects at once and the listener fires per +object, so without coalescing a payload of forty skills would run forty +reconciles against one root. Half a second is far below the seconds-scale +latency this feature is trying to achieve and far above the microseconds a +commit's listener calls take. +""" + + +class SkillWatcher: + """ + A running re-reconcile. Returned by ``watch_skills``; stop it with ``close``. + + One watcher owns one root. **Do not point two watchers at the same root**, + and do not run ``write_skills`` against a watched root concurrently: the + reconcile's own contract is one root, one reconcile at a time, because two + interleaved runs lose the loser's manifest entries and leave the files it + wrote unmanaged. This class enforces that for its *own* reconciles — they run + on a single worker thread, serialised — and cannot enforce it against a + caller who reconciles the same root by hand. + + The watcher owns its registration on *store*: it registers ``notify`` when + constructed and unregisters it in ``close``, so a closed watcher is no longer + reachable from the store and can be collected. *store* must implement + ``add_listener``; ``remove_listener`` is probed for and, when the store does + not offer it, the listener stays registered for the store's lifetime. + """ + + def __init__( + self, + request: Sequence[Skill | SkillReference | str] | str, + root: str | os.PathLike[str], + store: Any, + *, + prune: bool, + timeout: float, + on_unavailable: OnUnavailable, + debounce: float, + on_reconcile: Callable[[ReconcileReport], Any] | None, + ) -> None: + self._request = request + self._root = root + self._prune = prune + self._timeout = timeout + self._on_unavailable = on_unavailable + self._debounce = debounce + self._on_reconcile = on_reconcile + + self._wake = threading.Event() + self._stop = threading.Event() + self._reconciles = 0 + self._lock = threading.Lock() + self._thread = threading.Thread( + target=self._run, name="ld-ai-skills-reconcile", daemon=True + ) + + # Register before the initial reconcile, and leave the worker unstarted + # until ``start``. ``notify`` only sets an event, so a change that lands + # while that reconcile is still running is recorded rather than lost, and + # the worker cannot reconcile the root while the caller's own reconcile is + # in flight. A store whose ``add_listener`` raises leaves no thread behind. + self._store = store + self._registered = False + store.add_listener(SKILL_OBJECT_KIND, self.notify) + self._registered = True + + def _start(self) -> None: + """ + Starts the worker. ``watch_skills`` calls this once, after the initial + reconcile; it is not part of the caller-facing interface. + + Split from construction so registration and reconciling can be ordered + independently: the listener attaches first, so no change is missed, while + the first re-reconcile waits for the initial one to finish, so a root only + ever has one reconcile running at a time. + """ + self._thread.start() + + # -- the listener the store calls ------------------------------------- + + def notify(self, _raw: Any = None) -> None: + """ + The store's change listener. Records that something changed; runs nothing. + + Deliberately trivial. It is called on the delivery thread, where a + reconcile — which does synchronous filesystem I/O, an fsync per file, and + a manifest rewrite — would stall event processing for the duration and, + on a stream, let the connection's read buffer back up behind a disk write. + The argument is ignored: a put's raw object and a revocation's tombstone + both mean the same thing here, which is "the store is not what it was". + """ + self._wake.set() + + # -- the worker -------------------------------------------------------- + + def _run(self) -> None: + while not self._stop.is_set(): + if not self._wake.wait(timeout=0.5): + continue + if self._stop.is_set(): + return + # Coalesce the rest of the burst. Clearing *before* the sleep rather + # than after is what makes a change arriving mid-debounce trigger the + # next pass instead of being swallowed by this one. + self._wake.clear() + if self._stop.wait(self._debounce): + return + self._reconcile_once() + + def _reconcile_once(self) -> None: + try: + report = asyncio.run( + write_skills( + self._request, + self._root, + prune=self._prune, + timeout=self._timeout, + on_unavailable=self._on_unavailable, + ) + ) + except Exception: + # A watcher that died on one bad reconcile would silently stop + # tracking revocations, which is worse than a noisy one. + logger.error( + "A skill re-reconcile raised; the watcher continues", exc_info=True + ) + return + + with self._lock: + self._reconciles += 1 + changed = [ + action + for action in report.actions + if action.action in ("written", "updated", "removed", "error") + ] + if changed: + logger.info( + "Re-reconciled skills after a delivery change: %d action(s) of note", + len(changed), + ) + if self._on_reconcile is not None: + try: + self._on_reconcile(report) + except Exception: + logger.error("A watch_skills callback raised", exc_info=True) + + # -- lifecycle --------------------------------------------------------- + + @property + def reconciles(self) -> int: + """How many re-reconciles have completed since the watcher started. + + Excludes the initial reconcile ``watch_skills`` awaits, which is the + caller's own result.""" + with self._lock: + return self._reconciles + + def close(self, timeout: float = 15.0) -> None: + """ + Stops watching. Idempotent. Does not undo anything already on disk. + + Waits out an in-flight reconcile rather than interrupting one, because a + reconcile killed between its content writes and its manifest rewrite is + the one case the manifest format has to recover from — worth avoiding + where the timing is under the SDK's control. + + Detaches ``notify`` from the store first, so no further change reaches a + watcher that is shutting down and the store no longer holds a reference to + it. A store without the optional ``remove_listener`` is left as it is + rather than failing the close. + """ + self._detach() + self._stop.set() + self._wake.set() + if self._thread.is_alive() and self._thread is not threading.current_thread(): + self._thread.join(timeout=timeout) + + def _detach(self) -> None: + with self._lock: + if not self._registered: + return + self._registered = False + remove_listener = getattr(self._store, "remove_listener", None) + if callable(remove_listener): + remove_listener(SKILL_OBJECT_KIND, self.notify) + + def __enter__(self) -> SkillWatcher: + return self + + def __exit__(self, *_exc: Any) -> None: + self.close() + + +async def watch_skills( + skills: Sequence[Skill | SkillReference | str] | str, + root: str | os.PathLike[str], + *, + prune: bool = True, + timeout: float = 10.0, + on_unavailable: OnUnavailable = "keep", + debounce: float = DEFAULT_DEBOUNCE_SECONDS, + on_reconcile: Callable[[ReconcileReport], Any] | None = None, +) -> tuple[ReconcileReport, SkillWatcher]: + """ + Reconciles now, then re-reconciles whenever delivery changes. + + Every argument that ``write_skills`` takes means the same thing here and is + passed straight through; the reconcile's semantics are untouched. Returns the + initial reconcile's report — so a caller can fail fast on a bad root or a + corrupt manifest exactly as they would with ``write_skills`` — paired with a + ``SkillWatcher`` to close when the process is done:: + + report, watcher = await watch_skills("*", "/etc/agent/skills") + try: + ... + finally: + watcher.close() + + A revocation delivered over a streaming connection then prunes the skill's + files within ``debounce`` of arriving, rather than at the next restart. + + Requires a store that implements the optional ``add_listener`` half of the + ``SkillStore`` interface. Raises ``RuntimeError`` when no store is configured, + and when the configured store has no ``add_listener`` — the second case + failing loudly rather than degrading to a one-shot reconcile, because a + watcher that silently never fires looks exactly like a watcher whose skills + never changed. The optional ``remove_listener`` lets ``SkillWatcher.close`` + detach from the store; a store without it still works, but each closed + watcher then stays registered for the store's lifetime. + """ + store = get_store() + if store is None: + raise RuntimeError( + "watch_skills needs a configured skill store. Configure one with " + 'init_client(options={"skillStore": store}).' + ) + add_listener = getattr(store, "add_listener", None) + if not callable(add_listener): + raise RuntimeError( + "watch_skills needs a skill store that implements add_listener(kind, " + "fn); the configured store does not, so delivery changes cannot be " + "observed. Use write_skills for a one-shot reconcile, or configure a " + "store with a delivery transport (FDv2SkillStore)." + ) + if debounce < 0: + raise ValueError(f"debounce must not be negative, got {debounce!r}") + + # The watcher attaches its listener before the initial reconcile, not after. + # The reconcile snapshots the store as its first step and then spends the + # rest of its time on the filesystem — a write and an fsync per skill, the + # prune, the manifest rewrite — so a change delivered after that snapshot + # needs something already listening to be seen at all. Nothing re-reconciles + # on a timer, so a revocation that landed unobserved would wait for the next + # unrelated change, which on a quiet root means the next restart. + watcher = SkillWatcher( + skills, + root, + store, + prune=prune, + timeout=timeout, + on_unavailable=on_unavailable, + debounce=debounce, + on_reconcile=on_reconcile, + ) + try: + # The initial reconcile runs on the caller's thread, so its report is the + # caller's to inspect and a bad root raises out of `watch_skills` rather + # than into a worker thread's log. + report = await write_skills( + skills, root, prune=prune, timeout=timeout, on_unavailable=on_unavailable + ) + except BaseException: + # The listener is already attached, so a reconcile that raises must not + # leave it on the store: the caller has no watcher to close. + watcher.close() + raise + + # Only now start the worker. A change that arrived during the reconcile has + # already set the wake event, so the worker's first pass picks it up; one that + # arrived before the reconcile's snapshot is already on disk, and the + # redundant pass it triggers converges on the same state. + watcher._start() + return report, watcher diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index f3505364..ba5790f8 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -92,7 +92,8 @@ class Message: AiConfigRep = dict[str, Any] """ Raw AI config dict as returned by ``parse_ai_config``. Fields include -``model``, ``provider``, and at least one of ``instructions`` / ``messages``. +``model``, ``provider``, at least one of ``instructions`` / ``messages``, and an +optional ``skills`` array of ``{key, version}`` references (see ``skill_refs``). """ VariationMeta = dict[str, Any] @@ -420,6 +421,156 @@ class ProviderGraphResponse: """Results from a graph-level judge, if configured.""" +# --------------------------------------------------------------------------- +# Agent Skills +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class SkillReference: + """A version-pinned pointer to a skill, as attached to an AI Config variation.""" + + key: str + """Immutable skill key — ``^[a-z0-9][a-z0-9-]*$``, at most 256 characters.""" + version: int + """Immutable skill version — an integer >= 1.""" + + +@dataclass(frozen=True) +class Skill: + """ + A single verbatim ``SKILL.md`` document. + + Only ever constructed after integrity verification passes, so ``content`` + holds the exact byte sequence LaunchDarkly delivered and ``content_hash`` + is its sha256. Instances are immutable. + """ + + key: str + version: int + content: bytes + """The verified verbatim bytes, exactly as LaunchDarkly delivered and + hashed them. Opaque to this SDK: no encoding is claimed and nothing here + ever parses or interprets them.""" + content_hash: str + """sha256, lowercase hex, over the verbatim bytes of ``content``.""" + name: str | None = None + """Display name from LaunchDarkly metadata; never parsed from the content.""" + description: str | None = None + """Description from LaunchDarkly metadata; never parsed from the content.""" + + +SkillOutcomeReason = Literal[ + "absent", "integrity_failure", "ok", "store_unavailable", "wrong_version" +] +""" +The closed set of outcomes ``get_skill_result`` reports. + +Each token is a distinct *decision* a caller can make, which is the point of the +type: ``absent`` is a skill the store does not hold, ``integrity_failure`` is +content that was delivered and did not verify, and a caller that wants to fail +closed on suspected tampering while tolerating a merely-absent skill needs the +two told apart. + +- ``ok`` — a verified skill was returned. +- ``absent`` — the store answered, and does not hold the key. +- ``integrity_failure`` — content was delivered and failed verification; it was + withheld. The one token worth failing closed on. +- ``store_unavailable`` — the store itself could not answer: it raised. + Deliberately distinct from ``absent``, because an outage is not a deletion. +- ``wrong_version`` — the store answered with a version other than the one + asked for, so the answer was withheld. +""" + + +@dataclass(frozen=True) +class SkillOutcome: + """ + Why one retrieval returned what it did — the reported form of ``get_skill``. + + ``get_skill`` collapses every failure to ``None``, which is the right shape + for a caller that only wants content and cannot act on the difference. This + is the shape for a caller that can: ``reason`` names which of the five + outcomes happened, so an integrity failure is distinguishable from a skill + that simply is not configured. The two accessors differ only in what they + report — the retrieval, the verification, and the telemetry are the same + code path, run once. + + Instances are immutable. + """ + + skill: Skill | None + """The verified skill, and only ever populated when ``reason == "ok"``.""" + reason: SkillOutcomeReason + """Which outcome happened. A closed set — see ``SkillOutcomeReason``.""" + detail: str | None + """ + Human-readable detail, set for every reason except ``ok``. + + Safe to log or surface to an operator: it carries the skill key and the + failure mode, and never any skill content or filesystem path. Intended for a + human, not for matching on — branch on ``reason``. + """ + + +ReconcileActionKind = Literal[ + "written", "updated", "skipped_current", "removed", "error" +] +""" +The closed set of outcomes ``write_skills`` reports. + +- ``written`` — the file did not exist and now holds the resolved content. +- ``updated`` — a managed file held different bytes and was overwritten. +- ``skipped_current`` — the bytes on disk already are the resolved content. +- ``removed`` — the skill is no longer managed and is not on disk. Reported + whether or not this run was the one that deleted the file, since a formerly + managed file a caller had already removed by hand reaches the same end state. +- ``error`` — the outcome was refused or failed; see ``ReconcileAction.error``. +""" + + +@dataclass(frozen=True) +class ReconcileAction: + """What ``write_skills`` did — or refused to do — for one skill.""" + + key: str + """ + The skill key, or the **empty string** for a failure that belongs to the run + rather than to one skill — a corrupt manifest, a manifest that could not be + rewritten, a retrieval that failed before any key was known. Callers grouping + a report by key need to expect that sentinel; a report may carry both kinds. + """ + action: ReconcileActionKind + version: int | None = None + path: str | None = None + """Canonical resolved path, when one was determined.""" + error: str | None = None + """Failure detail, set only when ``action == "error"``.""" + + +@dataclass(frozen=True) +class ReconcileReport: + """The result of a ``write_skills`` run — every outcome is visible here.""" + + actions: list[ReconcileAction] = field(default_factory=list) + + @property + def ok(self) -> bool: + """``True`` iff no action is an ``error``.""" + return not self.errors + + @property + def errors(self) -> list[ReconcileAction]: + """ + The ``error`` actions, in ``actions`` order. + + Exposed so callers never re-derive it — filtering ``actions`` is + boilerplate that otherwise reappears in every consumer. ``ok`` is defined + in terms of this, so the two can never disagree. + """ + return [a for a in self.actions if a.action == "error"] + + # --------------------------------------------------------------------------- # Model / graph options # --------------------------------------------------------------------------- diff --git a/packages/client/src/launchdarkly_ai_server/types_validation.py b/packages/client/src/launchdarkly_ai_server/types_validation.py index acdcce42..37cd56bf 100644 --- a/packages/client/src/launchdarkly_ai_server/types_validation.py +++ b/packages/client/src/launchdarkly_ai_server/types_validation.py @@ -1,16 +1,62 @@ from __future__ import annotations -from typing import Any +import re +from typing import Any, TypeGuard from .types import ParseFailure, ParseResult, ParseSuccess _VALID_ROLES = {"user", "assistant", "system"} +SKILL_KEY_GRAMMAR = "^[a-z0-9][a-z0-9-]*$" +""" +The skill key grammar, as a string, so every message that has to explain a +rejection quotes the rule rather than restating it. Tightening the pattern below +then cannot leave an error message describing the old grammar. +""" + +_SKILL_KEY_PATTERN = re.compile(r"\A[a-z0-9][a-z0-9-]*\Z") +""" +``SKILL_KEY_GRAMMAR``, anchored with ``\\A``/``\\Z`` rather than ``^``/``$`` +because ``$`` also matches immediately before a trailing newline, which would +let ``"pdf-extraction\\n"`` through as a directory name. +""" + +SKILL_KEY_MAX_LENGTH = 256 +"""Longest key the data model permits. Note that no mainstream filesystem allows +a 256-byte path component, so ``write_skills`` applies a tighter bound of its own.""" + def _is_object(v: Any) -> bool: return isinstance(v, dict) +def skill_key_rejection_reason(key: Any) -> str | None: + """ + Why *key* is not a valid skill key, or ``None`` when it is. + + The canonical explanation, so the config parser, the filesystem layer and + the reference projection all reject a key for the same stated reason. + ``is_valid_skill_key`` is this predicate with the reason discarded. + """ + if not isinstance(key, str): + return "must be a string" + if len(key) > SKILL_KEY_MAX_LENGTH: + return f"must be at most {SKILL_KEY_MAX_LENGTH} characters" + if _SKILL_KEY_PATTERN.match(key) is None: + return f"must match {SKILL_KEY_GRAMMAR}" + return None + + +def is_valid_skill_key(key: Any) -> TypeGuard[str]: + """Skill keys are untrusted input everywhere they appear — validate every time.""" + return isinstance(key, str) and skill_key_rejection_reason(key) is None + + +def is_valid_skill_version(version: Any) -> TypeGuard[int]: + """Skill versions are integers >= 1. ``bool`` is not an acceptable integer.""" + return isinstance(version, int) and not isinstance(version, bool) and version >= 1 + + def _parse_tool(raw: Any, key: str) -> str | None: """Returns an error message string or ``None`` on success.""" if not _is_object(raw): @@ -24,6 +70,28 @@ def _parse_tool(raw: Any, key: str) -> str | None: return None +def _parse_skills(raw: Any) -> str | None: + """ + Validates the optional ``skills`` array. Returns an error message or ``None``. + + Fail closed: a malformed reference makes the whole config malformed, because + an SDK that silently dropped a bad reference would materialize a partial + skill set without telling anyone. + """ + if not isinstance(raw, list): + return "skills must be an array of {key, version} objects" + + for index, entry in enumerate(raw): + if not _is_object(entry): + return f"skills[{index}] must be an object with key and version" + key_rejection = skill_key_rejection_reason(entry.get("key")) + if key_rejection is not None: + return f"skills[{index}].key {key_rejection}" + if not is_valid_skill_version(entry.get("version")): + return f"skills[{index}].version must be an integer >= 1" + return None + + def parse_ai_config(raw: Any) -> ParseResult: """ Validates a raw LaunchDarkly flag variation as an ``AiConfigRep``. @@ -88,4 +156,10 @@ def parse_ai_config(raw: Any) -> ParseResult: error={"message": "outputFormat must be an object (JSON Schema)"}, ) + skills = raw.get("skills") + if skills is not None: + err = _parse_skills(skills) + if err: + return ParseFailure(success=False, error={"message": err}) + return ParseSuccess(success=True, data=raw) diff --git a/packages/client/tests/conftest.py b/packages/client/tests/conftest.py index 6c140200..17fbb68b 100644 --- a/packages/client/tests/conftest.py +++ b/packages/client/tests/conftest.py @@ -1,7 +1,14 @@ +import hashlib +from collections.abc import Iterator +from typing import Any from unittest.mock import AsyncMock, MagicMock import pytest +import launchdarkly_ai_server.lifecycle as lifecycle_module +import launchdarkly_ai_server.skills as skills_module +from launchdarkly_ai_server import InMemorySkillStore + @pytest.fixture def mock_ld_client() -> MagicMock: @@ -36,3 +43,108 @@ def mock_tracer(mock_span: MagicMock) -> MagicMock: tracer.start_as_current_span.return_value.__exit__ = MagicMock(return_value=False) tracer.start_span.return_value = mock_span return tracer + + +# --------------------------------------------------------------------------- +# Agent Skills helpers +# +# Exposed as fixtures rather than importable module-level helpers: pytest runs +# with --import-mode=importlib and the tests directory is not a package, so +# sibling imports from conftest are not reliable. +# --------------------------------------------------------------------------- + +SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" + + +class _RecordingEmitter: + """Telemetry seam double — records (signal, properties) pairs.""" + + def __init__(self) -> None: + self.records: list[tuple[str, dict[str, Any]]] = [] + + def record(self, signal: str, properties: dict[str, Any]) -> None: + self.records.append((signal, properties)) + + def signals(self, name: str) -> list[dict[str, Any]]: + return [props for sig, props in self.records if sig == name] + + +class _ThrowingEmitter: + """Telemetry seam double whose record() always raises.""" + + def record(self, signal: str, properties: dict[str, Any]) -> None: + raise RuntimeError("emitter exploded") + + +@pytest.fixture +def make_raw_skill() -> Any: + """Factory for wire-shaped raw store objects with a correct contentHash.""" + + def _make( + key: str = "test-skill", + version: int = 1, + content: str = SKILL_BODY, + **overrides: Any, + ) -> dict[str, Any]: + obj: dict[str, Any] = { + "key": key, + "version": version, + "content": content, + "contentHash": hashlib.sha256(content.encode("utf-8")).hexdigest(), + "name": "Test Skill", + "description": "A skill used in tests.", + } + obj.update(overrides) + return obj + + return _make + + +@pytest.fixture +def store() -> InMemorySkillStore: + """An in-memory store, wired in as the configured store for the test.""" + s = InMemorySkillStore() + skills_module._set_store(s) + return s + + +class _ExplodingStore: + """Store double whose every read raises — the "transport is down" case.""" + + def get_object(self, kind: str, key: str) -> dict[str, Any] | None: + raise RuntimeError("transport failure") + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + raise RuntimeError("transport failure") + + +@pytest.fixture +def exploding_store() -> _ExplodingStore: + """A raising store, wired in as the configured store for the test.""" + s = _ExplodingStore() + skills_module._set_store(s) + return s + + +@pytest.fixture +def reset_skill_state() -> Iterator[None]: + """Clears client, store, and emitter module state around one test. + + Opted into per module with ``pytestmark = pytest.mark.usefixtures(...)`` + rather than being autouse here: autouse would newly reset lifecycle state + for every test in every module in this directory, which is a behaviour change + well outside the skills tests. + """ + lifecycle_module._reset_for_testing() + yield + lifecycle_module._reset_for_testing() + + +@pytest.fixture +def recording_emitter() -> _RecordingEmitter: + return _RecordingEmitter() + + +@pytest.fixture +def throwing_emitter() -> _ThrowingEmitter: + return _ThrowingEmitter() diff --git a/packages/client/tests/test_safe_fs.py b/packages/client/tests/test_safe_fs.py new file mode 100644 index 00000000..9e622886 --- /dev/null +++ b/packages/client/tests/test_safe_fs.py @@ -0,0 +1,261 @@ +""" +Tests for the descriptor-pinned filesystem primitives. + +These exercise ``safe_fs`` directly, on its own terms — the module knows nothing +about skills, and its guarantees are worth asserting without a caller in the way. +The TOCTOU races these primitives exist to close are proved through the +materialization layer, which is what actually holds a descriptor across a +sequence of operations. +""" + +from __future__ import annotations + +import os +import stat +from pathlib import Path + +import pytest + +import launchdarkly_ai_server.safe_fs as safe_fs_module +from launchdarkly_ai_server.safe_fs import ( + SymlinkRefused, + atomic_write, + atomic_write_in, + open_directory_nofollow, + open_or_create_directory, + pinned_directory, + unlink_file, +) + + +class TestOpenDirectory: + """Pinning a directory, and refusing anything that is not one.""" + + def test_opens_a_real_directory(self, tmp_path: Path) -> None: + with pinned_directory(tmp_path) as dir_fd: + if dir_fd is None: + pytest.skip("no *at() family on this platform") + assert stat.S_ISDIR(os.fstat(dir_fd).st_mode) + + def test_refuses_a_symlink_to_a_directory(self, tmp_path: Path) -> None: + real = tmp_path / "real" + real.mkdir() + link = tmp_path / "link" + link.symlink_to(real, target_is_directory=True) + with pytest.raises(ValueError): + open_directory_nofollow(link) + + def test_refuses_a_regular_file(self, tmp_path: Path) -> None: + target = tmp_path / "file" + target.write_text("not a directory") + with pytest.raises(ValueError): + open_directory_nofollow(target) + + def test_refuses_an_absent_path(self, tmp_path: Path) -> None: + with pytest.raises(ValueError): + open_directory_nofollow(tmp_path / "nope") + + def test_create_makes_the_directory(self, tmp_path: Path) -> None: + target = tmp_path / "new" + fd = open_or_create_directory(target) + try: + assert target.is_dir() + finally: + if fd is not None: + os.close(fd) + + def test_create_refuses_an_existing_symlink(self, tmp_path: Path) -> None: + """``Path.mkdir(exist_ok=True)`` would accept this and reopen the hole. + + A symlink-to-directory already present reads as "already there" to + ``exist_ok``, so the caller's containment check would be bypassed by + something that was never checked. ``os.mkdir`` plus an ``lstat`` on the + ``FileExistsError`` path refuses it. + """ + real = tmp_path / "real" + real.mkdir() + link = tmp_path / "link" + link.symlink_to(real, target_is_directory=True) + with pytest.raises(ValueError): + open_or_create_directory(link) + + def test_pinned_directory_closes_the_descriptor(self, tmp_path: Path) -> None: + with pinned_directory(tmp_path) as dir_fd: + if dir_fd is None: + pytest.skip("no *at() family on this platform") + held = dir_fd + with pytest.raises(OSError): + os.fstat(held) + + +class TestAtomicWrite: + """Explicit mode, no observable partial file, and one rename call site.""" + + def test_writes_the_bytes(self, tmp_path: Path) -> None: + with pinned_directory(tmp_path) as dir_fd: + atomic_write(tmp_path, "f.txt", b"hello", dir_fd=dir_fd) + assert (tmp_path / "f.txt").read_bytes() == b"hello" + + def test_mode_is_0644_and_never_executable(self, tmp_path: Path) -> None: + """Set explicitly on the descriptor, so the process umask cannot widen or + narrow it and the execute bit is never inherited.""" + previous = os.umask(0o077) + try: + with pinned_directory(tmp_path) as dir_fd: + atomic_write(tmp_path, "f.txt", b"x", dir_fd=dir_fd) + finally: + os.umask(previous) + mode = (tmp_path / "f.txt").stat().st_mode + assert stat.S_IMODE(mode) == 0o644 + assert not mode & stat.S_IXUSR + + def test_overwrites_an_existing_file(self, tmp_path: Path) -> None: + (tmp_path / "f.txt").write_bytes(b"old") + with pinned_directory(tmp_path) as dir_fd: + atomic_write(tmp_path, "f.txt", b"new", dir_fd=dir_fd) + assert (tmp_path / "f.txt").read_bytes() == b"new" + + def test_leaves_no_temp_file_behind(self, tmp_path: Path) -> None: + with pinned_directory(tmp_path) as dir_fd: + atomic_write(tmp_path, "f.txt", b"x", dir_fd=dir_fd) + assert [p.name for p in tmp_path.iterdir()] == ["f.txt"] + + def test_a_failed_rename_removes_the_temp_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A crash between write and rename must not leave a partial file, and + must not leave the temp file either.""" + + def _boom(*args: object, **kwargs: object) -> None: + raise OSError("injected rename failure") + + monkeypatch.setattr(os, "replace", _boom) + with pinned_directory(tmp_path) as dir_fd: + with pytest.raises(OSError, match="injected"): + atomic_write(tmp_path, "f.txt", b"x", dir_fd=dir_fd) + assert list(tmp_path.iterdir()) == [] + + def test_rename_goes_through_os_replace( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """``os.replace`` is the single rename call site. + + ``os.rename`` must not be substituted for it: it is the only one with + defined overwrite semantics on Windows, and it is the seam the + materialization tests intercept to prove atomicity. + """ + calls: list[object] = [] + real = os.replace + + def _spy(src: object, dst: object, **kwargs: object) -> None: + calls.append(dst) + real(src, dst, **kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(os, "replace", _spy) + with pinned_directory(tmp_path) as dir_fd: + atomic_write(tmp_path, "f.txt", b"x", dir_fd=dir_fd) + assert len(calls) == 1 + assert os.path.basename(str(calls[0])) == "f.txt" + + def test_write_without_a_descriptor_uses_the_path_fallback( + self, tmp_path: Path + ) -> None: + """The no-``*at()`` shape must produce an identical result. + + Windows takes this path for every write, so it is not a degenerate case — + the file, its mode, and the absence of a temp file all have to match. + """ + atomic_write(tmp_path, "f.txt", b"fallback", dir_fd=None) + assert (tmp_path / "f.txt").read_bytes() == b"fallback" + assert stat.S_IMODE((tmp_path / "f.txt").stat().st_mode) == 0o644 + assert [p.name for p in tmp_path.iterdir()] == ["f.txt"] + + def test_the_fallback_works_without_fchmod( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Windows before CPython 3.13 has no ``os.fchmod``, and 3.12 is supported. + + The mode goes on the temp path there instead; what must not happen is an + ``AttributeError`` out of the branch every Windows write takes. + """ + monkeypatch.delattr(os, "fchmod") + monkeypatch.setattr(safe_fs_module, "_SUPPORTS_FCHMOD", False) + atomic_write(tmp_path, "f.txt", b"fallback", dir_fd=None) + assert (tmp_path / "f.txt").read_bytes() == b"fallback" + assert stat.S_IMODE((tmp_path / "f.txt").stat().st_mode) == 0o644 + assert [p.name for p in tmp_path.iterdir()] == ["f.txt"] + + def test_write_in_pins_the_directory_itself(self, tmp_path: Path) -> None: + atomic_write_in(tmp_path, "f.txt", b"x") + assert (tmp_path / "f.txt").read_bytes() == b"x" + + def test_write_in_refuses_a_symlinked_directory(self, tmp_path: Path) -> None: + real = tmp_path / "real" + real.mkdir() + link = tmp_path / "link" + link.symlink_to(real, target_is_directory=True) + with pytest.raises(ValueError): + atomic_write_in(link, "f.txt", b"x") + + +class TestUnlinkFile: + """Removing only a real file, and refusing a link found in its place.""" + + def test_removes_a_regular_file(self, tmp_path: Path) -> None: + (tmp_path / "f.txt").write_text("x") + with pinned_directory(tmp_path) as dir_fd: + unlink_file(tmp_path, "f.txt", dir_fd=dir_fd) + assert not (tmp_path / "f.txt").exists() + + def test_refuses_a_symlink_rather_than_removing_it(self, tmp_path: Path) -> None: + """It refuses rather than tidies. + + ``unlink`` would happily delete the link itself, but a link where this SDK + expects its own file means the state on disk is not what the manifest + describes — the caller's to report, not this module's to clean up. + """ + outside = tmp_path / "outside.txt" + outside.write_text("do not touch") + link = tmp_path / "f.txt" + link.symlink_to(outside) + + with pinned_directory(tmp_path) as dir_fd: + with pytest.raises(SymlinkRefused): + unlink_file(tmp_path, "f.txt", dir_fd=dir_fd) + + assert link.is_symlink() + assert outside.read_text() == "do not touch" + + def test_refuses_a_symlink_on_the_path_fallback(self, tmp_path: Path) -> None: + outside = tmp_path / "outside.txt" + outside.write_text("do not touch") + (tmp_path / "f.txt").symlink_to(outside) + with pytest.raises(SymlinkRefused): + unlink_file(tmp_path, "f.txt", dir_fd=None) + assert outside.exists() + + def test_symlink_refused_is_an_oserror(self) -> None: + """A caller that only cares the removal failed keeps its single + ``except OSError``; one that must report this refusal specifically does + not have to match on a message.""" + assert issubclass(SymlinkRefused, OSError) + + +class TestDirFdProbe: + """The capability probe names the advertised twins, not the calls made.""" + + def test_probe_names_the_syscalls_python_advertises(self) -> None: + """``os.supports_dir_fd`` is populated per underlying syscall, and CPython + registers ``renameat`` under ``os.rename`` and ``fstatat`` under + ``os.stat``. Probing ``os.replace`` and ``os.lstat`` — the names this + module actually calls — reports "unsupported" on every POSIX platform and + would silently disable the defense. + """ + expected = os.supports_dir_fd.issuperset( + {os.rename, os.open, os.unlink, os.stat} + ) + assert safe_fs_module.SUPPORTS_DIR_FD is expected + + @pytest.mark.skipif(os.name == "nt", reason="POSIX advertises the *at() family") + def test_posix_has_the_family(self) -> None: + assert safe_fs_module.SUPPORTS_DIR_FD is True diff --git a/packages/client/tests/test_schema.py b/packages/client/tests/test_schema.py index fad75cf0..3be146d3 100644 --- a/packages/client/tests/test_schema.py +++ b/packages/client/tests/test_schema.py @@ -1,7 +1,8 @@ -""" -Tests for §3.5 parse_ai_config (AiConfig validation). -Reference: TESTING.md §3.5 -""" +"""Tests for ``parse_ai_config`` — AI Config variation validation.""" + +from typing import Any + +import pytest from launchdarkly_ai_server import parse_ai_config @@ -81,3 +82,97 @@ def test_output_format_accepted(self) -> None: raw["outputFormat"] = {"type": "object", "properties": {}} result = parse_ai_config(raw) assert result.success is True + + +class TestParseAiConfigSkills: + """ + Fail-closed validation of the optional ``skills`` array. + """ + + def _base(self, **extra: Any) -> dict[str, Any]: + raw: dict[str, Any] = { + "model": {"name": "claude-3"}, + "provider": {"name": "Anthropic"}, + "instructions": "You are helpful.", + } + raw.update(extra) + return raw + + def test_absent_skills_is_valid(self) -> None: + assert parse_ai_config(self._base()).success is True + + def test_empty_skills_is_valid(self) -> None: + assert parse_ai_config(self._base(skills=[])).success is True + + def test_valid_entries_accepted(self) -> None: + raw = self._base(skills=[{"key": "pdf-extraction", "version": 2}]) + result = parse_ai_config(raw) + assert result.success is True + assert result.data["skills"] == [{"key": "pdf-extraction", "version": 2}] + + def test_multiple_valid_entries_accepted(self) -> None: + raw = self._base( + skills=[{"key": "a", "version": 1}, {"key": "b-2", "version": 10}] + ) + assert parse_ai_config(raw).success is True + + def test_key_at_length_bound_accepted(self) -> None: + raw = self._base(skills=[{"key": "a" * 256, "version": 1}]) + assert parse_ai_config(raw).success is True + + @pytest.mark.parametrize("bad_skills", ["pdf", {"key": "a"}, 3, True]) + def test_non_array_skills_fails(self, bad_skills: Any) -> None: + assert parse_ai_config(self._base(skills=bad_skills)).success is False + + @pytest.mark.parametrize("entry", ["pdf-extraction", 1, None, ["a", 1]]) + def test_non_object_entry_fails(self, entry: Any) -> None: + assert parse_ai_config(self._base(skills=[entry])).success is False + + @pytest.mark.parametrize("bad_key", [None, 1, True, {"a": 1}, ["a"]]) + def test_missing_or_non_string_key_fails(self, bad_key: Any) -> None: + raw = self._base(skills=[{"key": bad_key, "version": 1}]) + assert parse_ai_config(raw).success is False + + def test_absent_key_fails(self) -> None: + assert parse_ai_config(self._base(skills=[{"version": 1}])).success is False + + @pytest.mark.parametrize( + "bad_key", + [ + "", + "Evil", + "-leading-dash", + ".hidden", + "_underscore", + "has space", + "a/b", + "a\\b", + "../escape", + "trailing-space ", + "under_score", + "a" * 257, + ], + ) + def test_pattern_and_length_violations_fail(self, bad_key: str) -> None: + raw = self._base(skills=[{"key": bad_key, "version": 1}]) + assert parse_ai_config(raw).success is False + + @pytest.mark.parametrize("bad_version", [0, -1, 2.5, "2", None, True, [1]]) + def test_invalid_version_fails(self, bad_version: Any) -> None: + raw = self._base(skills=[{"key": "a", "version": bad_version}]) + assert parse_ai_config(raw).success is False + + def test_absent_version_fails(self) -> None: + assert parse_ai_config(self._base(skills=[{"key": "a"}])).success is False + + def test_one_bad_entry_fails_the_whole_config(self) -> None: + raw = self._base( + skills=[{"key": "good", "version": 1}, {"key": "../bad", "version": 1}] + ) + assert parse_ai_config(raw).success is False + + def test_error_message_mentions_skills(self) -> None: + raw = self._base(skills=[{"key": "../bad", "version": 1}]) + result = parse_ai_config(raw) + assert result.success is False + assert "skills" in result.error["message"] diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py new file mode 100644 index 00000000..1b8ebd58 --- /dev/null +++ b/packages/client/tests/test_skills.py @@ -0,0 +1,2017 @@ +""" +Tests for Agent Skills types, reference discovery, content accessors, +integrity verification, and the telemetry seam. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +from typing import Any +from unittest.mock import MagicMock + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +import launchdarkly_ai_server.skills as skills_module +from launchdarkly_ai_server import ( + InMemorySkillStore, + ReconcileAction, + ReconcileReport, + Skill, + SkillOutcome, + SkillReference, + all_skills, + get_client, + get_skill, + get_skill_result, + get_skills, + init_client, + shutdown, + skill_refs, +) +from launchdarkly_ai_server.skills_core import list_raw_objects, require_store + +SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" + +INTEGRITY_SIGNAL = "AgentControl Skill Integrity Failure" +MATERIALIZED_SIGNAL = "AgentControl Skill Materialized" +REVOKED_SIGNAL = "AgentControl Skill Revoked Received" + +# The three signal names are an allowlist, not a floor: any +# other name reaching the emitter is a regression. +APPROVED_SIGNALS = frozenset({INTEGRITY_SIGNAL, MATERIALIZED_SIGNAL, REVOKED_SIGNAL}) + +# These two were considered and deliberately excluded from SDK emission — +# named explicitly rather than relying on the subset check to be read as +# covering them. +REMOVED_SIGNALS = frozenset( + { + "AgentControl Skill SDK Reference Returned", + "AgentControl Skill Content Retrieved", + } +) + + +pytestmark = pytest.mark.usefixtures("reset_skill_state") +"""Every test in this module runs against freshly cleared module state.""" + + +def _hash(content: str) -> str: + """sha256, lowercase hex, over verbatim utf-8 bytes.""" + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +def _skill( + key: str = "test-skill", + version: int = 1, + content: str = SKILL_BODY, + content_hash: str | None = None, +) -> Skill: + """Build a verified-shaped Skill directly (bypasses the accessors).""" + return Skill( + key=key, + version=version, + content=content.encode("utf-8"), + content_hash=content_hash if content_hash is not None else _hash(content), + ) + + +# --------------------------------------------------------------------------- + +UNENCODABLE_BODIES = ( + json.loads(r'"hi \ud800 there"'), # lone high surrogate + # A lone *low* surrogate, which is the one range errors="surrogateescape" + # smuggles through (as a raw 0x80 byte) while raising on everything else. + json.loads(r'"hi \udc80 there"'), +) + +NON_STRICT_HANDLERS = ( + "surrogatepass", + "surrogateescape", + "replace", + "ignore", + "backslashreplace", + "xmlcharrefreplace", + "namereplace", +) +"""Every ``str.encode`` error handler that is not ``strict``. + +``verified_bytes`` must use none of them: each one *fabricates* bytes for input +that has no encoding, and fabricated bytes can satisfy the hash comparison. +""" + + +def _fabricated_hash_cases() -> list[Any]: + """One case per (body, handler) pair the handler can actually encode. + + Each carries the sha256 of the bytes *that* handler would have produced, so + the case is not vacuous: an implementation that reached for the handler + would encode successfully, match the pinned hash, and return content + LaunchDarkly never delivered. Handlers that raise on a given body are + skipped — for that input they are as strict as ``strict``, so there is + nothing to detect. + """ + cases: list[Any] = [] + for index, body in enumerate(UNENCODABLE_BODIES): + for handler in NON_STRICT_HANDLERS: + try: + fabricated = body.encode("utf-8", errors=handler) + except UnicodeEncodeError: + continue + cases.append( + pytest.param( + body, + hashlib.sha256(fabricated).hexdigest(), + id=f"body{index}-{handler}", + ) + ) + return cases + + +FABRICATED_HASH_CASES = _fabricated_hash_cases() + + +class TestSkillTypes: + """Immutability, optional metadata, and ``ReconcileReport.ok``.""" + + def test_skill_reference_is_immutable(self) -> None: + ref = SkillReference(key="pdf-extraction", version=2) + with pytest.raises(dataclasses.FrozenInstanceError): + ref.version = 3 # type: ignore[misc] + + def test_skill_is_immutable(self) -> None: + skill = _skill() + with pytest.raises(dataclasses.FrozenInstanceError): + skill.content = b"tampered" # type: ignore[misc] + + def test_skill_outcome_is_immutable(self) -> None: + """A reported outcome is a value, like every other public skills type. + + Matters more here than for the others: a caller that fails closed on + ``reason`` must not be handed something a later layer can rewrite. + """ + outcome = SkillOutcome(skill=None, reason="integrity_failure", detail="nope") + with pytest.raises(dataclasses.FrozenInstanceError): + outcome.reason = "ok" # type: ignore[misc] + + def test_skill_content_is_bytes(self) -> None: + """Content is the verified verbatim bytes — opaque, never text.""" + skill = _skill() + assert isinstance(skill.content, bytes) + assert skill.content == SKILL_BODY.encode("utf-8") + + def test_skill_carries_optional_metadata(self) -> None: + skill = Skill( + key="a", + version=1, + content=SKILL_BODY.encode("utf-8"), + content_hash=_hash(SKILL_BODY), + name="A Skill", + description="does things", + ) + assert skill.name == "A Skill" + assert skill.description == "does things" + + def test_skill_metadata_defaults_to_none(self) -> None: + skill = _skill() + assert skill.name is None + assert skill.description is None + + def test_report_ok_true_when_no_error_action(self) -> None: + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="skipped_current", version=2), + ReconcileAction(key="c", action="removed"), + ReconcileAction(key="d", action="updated", version=3), + ] + ) + assert report.ok is True + + def test_report_ok_false_with_error_action(self) -> None: + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="error", error="nope"), + ] + ) + assert report.ok is False + + def test_empty_report_is_ok(self) -> None: + assert ReconcileReport(actions=[]).ok is True + + def test_report_errors_lists_error_actions_in_order(self) -> None: + """The report exposes its error actions itself.""" + first = ReconcileAction(key="b", action="error", error="first") + second = ReconcileAction(key="d", action="error", error="second") + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + first, + ReconcileAction(key="c", action="skipped_current", version=2), + second, + ReconcileAction(key="e", action="removed"), + ] + ) + assert report.errors == [first, second] + + def test_report_errors_empty_when_no_error_action(self) -> None: + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="removed"), + ] + ) + assert report.errors == [] + + def test_empty_report_has_no_errors(self) -> None: + assert ReconcileReport(actions=[]).errors == [] + + def test_report_ok_and_errors_always_agree(self) -> None: + """``ok`` is true iff ``errors`` is empty, on the same objects.""" + clean = ReconcileReport( + actions=[ReconcileAction(key="a", action="written", version=1)] + ) + failed = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="error", error="nope"), + ] + ) + for report in (clean, failed, ReconcileReport(actions=[])): + assert report.ok is (report.errors == []) + + +class TestSkillRefs: + """Pure projection of the config's skills array.""" + + def _config(self, **extra: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "model": {"name": "claude-3"}, + "provider": {"name": "Anthropic"}, + "instructions": "hi", + } + base.update(extra) + return base + + def test_absent_skills_returns_empty_list(self) -> None: + assert skill_refs(self._config()) == [] + + def test_empty_skills_returns_empty_list(self) -> None: + assert skill_refs(self._config(skills=[])) == [] + + def test_returns_typed_references_in_order(self) -> None: + config = self._config( + skills=[{"key": "a", "version": 1}, {"key": "b", "version": 3}] + ) + refs = skill_refs(config) + assert refs == [ + SkillReference(key="a", version=1), + SkillReference(key="b", version=3), + ] + assert all(isinstance(r, SkillReference) for r in refs) + + def test_emits_no_telemetry(self, recording_emitter: Any) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + skill_refs(self._config(skills=[{"key": "a", "version": 1}])) + assert recording_emitter.records == [] + + def test_dropped_entries_are_logged(self, caplog: pytest.LogCaptureFixture) -> None: + """A shortened projection is never silent. + + ``parse_ai_config`` fails the whole config closed on a malformed + reference, so a config that reached here through it cannot contain one. + A hand-built dict can, and feeding the shortened list to + ``write_skills`` would prune the dropped skill's on-disk copy — so the + drop is observable rather than silent. + """ + config = self._config( + skills=[ + {"key": "good", "version": 1}, + {"key": "bad", "version": 0}, + {"key": "Bad-Key", "version": 1}, + "not-an-object", + ] + ) + + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills"): + refs = skill_refs(config) + + assert refs == [SkillReference(key="good", version=1)] + assert len(caplog.records) == 3 + # The body is never echoed, and neither is the invalid key. + assert all("skills[" in r.getMessage() for r in caplog.records) + + def test_requires_no_client_or_store(self, mock_ld_client: Any) -> None: + """No store configured, no client initialized — still a pure projection.""" + refs = skill_refs(self._config(skills=[{"key": "a", "version": 2}])) + assert refs == [SkillReference(key="a", version=2)] + mock_ld_client.track.assert_not_called() + + +class TestPackageExports: + """ + What is and is not part of the public surface. + + The literal values are spelled out on purpose: this is the one place the + constants themselves are asserted, so importing them to build the + expectation would make the assertion circular. + """ + + def test_content_cap_is_not_public_api(self) -> None: + """The content cap stays internal to ``skills_core`` — see the + ``MAX_SKILL_CONTENT_BYTES`` docstring there for why it is not exported.""" + import launchdarkly_ai_server as package + from launchdarkly_ai_server import skills_core + + assert skills_core.MAX_SKILL_CONTENT_BYTES == 10485760 + assert "MAX_SKILL_CONTENT_BYTES" not in package.__all__ + assert not hasattr(package, "MAX_SKILL_CONTENT_BYTES") + + def test_object_kind_is_not_public_api(self) -> None: + """The kind is an SDK-side seam value, not the wire contract. + + A store adapter maps whatever the transport calls a skill onto the value + this SDK passes it, so publishing the string would advertise a contract + this side does not own — and one that would be hard to walk back once a + caller depended on it. It stays reachable through ``skills_core`` for the + adapter that needs to agree with it. + """ + import launchdarkly_ai_server as package + from launchdarkly_ai_server import skills_core + + assert skills_core.SKILL_OBJECT_KIND == "skill" + assert "SKILL_OBJECT_KIND" not in package.__all__ + assert not hasattr(package, "SKILL_OBJECT_KIND") + + def test_constants_are_exported_from_the_package_root(self) -> None: + import launchdarkly_ai_server as package + + assert package.SKILL_FILENAME == "SKILL.md" + assert package.MANIFEST_FILENAME == ".launchdarkly-skills.json" + assert package.MANIFEST_VERSION == 1 + + def test_constants_are_listed_in_dunder_all(self) -> None: + """A name absent from ``__all__`` is not part of the public surface.""" + import launchdarkly_ai_server as package + + expected = { + "SKILL_FILENAME", + "MANIFEST_FILENAME", + "MANIFEST_VERSION", + } + assert expected <= set(package.__all__) + + def test_closed_set_types_are_exported_from_the_package_root(self) -> None: + """The two closed-set unions are public API, not implementation detail. + + ``ReconcileActionKind`` types the ``ReconcileAction.action`` field every + consumer of a report reads and switches on, and ``OnUnavailable`` types + a public keyword argument of ``write_skills``. ``agents.md`` forbids + handler packages from importing sub-path modules, so a name exported + only from the implementation module has no supported import path. + """ + import launchdarkly_ai_server as package + + assert hasattr(package, "ReconcileActionKind") + assert hasattr(package, "OnUnavailable") + assert {"ReconcileActionKind", "OnUnavailable"} <= set(package.__all__) + + def test_exported_action_union_admits_exactly_the_five_actions(self) -> None: + """The union must match the actions a report can actually carry. + + Spelled out rather than imported from the implementation for the same + reason as the constants above: deriving the expectation from the thing + under test would make the assertion circular. + """ + import typing + + import launchdarkly_ai_server as package + + assert set(typing.get_args(package.ReconcileActionKind)) == { + "written", + "updated", + "skipped_current", + "removed", + "error", + } + assert set(typing.get_args(package.OnUnavailable)) == {"keep", "raise"} + assert set(typing.get_args(package.SkillOutcomeReason)) == { + "absent", + "integrity_failure", + "ok", + "store_unavailable", + "wrong_version", + } + + def test_retrieval_surface_is_exported_from_the_package_root(self) -> None: + """A name absent from ``__all__`` is not part of the public surface.""" + import launchdarkly_ai_server as package + + expected = { + "skill_refs", + "get_skill", + "get_skill_result", + "get_skills", + "all_skills", + "SkillStore", + "InMemorySkillStore", + "Skill", + "SkillOutcome", + "SkillOutcomeReason", + "SkillReference", + } + assert expected <= set(package.__all__) + + +class TestInMemorySkillStore: + """The public in-memory store implementation.""" + + def test_get_object_round_trips(self, make_raw_skill: Any) -> None: + raw = make_raw_skill(key="pdf-extraction", version=2) + s = InMemorySkillStore({"pdf-extraction": raw}) + assert s.get_object("skill", "pdf-extraction") == raw + + def test_get_object_unknown_key_returns_none(self) -> None: + assert InMemorySkillStore().get_object("skill", "nope") is None + + def test_put_then_get(self, make_raw_skill: Any) -> None: + s = InMemorySkillStore() + raw = make_raw_skill(key="a") + s.put(raw) + assert s.get_object("skill", "a") == raw + + def test_all_objects_returns_everything(self, make_raw_skill: Any) -> None: + """Asserted on the object bodies, not the dict keys. + + ``all_objects`` keys are opaque store-internal identifiers — the seam + documents them as such — so a test that pinned their spelling would be + asserting an implementation detail the contract disclaims. + """ + s = InMemorySkillStore() + first = make_raw_skill(key="a") + second = make_raw_skill(key="b") + s.put(first) + s.put(second) + held = s.all_objects("skill").values() + assert len(held) == 2 + assert first in held + assert second in held + + def test_all_objects_holds_every_version_of_one_key( + self, make_raw_skill: Any + ) -> None: + s = InMemorySkillStore() + v1 = make_raw_skill(key="a", version=1, content="one\n") + v2 = make_raw_skill(key="a", version=2, content="two\n") + s.put(v1) + s.put(v2) + held = list(s.all_objects("skill").values()) + assert len(held) == 2 + assert v1 in held + assert v2 in held + + def test_put_replaces_only_the_same_key_and_version( + self, make_raw_skill: Any + ) -> None: + s = InMemorySkillStore() + s.put(make_raw_skill(key="a", version=1, content="first\n")) + replacement = make_raw_skill(key="a", version=1, content="second\n") + s.put(replacement) + assert list(s.all_objects("skill").values()) == [replacement] + + def test_get_object_with_a_version_selects_that_version( + self, make_raw_skill: Any + ) -> None: + s = InMemorySkillStore() + v1 = make_raw_skill(key="a", version=1, content="one\n") + v2 = make_raw_skill(key="a", version=3, content="three\n") + s.put(v1) + s.put(v2) + assert s.get_object("skill", "a", 1) == v1 + assert s.get_object("skill", "a", 3) == v2 + + def test_get_object_without_a_version_selects_the_newest( + self, make_raw_skill: Any + ) -> None: + s = InMemorySkillStore() + s.put(make_raw_skill(key="a", version=1, content="one\n")) + newest = make_raw_skill(key="a", version=7, content="seven\n") + s.put(newest) + s.put(make_raw_skill(key="a", version=4, content="four\n")) + assert s.get_object("skill", "a") == newest + + def test_get_object_unknown_version_falls_back_to_a_malformed_object( + self, make_raw_skill: Any + ) -> None: + """A malformed object must reach verification, not read as absent. + + An object whose ``version`` is unusable is filed under its key alone. A + pinned lookup that finds nothing well-formed serves it anyway, so + verification withholds it and records an integrity signal — a store that + returned ``None`` here would make tampering indistinguishable from a + skill that was never delivered. + """ + s = InMemorySkillStore() + malformed = make_raw_skill(key="a", version="two") + s.put(malformed) + assert s.get_object("skill", "a", 2) == malformed + assert s.get_object("skill", "a") == malformed + + def test_get_object_unknown_version_beside_well_formed_ones_is_absent( + self, make_raw_skill: Any + ) -> None: + """A pin miss is a miss, not an integrity failure. + + The fall-back above applies only when nothing well-formed is filed under + the key. Once well-formed versions are held, a leftover malformed object + must not answer for a version that was never delivered: verification + would withhold it and record an integrity failure against a skill whose + integrity is not in question. + """ + s = InMemorySkillStore() + s.put(make_raw_skill(key="a", version=1, content="one\n")) + s.put(make_raw_skill(key="a", version=2, content="two\n")) + s.put(make_raw_skill(key="a", version="two")) + assert s.get_object("skill", "a", 5) is None + + def test_all_objects_unknown_kind_is_empty(self, make_raw_skill: Any) -> None: + s = InMemorySkillStore() + s.put(make_raw_skill(key="a")) + assert s.all_objects("flag") == {} + + def test_put_notifies_skill_kind_listeners(self, make_raw_skill: Any) -> None: + """``add_listener`` is part of the seam, so its one + implementation carries a smoke test for the callback contract: the raw + object, verbatim and unverified, as a single positional argument.""" + s = InMemorySkillStore() + seen: list[dict[str, Any]] = [] + s.add_listener("skill", seen.append) + raw = make_raw_skill(key="a") + + s.put(raw) + + assert seen == [raw] + + def test_put_does_not_notify_other_kind_listeners( + self, make_raw_skill: Any + ) -> None: + s = InMemorySkillStore() + seen: list[dict[str, Any]] = [] + s.add_listener("flag", seen.append) + + s.put(make_raw_skill(key="a")) + + assert seen == [] + + def test_remove_listener_stops_put_notifying_it(self, make_raw_skill: Any) -> None: + s = InMemorySkillStore() + seen: list[dict[str, Any]] = [] + s.add_listener("skill", seen.append) + s.remove_listener("skill", seen.append) + + s.put(make_raw_skill(key="a")) + + assert seen == [] + + def test_remove_listener_removes_one_occurrence(self, make_raw_skill: Any) -> None: + s = InMemorySkillStore() + seen: list[dict[str, Any]] = [] + s.add_listener("skill", seen.append) + s.add_listener("skill", seen.append) + s.remove_listener("skill", seen.append) + + s.put(make_raw_skill(key="a")) + + assert len(seen) == 1 + + def test_remove_listener_of_an_unregistered_callable_is_a_no_op(self) -> None: + s = InMemorySkillStore() + s.remove_listener("skill", print) + s.add_listener("skill", print) + s.remove_listener("flag", print) + s.remove_listener("skill", print) + s.remove_listener("skill", print) + + +class TestStoreConfiguration: + """Store wiring on the lifecycle layer.""" + + async def test_configured_via_init_client_option( + self, make_raw_skill: Any, mock_ld_client: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + await init_client(options={"skillStore": store}, client=mock_ld_client) + skill = await get_skill("a") + assert skill is not None + assert skill.key == "a" + + async def test_get_skill_raises_actionably_when_no_store(self) -> None: + with pytest.raises(RuntimeError, match="skill store"): + await get_skill("a") + + async def test_get_skills_raises_actionably_when_no_store(self) -> None: + with pytest.raises(RuntimeError, match="skill store"): + await get_skills([SkillReference(key="a", version=1)]) + + async def test_all_skills_raises_actionably_when_no_store(self) -> None: + with pytest.raises(RuntimeError, match="skill store"): + await all_skills() + + async def test_the_no_store_message_names_the_delivery_store_first(self) -> None: + """ + A deployment that hits this message must be pointed at the store that + receives content from LaunchDarkly, not only at the development one. + """ + with pytest.raises(RuntimeError) as reported: + await get_skill("a") + message = str(reported.value) + assert "FDv2SkillStore" in message + assert "InMemorySkillStore" in message + assert message.index("FDv2SkillStore") < message.index("InMemorySkillStore") + + async def test_shutdown_clears_the_store( + self, make_raw_skill: Any, mock_ld_client: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + await init_client(options={"skillStore": store}, client=mock_ld_client) + assert await get_skill("a") is not None + + await shutdown() + + with pytest.raises(RuntimeError, match="skill store"): + await get_skill("a") + + async def test_skill_store_is_applied_on_every_init_client_call( + self, make_raw_skill: Any + ) -> None: + """``skillStore`` is the one option a second call applies. + + ``init_client`` is idempotent for the client singleton, and on a second + call every other option is ignored. ``skillStore`` is applied anyway, + on purpose: it is what lets a client that was lazily auto-initialized, + or initialized without a store, be given one afterwards. Both halves are + asserted on the same pair of calls, because each is meaningless without + the other. + """ + first_store = InMemorySkillStore() + first_store.put(make_raw_skill(key="first")) + second_store = InMemorySkillStore() + second_store.put(make_raw_skill(key="second")) + + first_client = MagicMock() + second_client = MagicMock() + + await init_client(options={"skillStore": first_store}, client=first_client) + await init_client(options={"skillStore": second_store}, client=second_client) + + # Half one: the client singleton is unchanged — the second call is a + # no-op for it, so the second client was discarded. + assert get_client() is first_client + + # Half two: the store was nevertheless swapped. + assert await get_skill("second") is not None + assert await get_skill("first") is None + + async def test_init_client_without_a_store_leaves_the_configured_one( + self, make_raw_skill: Any + ) -> None: + """Only a non-None ``skillStore`` replaces the configured store. + + Otherwise a bare ``init_client()`` from an unrelated code path — the + lazy auto-init, say — would silently unconfigure skills. + """ + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + await init_client(options={"skillStore": store}, client=MagicMock()) + + await init_client(client=MagicMock()) + + assert await get_skill("a") is not None + + async def test_failed_init_client_leaves_no_store_configured( + self, monkeypatch: pytest.MonkeyPatch, make_raw_skill: Any + ) -> None: + """A raising ``init_client`` must not leave global state behind. + + Installing the store before the SDK-key check would leave the accessors + working against a store the application believes was never installed, + masking a failed initialization. + """ + monkeypatch.delenv("LD_SDK_KEY", raising=False) + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + + with pytest.raises(RuntimeError, match="No LaunchDarkly SDK key"): + await init_client(options={"skillStore": store}) + + with pytest.raises(RuntimeError, match="skill store"): + await get_skill("a") + + async def test_reset_for_testing_clears_the_store( + self, make_raw_skill: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + skills_module._set_store(store) + assert await get_skill("a") is not None + + lifecycle_module._reset_for_testing() + + with pytest.raises(RuntimeError, match="skill store"): + await get_skill("a") + + +class TestAccessorArgumentErrors: + """A bare string is a type error, not a reference. + + ``str`` satisfies ``Sequence[str]``, so the annotation on ``get_skills`` + admits a bare string and only this runtime guard catches it; iterating one + would look up a skill per character. Deliberately a *different* class from + ``write_skills``'s bare-string rejection, which is a ``ValueError`` because + a string is an accepted argument type there. + """ + + async def test_bare_string_raises_type_error( + self, store: InMemorySkillStore + ) -> None: + with pytest.raises(TypeError) as excinfo: + await get_skills("pdf-extraction") # type: ignore[arg-type] + + # The message has to name the fix, not merely reject the input. + assert "[key]" in str(excinfo.value) + + async def test_bare_string_is_rejected_before_the_store_is_consulted( + self, make_raw_skill: Any + ) -> None: + """The guard is an argument check, so it precedes store resolution. + + Asserting the raise alone would also pass if the string were iterated + into single-character lookups that all missed, so pin that no lookup + happened at all. + """ + looked_up: list[str] = [] + + class _RecordingStore: + def get_object(self, kind: str, key: str) -> dict[str, Any] | None: + looked_up.append(key) + return None + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + return {} + + skills_module._set_store(_RecordingStore()) + + with pytest.raises(TypeError): + await get_skills("abc") # type: ignore[arg-type] + + assert looked_up == [] + + +class TestGetSkill: + """Single-skill accessor.""" + + async def test_returns_verified_skill( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="pdf-extraction", version=2)) + skill = await get_skill("pdf-extraction") + assert skill is not None + assert skill.key == "pdf-extraction" + assert skill.version == 2 + assert skill.content == SKILL_BODY.encode("utf-8") + assert skill.content_hash == _hash(SKILL_BODY) + assert skill.name == "Test Skill" + assert skill.description == "A skill used in tests." + + async def test_version_omitted_returns_newest_available( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=7)) + skill = await get_skill("a") + assert skill is not None + assert skill.version == 7 + + async def test_exact_version_match_returns_skill( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=3)) + skill = await get_skill("a", version=3) + assert skill is not None + assert skill.version == 3 + + async def test_other_version_returns_none( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=3)) + assert await get_skill("a", version=2) is None + assert await get_skill("a", version=4) is None + + async def test_missing_key_returns_none(self, store: InMemorySkillStore) -> None: + assert await get_skill("nope") is None + + async def test_a_store_answering_under_a_different_key_is_withheld( + self, make_raw_skill: Any + ) -> None: + """The key needs the same post-fetch defense the version already has. + + Identity is read off the object itself, and the store is untrusted. An + answer served under a different key would otherwise be handed back + under the key the caller asked for while carrying its own. + """ + + class _AliasingStore: + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return make_raw_skill(key="other-key") + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + skills_module._set_store(_AliasingStore()) + assert await get_skill("asked-for") is None + + async def test_multibyte_content_verifies( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + content = "---\nname: emoji\n---\n🚀 unicode ✅ body\n" + store.put(make_raw_skill(key="a", content=content)) + skill = await get_skill("a") + assert skill is not None + assert skill.content == content.encode("utf-8") + + +class _RaisingStore: + """A store whose reads raise — the "the transport is down" case. + + Declared with the full ``get_object`` signature on purpose. A double missing + the ``version`` parameter would also produce a raise here, but a + ``TypeError`` from the call itself rather than from the store, and the test + would then pass without the store ever having been consulted. + """ + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + raise RuntimeError("transport failure") + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + raise RuntimeError("transport failure") + + +class _WrongVersionAnsweringStore: + """A store that answers a pinned lookup with some other version.""" + + def __init__(self, make_raw_skill: Any, answered_version: int = 99) -> None: + self._make = make_raw_skill + self._answered_version = answered_version + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + answer: dict[str, Any] = self._make(key=key, version=self._answered_version) + return answer + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + return {} + + +class TestGetSkillResult: + """ + The reported accessor — one token per outcome a retrieval can have. + + ``get_skill`` collapses four distinct failures to ``None``, which leaves a + caller unable to fail closed on suspected tampering while tolerating a skill + that is merely not configured. These tests pin that the five outcomes are + told apart, and that reporting them changed nothing about ``get_skill``. + """ + + def _tampered(self, make_raw_skill: Any, key: str = "a") -> dict[str, Any]: + raw: dict[str, Any] = make_raw_skill(key=key) + raw["contentHash"] = "0" * 64 + return raw + + async def test_ok_carries_the_skill_and_no_detail( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="pdf-extraction", version=2)) + + outcome = await get_skill_result("pdf-extraction") + + assert outcome.reason == "ok" + assert outcome.detail is None + assert outcome.skill is not None + assert outcome.skill.key == "pdf-extraction" + assert outcome.skill.version == 2 + assert outcome.skill.content == SKILL_BODY.encode("utf-8") + + async def test_absent_when_the_store_does_not_hold_the_key( + self, store: InMemorySkillStore + ) -> None: + outcome = await get_skill_result("nope") + + assert outcome.reason == "absent" + assert outcome.skill is None + assert outcome.detail + assert "'nope'" in outcome.detail + + async def test_integrity_failure_when_content_does_not_verify( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """The one outcome a caller is expected to fail closed on.""" + store.put(self._tampered(make_raw_skill, key="a")) + + outcome = await get_skill_result("a") + + assert outcome.reason == "integrity_failure" + assert outcome.skill is None + assert outcome.detail + # The quoted form, so the assertion is about the key and not about a + # letter that appears in half the words in the message. + assert "'a'" in outcome.detail + + async def test_integrity_failure_when_the_store_answers_under_another_key( + self, make_raw_skill: Any + ) -> None: + """Identity is part of verification, so a substitution fails closed. + + Grouped with the tampered-content case above rather than with + ``absent``, and the grouping is the assertion: the store did answer, it + answered with a skill, and what disqualified the answer was its + identity. A caller that fails closed on suspected tampering has to see + that. Filing it as ``absent`` would put a store substituting one skill + for another in the bucket that same caller is invited to tolerate. + + Pinned in a test because the token is not recoverable from the message + and the construction site went without one long enough to raise + ``TypeError`` on this path. + """ + + class _AliasingStore: + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return make_raw_skill(key="other-key") + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + skills_module._set_store(_AliasingStore()) + + outcome = await get_skill_result("asked-for") + + assert outcome.reason == "integrity_failure" + assert outcome.skill is None + assert outcome.detail + # Both keys, since the detail is what makes a substitution diagnosable. + assert "'asked-for'" in outcome.detail + assert "'other-key'" in outcome.detail + + async def test_wrong_version_when_the_store_answers_with_another( + self, make_raw_skill: Any + ) -> None: + skills_module._set_store(_WrongVersionAnsweringStore(make_raw_skill)) + + outcome = await get_skill_result("a", version=1) + + assert outcome.reason == "wrong_version" + assert outcome.skill is None + assert outcome.detail + # The detail is what makes this actionable rather than merely negative: + # it names both the version asked for and the version held. + assert "version 1" in outcome.detail + assert "version 99" in outcome.detail + + async def test_store_unavailable_when_the_store_raises( + self, caplog: pytest.LogCaptureFixture + ) -> None: + skills_module._set_store(_RaisingStore()) + + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + outcome = await get_skill_result("a") + + assert outcome.reason == "store_unavailable" + assert outcome.skill is None + assert outcome.detail + assert "RuntimeError" in outcome.detail + + async def test_store_unavailable_is_distinct_from_absent( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """An outage is not a deletion, and the two must not read alike. + + This is the distinction ``write_skills`` already depends on to decide + whether pruning may run — only a raising store suppresses it — so + collapsing the two tokens here would put the public vocabulary at odds + with a policy the SDK already enforces internally. + """ + skills_module._set_store(_RaisingStore()) + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + raised = await get_skill_result("a") + + skills_module._set_store(InMemorySkillStore()) + empty = await get_skill_result("a") + + # Asserted as two named tokens rather than as an inequality: the type + # checker can already see that these two literals differ, so an + # inequality here would be dead weight. + assert raised.reason == "store_unavailable" + assert empty.reason == "absent" + + async def test_every_non_ok_outcome_carries_a_detail( + self, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """A reason with no detail leaves an operator nothing to act on. + + Swept over all four failures in one test rather than asserted per case + only, so a fifth failure path added later without a message is caught by + a test whose name says what it is about. + """ + stores: list[tuple[str, Any]] = [ + ("absent", InMemorySkillStore()), + ( + "integrity_failure", + InMemorySkillStore({"a": self._tampered(make_raw_skill)}), + ), + ("wrong_version", _WrongVersionAnsweringStore(make_raw_skill)), + ("store_unavailable", _RaisingStore()), + ] + + outcomes: list[SkillOutcome] = [] + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + for _expected, store_double in stores: + skills_module._set_store(store_double) + outcomes.append(await get_skill_result("a", version=1)) + + assert [o.reason for o in outcomes] == [expected for expected, _ in stores] + assert all(o.skill is None for o in outcomes) + assert all(o.detail for o in outcomes) + + async def test_detail_never_carries_the_skill_content( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """``detail`` is safe to log, so the body must not travel in it. + + Same rule the integrity log record follows, asserted separately here + because this string reaches the caller through a different surface. + """ + secret = "---\nname: Secret\n---\nSSN 000-00-0000 and an API key.\n" + store.put(make_raw_skill(key="a", content=secret, contentHash="0" * 64)) + + outcome = await get_skill_result("a") + + assert outcome.reason == "integrity_failure" + assert outcome.detail is not None + assert secret not in outcome.detail + assert "SSN" not in outcome.detail + assert "API key" not in outcome.detail + + async def test_raises_the_same_way_as_get_skill_with_no_store(self) -> None: + """Identical failure mode, down to the message. + + The two accessors differ only in what they report about a retrieval; a + missing store is a configuration error in both, so a caller cannot need + to handle it twice. + """ + with pytest.raises(RuntimeError, match="skill store") as reported: + await get_skill_result("a") + with pytest.raises(RuntimeError, match="skill store") as collapsed: + await get_skill("a") + + assert str(reported.value) == str(collapsed.value) + + async def test_records_no_second_integrity_signal( + self, + store: InMemorySkillStore, + make_raw_skill: Any, + recording_emitter: Any, + caplog: pytest.LogCaptureFixture, + ) -> None: + """One failed retrieval is one failure, on both surfaces. + + Verification already recorded the log record and the signal before + ``resolve_from_store`` returned, so reporting the reason must add + nothing: a second record would double-count one event in a SIEM and + inflate the product counter. ``_integrity_records`` is the shared parser + used by the log-record tests further down this module. + """ + skills_module._set_emitter_for_testing(recording_emitter) + store.put(self._tampered(make_raw_skill, key="a")) + + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + outcome = await get_skill_result("a") + + assert outcome.reason == "integrity_failure" + assert len(_integrity_records(caplog)) == 1 + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_get_skill_still_returns_none_for_every_failure( + self, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """The no-behaviour-change guarantee. + + ``get_skill``'s contract — ``None`` for every failure, and it never + raises for one — is documented in its docstring and in the README, and + every existing caller treats ``None`` as "no skill". Adding a reported + accessor beside it must not move that line, so the four failures are + driven through both accessors in one test: the reason is distinguishable + *and* the collapsed form still collapses. + """ + cases: list[tuple[str, Any]] = [ + ("absent", InMemorySkillStore()), + ( + "integrity_failure", + InMemorySkillStore({"a": self._tampered(make_raw_skill)}), + ), + ("wrong_version", _WrongVersionAnsweringStore(make_raw_skill)), + ("store_unavailable", _RaisingStore()), + ] + + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + for expected_reason, store_double in cases: + skills_module._set_store(store_double) + reported = await get_skill_result("a", version=1) + assert reported.reason == expected_reason + # No pytest.raises wrapper: an escaping exception fails the test + # here, which is the "never raises" half of the contract. + assert await get_skill("a", version=1) is None + + +class TestGetSkills: + """Batch accessor.""" + + async def test_mixed_refs_and_strings( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=1)) + store.put(make_raw_skill(key="b", version=5)) + result = await get_skills([SkillReference(key="a", version=1), "b"]) + assert [s.key for s in result] == ["a", "b"] + + async def test_preserves_input_order( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + for k in ("a", "b", "c"): + store.put(make_raw_skill(key=k)) + result = await get_skills(["c", "a", "b"]) + assert [s.key for s in result] == ["c", "a", "b"] + + async def test_missing_entries_are_omitted( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a")) + result = await get_skills(["a", "missing"]) + assert [s.key for s in result] == ["a"] + + async def test_version_mismatch_is_omitted( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=2)) + result = await get_skills([SkillReference(key="a", version=1)]) + assert result == [] + + async def test_empty_input_returns_empty_list( + self, store: InMemorySkillStore + ) -> None: + assert await get_skills([]) == [] + + async def test_integrity_failure_omitted_and_signalled( + self, + store: InMemorySkillStore, + make_raw_skill: Any, + recording_emitter: Any, + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="good-a")) + store.put(make_raw_skill(key="bad", contentHash="0" * 64)) + store.put(make_raw_skill(key="good-b")) + + result = await get_skills(["good-a", "bad", "good-b"]) + + assert [s.key for s in result] == ["good-a", "good-b"] + failures = recording_emitter.signals(INTEGRITY_SIGNAL) + assert len(failures) == 1 + assert failures[0]["skill_key"] == "bad" + + +class TestAllSkills: + """All_skills accessor.""" + + async def test_returns_every_verified_skill( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + for k in ("a", "b", "c"): + store.put(make_raw_skill(key=k)) + result = await all_skills() + assert {s.key for s in result} == {"a", "b", "c"} + + async def test_empty_store_returns_empty_list( + self, store: InMemorySkillStore + ) -> None: + assert await all_skills() == [] + + async def test_omits_skills_that_fail_verification( + self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="good")) + store.put(make_raw_skill(key="bad", contentHash="deadbeef")) + result = await all_skills() + assert {s.key for s in result} == {"good"} + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_a_non_mapping_listing_is_reported_as_a_broken_store(self) -> None: + """A listing that is not a mapping is a broken store, not an empty one. + + ``all_skills`` has no way to report the difference, so it returns an + empty list either way — but the reason has to reach the caller that + does act on it. Collapsing the answer to "no skills" reads downstream + as "every skill was revoked". + """ + + class _BrokenListingStore: + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return None + + def all_objects(self, kind: str) -> Any: + return None + + skills_module._set_store(_BrokenListingStore()) + + assert await all_skills() == [] + + objects, error = list_raw_objects(require_store()) + assert objects == {} + assert error is not None + assert "rather than an object" in error + + +class TestVersionPinning: + """ + A payload holds several versions of one key, and a pin has to resolve to the + version it names. + + Delivery serves the newest version of every skill *plus* every version any + variation currently pins, so this is the ordinary case rather than an edge + one. A seam keyed by key alone cannot express it: it answers with the newest + and the pin then reads as a missing skill. + """ + + async def _two_versions(self, store: Any, make_raw_skill: Any) -> None: + store.put(make_raw_skill(key="a", version=1, content="version one\n")) + store.put(make_raw_skill(key="a", version=2, content="version two\n")) + + async def test_pinned_old_version_resolves( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + skill = await get_skill("a", version=1) + assert skill is not None + assert skill.version == 1 + assert skill.content == b"version one\n" + + async def test_latest_resolves_alongside_the_pinned_old_version( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + skill = await get_skill("a") + assert skill is not None + assert skill.version == 2 + + async def test_both_lookups_succeed_against_one_store( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + pinned = await get_skill("a", version=1) + latest = await get_skill("a", version=2) + assert pinned is not None and pinned.version == 1 + assert latest is not None and latest.version == 2 + + async def test_get_skills_resolves_a_mix_of_pins( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + store.put(make_raw_skill(key="b", version=5, content="bee\n")) + skills = await get_skills( + [ + SkillReference(key="a", version=1), + SkillReference(key="b", version=5), + "a", + ] + ) + assert [(s.key, s.version) for s in skills] == [("a", 1), ("b", 5), ("a", 2)] + + async def test_pin_to_a_version_the_store_does_not_hold_returns_none( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + assert await get_skill("a", version=9) is None + + async def test_pin_miss_beside_a_malformed_object_records_no_failure( + self, + store: Any, + make_raw_skill: Any, + recording_emitter: Any, + ) -> None: + """An undelivered version must not raise an integrity alarm. + + A malformed object is filed under its key alone, and a pinned lookup + serves it when that is all the store holds — so verification withholds it + with a signal rather than letting tampering read as a skill that was + never delivered. Once well-formed versions are held, that reasoning no + longer applies: the pin is simply not there, and reporting an integrity + failure would point an alert at the wrong skill. + """ + skills_module._set_emitter_for_testing(recording_emitter) + await self._two_versions(store, make_raw_skill) + store.put(make_raw_skill(key="a", version="two")) + + assert await get_skill("a", version=9) is None + assert recording_emitter.signals(INTEGRITY_SIGNAL) == [] + + async def test_all_skills_returns_one_entry_per_key_at_the_newest_version( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + store.put(make_raw_skill(key="b", version=5, content="bee\n")) + skills = await all_skills() + assert sorted((s.key, s.version) for s in skills) == [("a", 2), ("b", 5)] + + async def test_a_store_answering_with_the_wrong_version_is_withheld( + self, make_raw_skill: Any + ) -> None: + """The post-fetch check is a defense, not the selection mechanism. + + The store is untrusted, so an answer that is not the version that was + asked for is withheld rather than returned. + """ + + class _WrongVersionStore: + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return make_raw_skill(key=key, version=99) + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + skills_module._set_store(_WrongVersionStore()) + assert await get_skill("a", version=1) is None + + +class TestWithholdingSummary: + """ + A run that withheld content says so at WARN, once. + + Every individual withholding already records an integrity signal and an + error line, but a caller reading logs at WARN sees neither — and the case + that matters most is a payload where *nothing* verifies, because the feature + then returns an empty result indistinguishable from "this project has no + skills". + """ + + def _tampered(self, make_raw_skill: Any, key: str = "a") -> dict[str, Any]: + raw: dict[str, Any] = make_raw_skill(key=key) + raw["contentHash"] = "0" * 64 + return raw + + async def test_total_withholding_warns_and_names_the_hash( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + store.put(self._tampered(make_raw_skill)) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + assert await all_skills() == [] + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + assert "contentHash" in warnings[0].getMessage() + + async def test_partial_withholding_warns_with_the_counts( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + store.put(make_raw_skill(key="good")) + store.put(self._tampered(make_raw_skill, key="bad")) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + skills = await all_skills() + assert [s.key for s in skills] == ["good"] + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + assert "1 of 2" in warnings[0].getMessage() + + async def test_get_skills_warns_once_per_batch( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + store.put(self._tampered(make_raw_skill, key="a")) + store.put(self._tampered(make_raw_skill, key="b")) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + assert await get_skills(["a", "b"]) == [] + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + + async def test_a_key_that_resolved_is_not_also_reported_withheld( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """ + A store may hold a malformed object beside a well-formed version of the + same key. The well-formed one resolves, so the key is not withheld and + must not be counted as though it were. + """ + store.put(make_raw_skill(key="a", version=1)) + store.put(make_raw_skill(key="a", version="not-a-version")) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + skills = await all_skills() + assert [s.key for s in skills] == ["a"] + assert [r for r in caplog.records if r.levelname == "WARNING"] == [] + + async def test_a_key_with_no_usable_version_is_still_withheld( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """The converse: nothing resolved the key, so the withholding stands.""" + store.put(make_raw_skill(key="a", version="not-a-version")) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + assert await all_skills() == [] + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1 + + async def test_a_fully_resolved_run_is_silent( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + store.put(make_raw_skill(key="a")) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + assert len(await all_skills()) == 1 + assert len(await get_skills(["a"])) == 1 + assert [r for r in caplog.records if r.levelname == "WARNING"] == [] + + +class TestIntegrityVerification: + """Mandatory verification at the accessor boundary.""" + + async def test_hash_mismatch_withholds_skill( + self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="a", contentHash="a" * 64)) + assert await get_skill("a") is None + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_tampered_content_withholds_skill( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + raw = make_raw_skill(key="a") + raw["content"] = raw["content"] + "x" # hash now stale by one byte + store.put(raw) + assert await get_skill("a") is None + + async def test_oversize_content_rejected( + self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + oversize = "x" * (10 * 1024 * 1024 + 1) + store.put(make_raw_skill(key="a", content=oversize)) + assert await get_skill("a") is None + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_content_at_size_cap_is_accepted( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + at_cap = "x" * (10 * 1024 * 1024) + store.put(make_raw_skill(key="a", content=at_cap)) + skill = await get_skill("a") + assert skill is not None + assert len(skill.content) == 10 * 1024 * 1024 + + async def test_key_at_length_bound_from_store_accepted( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """The accepting side of the <= 256 bound. + + ``write_skills`` cannot reach this bound (a key is one directory name and + NAME_MAX is 255), so config validation and this accessor-side + revalidation are the only two layers where 256 is observable at all. The + rejecting side is ``test_invalid_key_from_store_rejected["x" * 257]``. + """ + key = "a" * 256 + store.put(make_raw_skill(key=key)) + skill = await get_skill(key) + assert skill is not None + assert skill.key == key + + @pytest.mark.parametrize( + "bad_key", + [ + "Evil", + "-leading-dash", + ".hidden", + "has space", + "a/b", + "../escape", + "", + "x" * 257, + ], + ) + async def test_invalid_key_from_store_rejected( + self, make_raw_skill: Any, bad_key: str + ) -> None: + """A hostile store may serve any key — the accessor revalidates.""" + raw = make_raw_skill(key="placeholder") + raw["key"] = bad_key + skills_module._set_store(InMemorySkillStore({bad_key: raw})) + assert await get_skill(bad_key) is None + + @pytest.mark.parametrize("bad_version", [0, -1, 2.5, "2", None, True]) + async def test_invalid_version_from_store_rejected( + self, make_raw_skill: Any, bad_version: Any + ) -> None: + raw = make_raw_skill(key="a", version=bad_version) + skills_module._set_store(InMemorySkillStore({"a": raw})) + assert await get_skill("a") is None + + async def test_missing_content_rejected(self, make_raw_skill: Any) -> None: + raw = make_raw_skill(key="a") + del raw["content"] + skills_module._set_store(InMemorySkillStore({"a": raw})) + assert await get_skill("a") is None + + async def test_missing_content_hash_rejected(self, make_raw_skill: Any) -> None: + raw = make_raw_skill(key="a") + del raw["contentHash"] + skills_module._set_store(InMemorySkillStore({"a": raw})) + assert await get_skill("a") is None + + async def test_uppercase_hash_rejected( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """Hashes are lowercase hex; a non-canonical hash is not authentic.""" + store.put(make_raw_skill(key="a", contentHash=_hash(SKILL_BODY).upper())) + assert await get_skill("a") is None + + @pytest.mark.parametrize(("body", "fabricated_hash"), FABRICATED_HASH_CASES) + async def test_unencodable_content_withheld( + self, recording_emitter: Any, body: str, fabricated_hash: str + ) -> None: + """Content with no UTF-8 encoding is withheld, and the signal recorded. + + ``str.encode`` raises on a lone surrogate, so ``verified_bytes`` has an + exception to catch and never sees bytes for this content at all. + + The parametrization is what makes that observable. The guard must never + pass ``errors="surrogatepass"``, or any other non-strict handler: each + of them fabricates bytes for input that has no encoding, and fabricated + bytes can satisfy the hash comparison. Every case here supplies the + sha256 of the bytes one such handler would have produced, so an + implementation that reached for one would verify this object + successfully and hand back content LaunchDarkly never sent. An + arbitrary wrong hash would not catch that — the mismatch check would + reject the input before the encoder guard was reached. + """ + with pytest.raises(UnicodeEncodeError): + body.encode("utf-8") # the premise: there is no encoding to hash + + skills_module._set_emitter_for_testing(recording_emitter) + skills_module._set_store( + InMemorySkillStore( + { + "a": { + "key": "a", + "version": 1, + "content": body, + "contentHash": fabricated_hash, + } + } + ) + ) + + assert await get_skill("a") is None + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_store_error_does_not_leak_content( + self, exploding_store: Any + ) -> None: + assert await get_skill("a") is None + assert await get_skills(["a"]) == [] + assert await all_skills() == [] + + +# --------------------------------------------------------------------------- +# Gap 1 — the local integrity-failure log record +# --------------------------------------------------------------------------- + +INTEGRITY_EVENT = "ld.skills.integrity_failure" +"""The stable event name, spelled out rather than imported. + +The name is a compatibility surface documented for customers to match on in a +SIEM, so the test has to fail when it is renamed. Importing the constant would +rename in lockstep and assert nothing. +""" + +LOGGED_BODY = "UNIQUE-SECRET-BODY-THAT-MUST-NOT-BE-LOGGED" + + +def _raw_object(**overrides: Any) -> dict[str, Any]: + """A wire-shaped raw object with a correct ``contentHash``. + + The ``make_raw_skill`` fixture is the same factory, but a module-level + ``parametrize`` table cannot reach a fixture. + """ + raw: dict[str, Any] = { + "key": "a", + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + raw.update(overrides) + return raw + + +def _raw_without(field: str) -> dict[str, Any]: + raw = _raw_object() + del raw[field] + return raw + + +_OVERSIZE = "x" * (10 * 1024 * 1024 + 1) + +REASON_CODE_CASES = [ + # Not a dict at all. Reachable through ``all_skills`` and not through + # ``get_skill``, which rejects a non-dict answer before verification. + pytest.param("not-an-object-at-all", "not_an_object", id="not_an_object"), + pytest.param(_raw_object(key="Evil/../x"), "invalid_key", id="invalid_key"), + pytest.param(_raw_object(version=0), "invalid_version", id="invalid_version"), + pytest.param(_raw_without("content"), "missing_content", id="missing_content"), + pytest.param( + _raw_without("contentHash"), "missing_content_hash", id="missing_content_hash" + ), + # A lone surrogate has no UTF-8 encoding. Only reachable on the wire-``str`` + # path: a ``Skill`` already holds bytes and skips the encode. + pytest.param( + _raw_object(content=json.loads(r'"hi \ud800 there"')), "not_utf8", id="not_utf8" + ), + # Correct hash for the oversize body, so the cap is what withheld it. + pytest.param( + _raw_object(content=_OVERSIZE, contentHash=_hash(_OVERSIZE)), + "over_size_cap", + id="over_size_cap", + ), + pytest.param( + _raw_object(contentHash="0" * 64), "hash_mismatch", id="hash_mismatch" + ), +] +"""One case per ``reason_code`` token, driven end to end through ``all_skills``. + +``all_skills`` rather than ``get_skill`` for every case so the table is uniform: +it verifies every object the store holds, including the ones too malformed to +carry a usable key, which is the only accessor path a non-dict reaches. +""" + + +def _integrity_records(caplog: pytest.LogCaptureFixture) -> list[dict[str, Any]]: + """Every integrity-failure record in *caplog*, parsed out of the message text. + + Read off the message rather than off ``record.ld_skills`` deliberately: the + message is what a customer sees under a plain ``logging.basicConfig()``, and + it is the surface the documented contract is about. The structured mirror is + asserted separately, against this. + """ + parsed: list[dict[str, Any]] = [] + for entry in caplog.records: + message = entry.getMessage() + if not message.startswith(f"{INTEGRITY_EVENT} "): + continue + parsed.append(json.loads(message[len(INTEGRITY_EVENT) + 1 :])) + return parsed + + +class TestIntegrityFailureLogRecord: + """ + The local log record is a documented detection surface, not a debugging aid. + + It is the only integrity signal that survives telemetry being switched off, + and the only one that exists at all in an instance with no telemetry + destination, so its shape is a contract: a stable event name in the message + text, a closed ``reason_code`` vocabulary, and no field a hostile store can + dictate. + """ + + @pytest.fixture(autouse=True) + def _capture_errors(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level("ERROR", logger="launchdarkly_ai_server.skills_core") + + async def _withhold(self, objects: dict[str, Any]) -> None: + skills_module._set_store(InMemorySkillStore(objects)) + assert await all_skills() == [] + + @pytest.mark.parametrize(("raw", "expected_code"), REASON_CODE_CASES) + async def test_one_record_per_reason_code( + self, caplog: pytest.LogCaptureFixture, raw: Any, expected_code: str + ) -> None: + await self._withhold({"a": raw}) + + records = _integrity_records(caplog) + assert len(records) == 1 + record = records[0] + assert record["reason_code"] == expected_code + assert record["event"] == INTEGRITY_EVENT + assert record["action"] == "withheld" + assert record["language"] == "python" + assert record["reason"] # the human-readable half, carrying byte counts + # Absent optional fields are omitted, never nulled: a SIEM field + # existence check has to mean something. + assert None not in record.values() + # The body never reaches a log line. Swept over every failure mode here; + # the two cases below are the ones that make the rule observable, since + # a well-formed key and digest never enter a redaction branch. + assert "Do the thing." not in json.dumps(record) + + def test_the_case_table_exhausts_the_vocabulary(self) -> None: + """The vocabulary is closed, and every token in it is reachable. + + Both directions matter. A ninth token added to the source without a call + site fails here, and so does a ninth call site that invented a token the + table does not cover — which is what keeps the Python and TypeScript + vocabularies from drifting apart one edit at a time. + """ + from launchdarkly_ai_server import skills_core + + covered = {case.values[1] for case in REASON_CODE_CASES} + assert covered == skills_core.INTEGRITY_REASON_CODES + assert len(covered) == 8 + + async def test_the_event_name_is_in_the_message_text( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Severity alone cannot discriminate, so the name has to be in the line. + + ``resolve_from_store`` and ``list_raw_objects`` also log ERROR from this + logger when a store raises, and the stdlib's default formatter drops + ``extra`` entirely — an ``extra``-only record would be invisible to a + customer running ``logging.basicConfig()``. + """ + await self._withhold({"a": _raw_object(contentHash="0" * 64)}) + + errors = [r for r in caplog.records if r.levelname == "ERROR"] + assert len(errors) == 1 + assert errors[0].getMessage().startswith(f"{INTEGRITY_EVENT} ") + + async def test_structured_handlers_get_the_same_record( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """``extra`` carries the record unflattened, and says the same thing.""" + await self._withhold({"a": _raw_object(contentHash="0" * 64)}) + + errors = [r for r in caplog.records if r.levelname == "ERROR"] + # extra lands in the record's __dict__, which is where a + # structured handler reads it from. + assert errors[0].__dict__["ld_skills"] == _integrity_records(caplog)[0] + + async def test_redaction_survives_into_the_record( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Both wire-sourced fields are still redacted in the JSON payload. + + ``key`` and ``contentHash`` are attacker-controlled, so a store can put + the skill body in either. The record must not have reopened a leak the + signal already closed — same treatment, same placeholders. + """ + await self._withhold( + { + "a": { + "key": f"{LOGGED_BODY}/../x", + "version": 1, + "content": LOGGED_BODY, + "contentHash": LOGGED_BODY, + } + } + ) + + record = _integrity_records(caplog)[0] + assert record["skill_key"] == "" + assert LOGGED_BODY not in json.dumps(record) + + async def test_a_non_sha256_expected_hash_is_redacted( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """The key is valid here, so ``expected_hash`` is the field under test.""" + await self._withhold({"a": _raw_object(contentHash=LOGGED_BODY)}) + + record = _integrity_records(caplog)[0] + assert record["expected_hash"] == "" + assert LOGGED_BODY not in json.dumps(record) + + async def test_observed_hash_is_absent_before_hashing( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Nothing was hashed, so there is no observed value to report.""" + await self._withhold({"a": _raw_without("contentHash")}) + + assert "observed_hash" not in _integrity_records(caplog)[0] + + async def test_observed_hash_is_present_on_a_mismatch( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """The one case that is a possible active-tampering signal. + + Positive control for the test above: an implementation that never + populated ``observed_hash`` would satisfy it vacuously. + """ + await self._withhold({"a": _raw_object(contentHash="0" * 64)}) + + record = _integrity_records(caplog)[0] + assert record["observed_hash"] == _hash(SKILL_BODY) + assert record["expected_hash"] == "0" * 64 + + async def test_the_serialized_payload_is_compact_and_key_sorted( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Sorted keys are what make the line byte-identical across SDKs. + + The other language implementations build this object in alphabetical + order, so a Python line following insertion order would differ byte for + byte on identical input. Compact separators are asserted alongside + because the two together are the serialization contract. + """ + await self._withhold({"a": _raw_object(contentHash="0" * 64)}) + + errors = [r for r in caplog.records if r.levelname == "ERROR"] + payload = errors[0].getMessage()[len(INTEGRITY_EVENT) + 1 :] + # json.loads preserves the document's order, so this is what was written. + keys = list(json.loads(payload)) + assert keys == sorted(keys) + assert ", " not in payload and ": " not in payload + + async def test_reason_code_stays_out_of_the_telemetry_signal( + self, recording_emitter: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """The signal's property set is an allowlist and does not grow. + + The record is the customer-owned detection path and carries the new + vocabulary; the LD-side counter is product telemetry and was left + exactly as designed. + """ + skills_module._set_emitter_for_testing(recording_emitter) + await self._withhold({"a": _raw_object(contentHash="0" * 64)}) + + props = recording_emitter.signals(INTEGRITY_SIGNAL)[0] + assert set(props) == { + "skill_key", + "version", + "expected_hash", + "observed_hash", + "language", + } + assert _integrity_records(caplog)[0]["reason_code"] == "hash_mismatch" + + +class TestTelemetrySeam: + """Internal emitter seam, no client.track, no context.""" + + async def test_default_emitter_is_noop( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", contentHash="0" * 64)) + assert await get_skill("a") is None # no emitter injected, no raise + + async def test_integrity_signal_properties( + self, + store: InMemorySkillStore, + make_raw_skill: Any, + recording_emitter: Any, + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="a", version=4, contentHash="b" * 64)) + + await get_skill("a") + + props = recording_emitter.signals(INTEGRITY_SIGNAL)[0] + assert props["skill_key"] == "a" + assert props["version"] == 4 + assert props["expected_hash"] == "b" * 64 + assert props["observed_hash"] == _hash(SKILL_BODY) + assert props["language"] == "python" + + async def test_skill_body_never_appears_in_signals( + self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="a", contentHash="c" * 64)) + + await get_skill("a") + + for _signal, props in recording_emitter.records: + for value in props.values(): + assert "Do the thing." not in str(value) + + # ``skill_key`` and ``expected_hash`` are copied off the wire, so a hostile + # store can smuggle the body through either one and publish it in a signal + # that is otherwise body-free. The sweep above cannot see that: it serves a + # well-formed 64-character digest under a valid key, so neither replacement + # branch ever runs, and it passes even against an implementation that copies + # both fields verbatim. These two cases are what make the rule observable. + # Both assert the body's *absence* rather than the placeholder's exact + # spelling, which is not part of the contract. + + async def test_body_smuggled_through_content_hash_is_redacted( + self, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + body = "UNIQUE-SECRET-BODY-VIA-HASH" + skills_module._set_store( + InMemorySkillStore( + {"a": {"key": "a", "version": 1, "content": body, "contentHash": body}} + ) + ) + + assert await get_skill("a") is None + + signals = recording_emitter.signals(INTEGRITY_SIGNAL) + assert len(signals) == 1 + for value in signals[0].values(): + assert body not in str(value) + + async def test_body_smuggled_through_the_key_is_redacted( + self, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + # Uppercase and a path separator, so this is not a valid skill key and + # the invalid-key branch is the one that has to redact it. + body = "UNIQUE-SECRET-BODY-VIA-KEY/../x" + skills_module._set_store( + InMemorySkillStore( + {body: {"key": body, "version": 1, "content": "x", "contentHash": "y"}} + ) + ) + + assert await get_skill(body) is None + + signals = recording_emitter.signals(INTEGRITY_SIGNAL) + assert len(signals) == 1 + for value in signals[0].values(): + assert body not in str(value) + + async def test_no_ld_track_calls_from_accessors( + self, make_raw_skill: Any, mock_ld_client: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + store.put(make_raw_skill(key="bad", contentHash="0" * 64)) + await init_client(options={"skillStore": store}, client=mock_ld_client) + # See the note in test_no_ld_track_calls_from_write_skills: the + # sdk-info flush belongs to init_client, not to the accessors. + mock_ld_client.track.reset_mock() + + await get_skill("a") + await get_skill("bad") + await get_skills(["a"]) + await all_skills() + + mock_ld_client.track.assert_not_called() + + async def test_throwing_emitter_never_breaks_the_operation( + self, store: InMemorySkillStore, make_raw_skill: Any, throwing_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(throwing_emitter) + store.put(make_raw_skill(key="bad", contentHash="0" * 64)) + store.put(make_raw_skill(key="good")) + + assert await get_skill("bad") is None + good = await get_skill("good") + assert good is not None + assert good.key == "good" + + async def test_accessors_record_no_signal_outside_the_approved_set( + self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any + ) -> None: + """The three names are an allowlist, not a floor. + + Asserted over the recorded strings, so nothing here mandates a + particular module-level constant. The write-side half of this sweep is + ``test_write_skills_records_no_signal_outside_the_approved_set`` in + test_skills_fs.py, where all four reconcile actions can be staged. + + Guards the most likely regression: an implementation that also emits + ``AgentControl Skill Content Retrieved`` from ``get_skill``, or + ``AgentControl Skill SDK Reference Returned`` from ``skill_refs``, + passes every other test in this class. + """ + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="good")) + store.put(make_raw_skill(key="tampered", contentHash="0" * 64)) + + assert await get_skill("good") is not None + assert await get_skill("tampered") is None + await get_skills(["good", "tampered"]) + await all_skills() + skill_refs({"skills": [{"key": "good", "version": 1}]}) + + recorded = {signal for signal, _props in recording_emitter.records} + assert recorded <= APPROVED_SIGNALS, ( + f"unapproved signal(s): {sorted(recorded - APPROVED_SIGNALS)}" + ) + assert not recorded & REMOVED_SIGNALS + # Positive control: a subset assertion is satisfied vacuously by an + # implementation that records nothing at all. + assert INTEGRITY_SIGNAL in recorded diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py new file mode 100644 index 00000000..d7affd6c --- /dev/null +++ b/packages/client/tests/test_skills_fdv2.py @@ -0,0 +1,3087 @@ +""" +Tests for the FDv2 skill delivery transport. + +Two layers, deliberately: + +- **A real fake endpoint.** ``_FakeFDv2Endpoint`` is an in-process + ``ThreadingHTTPServer`` that implements the wire contract — the ``basis`` + query parameter, ``Authorization``, ``If-None-Match``/304, the + ``{"events": [...]}`` polling envelope, and SSE for streaming. The store under + test opens real sockets against it, so request construction and header + handling are exercised rather than mocked. +- **The protocol reader driven directly.** Wire semantics — which objects are + skills, the skill's version in the wire ``key`` versus the payload's in + ``version``, revocation, mixed payloads — are + asserted against ``_ProtocolReader``, which has no I/O, so those cases read as + the contract they are instead of as a server script. +""" + +from __future__ import annotations + +import hashlib +import json +import socket +import threading +import time +from http.client import IncompleteRead +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, ClassVar +from urllib.parse import parse_qs, urlparse + +import pytest + +from launchdarkly_ai_server import ( + FDv2SkillStore, + InMemorySkillStore, + all_skills, + get_skill, + get_skill_result, + init_client, + skills_fdv2, + watch_skills, +) +from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND +from launchdarkly_ai_server.skills_fdv2 import ( + DEFAULT_POLL_TIMEOUT, + DEFAULT_STREAM_READ_TIMEOUT, + FDV2_KEY_DELIMITER, + FDV2_OBJECT_KIND, + MAX_RESPONSE_BYTES, + _backoff_delay, + _FatalTransportError, + _is_skill_event, + _iter_sse, + _ProtocolReader, + _RecoverableTransportError, + _Requester, + _retry_after_seconds, + _SkillObjectSet, + _store_object_from_put, + _StreamConnection, + _tombstone_from_delete, +) + +pytestmark = pytest.mark.usefixtures("reset_skill_state") + +SDK_KEY = "sdk-00000000-0000-4000-8000-000000000000" +SKILL_BODY = "---\nname: PDF Extraction\n---\nExtract text from PDFs.\n" + + +def _hash(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +# --------------------------------------------------------------------------- +# Wire builders — one place that knows the shape, so a contract change is one edit +# --------------------------------------------------------------------------- + + +def wire_key(key: str, object_version: Any) -> str: + """ + The wire ``key`` of one skill object: ``:``. + + ``None`` builds a key with no version at all, which is how the tests spell a + malformed object; anything else is spelled after the delimiter verbatim. + """ + if object_version is None: + return key + return f"{key}{FDV2_KEY_DELIMITER}{object_version}" + + +def put_skill( + key: str = "pdf-extraction", + *, + object_version: Any = 3, + payload_version: int = 42, + content: str = SKILL_BODY, + content_hash: Any = None, + omit_hash: bool = False, + name: str = "PDF Extraction", +) -> dict[str, Any]: + """One skill ``put-object`` event's data, in the shape the wire delivers it.""" + envelope: dict[str, Any] = { + "contentType": "text/markdown", + "content": content, + "name": name, + "description": "Extracts text", + } + if not omit_hash: + envelope["contentHash"] = ( + content_hash if content_hash is not None else _hash(content) + ) + return { + "key": wire_key(key, object_version), + "kind": FDV2_OBJECT_KIND, + "version": payload_version, + "object": envelope, + } + + +def delete_skill( + key: str = "pdf-extraction", *, object_version: Any = 3, payload_version: int = 43 +) -> dict[str, Any]: + return { + "key": wire_key(key, object_version), + "kind": FDV2_OBJECT_KIND, + "version": payload_version, + } + + +def put_flag(key: str = "my-flag", version: int = 17) -> dict[str, Any]: + """A flag ``put-object``: the same envelope fields, a different ``kind``.""" + return { + "key": key, + "kind": "flag", + "version": version, + "object": { + "key": key, + "version": version, + "on": True, + "variations": [True, False], + }, + } + + +def put_segment(key: str = "beta-users", version: int = 4) -> dict[str, Any]: + return { + "key": key, + "kind": "segment", + "version": version, + "object": {"key": key, "version": version, "included": []}, + } + + +def server_intent( + code: str = "xfer-full", payload_id: str = "agent-skill" +) -> dict[str, Any]: + return { + "payloads": [ + {"id": payload_id, "target": 1, "intentCode": code, "reason": "test"} + ] + } + + +def transferred(state: str = "basis-1", version: int = 42) -> dict[str, Any]: + return {"state": state, "version": version} + + +def events(*pairs: tuple[str, Any]) -> list[dict[str, Any]]: + return [{"event": name, "data": data} for name, data in pairs] + + +def full_payload( + *object_events: tuple[str, Any], state: str = "basis-1" +) -> list[dict[str, Any]]: + return events( + ("server-intent", server_intent("xfer-full")), + *object_events, + ("payload-transferred", transferred(state)), + ) + + +# --------------------------------------------------------------------------- +# The fake endpoint +# --------------------------------------------------------------------------- + + +class _FakeFDv2Endpoint: + """ + An in-process server implementing the SDK-facing FDv2 contract. + + Scripted per request: ``queue_poll`` appends a response for the next + ``/sdk/poll``, ``queue_stream`` appends a sequence of SSE events for the next + ``/sdk/stream``. Every request's method, path, query and headers are recorded + in ``requests`` so the tests can assert on what the store actually sent — + which is the only way ``basis`` round-tripping and ``If-None-Match`` can be + checked at all. + """ + + def __init__(self) -> None: + self.requests: list[dict[str, Any]] = [] + self._polls: list[dict[str, Any]] = [] + self._streams: list[list[dict[str, Any]]] = [] + self._lock = threading.Lock() + self.hold_stream_open = False + # When set, every ``/sdk/stream`` answers 307 to this URL instead of + # streaming, so a test can check that the store refuses to follow it. + self.redirect_stream_to: str | None = None + self._release = threading.Event() + + endpoint = self + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args: Any) -> None: + return + + def do_GET(self) -> None: + parsed = urlparse(self.path) + query = {k: v[0] for k, v in parse_qs(parsed.query).items()} + with endpoint._lock: + endpoint.requests.append( + { + "path": parsed.path, + "query": query, + "authorization": self.headers.get("Authorization"), + "if_none_match": self.headers.get("If-None-Match"), + "accept": self.headers.get("Accept"), + } + ) + if parsed.path == "/sdk/poll": + endpoint._serve_poll(self) + elif parsed.path == "/sdk/stream": + endpoint._serve_stream(self) + else: + self.send_response(404) + self.send_header("Content-Length", "0") + self.end_headers() + + class Server(ThreadingHTTPServer): + # Handler threads are not joined on shutdown: a test that ends while + # a stream is deliberately held open should not pay for the hold. + daemon_threads = True + + self._server = Server(("127.0.0.1", 0), Handler) + # A short poll interval so `shutdown` is prompt: the default 0.5s is + # paid at the teardown of every test that touches the endpoint. + self._thread = threading.Thread( + target=lambda: self._server.serve_forever(poll_interval=0.01), daemon=True + ) + self._thread.start() + + @property + def base_uri(self) -> str: + host, port = self._server.server_address[:2] + return f"http://{host}:{port}" + + # -- scripting --------------------------------------------------------- + + def queue_poll( + self, + payload_events: list[dict[str, Any]] | None = None, + *, + status: int = 200, + etag: str | None = None, + retry_after: str | None = None, + location: str | None = None, + ) -> None: + with self._lock: + self._polls.append( + { + "status": status, + "events": payload_events or [], + "etag": etag, + "retry_after": retry_after, + "location": location, + } + ) + + def queue_stream(self, payload_events: list[dict[str, Any]]) -> None: + with self._lock: + self._streams.append(payload_events) + + # -- serving ----------------------------------------------------------- + + def _serve_poll(self, handler: BaseHTTPRequestHandler) -> None: + with self._lock: + response = ( + self._polls.pop(0) if self._polls else {"status": 304, "events": []} + ) + status = response["status"] + handler.send_response(status) + if response.get("etag"): + handler.send_header("ETag", response["etag"]) + if response.get("retry_after"): + handler.send_header("Retry-After", response["retry_after"]) + if response.get("location"): + handler.send_header("Location", response["location"]) + if status in (200,): + body = json.dumps({"events": response["events"]}).encode("utf-8") + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + return + handler.send_header("Content-Length", "0") + handler.end_headers() + + def _serve_stream(self, handler: BaseHTTPRequestHandler) -> None: + with self._lock: + payload_events = self._streams.pop(0) if self._streams else [] + if self.redirect_stream_to: + handler.send_response(307) + handler.send_header("Location", self.redirect_stream_to) + handler.send_header("Content-Length", "0") + handler.end_headers() + return + handler.send_response(200) + handler.send_header("Content-Type", "text/event-stream") + handler.send_header("Cache-Control", "no-cache") + handler.send_header("Transfer-Encoding", "chunked") + handler.end_headers() + for event in payload_events: + chunk = ( + f"event: {event['event']}\ndata: {json.dumps(event.get('data'))}\n\n" + ).encode() + handler.wfile.write(f"{len(chunk):X}\r\n".encode() + chunk + b"\r\n") + handler.wfile.flush() + if self.hold_stream_open: + # Keeps the connection up so a test can assert on the store's state + # without racing the reconnect path. Released on ``close`` so the + # hold costs the suite nothing once the test is done with it. + self._release.wait(timeout=10) + handler.wfile.write(b"0\r\n\r\n") + + def close(self) -> None: + self._release.set() + self._server.shutdown() + self._server.server_close() + + +@pytest.fixture +def endpoint() -> Any: + server = _FakeFDv2Endpoint() + yield server + server.close() + + +def poll_store(endpoint: Any, **kwargs: Any) -> FDv2SkillStore: + return FDv2SkillStore( + SDK_KEY, + base_uri=endpoint.base_uri, + mode="poll", + poll_interval=kwargs.pop("poll_interval", 0.05), + initial_backoff=kwargs.pop("initial_backoff", 0.01), + max_backoff=kwargs.pop("max_backoff", 0.05), + read_timeout=kwargs.pop("read_timeout", 5.0), + **kwargs, + ) + + +def wait_until(predicate: Any, timeout: float = 5.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.01) + return predicate() + + +# --------------------------------------------------------------------------- +# Identifying skill objects, and ignoring everything else +# --------------------------------------------------------------------------- + + +class TestObjectIdentification: + def test_the_kind_alone_identifies_a_skill(self) -> None: + assert _is_skill_event(put_skill()) is True + + def test_the_kind_is_the_bare_category_name(self) -> None: + """ + Object kinds on the channel are open strings and the agent-skill payload + is ``generic``, so a skill arrives under the kind its producer + registered — ``skill`` — not under a broader wrapper kind. + """ + assert FDV2_OBJECT_KIND == "skill" + + def test_a_flag_is_not_a_skill(self) -> None: + assert _is_skill_event(put_flag()) is False + + def test_a_segment_is_not_a_skill(self) -> None: + assert _is_skill_event(put_segment()) is False + + def test_another_generic_kind_is_not_a_skill(self) -> None: + """A generic payload may carry other registered kinds one day.""" + other = put_skill() + other["kind"] = "prompt-template" + assert _is_skill_event(other) is False + + def test_a_skill_shaped_envelope_under_another_kind_is_not_a_skill(self) -> None: + other = put_skill() + other["kind"] = "some-future-kind" + assert _is_skill_event(other) is False + + def test_nothing_but_the_kind_is_consulted(self) -> None: + """No secondary field narrows the kind, and none may be required.""" + assert set(put_skill()) == {"key", "kind", "version", "object"} + + @pytest.mark.parametrize("value", [None, "skill", 3, [], ()]) + def test_non_dict_events_are_not_skills(self, value: Any) -> None: + assert _is_skill_event(value) is False + + +# --------------------------------------------------------------------------- +# The skill's version is in the wire key; `version` is the payload's +# --------------------------------------------------------------------------- + + +class TestVersionTranslation: + def test_the_wire_key_is_key_colon_version(self) -> None: + assert ( + put_skill("pdf-extraction", object_version=3)["key"] == "pdf-extraction:3" + ) + + def test_the_version_after_the_delimiter_becomes_the_seam_version(self) -> None: + raw = _store_object_from_put(put_skill(object_version=3, payload_version=42)) + assert raw is not None + assert raw["version"] == 3 + assert isinstance(raw["version"], int) + + def test_the_key_before_the_delimiter_becomes_the_seam_key(self) -> None: + """A caller asks for ``pdf-extraction``, never for ``pdf-extraction:3``.""" + raw = _store_object_from_put(put_skill("pdf-extraction", object_version=3)) + assert raw is not None + assert raw["key"] == "pdf-extraction" + + def test_the_payload_version_never_reaches_the_seam(self) -> None: + """ + The failure this asserts against is silent: a store that read ``version`` + would serve verifiable content under a version number that means nothing, + and every pinned reference would resolve to the wrong thing with no error. + """ + raw = _store_object_from_put(put_skill(object_version=3, payload_version=42)) + assert raw is not None + assert raw["version"] != 42 + assert 42 not in raw.values() + + def test_the_two_are_distinguished_even_when_the_payload_version_is_lower( + self, + ) -> None: + raw = _store_object_from_put(put_skill(object_version=99, payload_version=1)) + assert raw is not None + assert raw["version"] == 99 + + def test_a_key_with_no_delimiter_is_held_version_less(self) -> None: + """Not defaulted from the payload version, and not dropped: verification + reports ``invalid_version`` under a key the caller recognises.""" + raw = _store_object_from_put(put_skill(object_version=None)) + assert raw is not None + assert raw["key"] == "pdf-extraction" + assert "version" not in raw + + @pytest.mark.parametrize("spelling", ["latest", "", "3.0", "-1", "1:2", "３"]) + def test_a_version_that_is_not_digits_is_carried_through_as_invalid( + self, spelling: str + ) -> None: + """Carried, not invented: verification reports ``invalid_version`` for + the object rather than the transport reporting it absent.""" + raw = _store_object_from_put(put_skill(object_version=spelling)) + assert raw is not None + assert raw["key"] == "pdf-extraction" + assert raw["version"] == spelling + + def test_leading_zeros_spell_the_same_version(self) -> None: + raw = _store_object_from_put(put_skill(object_version="03")) + assert raw is not None + assert raw["version"] == 3 + + def test_a_delete_reads_the_wire_key_the_same_way(self) -> None: + tombstone = _tombstone_from_delete( + delete_skill(object_version=3, payload_version=43) + ) + assert tombstone is not None + assert tombstone.key == "pdf-extraction" + assert tombstone.object_version == 3 + + @pytest.mark.parametrize("spelling", [None, "latest", "0"]) + def test_a_delete_with_no_usable_version_revokes_every_version( + self, spelling: Any + ) -> None: + tombstone = _tombstone_from_delete(delete_skill(object_version=spelling)) + assert tombstone is not None + assert tombstone.key == "pdf-extraction" + assert tombstone.object_version is None + + @pytest.mark.parametrize("bad_key", [":3", "", None, 3]) + def test_a_put_with_no_skill_key_is_dropped_because_it_has_no_identity( + self, bad_key: Any + ) -> None: + wire = put_skill() + wire["key"] = bad_key + assert _store_object_from_put(wire) is None + + def test_a_keyless_put_is_dropped_because_it_has_no_identity(self) -> None: + wire = put_skill() + del wire["key"] + assert _store_object_from_put(wire) is None + + def test_a_delete_with_no_skill_key_is_ignored(self) -> None: + wire = delete_skill() + wire["key"] = ":3" + assert _tombstone_from_delete(wire) is None + + def test_the_stored_identity_round_trips_to_the_wire_key(self) -> None: + """``_SkillObjectSet.snapshot`` spells its opaque keys the way the wire + does, so a held object can be matched back to the event that carried it.""" + held = _SkillObjectSet() + wire = put_skill("pdf-extraction", object_version=3) + raw = _store_object_from_put(wire) + assert raw is not None + held.put(raw) + assert set(held.snapshot()) == {wire["key"]} + + def test_the_envelope_is_copied_verbatim(self) -> None: + raw = _store_object_from_put(put_skill()) + assert raw is not None + assert raw["content"] == SKILL_BODY + assert raw["contentHash"] == _hash(SKILL_BODY) + assert raw["name"] == "PDF Extraction" + assert raw["contentType"] == "text/markdown" + + def test_an_absent_envelope_field_is_absent_rather_than_defaulted(self) -> None: + wire = put_skill() + del wire["object"]["name"] + raw = _store_object_from_put(wire) + assert raw is not None + assert "name" not in raw + + +# --------------------------------------------------------------------------- +# The protocol reader +# --------------------------------------------------------------------------- + + +def drive(reader: _ProtocolReader, payload_events: list[dict[str, Any]]) -> list[Any]: + return [reader.handle(e["event"], e.get("data")) for e in payload_events] + + +class TestProtocolReader: + def test_a_full_transfer_commits_at_payload_transferred(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + outcomes = drive(reader, full_payload(("put-object", put_skill()))) + assert len(held) == 1 + assert outcomes[-1].committed is True + assert outcomes[-1].basis == "basis-1" + + def test_an_up_to_date_intent_is_reported_as_such(self) -> None: + """``intentCode: "none"`` is the stream's 304: current, nothing to send.""" + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle("server-intent", server_intent("none")) + assert outcome.up_to_date is True + assert outcome.committed is False + assert outcome.disconnect is None + # A transfer intent is a promise of content, not an up-to-date answer. + transfer = reader.handle("server-intent", server_intent("xfer-full")) + assert transfer.up_to_date is False + + def test_nothing_is_visible_before_payload_transferred(self) -> None: + """A payload version is the unit of consistency; half of one is not a state.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill()), + ), + ) + assert len(held) == 0 + + def test_an_interrupted_full_transfer_leaves_last_known_good_intact(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill(object_version=1)))) + assert held.get("pdf-extraction", None) is not None + + # A second full transfer starts and never completes. + drive( + reader, + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill(object_version=2)), + ), + ) + still_held = held.get("pdf-extraction", None) + assert still_held is not None + assert still_held["version"] == 1 + + def test_a_full_transfer_replaces_rather_than_merges(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill("first")))) + drive( + reader, full_payload(("put-object", put_skill("second")), state="basis-2") + ) + assert held.get("first", None) is None + assert held.get("second", None) is not None + + def test_a_change_transfer_applies_deltas_over_what_is_held(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill("first")))) + drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("second")), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert held.get("first", None) is not None + assert held.get("second", None) is not None + + def test_a_delete_object_revokes_the_skill(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill(object_version=3)))) + drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill(object_version=3)), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert held.get("pdf-extraction", None) is None + assert reader.diagnostics.objects_revoked == 1 + + def test_a_delete_notifies_with_a_tombstone_carrying_no_content(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill()))) + outcomes = drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ), + ) + (change,) = outcomes[-1].changes + assert change == {"key": "pdf-extraction", "version": 3} + assert "content" not in change + + def test_a_delete_for_one_version_leaves_the_other_held(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + full_payload( + ("put-object", put_skill(object_version=2)), + ("put-object", put_skill(object_version=3)), + ), + ) + drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill(object_version=3)), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert held.get("pdf-extraction", 2) is not None + assert held.get("pdf-extraction", None)["version"] == 2 + + def test_flag_and_segment_objects_are_skipped_cleanly(self) -> None: + """ + The mixed payload is the normal case, not an edge one: an environment's + assignment carries its flag payload alongside its agent-skill payload. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + outcomes = drive( + reader, + full_payload( + ("put-object", put_flag("flag-a")), + ("put-object", put_skill("pdf-extraction")), + ("put-object", put_segment("beta-users")), + ("put-object", put_flag("flag-b")), + ("delete-object", put_flag("flag-c")), + ), + ) + assert len(held) == 1 + assert held.get("pdf-extraction", None) is not None + assert reader.diagnostics.objects_ignored == 4 + assert reader.diagnostics.skill_objects_received == 1 + assert all(o.fatal is None and o.disconnect is None for o in outcomes) + + def test_an_unknown_kind_is_ignored_rather_than_fatal(self) -> None: + """ + Erroring on an unrecognised kind would turn a normal payload into a + permanent reconnect loop — a flag-delivery outage caused by a skills + rollout. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + exotic = { + "key": "x", + "kind": "quantum-widget", + "version": 1, + "object": {"a": 1}, + } + outcomes = drive(reader, full_payload(("put-object", exotic))) + assert len(held) == 0 + assert all(o.fatal is None and o.disconnect is None for o in outcomes) + + def test_an_unknown_event_name_is_ignored(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + outcome = reader.handle("some-future-event", {"anything": True}) + assert outcome.fatal is None + assert outcome.disconnect is None + + def test_a_heartbeat_does_nothing(self) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle("heart-beat", None) + assert outcome == type(outcome)() + + def test_an_error_event_abandons_the_in_flight_payload(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill(object_version=1)))) + outcomes = drive( + reader, + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill(object_version=2)), + ( + "error", + {"payloadId": "agent-skill", "reason": "backend unavailable"}, + ), + ), + ) + assert outcomes[-1].disconnect is not None + assert held.get("pdf-extraction", None)["version"] == 1 + + def test_a_goodbye_asks_for_a_reconnect(self) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle("goodbye", {"reason": "rebalancing", "silent": False}) + assert outcome.disconnect is not None + assert outcome.fatal is None + + def test_a_catastrophic_goodbye_is_fatal(self) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle( + "goodbye", {"reason": "no", "silent": False, "catastrophe": True} + ) + assert outcome.fatal is not None + + def test_transfer_none_holds_everything_and_commits(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill()))) + drive( + reader, + events( + ("server-intent", server_intent("none")), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert len(held) == 1 + + def test_an_object_arriving_with_no_intent_is_treated_as_a_delta(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + events( + ("put-object", put_skill()), + ("payload-transferred", transferred("basis-1")), + ), + ) + assert len(held) == 1 + + +# --------------------------------------------------------------------------- +# Which payload a transfer completed +# --------------------------------------------------------------------------- + + +def _payload_warnings(caplog: Any, fragment: str) -> list[Any]: + return [ + r + for r in caplog.records + if r.levelname == "WARNING" and fragment in r.getMessage() + ] + + +def skill_payload( + *object_events: tuple[str, Any], + payload_id: str = "agent-skill", + code: str = "xfer-full", + state: str = "basis-1", +) -> list[dict[str, Any]]: + """One payload's events, with the payload it belongs to named explicitly.""" + return events( + ("server-intent", server_intent(code, payload_id)), + *object_events, + ("payload-transferred", transferred(state)), + ) + + +class TestPayloadIdentity: + """ + Which payload a transfer completed, and why this layer tracks it at all. + + Delivery provides one payload per credential and the protocol requires a + client to read only the first payload intent, so today the payload read is + the payload skills arrive on. These assert the behaviour that survives if + the first of those stops holding: another payload's ``xfer-full`` must not + publish an empty skill set, because with pruning on that deletes a + customer's materialized files. + """ + + def test_only_the_first_payload_intent_is_read(self) -> None: + """Reading only the first is what the protocol asks for, however many + arrive — the point of the rest of this class is to make that safe.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + events( + ( + "server-intent", + { + "payloads": [ + { + "id": "agent-skill", + "target": 1, + "intentCode": "xfer-full", + }, + {"id": "env-flags", "target": 2, "intentCode": "none"}, + ] + }, + ), + ("put-object", put_skill()), + ("payload-transferred", transferred()), + ), + ) + assert len(held) == 1 + + def test_more_than_one_payload_intent_warns_once(self, caplog: Any) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + intent = { + "payloads": [ + {"id": "env-flags", "target": 1, "intentCode": "xfer-changes"}, + {"id": "agent-skill", "target": 2, "intentCode": "xfer-changes"}, + ] + } + with caplog.at_level("WARNING"): + reader.handle("server-intent", intent) + reader.handle("server-intent", intent) + assert len(_payload_warnings(caplog, "described 2 payloads")) == 1 + + def test_one_payload_intent_warns_about_nothing(self, caplog: Any) -> None: + with caplog.at_level("WARNING"): + drive( + _ProtocolReader(_SkillObjectSet()), + skill_payload(("put-object", put_skill())), + ) + assert _payload_warnings(caplog, "payload") == [] + + def test_another_payloads_full_transfer_does_not_empty_the_skills_held( + self, caplog: Any + ) -> None: + """ + The case this guard exists for. A flag payload's ``xfer-full`` starts an + empty pending set; applying it at ``payload-transferred`` would publish + every skill as revoked, which a reconcile with pruning on reads as + "delete these files". + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, skill_payload(("put-object", put_skill()))) + with caplog.at_level("WARNING"): + outcomes = drive( + reader, + skill_payload( + ("put-object", put_flag()), payload_id="env-flags", state="basis-2" + ), + ) + assert held.get("pdf-extraction", None) is not None + assert reader.diagnostics.payloads_ignored == 1 + assert len(_payload_warnings(caplog, "was not applied")) == 1 + # Nothing changed, so no listener is woken to reconcile against it. + assert outcomes[-1].changes == [] + + def test_a_declined_transfer_warns_once_however_often_it_repeats( + self, caplog: Any + ) -> None: + """A polling connection sees the other payload on every poll.""" + reader = _ProtocolReader(_SkillObjectSet()) + drive(reader, skill_payload(("put-object", put_skill()))) + foreign = skill_payload(("put-object", put_flag()), payload_id="env-flags") + with caplog.at_level("WARNING"): + drive(reader, foreign) + drive(reader, foreign) + assert len(_payload_warnings(caplog, "was not applied")) == 1 + assert reader.diagnostics.payloads_ignored == 2 + + def test_a_full_transfer_of_the_skill_payload_still_empties_it(self) -> None: + """Every skill deleted is a real state, and the guard must not mask it.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, skill_payload(("put-object", put_skill()))) + drive(reader, skill_payload(state="basis-2")) + assert len(held) == 0 + assert reader.diagnostics.payloads_ignored == 0 + + def test_a_revocation_identifies_the_payload_as_the_skill_payload(self) -> None: + """A payload that only revokes is still a payload skills arrive on.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + skill_payload(("delete-object", delete_skill()), code="xfer-changes"), + ) + drive( + reader, skill_payload(("put-object", put_skill()), payload_id="env-flags") + ) + assert reader.diagnostics.payloads_ignored == 1 + + def test_the_payload_is_identified_from_the_selector_when_no_id_is_named( + self, + ) -> None: + """``payload-transferred``'s selector is the only other place a completed + transfer names its payload.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + unnamed = {"payloads": [{"target": 1, "intentCode": "xfer-full"}]} + drive( + reader, + events( + ("server-intent", unnamed), + ("put-object", put_skill()), + ("payload-transferred", transferred("(p:agent-skill:53)")), + ), + ) + drive( + reader, + events( + ("server-intent", unnamed), + ("put-object", put_flag()), + ("payload-transferred", transferred("(p:env-flags:12)")), + ), + ) + assert held.get("pdf-extraction", None) is not None + assert reader.diagnostics.payloads_ignored == 1 + + def test_an_unidentifiable_payload_is_applied_rather_than_withheld(self) -> None: + """ + A transfer naming no payload at all is the store's own, since delivery + sends it one payload. Withholding it would break the common case to + defend against a hypothetical one. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, skill_payload(("put-object", put_skill()))) + drive( + reader, + events( + ("server-intent", {"payloads": [{"intentCode": "xfer-full"}]}), + ("put-object", put_skill(object_version=4)), + ("payload-transferred", {"version": 44}), + ), + ) + assert held.get("pdf-extraction", None)["version"] == 4 + assert reader.diagnostics.payloads_ignored == 0 + + def test_the_first_transfer_of_a_connection_is_the_residual( + self, caplog: Any + ) -> None: + """ + Before a skill has arrived there is nothing to compare a payload + against, so another payload's ``xfer-full`` arriving first cannot be + told apart. The multiple-payload WARNING is the only signal there is, + which is why it exists. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + with caplog.at_level("WARNING"): + drive( + reader, + events( + ( + "server-intent", + { + "payloads": [ + {"id": "env-flags", "intentCode": "xfer-full"}, + {"id": "agent-skill", "intentCode": "xfer-full"}, + ] + }, + ), + ("put-object", put_flag()), + ("payload-transferred", transferred()), + ), + ) + assert len(held) == 0 + assert len(_payload_warnings(caplog, "described 2 payloads")) == 1 + + +# --------------------------------------------------------------------------- +# Interface parity with InMemorySkillStore +# --------------------------------------------------------------------------- + + +class TestInterfaceParity: + """ + The two stores must resolve identically. ``_SkillObjectSet`` reimplements the + lookup rather than inheriting it — see its docstring for why — so this is the + test that stops the two from drifting. + """ + + RAWS: ClassVar[list[dict[str, Any]]] = [ + {"key": "a", "version": 1, "content": "x", "contentHash": _hash("x")}, + {"key": "a", "version": 4, "content": "y", "contentHash": _hash("y")}, + {"key": "b", "version": 2, "content": "z", "contentHash": _hash("z")}, + {"key": "malformed", "version": "not-a-version", "content": "q"}, + # A key holding a well-formed version *and* a version-less entry. The + # quadrant the fixtures above miss: "malformed" has no usable version + # at all, and "a"/"b" have no version-less entry, so neither exercises + # what happens when a pin misses a key that has both. + {"key": "mixed", "version": 2, "content": "m", "contentHash": _hash("m")}, + {"key": "mixed", "version": "not-a-version", "content": "n"}, + ] + + def _both(self) -> tuple[InMemorySkillStore, _SkillObjectSet]: + memory = InMemorySkillStore() + objects = _SkillObjectSet() + for raw in self.RAWS: + memory.put(dict(raw)) + objects.put(dict(raw)) + return memory, objects + + @pytest.mark.parametrize( + "key,version", + [ + ("a", None), + ("a", 1), + ("a", 4), + ("a", 9), + ("b", 2), + ("b", None), + ("missing", None), + ("missing", 1), + ("malformed", None), + ("malformed", 7), + ("mixed", None), + ("mixed", 2), + ("mixed", 7), + ], + ) + def test_get_agrees(self, key: str, version: int | None) -> None: + memory, objects = self._both() + assert memory.get_object(SKILL_OBJECT_KIND, key, version) == objects.get( + key, version + ) + + def test_snapshot_agrees(self) -> None: + memory, objects = self._both() + assert memory.all_objects(SKILL_OBJECT_KIND) == objects.snapshot() + + +# --------------------------------------------------------------------------- +# The store against the fake endpoint +# --------------------------------------------------------------------------- + + +class TestPollingAgainstTheEndpoint: + def test_a_polled_skill_becomes_retrievable_through_the_accessors( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + raw = store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") + assert raw is not None + assert raw["version"] == 3 + + def test_the_request_carries_the_sdk_key_and_no_data_model_version( + self, endpoint: Any + ) -> None: + """ + No ``mv``: that parameter selects the *flag* data model, the connection + rejects any value but the flag default, and the generic agent-skill + payload is served regardless of it. Sending ``mv=1`` — the skill + payload's own model version — gets the whole connection refused. + """ + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + first = endpoint.requests[0] + assert first["path"] == "/sdk/poll" + assert first["authorization"] == SDK_KEY + assert "mv" not in first["query"] + + def test_the_first_request_sends_no_basis(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert "basis" not in endpoint.requests[0]["query"] + + def test_the_basis_from_payload_transferred_is_echoed_on_the_next_request( + self, endpoint: Any + ) -> None: + endpoint.queue_poll( + full_payload(("put-object", put_skill()), state="selector-abc") + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert endpoint.requests[1]["query"]["basis"] == "selector-abc" + + def test_the_basis_advances_across_successive_payloads(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()), state="basis-1")) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("second")), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint): + assert wait_until(lambda: len(endpoint.requests) >= 3) + bases = [r["query"].get("basis") for r in endpoint.requests[:3]] + assert bases == [None, "basis-1", "basis-2"] + + def test_an_etag_is_returned_as_if_none_match(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill())), etag='W/"v1"') + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert endpoint.requests[1]["if_none_match"] == 'W/"v1"' + + def test_a_304_keeps_the_held_content(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill())), etag='W/"v1"') + endpoint.queue_poll(status=304) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert wait_until(lambda: len(endpoint.requests) >= 3) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + assert store.diagnostics.payloads_transferred == 1 + assert store.failed is None + + def test_a_304_before_any_payload_still_releases_wait_for_skills( + self, endpoint: Any + ) -> None: + """A reconnect with a cached basis has nothing to transfer; boot must not + block on a payload the server has no reason to send.""" + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + + def test_a_mixed_payload_over_the_wire_yields_only_the_skill( + self, endpoint: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_flag("flag-a")), + ("put-object", put_segment("beta")), + ("put-object", put_skill("pdf-extraction")), + ("put-object", put_flag("flag-b")), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + held = store.all_objects(SKILL_OBJECT_KIND) + assert len(held) == 1 + assert next(iter(held.values()))["key"] == "pdf-extraction" + assert store.diagnostics.objects_ignored == 3 + + def test_a_revocation_over_the_wire_removes_the_skill(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + assert wait_until( + lambda: ( + store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is None + and store.diagnostics.objects_revoked == 1 + ) + ) + + def test_the_store_asks_for_only_the_kind_it_serves(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert store.get_object("flag", "pdf-extraction") is None + assert store.all_objects("flag") == {} + + +class TestStreamingAgainstTheEndpoint: + def test_a_streamed_payload_lands(self, endpoint: Any) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore( + SDK_KEY, base_uri=endpoint.base_uri, mode="stream", initial_backoff=0.01 + ) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + finally: + store.close() + + def test_the_stream_request_advertises_event_stream(self, endpoint: Any) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + try: + store.start() + store.wait_for_skills(timeout=5) + finally: + store.close() + assert endpoint.requests[0]["path"] == "/sdk/stream" + assert endpoint.requests[0]["accept"] == "text/event-stream" + + def test_a_streamed_revocation_arrives_without_a_restart( + self, endpoint: Any + ) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream( + full_payload(("put-object", put_skill())) + + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ) + ) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + try: + store.start() + assert wait_until( + lambda: ( + store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is None + and store.diagnostics.objects_revoked == 1 + ) + ) + finally: + store.close() + + def test_a_dropped_stream_reconnects_with_the_basis_it_reached( + self, endpoint: Any + ) -> None: + endpoint.queue_stream( + full_payload(("put-object", put_skill()), state="basis-1") + ) + endpoint.queue_stream(events(("heart-beat", None))) + store = FDv2SkillStore( + SDK_KEY, + base_uri=endpoint.base_uri, + mode="stream", + initial_backoff=0.01, + max_backoff=0.05, + ) + try: + store.start() + assert wait_until(lambda: len(endpoint.requests) >= 2) + finally: + store.close() + assert endpoint.requests[1]["query"]["basis"] == "basis-1" + + def test_close_returns_promptly_while_a_stream_is_open(self, endpoint: Any) -> None: + """ + The delivery thread is blocked in a socket read that no stop flag can + reach, so ``close`` closes the connection under it. Without that, every + shutdown of a healthy stream waits out the join timeout. + """ + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + store.start() + assert store.wait_for_skills(timeout=5) is True + started = time.monotonic() + store.close(timeout=5.0) + assert time.monotonic() - started < 1.0 + + def test_an_interrupted_stream_is_not_reported_as_a_failure( + self, endpoint: Any + ) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + store.start() + store.wait_for_skills(timeout=5) + store.close() + assert store.failed is None + + def test_content_survives_a_reconnect(self, endpoint: Any) -> None: + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + endpoint.queue_stream(events(("heart-beat", None))) + store = FDv2SkillStore( + SDK_KEY, base_uri=endpoint.base_uri, mode="stream", initial_backoff=0.01 + ) + try: + store.start() + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + finally: + store.close() + + +# --------------------------------------------------------------------------- +# Failure handling +# --------------------------------------------------------------------------- + + +class _DyingResponse: + """ + A streaming body that transfers a payload and then fails mid-read. + + This is how a live stream actually ends: not with a clean end of body but + with a read timeout on a stream that went quiet, or a reset from the server + or a proxy in between. + """ + + def __init__(self, exc: BaseException) -> None: + self._exc = exc + self._lines: list[bytes] = [] + for event in full_payload(("put-object", put_skill())): + self._lines.append(f"event: {event['event']}\n".encode()) + self._lines.append(f"data: {json.dumps(event['data'])}\n".encode()) + self._lines.append(b"\n") + + def readline(self, size: int = -1) -> bytes: + if self._lines: + return self._lines.pop(0) + raise self._exc + + def close(self) -> None: + pass + + +class _FakeRequester: + """ + Base for the requester fakes: supplies the ``interrupt`` the store calls on + ``close``, so each fake only scripts the part it is about. + """ + + def interrupt(self) -> None: + """No real socket to reach; these fakes end their own connections.""" + + +class _DyingStreamRequester(_FakeRequester): + """Every connection transfers a payload, then dies with *exc* mid-read.""" + + def __init__(self, exc: BaseException) -> None: + self.connections = 0 + self._exc = exc + + def stream(self, basis: str | None) -> Any: + self.connections += 1 + return _StreamConnection(_DyingResponse(self._exc)) + + +class _ScriptedConnection: + """Stands in for ``_StreamConnection``: an event iterator plus a close.""" + + def __init__(self, payload_events: Any) -> None: + self.events = iter(payload_events) + self.closed = False + + def close(self) -> None: + self.closed = True + + +class _ScriptedRequester(_FakeRequester): + """Raises a scripted sequence, so backoff is asserted without real sockets.""" + + def __init__(self, *outcomes: Any) -> None: + self.outcomes = list(outcomes) + self.calls: list[tuple[str | None, str | None]] = [] + + def poll(self, basis: str | None, etag: str | None) -> Any: + self.calls.append((basis, etag)) + outcome = ( + self.outcomes.pop(0) if self.outcomes else _RecoverableTransportError("x") + ) + if isinstance(outcome, Exception): + raise outcome + return outcome + + def stream(self, basis: str | None) -> Any: + self.calls.append((basis, None)) + outcome = ( + self.outcomes.pop(0) if self.outcomes else _RecoverableTransportError("x") + ) + if isinstance(outcome, Exception): + raise outcome + return _ScriptedConnection(outcome) + + +class _RecyclingRequester(_FakeRequester): + """ + A healthy server that recycles connections: every ``stream`` call succeeds, + transfers a full payload, and then ends the connection, as LaunchDarkly and + any proxy in between do to a long-lived stream. + """ + + def __init__(self) -> None: + self.connections = 0 + + def stream(self, basis: str | None) -> Any: + self.connections += 1 + return _ScriptedConnection( + [ + (e["event"], e["data"]) + for e in full_payload( + ("put-object", put_skill()), state=f"basis-{self.connections}" + ) + ] + ) + + +class _UpToDateRecyclingRequester(_FakeRequester): + """ + A healthy server with nothing new to say: every connection answers + ``intentCode: "none"`` — the stream's equivalent of a 304 — transfers + nothing, and is then recycled. This is the steady state of an environment + whose skills are not changing, which is most environments most of the time. + """ + + def __init__(self, farewell: bool = False) -> None: + self.connections = 0 + self._farewell = farewell + + def stream(self, basis: str | None) -> Any: + self.connections += 1 + script: list[tuple[str, Any]] = [ + ("server-intent", server_intent("none")), + ("heart-beat", {}), + ] + if self._farewell: + # A recycle is often announced rather than abrupt. + script.append(("goodbye", {"reason": "connection recycled"})) + return _ScriptedConnection(script) + + +class _SlowPollRequester(_FakeRequester): + """ + A poll whose request does not return until the test releases it, standing in + for one blocked where no interrupt can reach: inside its connect. + """ + + def __init__(self) -> None: + self.entered = threading.Event() + self.release = threading.Event() + + def poll(self, basis: str | None, etag: str | None) -> Any: + self.entered.set() + self.release.wait(timeout=10) + raise _RecoverableTransportError("released") + + +class _SilentStreamRequester(_FakeRequester): + """A stream that connects and then delivers nothing until it is closed.""" + + def stream(self, basis: str | None) -> Any: + return _BlockingConnection() + + +class _BlockingConnection: + """A stream that never produces an event until it is closed.""" + + def __init__(self) -> None: + self._closed = threading.Event() + + @property + def events(self) -> Any: + self._closed.wait() + return iter(()) + + def close(self) -> None: + self._closed.set() + + +class _SlowConnectRequester(_FakeRequester): + """ + A ``stream`` whose connect does not return until the test releases it, + standing in for a slow TLS handshake, followed by a read that never yields. + """ + + def __init__(self) -> None: + self.entered = threading.Event() + self.release = threading.Event() + + def stream(self, basis: str | None) -> Any: + self.entered.set() + self.release.wait(timeout=10) + return _BlockingConnection() + + +def stream_store(**kwargs: Any) -> FDv2SkillStore: + return FDv2SkillStore( + SDK_KEY, + mode="stream", + initial_backoff=kwargs.pop("initial_backoff", 0.001), + max_backoff=kwargs.pop("max_backoff", 0.002), + **kwargs, + ) + + +class TestFailureHandling: + def test_a_403_stops_delivery_and_explains_why( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll(status=403) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert "403" in store.failed + assert "opt-in" in store.failed + assert any("opt-in" in r.getMessage() for r in caplog.records) + + def test_a_restarted_store_does_not_report_the_old_failure( + self, endpoint: Any + ) -> None: + """``failed`` says why delivery stopped *for good*. + + A store started again is delivering, so the terminal reason from the + previous run is no longer true of it. Leaving it would have a healthy + store reporting a failure it has recovered from. + """ + endpoint.queue_poll(status=401) + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert "401" in store.failed + + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store.start() + assert store.wait_for_skills(timeout=5) is True + assert store.failed is None + + def test_a_restart_during_the_give_up_still_delivers( + self, endpoint: Any, monkeypatch: Any + ) -> None: + """A restart that races the dying thread must not adopt it. + + ``failed`` becomes readable while the delivery thread is still alive and + winding down. A ``start`` in that window has to spawn a replacement: a + live thread is not by itself a delivering one, and treating it as one + leaves a store reporting no failure with nothing left to deliver. + + The window is held open by blocking the give-up log line, which the + dying thread emits after publishing the reason. + """ + giving_up = threading.Event() + may_finish = threading.Event() + real_error = skills_fdv2.logger.error + + def blocking_error(msg: Any, *args: Any, **kwargs: Any) -> None: + if isinstance(msg, str) and msg.startswith("Skill delivery has stopped"): + giving_up.set() + may_finish.wait(timeout=5) + real_error(msg, *args, **kwargs) + + monkeypatch.setattr(skills_fdv2.logger, "error", blocking_error) + + endpoint.queue_poll(status=401) + with poll_store(endpoint) as store: + assert giving_up.wait(timeout=5) + # The premise of the test: the reason is readable and the thread + # that published it has not returned yet. + assert store.failed is not None + assert store._thread is not None + assert store._thread.is_alive() + + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store.start() + may_finish.set() + + assert store.wait_for_skills(timeout=5) is True + assert store.failed is None + + def test_a_restart_returns_the_retry_budget(self, endpoint: Any) -> None: + """The retry budget belongs to the run that spent it. + + A store that gave up at its failure limit would otherwise carry the + spent count into the restarted run and give up again on its first + recoverable failure, without retrying once. + """ + store = poll_store(endpoint, max_consecutive_failures=1) + # One over the limit, so the first run retries once and then gives up. + endpoint.queue_poll(status=500) + endpoint.queue_poll(status=500) + with store: + assert wait_until(lambda: store.failed is not None) + + endpoint.queue_poll(status=500) + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store.start() + assert store.wait_for_skills(timeout=5) is True + assert store.failed is None + + def test_a_401_stops_delivery(self, endpoint: Any) -> None: + endpoint.queue_poll(status=401) + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert "401" in store.failed + + def test_a_fatal_failure_releases_wait_for_skills_rather_than_hanging( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(status=401) + with poll_store(endpoint) as store: + started = time.monotonic() + # Released promptly, and ``False``: no payload arrived, and saying + # otherwise would send a caller on to read a store holding nothing. + assert store.wait_for_skills(timeout=5) is False + assert time.monotonic() - started < 2.0 + assert store.failed is not None + + def test_a_fatal_failure_keeps_last_known_good_servable( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll(status=403) + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + + def test_a_500_is_retried(self, endpoint: Any) -> None: + endpoint.queue_poll(status=500) + endpoint.queue_poll(status=503) + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + assert store.failed is None + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + + def test_a_retry_resets_the_failure_count_on_success(self, endpoint: Any) -> None: + endpoint.queue_poll(status=500) + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) + assert wait_until(lambda: store.diagnostics.connection_failures == 0) + + def test_retries_are_bounded(self) -> None: + store = FDv2SkillStore( + SDK_KEY, + mode="poll", + poll_interval=0.01, + initial_backoff=0.001, + max_backoff=0.002, + max_consecutive_failures=3, + _requester=_ScriptedRequester(), + ) + try: + store.start() + assert wait_until(lambda: store.failed is not None) + # Four, not three: the bound is the number of failures *tolerated*, + # so the run that exceeds it is the one that gives up. + assert "gave up after 4 consecutive failures" in store.failed + finally: + store.close() + + def test_recycled_stream_connections_are_not_failures(self) -> None: + # A streaming connection only ever ends by being dropped, so a loop + # that counted every drop as a failure would give up on a healthy + # server after max_consecutive_failures + 1 recycles, and delivery + # (including revocation) would silently stop for the process lifetime. + requester = _RecyclingRequester() + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert wait_until(lambda: requester.connections >= 8) + assert store.failed is None + assert store.diagnostics.payloads_transferred >= 8 + # A drop is a failure until the next commit clears it, so the count + # may read 1 mid-reconnect. What it must never do is climb. + assert store.diagnostics.connection_failures <= 1 + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + finally: + store.close() + + @pytest.mark.parametrize("farewell", [False, True], ids=["dropped", "goodbye"]) + def test_an_up_to_date_recycled_stream_is_not_a_failure( + self, farewell: bool + ) -> None: + # Resetting at a commit covers only a connection that carried new + # content. An environment whose skills are not changing answers every + # reconnect with ``intentCode: "none"`` and transfers nothing, so a loop + # that counted those drops would give up on a *healthy* idle stream + # after max_consecutive_failures + 1 recycles — and revocation, the one + # thing streaming exists to deliver promptly, would never arrive again. + requester = _UpToDateRecyclingRequester(farewell=farewell) + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert wait_until(lambda: requester.connections >= 8) + assert store.failed is None + # As with a payload-carrying recycle, the count may read 1 + # mid-reconnect. What it must never do is climb. + assert store.diagnostics.connection_failures <= 1 + finally: + store.close() + + def test_a_recycled_connection_reconnects_quietly(self, caplog: Any) -> None: + # A healthy idle stream reconnects for as long as the process runs, so + # warning on each one would fill a customer's logs with a fault they do + # not have and teach them to ignore the level that means something. + requester = _UpToDateRecyclingRequester() + store = stream_store(_requester=requester) + with caplog.at_level("DEBUG", logger="launchdarkly_ai_server.skills_fdv2"): + try: + store.start() + assert wait_until(lambda: requester.connections >= 5) + finally: + store.close() + assert store.failed is None + assert not [r for r in caplog.records if r.levelname == "WARNING"] + assert [r for r in caplog.records if "reconnecting in" in r.getMessage()] + + def test_a_connection_that_never_answered_still_warns(self, caplog: Any) -> None: + # The quiet path is earned by answering. A connection that failed before + # it told us anything is the case the warning exists for. + store = stream_store( + max_consecutive_failures=10, _requester=_ScriptedRequester() + ) + with caplog.at_level("DEBUG", logger="launchdarkly_ai_server.skills_fdv2"): + try: + store.start() + assert wait_until(lambda: store.diagnostics.connection_failures >= 3) + finally: + store.close() + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert warnings + assert all("Skill delivery failed" in r.getMessage() for r in warnings) + + @pytest.mark.parametrize( + "exc", + [ + TimeoutError("timed out"), + ConnectionResetError(54, "Connection reset by peer"), + IncompleteRead(b"partial"), + ], + ids=["read timeout", "reset", "truncated body"], + ) + def test_a_stream_that_dies_mid_read_reconnects(self, exc: BaseException) -> None: + # A stream fails in its body far more often than at its connect, and + # ``read_timeout`` exists to bound one that has gone quiet. Treating + # such a failure as unexpected would stop delivery — including + # revocation — for the process lifetime the first time a socket died. + requester = _DyingStreamRequester(exc) + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + assert wait_until(lambda: requester.connections >= 5) + assert store.failed is None + finally: + store.close() + + def test_a_stream_commit_resets_the_failure_count(self) -> None: + payload = [ + (e["event"], e["data"]) for e in full_payload(("put-object", put_skill())) + ] + requester = _ScriptedRequester( + _RecoverableTransportError("x"), + _RecoverableTransportError("x"), + _RecoverableTransportError("x"), + payload, + ) + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=5) + # Three failures reach the bound, then a commit, then the exhausted + # requester fails on every reconnect. The count must start again at + # the commit: the stream's own drop is failure one, and three more + # connects are owed before giving up. Carrying the three over would + # give up on the drop itself, with no further connect at all. + assert wait_until(lambda: store.failed is not None) + assert "gave up after 4 consecutive failures" in store.failed + assert "last error: x" in store.failed + assert len(requester.calls) == 7 + finally: + store.close() + + def test_stream_retries_are_bounded(self) -> None: + store = stream_store( + max_consecutive_failures=3, _requester=_ScriptedRequester() + ) + try: + store.start() + assert wait_until(lambda: store.failed is not None) + assert "gave up after 4 consecutive failures" in store.failed + finally: + store.close() + + def test_a_retry_after_header_is_honoured(self) -> None: + requester = _ScriptedRequester( + _RecoverableTransportError("slow down", retry_after=0.25), + ) + store = FDv2SkillStore( + SDK_KEY, + mode="poll", + poll_interval=10.0, + initial_backoff=5.0, + _requester=requester, + ) + try: + started = time.monotonic() + store.start() + assert wait_until(lambda: len(requester.calls) >= 2, timeout=3) + elapsed = time.monotonic() - started + # The server asked for 0.25s; our own backoff would have been 5s. + assert 0.2 <= elapsed < 3.0 + finally: + store.close() + + def test_a_retry_after_header_is_parsed_off_the_wire(self, endpoint: Any) -> None: + endpoint.queue_poll(status=429, retry_after="0") + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint, initial_backoff=5.0) as store: + # If Retry-After were ignored the 5s backoff would blow the timeout. + assert store.wait_for_skills(timeout=3) is True + + @pytest.mark.parametrize("raw", ["inf", "Infinity", "-inf", "nan", "1e309"]) + def test_a_non_finite_retry_after_is_ignored(self, raw: str) -> None: + assert _retry_after_seconds({"Retry-After": raw}) is None + + def test_retry_after_parsing_keeps_its_edges(self) -> None: + assert _retry_after_seconds({"Retry-After": "0"}) == 0.0 + assert _retry_after_seconds({"Retry-After": "-5"}) == 0.0 + assert ( + _retry_after_seconds({"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}) + is None + ) + assert _retry_after_seconds({"Retry-After": "2.5"}) == 2.5 + + @pytest.mark.parametrize("retry_after", [float("inf"), float("nan"), 86400.0]) + def test_an_unreasonable_retry_after_neither_kills_delivery_nor_parks_it( + self, retry_after: float + ) -> None: + # An infinite wait would overflow inside the retry handler and kill the + # thread with `failed` still None; a day-long one would be honoured to + # the second. Both must fall back to the max_backoff cap and carry on. + requester = _ScriptedRequester( + _RecoverableTransportError("slow down", retry_after=retry_after), + [ + (e["event"], e["data"]) + for e in full_payload(("put-object", put_skill())) + ], + ) + store = stream_store(max_backoff=0.05, _requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=3) is True + assert store.failed is None + assert store._thread is not None and store._thread.is_alive() + finally: + store.close() + + def test_a_non_finite_retry_after_off_the_wire_falls_back_to_backoff( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(status=429, retry_after="inf") + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=3) is True + assert store.failed is None + + def test_backoff_is_exponential_and_capped(self) -> None: + assert _backoff_delay(1, base=1.0, maximum=30.0, jitter=0.0) == 1.0 + assert _backoff_delay(2, base=1.0, maximum=30.0, jitter=0.0) == 2.0 + assert _backoff_delay(3, base=1.0, maximum=30.0, jitter=0.0) == 4.0 + assert _backoff_delay(20, base=1.0, maximum=30.0, jitter=0.0) == 30.0 + + def test_jitter_never_exceeds_the_cap(self) -> None: + for attempt in range(1, 12): + for _ in range(50): + assert 0.0 <= _backoff_delay(attempt, base=1.0, maximum=5.0) <= 5.0 + + def test_a_malformed_polling_envelope_is_recoverable_not_fatal( + self, endpoint: Any + ) -> None: + store = FDv2SkillStore( + SDK_KEY, + mode="poll", + poll_interval=0.01, + initial_backoff=0.001, + _requester=_ScriptedRequester( + _RecoverableTransportError("polling response had no 'events' array") + ), + ) + try: + store.start() + assert wait_until(lambda: store.diagnostics.connection_failures >= 1) + assert store.failed is None + finally: + store.close() + + def test_a_listener_that_raises_does_not_kill_delivery(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill("first")))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("second")), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.add_listener(SKILL_OBJECT_KIND, lambda _raw: 1 / 0) + assert wait_until( + lambda: store.get_object(SKILL_OBJECT_KIND, "second") is not None + ) + assert store.failed is None + + +# --------------------------------------------------------------------------- +# The contentHash gap +# --------------------------------------------------------------------------- + + +# --------------------------------------------------------------------------- +# Redirects are refused +# --------------------------------------------------------------------------- + + +class _LineSource: + """A streaming body served from bytes, with the ``readline`` the parser uses.""" + + def __init__(self, data: bytes) -> None: + self._buf = data + self.closed = False + + def readline(self, size: int = -1) -> bytes: + if size < 0: + size = len(self._buf) + newline = self._buf.find(b"\n", 0, size) + end = size if newline < 0 else newline + 1 + line, self._buf = self._buf[:end], self._buf[end:] + return line + + def close(self) -> None: + self.closed = True + + +class TestTransportMemoryBound: + """ + ``MAX_RESPONSE_BYTES`` bounds what one response may put in memory before + verification's per-skill content cap can see any of it. Disk was already + bounded; this is what bounds memory. The cap is patched small here so the + suite does not have to move 64 MiB to prove it. + """ + + def test_the_bound_is_far_above_any_legitimate_payload(self) -> None: + assert MAX_RESPONSE_BYTES == 64 * 1024 * 1024 + + def test_an_over_cap_poll_body_is_not_applied_and_is_retried( + self, endpoint: Any, monkeypatch: Any + ) -> None: + monkeypatch.setattr(skills_fdv2, "MAX_RESPONSE_BYTES", 2048) + endpoint.queue_poll(full_payload(("put-object", put_skill(content="x" * 8192)))) + # A long enough backoff to observe the failure before the retry lands. + with poll_store(endpoint, initial_backoff=0.3, max_backoff=0.3) as store: + assert wait_until(lambda: store.diagnostics.connection_failures == 1) + assert "2048-byte transport bound" in (store.diagnostics.last_error or "") + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is None + assert store.diagnostics.payloads_transferred == 0 + assert store.diagnostics.skill_objects_received == 0 + assert store.failed is None + # The retry is an ordinary poll; the endpoint answers it 304. + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert store.wait_for_skills(timeout=5) is True + assert wait_until(lambda: store.diagnostics.connection_failures == 0) + assert "2048-byte transport bound" in (store.diagnostics.last_error or "") + assert all(r["path"] == "/sdk/poll" for r in endpoint.requests) + + def test_a_poll_body_exactly_at_the_cap_is_accepted( + self, endpoint: Any, monkeypatch: Any + ) -> None: + payload = full_payload(("put-object", put_skill())) + body = json.dumps({"events": payload}).encode("utf-8") + monkeypatch.setattr(skills_fdv2, "MAX_RESPONSE_BYTES", len(body)) + endpoint.queue_poll(payload) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + assert store.diagnostics.connection_failures == 0 + assert store.diagnostics.last_error is None + + def test_a_poll_body_one_byte_over_the_cap_is_refused( + self, endpoint: Any, monkeypatch: Any + ) -> None: + payload = full_payload(("put-object", put_skill())) + body = json.dumps({"events": payload}).encode("utf-8") + monkeypatch.setattr(skills_fdv2, "MAX_RESPONSE_BYTES", len(body) - 1) + endpoint.queue_poll(payload) + with poll_store(endpoint, max_consecutive_failures=0) as store: + assert wait_until(lambda: store.failed is not None) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is None + assert f"{len(body) - 1}-byte transport bound" in store.failed + assert f"at least {len(body)} bytes received" in store.failed + + def test_the_default_cap_leaves_ordinary_payloads_alone( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + assert store.diagnostics.connection_failures == 0 + assert store.diagnostics.last_error is None + + def test_an_over_cap_stream_event_abandons_the_payload_in_flight( + self, endpoint: Any, monkeypatch: Any + ) -> None: + """ + The first payload commits. The second starts, then carries an event + over the cap: that connection is dropped, the half-received payload is + never committed, and the reconnect finds the committed set intact. + """ + monkeypatch.setattr(skills_fdv2, "MAX_RESPONSE_BYTES", 2048) + endpoint.queue_stream( + full_payload(("put-object", put_skill())) + + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("oversized", content="x" * 8192)), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.hold_stream_open = True + endpoint.queue_stream(events(("server-intent", server_intent("none")))) + store = FDv2SkillStore( + SDK_KEY, + base_uri=endpoint.base_uri, + mode="stream", + initial_backoff=0.01, + max_backoff=0.02, + ) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + assert wait_until( + lambda: "transport bound" in (store.diagnostics.last_error or "") + ) + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + assert store.get_object(SKILL_OBJECT_KIND, "oversized") is None + assert store.diagnostics.payloads_transferred == 1 + assert store.diagnostics.skill_objects_received == 1 + assert store.failed is None + assert endpoint.requests[1]["query"].get("basis") == "basis-1" + finally: + store.close() + + def test_a_stream_line_that_never_ends_is_refused_and_the_body_closed( + self, monkeypatch: Any + ) -> None: + monkeypatch.setattr(skills_fdv2, "MAX_RESPONSE_BYTES", 1024) + source = _LineSource(b"data: " + b"x" * 4096) + with pytest.raises(_RecoverableTransportError, match="1024-byte"): + list(_iter_sse(source)) + assert source.closed + + def test_an_event_is_measured_across_its_data_lines(self, monkeypatch: Any) -> None: + monkeypatch.setattr(skills_fdv2, "MAX_RESPONSE_BYTES", 1024) + lines = b"".join(b"data: " + b"x" * 500 + b"\n" for _ in range(3)) + source = _LineSource(b"event: put-object\n" + lines + b"\n") + with pytest.raises(_RecoverableTransportError, match="1024-byte"): + list(_iter_sse(source)) + assert source.closed + + def test_multi_line_data_under_the_cap_still_decodes(self) -> None: + source = _LineSource(b'event: put-object\ndata: {"a":\ndata: 1}\n\n') + assert list(_iter_sse(source)) == [("put-object", {"a": 1})] + assert source.closed + + +@pytest.fixture +def second_endpoint() -> Any: + """A second host, to stand for wherever a ``Location`` header points.""" + server = _FakeFDv2Endpoint() + yield server + server.close() + + +class TestRedirectsAreRefused: + """ + ``urllib``'s standard redirect handler copies every request header onto the + redirected request, ``Authorization`` included. The transport's opener + declines every redirect instead, so a 3xx is a fatal, non-retried failure + and the SDK key never reaches the host ``Location`` names. + """ + + @pytest.mark.parametrize("status", [301, 302, 307, 308]) + def test_a_poll_redirect_is_fatal_and_not_followed( + self, endpoint: Any, second_endpoint: Any, status: int + ) -> None: + target = second_endpoint.base_uri + "/sdk/poll" + endpoint.queue_poll(status=status, location=target) + requester = _Requester(SDK_KEY, endpoint.base_uri, read_timeout=5.0) + with pytest.raises(_FatalTransportError) as excinfo: + requester.poll(None, None) + assert str(status) in str(excinfo.value) + assert "not followed" in str(excinfo.value) + assert second_endpoint.requests == [] + + def test_a_stream_redirect_is_fatal_and_not_followed( + self, endpoint: Any, second_endpoint: Any + ) -> None: + endpoint.redirect_stream_to = second_endpoint.base_uri + "/sdk/stream" + requester = _Requester(SDK_KEY, endpoint.base_uri, read_timeout=5.0) + with pytest.raises(_FatalTransportError) as excinfo: + requester.stream(None) + assert "307" in str(excinfo.value) + assert second_endpoint.requests == [] + + def test_a_same_host_redirect_is_refused_too(self, endpoint: Any) -> None: + """The endpoints do not redirect, so there is nothing legitimate to follow.""" + endpoint.queue_poll(status=302, location=endpoint.base_uri + "/sdk/poll") + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + requester = _Requester(SDK_KEY, endpoint.base_uri, read_timeout=5.0) + with pytest.raises(_FatalTransportError): + requester.poll(None, None) + assert len(endpoint.requests) == 1 + + def test_the_sdk_key_never_reaches_the_second_host( + self, endpoint: Any, second_endpoint: Any + ) -> None: + """ + End to end through the store: the redirect stops delivery for good, + with no retry spent on it, and the second host sees no request at all — + so no ``Authorization`` header, since that is what following would + have forwarded. + """ + endpoint.queue_poll(status=301, location=second_endpoint.base_uri + "/sdk/poll") + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert store.wait_for_skills(timeout=5) is False + assert store.failed is not None + assert "301" in store.failed + assert "never forwarded" in store.failed + assert store.diagnostics.connection_failures == 0 + assert len(endpoint.requests) == 1 + assert [r["authorization"] for r in second_endpoint.requests] == [] + + def test_a_redirect_in_stream_mode_stops_delivery( + self, endpoint: Any, second_endpoint: Any + ) -> None: + endpoint.redirect_stream_to = second_endpoint.base_uri + "/sdk/stream" + store = FDv2SkillStore( + SDK_KEY, base_uri=endpoint.base_uri, mode="stream", initial_backoff=0.01 + ) + with store: + assert wait_until(lambda: store.failed is not None) + assert store.failed is not None + assert "307" in store.failed + assert second_endpoint.requests == [] + + def test_a_redirect_with_no_location_is_still_fatal(self, endpoint: Any) -> None: + endpoint.queue_poll(status=302) + requester = _Requester(SDK_KEY, endpoint.base_uri, read_timeout=5.0) + with pytest.raises(_FatalTransportError): + requester.poll(None, None) + + def test_a_304_is_not_a_redirect(self, endpoint: Any) -> None: + """The refusal must leave the poll's not-modified path exactly as it was.""" + endpoint.queue_poll(status=304) + requester = _Requester(SDK_KEY, endpoint.base_uri, read_timeout=5.0) + result = requester.poll(None, "etag-1") + assert result.not_modified is True + assert result.etag == "etag-1" + + +def _per_object_hashless_errors(caplog: Any) -> list[Any]: + """The per-object ERROR, as distinct from the whole-payload summary.""" + return [ + r + for r in caplog.records + if r.levelname == "ERROR" + and "arrived without a contentHash" in r.getMessage() + and "No skill content will resolve" not in r.getMessage() + ] + + +class TestMissingContentHash: + """ + A skill delivered without a ``contentHash``, asserted as behaviour. + + An envelope with no ``contentHash`` must produce a *withheld* skill with the + ``missing_content_hash`` reason — loudly, diagnosably, and without a crash. + There is deliberately no fallback that skips verification: a hash the SDK + computed from the content it was handed would certify the content against + itself and verify nothing. + """ + + async def test_a_hashless_skill_is_withheld_with_the_right_reason( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill(omit_hash=True)))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + outcome = await get_skill_result("pdf-extraction") + assert outcome.skill is None + assert outcome.reason == "integrity_failure" + assert await get_skill("pdf-extraction") is None + assert await all_skills() == [] + + async def test_the_object_is_still_held_so_the_outcome_is_not_absent( + self, endpoint: Any + ) -> None: + """ + Holding it is what makes the failure diagnosable. Dropping it at the + transport would report ``absent`` — indistinguishable from "no such + skill" — and would additionally let a prune delete the last known-good + copy already on disk. + """ + endpoint.queue_poll(full_payload(("put-object", put_skill(omit_hash=True)))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + raw = store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") + assert raw is not None + assert "contentHash" not in raw + await init_client(options={"skillStore": store}, client=object()) + assert (await get_skill_result("pdf-extraction")).reason != "absent" + + def test_the_store_counts_hashless_objects(self, endpoint: Any) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill("a", omit_hash=True)), + ("put-object", put_skill("b", omit_hash=True)), + ("put-object", put_skill("c")), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert store.diagnostics.hashless_objects == 2 + assert store.diagnostics.skill_objects_received == 3 + + def test_a_hashless_object_logs_an_error_naming_the_reason_code( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill(omit_hash=True)))) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + rendered = "\n".join(r.getMessage() for r in caplog.records) + assert "missing_content_hash" in rendered + assert "pdf-extraction" in rendered + assert "contentHash" in rendered + + def test_a_redelivered_hashless_object_logs_once_per_store( + self, caplog: Any + ) -> None: + """Re-delivering the same ``(key, version)`` to one store must not + multiply the ERROR: a polling store sees every object on every poll.""" + reader = _ProtocolReader(_SkillObjectSet()) + payload = full_payload(("put-object", put_skill(omit_hash=True))) + with caplog.at_level("ERROR"): + drive(reader, payload) + drive(reader, payload) + assert len(_per_object_hashless_errors(caplog)) == 1 + + def test_a_recreated_store_reports_the_same_hashless_object_again( + self, caplog: Any + ) -> None: + """ + The dedupe belongs to the store, not the process. A host that rebuilds + its store (reconnect wrapper, config reload, credential rotation) must + get the ERROR again, since it is the loudest signal that a deployment is + broken rather than empty by design. + """ + payload = full_payload(("put-object", put_skill(omit_hash=True))) + with caplog.at_level("ERROR"): + drive(_ProtocolReader(_SkillObjectSet()), payload) + first = len(_per_object_hashless_errors(caplog)) + drive(_ProtocolReader(_SkillObjectSet()), payload) + assert first == 1 + assert len(_per_object_hashless_errors(caplog)) == 2 + + def test_two_live_stores_do_not_suppress_each_other(self, caplog: Any) -> None: + """Two stores in one process (say, two environments) each report.""" + one = _ProtocolReader(_SkillObjectSet()) + two = _ProtocolReader(_SkillObjectSet()) + payload = full_payload(("put-object", put_skill(omit_hash=True))) + with caplog.at_level("ERROR"): + drive(one, payload) + drive(two, payload) + # And each still dedupes its own re-deliveries. + drive(one, payload) + drive(two, payload) + assert len(_per_object_hashless_errors(caplog)) == 2 + + def test_a_wholly_hashless_payload_says_so_once( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill("a", omit_hash=True)), + ("put-object", put_skill("b", omit_hash=True)), + ) + ) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + summaries = [ + r + for r in caplog.records + if "No skill content will resolve" in r.getMessage() + ] + assert len(summaries) == 1 + assert "All 2 skill object(s)" in summaries[0].getMessage() + + def test_a_partly_hashed_payload_does_not_claim_total_failure( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill("a", omit_hash=True)), + ("put-object", put_skill("b")), + ) + ) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + rendered = "\n".join(r.getMessage() for r in caplog.records) + assert "No skill content will resolve" not in rendered + + async def test_a_hash_that_does_not_match_is_a_different_failure( + self, endpoint: Any + ) -> None: + """``missing_content_hash`` and ``hash_mismatch`` must not collapse: one + means the envelope carried no hash, the other means the content did not + match the hash it carried.""" + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(content_hash=_hash("something else"))) + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert store.diagnostics.hashless_objects == 0 + await init_client(options={"skillStore": store}, client=object()) + assert ( + await get_skill_result("pdf-extraction") + ).reason == "integrity_failure" + + async def test_a_hashed_skill_resolves_end_to_end(self, endpoint: Any) -> None: + """The positive control: a well-formed envelope resolves end to end.""" + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + skill = await get_skill("pdf-extraction") + assert skill is not None + assert skill.key == "pdf-extraction" + assert skill.version == 3 + assert skill.content == SKILL_BODY.encode("utf-8") + assert skill.content_hash == _hash(SKILL_BODY) + assert skill.name == "PDF Extraction" + + async def test_a_pinned_reference_resolves_to_the_pinned_object_version( + self, endpoint: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(object_version=2, content="v2 body")), + ("put-object", put_skill(object_version=5, content="v5 body")), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + pinned = await get_skill("pdf-extraction", version=2) + assert pinned is not None + assert pinned.content == b"v2 body" + newest = await get_skill("pdf-extraction") + assert newest is not None + assert newest.version == 5 + + async def test_a_missed_pin_is_absent_even_beside_a_malformed_sibling( + self, endpoint: Any + ) -> None: + """ + A key can hold a well-formed version and a version-less entry at once — + a malformed object arrives with no version in its wire key, and is held + anyway so verification withholds it with a signal. + + A pin that misses is still a plain miss. Answering it with the + version-less entry would report ``integrity_failure`` for a skill whose + integrity is not in question, and that is the one reason callers are + told to fail closed on. + """ + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(object_version=3)), + ("put-object", put_skill(object_version=None)), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + missed = await get_skill_result("pdf-extraction", version=9) + assert missed.skill is None + assert missed.reason == "absent" + # The well-formed version still resolves, and the malformed sibling + # is still reachable to be withheld when nothing else answers. + assert await get_skill("pdf-extraction", version=3) is not None + + async def test_the_payload_version_is_not_resolvable_as_a_skill_version( + self, endpoint: Any + ) -> None: + """ + The end-to-end form of the wire-key/``version`` assertion. + + Asking for the payload version resolves nothing — reported ``absent``, + because the store answers "I hold no such version" rather than answering + with the wrong one. The version that *does* resolve is the one after the + delimiter in the object's wire ``key``. + """ + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(object_version=3, payload_version=42)) + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + by_payload_version = await get_skill_result("pdf-extraction", version=42) + assert by_payload_version.skill is None + assert by_payload_version.reason == "absent" + assert await get_skill("pdf-extraction", version=3) is not None + + +# --------------------------------------------------------------------------- +# Server-side only +# --------------------------------------------------------------------------- + + +class TestServerSideOnly: + def test_a_mobile_key_is_refused(self) -> None: + with pytest.raises(ValueError, match="mobile key"): + FDv2SkillStore("mob-00000000-0000-4000-8000-000000000000") + + def test_a_client_side_environment_id_is_refused(self) -> None: + with pytest.raises(ValueError, match="client-side"): + FDv2SkillStore("0123456789abcdef01234567") + + def test_an_empty_credential_is_refused(self) -> None: + with pytest.raises(ValueError, match="server-side SDK key"): + FDv2SkillStore(" ") + + def test_a_server_side_key_is_accepted(self) -> None: + assert FDv2SkillStore(SDK_KEY) is not None + + def test_an_unrecognised_credential_shape_warns_but_is_allowed( + self, caplog: Any + ) -> None: + """Private instances and test doubles issue keys without the public prefix.""" + with caplog.at_level("WARNING"): + FDv2SkillStore("my-private-instance-credential") + assert any("server-side SDK key" in r.message for r in caplog.records) + + def test_an_unknown_mode_is_refused(self) -> None: + with pytest.raises(ValueError, match="stream"): + FDv2SkillStore(SDK_KEY, mode="mobile") # type: ignore[arg-type] + + +class TestBaseUriScheme: + """ + Every request carries the SDK key in ``Authorization``, so the base URI is + ``https://`` only. Plain ``http://`` is allowed to a loopback host and + nowhere else: that is what this suite's own endpoints listen on, and it + never leaves the machine. + """ + + def test_a_plain_http_base_uri_is_refused(self) -> None: + with pytest.raises(ValueError, match="cleartext") as excinfo: + FDv2SkillStore(SDK_KEY, base_uri="http://sdk.launchdarkly.com") + assert "https://" in str(excinfo.value) + + def test_a_plain_http_base_uri_is_refused_even_with_a_requester_injected( + self, + ) -> None: + """The check is on the store, not on the socket it happens to open.""" + with pytest.raises(ValueError, match="cleartext"): + FDv2SkillStore( + SDK_KEY, + base_uri="http://relay.internal:8030", + _requester=_FakeRequester(), + ) + + @pytest.mark.parametrize( + "base_uri", + [ + "http://localhost:8030", + "http://127.0.0.1:8030", + "http://[::1]:8030", + "http://LOCALHOST/", + ], + ) + def test_plain_http_to_a_loopback_host_is_allowed(self, base_uri: str) -> None: + assert FDv2SkillStore(SDK_KEY, base_uri=base_uri) is not None + + def test_a_private_address_is_not_loopback(self) -> None: + """Only the machine itself is exempt; the LAN is not.""" + with pytest.raises(ValueError, match="cleartext"): + FDv2SkillStore(SDK_KEY, base_uri="http://10.0.0.5:8030") + + @pytest.mark.parametrize( + "base_uri", + ["", " ", "sdk.launchdarkly.com", "ftp://sdk.launchdarkly.com", "https://"], + ) + def test_anything_but_an_https_url_with_a_host_is_refused( + self, base_uri: str + ) -> None: + with pytest.raises(ValueError, match="https://"): + FDv2SkillStore(SDK_KEY, base_uri=base_uri) + + def test_https_is_accepted(self) -> None: + assert FDv2SkillStore(SDK_KEY, base_uri="https://sdk.example.com/") is not None + assert FDv2SkillStore(SDK_KEY) is not None + + +# --------------------------------------------------------------------------- +# The eager re-reconcile, end to end over the transport +# --------------------------------------------------------------------------- + + +class TestWatchSkillsOverTheTransport: + """ + ``watch_skills`` against a live ``FDv2SkillStore``. The watcher's own + behaviour — debounce, refusal of a store without ``add_listener``, detaching + on close — is covered in ``test_skills_watch.py`` against the in-memory + store; these are the cases that only mean something with a transport + underneath: a wire-level revocation, a new skill version, and an outage. + """ + + async def test_a_revocation_prunes_without_a_restart( + self, endpoint: Any, tmp_path: Any + ) -> None: + """ + The store's change listener drives the reconcile, so the file goes away + seconds after the ``delete-object`` rather than at the next process start. + """ + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + + with poll_store(endpoint, poll_interval=0.2) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + report, watcher = await watch_skills( + "*", tmp_path / "skills", debounce=0.05 + ) + try: + written = tmp_path / "skills" / "pdf-extraction" / "SKILL.md" + assert written.exists() + assert any(a.action == "written" for a in report.actions) + assert wait_until(lambda: not written.exists(), timeout=10) + finally: + watcher.close() + + async def test_a_new_version_is_rewritten_without_a_restart( + self, endpoint: Any, tmp_path: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill(content="first")))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill(object_version=4, content="second")), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + + with poll_store(endpoint, poll_interval=0.2) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + assert written.read_text() == "first" + assert wait_until(lambda: written.read_text() == "second", timeout=10) + finally: + watcher.close() + + async def test_the_default_keeps_last_known_good_during_an_outage( + self, endpoint: Any, tmp_path: Any + ) -> None: + """``on_unavailable="keep"`` is the default: an outage must not read as + "everything was revoked".""" + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll(status=500) + with poll_store(endpoint, poll_interval=0.05) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + assert written.exists() + # ``last_error`` rather than ``connection_failures``: the counter + # resets on the next successful poll, so asserting on it races + # the retry that is supposed to happen. + assert wait_until( + lambda: store.diagnostics.last_error is not None, timeout=10 + ) + time.sleep(0.3) + assert written.exists() + finally: + watcher.close() + + +# --------------------------------------------------------------------------- +# Listener registration +# --------------------------------------------------------------------------- + + +class TestListenerRegistration: + @staticmethod + def _skill_listeners(store: Any) -> list[Any]: + return list(store._listeners.get(SKILL_OBJECT_KIND, [])) + + def test_fdv2_remove_listener_of_an_unregistered_callable_is_a_no_op( + self, endpoint: Any + ) -> None: + with poll_store(endpoint) as store: + store.remove_listener(SKILL_OBJECT_KIND, print) + store.add_listener(SKILL_OBJECT_KIND, print) + store.remove_listener("flag", print) + store.remove_listener(SKILL_OBJECT_KIND, print) + store.remove_listener(SKILL_OBJECT_KIND, print) + assert self._skill_listeners(store) == [] + + +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + + +class TestLifecycle: + def test_start_is_idempotent(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store = poll_store(endpoint) + try: + assert store.start() is store + assert store.start() is store + assert store.wait_for_skills(timeout=5) + finally: + store.close() + + def test_close_is_idempotent(self, endpoint: Any) -> None: + store = poll_store(endpoint) + store.start() + store.close() + store.close() + + def test_close_during_a_slow_connect_returns_promptly(self) -> None: + # Before the connect returns there is no connection for close() to + # interrupt. If the delivery thread then enters the read anyway, close() + # sits out its whole join timeout on a stream that will never speak. + requester = _SlowConnectRequester() + store = stream_store(_requester=requester) + store.start() + assert requester.entered.wait(timeout=5) + threading.Timer(0.1, requester.release.set).start() + started = time.monotonic() + store.close(timeout=5.0) + elapsed = time.monotonic() - started + assert elapsed < 2.0 + assert store._thread is not None + assert not store._thread.is_alive() + + def test_a_closed_store_still_answers_from_what_it_received( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store = poll_store(endpoint) + store.start() + store.wait_for_skills(timeout=5) + store.close() + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + + def test_wait_for_skills_times_out_rather_than_hanging(self) -> None: + store = FDv2SkillStore( + SDK_KEY, mode="poll", poll_interval=60, _requester=_ScriptedRequester() + ) + try: + assert store.wait_for_skills(timeout=0.05) is False + finally: + store.close() + + def test_the_store_satisfies_the_seam_before_it_starts(self) -> None: + store = FDv2SkillStore(SDK_KEY) + assert store.get_object(SKILL_OBJECT_KIND, "anything") is None + assert store.all_objects(SKILL_OBJECT_KIND) == {} + + +# --------------------------------------------------------------------------- +# Timeouts +# --------------------------------------------------------------------------- + + +class _BlackHole: + """ + A listening socket that accepts connections and never sends a byte. + + This is the host ``read_timeout`` exists for: the TCP handshake completes, so + nothing fails fast, and then no response ever comes. A request against it can + only end by timing out, which makes the elapsed time a direct measurement of + the timeout actually applied. + """ + + def __init__(self) -> None: + self._listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._listener.bind(("127.0.0.1", 0)) + self._listener.listen(8) + self._accepted: list[socket.socket] = [] + self._stop = threading.Event() + self._thread = threading.Thread(target=self._accept_forever, daemon=True) + self._thread.start() + host, port = self._listener.getsockname() + self.base_uri = f"http://{host}:{port}" + + def _accept_forever(self) -> None: + self._listener.settimeout(0.05) + while not self._stop.is_set(): + try: + conn, _ = self._listener.accept() + except OSError: + continue + self._accepted.append(conn) + + def close(self) -> None: + self._stop.set() + self._thread.join(timeout=2) + for conn in self._accepted: + conn.close() + self._listener.close() + + +class _StalledBody: + """ + A listening socket that answers with headers and then stalls the body. + + Distinct from ``_BlackHole``: here the request succeeds far enough to hand + urllib a response, and the caller then parks in ``read``. That is the state + ``close`` has to interrupt — and, unlike a request still inside its connect, + the state an interrupt can actually reach. + """ + + def __init__(self) -> None: + self._listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._listener.bind(("127.0.0.1", 0)) + self._listener.listen(8) + self._accepted: list[socket.socket] = [] + self.serving = threading.Event() + self._stop = threading.Event() + self._thread = threading.Thread(target=self._serve_forever, daemon=True) + self._thread.start() + host, port = self._listener.getsockname() + self.base_uri = f"http://{host}:{port}" + + def _serve_forever(self) -> None: + self._listener.settimeout(0.05) + while not self._stop.is_set(): + try: + conn, _ = self._listener.accept() + except OSError: + continue + self._accepted.append(conn) + try: + conn.recv(4096) + # A length far longer than the body that follows, so the read + # blocks rather than seeing the end of the message. + conn.sendall( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" + b"Content-Length: 4096\r\n\r\n" + ) + except OSError: + continue + self.serving.set() + + def close(self) -> None: + self._stop.set() + self._thread.join(timeout=2) + for conn in self._accepted: + conn.close() + self._listener.close() + + +@pytest.fixture +def stalled_body() -> Any: + server = _StalledBody() + yield server + server.close() + + +@pytest.fixture +def black_hole() -> Any: + server = _BlackHole() + yield server + server.close() + + +class TestTimeouts: + """ + ``read_timeout`` is the only network timeout, and every request honours it. + + The bounds asserted here are loose on purpose: the point is that a request + against an unresponsive host fails in roughly ``read_timeout`` rather than in + minutes, and that a regression back to a much longer default fails this + suite quickly instead of hanging it. + """ + + def test_a_poll_against_an_unresponsive_host_fails_within_read_timeout( + self, black_hole: Any + ) -> None: + requester = _Requester(SDK_KEY, black_hole.base_uri, read_timeout=0.3) + started = time.monotonic() + with pytest.raises(_RecoverableTransportError) as excinfo: + requester.poll(None, None) + elapsed = time.monotonic() - started + assert 0.2 <= elapsed < 2.0 + assert "timed out" in str(excinfo.value) + + def test_a_stream_against_an_unresponsive_host_fails_within_read_timeout( + self, black_hole: Any + ) -> None: + requester = _Requester(SDK_KEY, black_hole.base_uri, read_timeout=0.3) + started = time.monotonic() + with pytest.raises(_RecoverableTransportError): + requester.stream(None) + assert time.monotonic() - started < 2.0 + + def test_the_store_reports_the_timeout_and_keeps_going( + self, black_hole: Any + ) -> None: + store = FDv2SkillStore( + SDK_KEY, + base_uri=black_hole.base_uri, + mode="poll", + poll_interval=0.05, + initial_backoff=0.01, + max_backoff=0.05, + read_timeout=0.3, + ) + try: + store.start() + assert wait_until(lambda: store.diagnostics.connection_failures >= 1) + assert store.failed is None + assert "timed out" in (store.diagnostics.last_error or "") + finally: + store.close() + + def test_the_default_bound_depends_on_the_mode(self) -> None: + assert DEFAULT_POLL_TIMEOUT == 10.0 + assert DEFAULT_STREAM_READ_TIMEOUT == 300.0 + polling = FDv2SkillStore(SDK_KEY, mode="poll") + streaming = FDv2SkillStore(SDK_KEY, mode="stream") + assert polling._requester._read_timeout == DEFAULT_POLL_TIMEOUT + assert streaming._requester._read_timeout == DEFAULT_STREAM_READ_TIMEOUT + + @pytest.mark.parametrize("mode", ["poll", "stream"]) + def test_an_explicit_read_timeout_overrides_the_default(self, mode: Any) -> None: + store = FDv2SkillStore(SDK_KEY, mode=mode, read_timeout=42.0) + assert store._requester._read_timeout == 42.0 + + @pytest.mark.parametrize("value", [0.0, -1.0, float("inf"), float("nan")]) + def test_a_non_positive_read_timeout_is_rejected(self, value: float) -> None: + with pytest.raises(ValueError, match="read_timeout"): + FDv2SkillStore(SDK_KEY, read_timeout=value) + + def test_there_is_no_separate_connect_timeout(self) -> None: + # ``urllib`` cannot bound the connect separately from the reads, so the + # constructor does not offer a parameter that would only pretend to. + with pytest.raises(TypeError): + FDv2SkillStore(SDK_KEY, connect_timeout=2.0) # type: ignore[call-arg] + + +class TestWaitingForSkills: + """ + ``wait_for_skills`` answers with what happened, and never outlives it. + + Its budget is a boot-ordering allowance, not a delay to spend: a store that + already knows no payload is coming owes the caller that answer immediately. + """ + + def test_close_releases_a_waiter_rather_than_leaving_it_parked(self) -> None: + # A shutdown racing a waiter is the ordinary case, not an exotic one: + # ``close`` on the main thread while a worker is still waiting for its + # first payload. Parking that worker for the rest of its timeout adds + # the whole budget to a process that has already decided to stop. + store = stream_store(_requester=_SilentStreamRequester()) + store.start() + answers: list[bool] = [] + waiter = threading.Thread( + target=lambda: answers.append(store.wait_for_skills(timeout=10)), + daemon=True, + ) + waiter.start() + time.sleep(0.2) + started = time.monotonic() + store.close() + waiter.join(timeout=5) + assert not waiter.is_alive() + assert time.monotonic() - started < 2.0 + assert answers == [False] + + def test_is_initialized_tracks_the_first_payload(self) -> None: + """The probe ``write_skills("*")`` reads to decide whether it may prune. + + Before the first payload, an empty store and an environment with no + skills are the same answer through ``all_objects``; this is what tells + them apart. + """ + store = stream_store(_requester=_SilentStreamRequester()) + try: + assert store.is_initialized() is False + store.start() + assert store.wait_for_skills(timeout=0.2) is False + assert store.is_initialized() is False + finally: + store.close() + + delivering = stream_store(_requester=_RecyclingRequester()) + try: + delivering.start() + assert delivering.wait_for_skills(timeout=5) is True + assert delivering.is_initialized() is True + finally: + delivering.close() + # Content outlives the connection, so the fact about it does too. + assert delivering.is_initialized() is True + + def test_a_payload_already_held_still_answers_true_after_close(self) -> None: + # ``close`` does not drop content, so it must not turn the answer about + # that content into a lie either. + requester = _RecyclingRequester() + store = stream_store(_requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + finally: + store.close() + assert store.wait_for_skills(timeout=5) is True + + def test_a_restarted_store_waits_again(self) -> None: + # The released flag is sticky by design, so a store closed before any + # payload and then started again has to re-arm: otherwise the next + # waiter is let go before delivery has had a chance to begin. + store = stream_store(_requester=_SilentStreamRequester()) + store.start() + store.close() + assert store.wait_for_skills(timeout=0.1) is False + store.start() + try: + started = time.monotonic() + assert store.wait_for_skills(timeout=0.5) is False + # Waited, rather than being released by the previous close. + assert time.monotonic() - started >= 0.4 + finally: + store.close() + + +class TestPollShutdown: + """ + ``close`` has to interrupt a poll in flight, as it already does a stream. + + Without it the delivery thread stays parked in its request and ``close`` + returns only when the join times out — on a 300s-class request, long after + the process meant to exit. The bound is loose on purpose: the point is + promptly rather than a particular number of milliseconds. + """ + + def test_interrupt_unblocks_a_poll_stalled_in_its_body( + self, stalled_body: Any + ) -> None: + requester = _Requester(SDK_KEY, stalled_body.base_uri, read_timeout=30.0) + raised: list[BaseException] = [] + + def poll_until_interrupted() -> None: + try: + requester.poll(None, None) + except BaseException as exc: + raised.append(exc) + + thread = threading.Thread(target=poll_until_interrupted, daemon=True) + thread.start() + assert stalled_body.serving.wait(timeout=5) + # The response is in hand; give the read a moment to park in it. + time.sleep(0.2) + started = time.monotonic() + requester.interrupt() + thread.join(timeout=5) + assert not thread.is_alive() + assert time.monotonic() - started < 2.0 + assert raised and isinstance(raised[0], _RecoverableTransportError) + + def test_close_during_a_stalled_poll_returns_promptly( + self, stalled_body: Any + ) -> None: + store = FDv2SkillStore( + SDK_KEY, + base_uri=stalled_body.base_uri, + mode="poll", + poll_interval=0.05, + read_timeout=30.0, + ) + store.start() + assert stalled_body.serving.wait(timeout=5) + time.sleep(0.2) + started = time.monotonic() + store.close(timeout=5.0) + assert time.monotonic() - started < 2.0 + assert store._thread is not None and not store._thread.is_alive() + + def test_a_poll_we_interrupted_is_not_a_delivery_failure( + self, stalled_body: Any + ) -> None: + # Our own shutdown is not an outage: counting it would spend a retry + # from the bounded budget and leave a misleading ``last_error`` behind + # on a store whose content is still perfectly good. + store = FDv2SkillStore( + SDK_KEY, + base_uri=stalled_body.base_uri, + mode="poll", + poll_interval=0.05, + read_timeout=30.0, + ) + store.start() + assert stalled_body.serving.wait(timeout=5) + time.sleep(0.2) + store.close(timeout=5.0) + assert store.diagnostics.connection_failures == 0 + assert store.diagnostics.last_error is None + assert store.failed is None + + def test_a_close_that_timed_out_leaves_the_store_restartable(self) -> None: + # A request blocked inside its connect is beyond any interrupt, so + # ``close`` can still return with the thread alive. ``start`` must not + # then find that thread and return with the stop flag set: the store + # would report itself started and never deliver again. + requester = _SlowPollRequester() + store = FDv2SkillStore( + SDK_KEY, mode="poll", poll_interval=0.01, _requester=requester + ) + try: + store.start() + assert requester.entered.wait(timeout=5) + store.close(timeout=0.2) + assert store._thread is not None and store._thread.is_alive() + store.start() + assert store._stop.is_set() is False + finally: + requester.release.set() + store.close(timeout=2) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py new file mode 100644 index 00000000..cd7e7105 --- /dev/null +++ b/packages/client/tests/test_skills_fs.py @@ -0,0 +1,3246 @@ +""" +Tests for ``write_skills`` — filesystem materialization, manifest reconcile +semantics, and the full security abuse matrix. + +Every test writes only inside pytest's ``tmp_path``. No network, no real +LaunchDarkly client, no real skill transport. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +from pathlib import Path +from typing import Any, NamedTuple + +import pytest + +import launchdarkly_ai_server.safe_fs as safe_fs_module +import launchdarkly_ai_server.skills as skills_module +import launchdarkly_ai_server.skills_core as skills_core_module +import launchdarkly_ai_server.skills_fs as skills_fs_module +from launchdarkly_ai_server import ( + InMemorySkillStore, + Skill, + SkillReference, + get_skill, + init_client, + parse_ai_config, + skill_refs, + write_skills, +) +from launchdarkly_ai_server.types_validation import is_valid_skill_key + +MANIFEST_NAME = ".launchdarkly-skills.json" +SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" + + +MATERIALIZED_SIGNAL = "AgentControl Skill Materialized" +REVOKED_SIGNAL = "AgentControl Skill Revoked Received" +INTEGRITY_SIGNAL = "AgentControl Skill Integrity Failure" + +# The three signal names are an allowlist, not a floor. +APPROVED_SIGNALS = frozenset({MATERIALIZED_SIGNAL, REVOKED_SIGNAL, INTEGRITY_SIGNAL}) + +# Considered and deliberately excluded from SDK emission — named explicitly +# so the regression is unmissable. +REMOVED_SIGNALS = frozenset( + { + "AgentControl Skill SDK Reference Returned", + "AgentControl Skill Content Retrieved", + } +) + + +pytestmark = pytest.mark.usefixtures("reset_skill_state") + + +_INJECTED = "simulated crash between write and rename" + + +def _dir_id(path: Path) -> tuple[int, int]: + """``(st_dev, st_ino)`` — a directory's identity, independent of its name.""" + info = os.stat(path) + return (info.st_dev, info.st_ino) + + +class _RenameCall(NamedTuple): + """One intercepted ``os.replace`` of a ``SKILL.md``. + + ``src``/``dst`` are exactly what the implementation passed. Where the rename + is ``dir_fd``-relative they are bare filenames and the location lives in the + descriptors, so ``*_dir_id`` carries each descriptor's ``(st_dev, st_ino)`` + resolved *at call time* — the implementation closes the descriptors as soon + as the write returns, so they cannot be resolved from the assertions. + """ + + src: str + dst: str + src_dir_fd: int | None + dst_dir_fd: int | None + src_dir_id: tuple[int, int] | None + dst_dir_id: tuple[int, int] | None + + +class _ReplaceSpy: + """Records — and optionally fails — every atomic rename of a ``SKILL.md``. + + Write/rename interception hook: the implementation performs + the final rename through a single ``os.replace`` call site, so patching the + attribute on the ``os`` module observes it. Destinations other than + ``SKILL.md`` (i.e. the manifest's own atomic write) pass straight through — + the filter holds for both call shapes, since the ``dir_fd``-relative form + passes ``"SKILL.md"`` itself as ``dst``. + + Used two ways: to prove an injected failure is what produced an ``error`` + action (atomicity), and to prove no write was *attempted* for a + rejected key — the OS would reject several hostile keys on its + own, so a failed write is not evidence of a defense. + """ + + def __init__(self, fail: bool = False) -> None: + self.calls: list[_RenameCall] = [] + self._fail = fail + self._real = os.replace + + def __call__(self, src: Any, dst: Any, **kwargs: Any) -> None: + if str(dst).endswith("SKILL.md"): + src_dir_fd = kwargs.get("src_dir_fd") + dst_dir_fd = kwargs.get("dst_dir_fd") + self.calls.append( + _RenameCall( + src=str(src), + dst=str(dst), + src_dir_fd=src_dir_fd, + dst_dir_fd=dst_dir_fd, + src_dir_id=None if src_dir_fd is None else _fd_id(src_dir_fd), + dst_dir_id=None if dst_dir_fd is None else _fd_id(dst_dir_fd), + ) + ) + if self._fail: + raise OSError(_INJECTED) + self._real(src, dst, **kwargs) + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _ReplaceSpy: + # The attribute is set on the shared ``os`` module, so the + # single ``os.replace`` call site in safe_fs is intercepted wherever it is + # reached from. Named through the calling module rather than an arbitrary + # one so the hook documents which code it covers. + monkeypatch.setattr(safe_fs_module.os, "replace", self) + return self + + +def _fd_id(fd: int) -> tuple[int, int]: + info = os.fstat(fd) + return (info.st_dev, info.st_ino) + + +def _assert_atomic_rename_of(spy: _ReplaceSpy, skill_dir: Path) -> None: + """Assert the one recorded rename put ``SKILL.md`` into *skill_dir*. + + The temp file must be created in the target's own + directory, so the rename is atomic rather than cross-device. Two call + shapes prove it. Where the platform has ``renameat`` + the rename is ``dir_fd``-relative and the property is asserted by descriptor + identity — one descriptor for both sides, resolving to *skill_dir*'s inode — + which is stronger than comparing path strings, because it also rules out the + descriptor having been redirected between the check and the rename. On the + ``lstat`` floor (Windows) the names are full paths and share a parent. + """ + assert len(spy.calls) == 1 + call = spy.calls[0] + + if safe_fs_module.SUPPORTS_DIR_FD: + assert call.dst == "SKILL.md" + assert call.src != "SKILL.md" + assert call.src_dir_fd is not None + assert call.src_dir_fd == call.dst_dir_fd + assert call.dst_dir_id == _dir_id(skill_dir) + else: + assert Path(call.dst) == skill_dir / "SKILL.md" + assert Path(call.src).parent == skill_dir + assert Path(call.src).name != "SKILL.md" + + +class _SwapDirectoryDuring: + """Fires the directory-swap race at the exact instant of an operation. + + Renames ``/`` aside and leaves a symlink to *outside* in its + place, then lets the intercepted call proceed — the narrowest possible + version of the window an attacker with write access to the managed root + would otherwise have to hit by timing. Both hooks are the + interception points (``os.replace`` for the write, ``os.unlink`` for the + prune), so no implementation internals are touched. + """ + + def __init__(self, attribute: str, skill_dir: Path, outside: Path) -> None: + self.attribute = attribute + self.skill_dir = skill_dir + self.moved_to = skill_dir.parent / f"{skill_dir.name}.real" + self.outside = outside + self.swapped = False + self._real = getattr(os, attribute) + + def __call__(self, first: Any, *args: Any, **kwargs: Any) -> Any: + named = args[0] if args else first + if str(named).endswith("SKILL.md") and not self.swapped: + os.rename(self.skill_dir, self.moved_to) + os.symlink(self.outside, self.skill_dir, target_is_directory=True) + self.swapped = True + return self._real(first, *args, **kwargs) + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _SwapDirectoryDuring: + # ``os.replace`` is called from safe_fs, ``os.unlink`` from skills_fs; both + # resolve to the same module object, so either name reaches both. + module = safe_fs_module if self.attribute == "replace" else skills_fs_module + monkeypatch.setattr(module.os, self.attribute, self) + return self + + +class _SwapRootDuring: + """Fires the *root*-swap race at the exact instant of an operation. + + ``_SwapDirectoryDuring`` one level up: renames the managed root itself aside + and leaves a symlink to *outside* in its place, then lets the intercepted + call proceed. ``O_NOFOLLOW`` guards only the final path component, so an + ``os.mkdir`` or ``os.open`` of ``/`` issued after the swap is + resolved by the kernel *through* the link: the descriptor that comes back is + pinned to ``/``, and every descriptor-relative step that + follows is relative to the wrong directory. + + The attacker capability is write permission on the root's *parent*, not on + the root. The README's privilege-separation checklist denies the agent + identity the root, the skill directories, the files and the manifest, and + says nothing about the parent; in the documented layout + (``/.claude/skills``) that parent is ``.claude``, which the agent + identity typically owns. + + Fires once, on the first call naming the skill directory — as the absolute + ``/`` or as the bare ```` — and every other call passes + straight through. + + Matching *both* spellings is what keeps this honest across the fix. Code + that opens ``/`` by path names it absolutely; code that opens it + relative to a held root descriptor passes the bare key. Triggering only on + the absolute form would mean that the moment the root is pinned the trigger + stops matching, the swap never fires, and all three tests below pass while + asserting nothing — the ``race.swapped is True`` guard would be the only + thing standing between a real fix and a vacuous one, and it would be + load-bearing for the wrong reason. Firing in both worlds is what makes these + tests fail before the fix and pass after it. + """ + + def __init__(self, attribute: str, root: Path, key: str, outside: Path) -> None: + self.attribute = attribute + self.root = Path(os.path.realpath(root)) + self.key = key + self.trigger = self.root / key + self.moved_to = self.root.parent / f"{self.root.name}.real" + self.outside = outside + self.swapped = False + self._real = getattr(os, attribute) + + def __call__(self, first: Any, *args: Any, **kwargs: Any) -> Any: + # ``os.mkdir(path, mode)`` and ``os.open(path, flags, ...)`` both take + # the path first, whether that path is absolute or a bare name resolved + # against a ``dir_fd``. + if not self.swapped and isinstance(first, (str, os.PathLike)): + named = os.fspath(first) + if named == self.key or Path(named) == self.trigger: + os.rename(self.root, self.moved_to) + os.symlink(self.outside, self.root, target_is_directory=True) + self.swapped = True + return self._real(first, *args, **kwargs) + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _SwapRootDuring: + monkeypatch.setattr(safe_fs_module.os, self.attribute, self) + return self + + +class _SwapRootBefore: + """Fires the *root*-swap race the instant before one decision read. + + ``_SwapRootDuring`` intercepts a destructive call. This intercepts a read + that decides whether a destructive call happens at all — the existence + probe before a prune, the compare read before a write, the listing before + the orphan sweep — and swaps the root the moment before it runs, once. Those + reads all come *after* the skill directory is pinned, so a swap here is the + latest possible one: every path check has already passed against the real + root, and the only thing left that could go wrong is the read itself + resolving through the link. + + *trigger* decides which call fires it, and must match both spellings — the + absolute path a path-based read passes and the bare name (or descriptor) a + pinned one passes — for the reason ``_SwapRootDuring`` gives: a trigger + that matched only the path would stop firing the moment the read was + pinned, and these tests would pass while asserting nothing. + """ + + def __init__( + self, + attribute: str, + root: Path, + outside: Path, + trigger: Any, + ) -> None: + self.attribute = attribute + self.root = Path(os.path.realpath(root)) + self.moved_to = self.root.parent / f"{self.root.name}.real" + self.outside = outside + self.trigger = trigger + self.swapped = False + self._real = getattr(os, attribute) + + def __call__(self, first: Any, *args: Any, **kwargs: Any) -> Any: + if not self.swapped and self.trigger(first, kwargs): + os.rename(self.root, self.moved_to) + os.symlink(self.outside, self.root, target_is_directory=True) + self.swapped = True + return self._real(first, *args, **kwargs) + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _SwapRootBefore: + monkeypatch.setattr(skills_fs_module.os, self.attribute, self) + return self + + +def _names_the_skill_file(skill_dir: Path) -> Any: + """A ``_SwapRootBefore`` trigger for a ``stat`` or ``open`` of ``SKILL.md``. + + The bare name is only accepted with a ``dir_fd``: a bare name without one + would resolve against the working directory, which is not a call this SDK + makes, and matching it would fire the swap on some unrelated read. + + The absolute form is only accepted when it follows symlinks. The path + check's ``is_symlink`` also stats the absolute ``SKILL.md``, as an + ``lstat``, and that runs *before* the pin as defense in depth — firing on + it would swap the root under the path checks themselves, which refuse, and + the decision read under test would never be reached. + """ + + def trigger(first: Any, kwargs: dict[str, Any]) -> bool: + if not isinstance(first, (str, os.PathLike)): + return False + named = os.fspath(first) + if named == "SKILL.md": + return kwargs.get("dir_fd") is not None + return ( + Path(named) == skill_dir / "SKILL.md" + and kwargs.get("follow_symlinks", True) is not False + ) + + return trigger + + +def _lists_the_skill_directory(skill_dir: Path) -> Any: + """A ``_SwapRootBefore`` trigger for the orphan sweep's ``listdir``. + + A pinned listing passes the descriptor itself; a path-based one passes + ``/``. + """ + + def trigger(first: Any, kwargs: dict[str, Any]) -> bool: + if isinstance(first, int): + return True + return isinstance(first, (str, os.PathLike)) and Path(first) == skill_dir + + return trigger + + +_needs_dir_fd = pytest.mark.skipif( + not safe_fs_module.SUPPORTS_DIR_FD, + reason="no *at() family on this platform; the per-component lstat floor applies", +) + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + r = tmp_path / "skills" + r.mkdir() + return r + + +pytestmark = pytest.mark.usefixtures("reset_skill_state") +"""Every test in this module runs against freshly cleared module state.""" + + +def _hash(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +def _skill( + key: str = "test-skill", version: int = 1, content: str = SKILL_BODY +) -> Skill: + return Skill( + key=key, + version=version, + content=content.encode("utf-8"), + content_hash=_hash(content), + ) + + +@pytest.fixture(scope="session") +def oversize_content() -> tuple[str, str]: + """Content one byte past the cap, with its hash — ``(content, sha256)``. + + Derived from ``MAX_SKILL_CONTENT_BYTES`` rather than written as a literal, + because a literal is how this went wrong: both consumers were pinned at the + old 64 KiB bound and stayed there when the cap moved to 10 MiB, which put + them *under* the limit and left them asserting that oversize content is + written and reported clean. Derived, they are one byte past whatever the + bound currently is, and moving it again cannot silently invert them. + + The bound's *value* is asserted once, in ``TestPackageExports``, which + spells the literal out on purpose — reading it from the module there would + make that assertion circular. These tests are about enforcement rather than + the number, so reading it here is the non-circular direction. + + Session-scoped: the string is cap-sized, so it is built and hashed once for + the run instead of per test. ``x`` encodes to one byte, so the character + count is the byte count. + """ + content = "x" * (skills_core_module.MAX_SKILL_CONTENT_BYTES + 1) + return content, _hash(content) + + +def _manifest_path(root: Path) -> Path: + return root / MANIFEST_NAME + + +def _read_manifest(root: Path) -> dict[str, Any]: + return json.loads(_manifest_path(root).read_text(encoding="utf-8")) + + +def _write_manifest(root: Path, raw: Any) -> None: + root.mkdir(parents=True, exist_ok=True) + _manifest_path(root).write_text( + raw if isinstance(raw, str) else json.dumps(raw), encoding="utf-8" + ) + + +def _entry(key: str, version: int, content: str) -> dict[str, Any]: + return { + "key": key, + "version": version, + "sha256": _hash(content), + "writtenAt": "2026-08-14T19:00:00Z", + } + + +def _place_managed(root: Path, key: str, content: str, version: int = 1) -> Path: + """Pre-create a file AND its manifest entry — i.e. an SDK-managed path.""" + target = root / key / "SKILL.md" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {f"{key}/SKILL.md": _entry(key, version, content)}, + }, + ) + return target + + +def _actions_by_key(report: Any) -> dict[str, Any]: + return {a.key: a for a in report.actions} + + +def _error_messages(report: Any) -> list[str]: + """All ``error`` action messages, regardless of which key they hang off. + + Run-level (manifest) errors have no well-defined ``key`` yet, so assertions + about them scan every error action rather than looking one up by key. + """ + return [a.error or "" for a in report.actions if a.action == "error"] + + +class TestBasicWrites: + """Basic writes and the returned report.""" + + async def test_new_skill_is_written_verbatim(self, root: Path) -> None: + report = await write_skills([_skill("pdf-extraction", 2)], root) + + target = root / "pdf-extraction" / "SKILL.md" + assert target.read_text(encoding="utf-8") == SKILL_BODY + assert report.ok is True + action = _actions_by_key(report)["pdf-extraction"] + assert action.action == "written" + assert action.version == 2 + assert action.path is not None + assert Path(action.path).resolve() == target.resolve() + assert action.error is None + + async def test_skill_inputs_need_no_store(self, root: Path) -> None: + report = await write_skills([_skill("a")], root) + assert report.ok is True + assert (root / "a" / "SKILL.md").exists() + + async def test_reference_inputs_resolve_through_store( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=3)) + report = await write_skills([SkillReference(key="a", version=3)], root) + assert report.ok is True + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_string_inputs_resolve_latest( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=9)) + report = await write_skills(["a"], root) + assert _actions_by_key(report)["a"].version == 9 + + async def test_star_writes_everything_in_the_store( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + for k in ("a", "b", "c"): + store.put(make_raw_skill(key=k)) + report = await write_skills("*", root) + assert report.ok is True + assert len([a for a in report.actions if a.action == "written"]) == 3 + for k in ("a", "b", "c"): + assert (root / k / "SKILL.md").exists() + + async def test_one_action_per_requested_skill(self, root: Path) -> None: + report = await write_skills([_skill("a"), _skill("b")], root) + assert sorted(a.key for a in report.actions) == ["a", "b"] + + async def test_empty_request_on_empty_root_is_ok(self, root: Path) -> None: + report = await write_skills([], root) + assert report.ok is True + assert report.actions == [] + + +class TestManifest: + """Manifest format and forward compatibility.""" + + async def test_manifest_format_is_exact(self, root: Path) -> None: + await write_skills([_skill("pdf-extraction", 2)], root) + + manifest = _read_manifest(root) + assert manifest["manifestVersion"] == 1 + entry = manifest["entries"]["pdf-extraction/SKILL.md"] + assert entry["key"] == "pdf-extraction" + assert entry["version"] == 2 + assert entry["sha256"] == _hash(SKILL_BODY) + assert isinstance(entry["writtenAt"], str) + + async def test_entry_paths_are_forward_slash_relative(self, root: Path) -> None: + await write_skills([_skill("a")], root) + keys = list(_read_manifest(root)["entries"].keys()) + assert keys == ["a/SKILL.md"] + assert "\\" not in keys[0] + assert not keys[0].startswith("/") + + @pytest.mark.parametrize("declared", [0, -1]) + async def test_manifest_version_below_one_is_corrupt( + self, root: Path, declared: int + ) -> None: + """The version gate is bounded below as well as above. + + No release ever wrote a version under 1, so one is not an older schema + this release can still read. Treating it as readable would run the + destructive steps against entries of unknown shape. + """ + target = _place_managed(root, "a", SKILL_BODY) + _write_manifest( + root, + { + "manifestVersion": declared, + "entries": {"a/SKILL.md": _entry("a", 1, SKILL_BODY)}, + }, + ) + + report = await write_skills([], root) + + assert report.ok is False + assert any("manifestVersion" in m for m in _error_messages(report)) + assert target.exists(), "pruned against a manifest schema that never existed" + # The manifest itself is left alone, as with any other corruption. + assert _read_manifest(root)["manifestVersion"] == declared + + async def test_unknown_fields_are_preserved_on_rewrite(self, root: Path) -> None: + entry = _entry("a", 1, SKILL_BODY) + entry["futureEntryField"] = "keep-me" + _write_manifest( + root, + { + "manifestVersion": 1, + "futureTopLevelField": {"keep": True}, + "entries": {"a/SKILL.md": entry}, + }, + ) + (root / "a").mkdir() + (root / "a" / "SKILL.md").write_text(SKILL_BODY, encoding="utf-8") + + await write_skills([_skill("a", 2, SKILL_BODY + "more\n")], root) + + manifest = _read_manifest(root) + assert manifest["futureTopLevelField"] == {"keep": True} + assert manifest["entries"]["a/SKILL.md"]["futureEntryField"] == "keep-me" + + +class TestReconcileSemantics: + """The reconcile state table.""" + + async def test_unchanged_managed_file_is_skipped_current(self, root: Path) -> None: + target = _place_managed(root, "a", SKILL_BODY) + before = target.stat().st_mtime_ns + + report = await write_skills([_skill("a")], root) + + assert _actions_by_key(report)["a"].action == "skipped_current" + assert target.read_text(encoding="utf-8") == SKILL_BODY + assert target.stat().st_mtime_ns == before + + async def test_new_version_updates(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY, version=1) + new_content = SKILL_BODY + "second version\n" + + report = await write_skills([_skill("a", 2, new_content)], root) + + action = _actions_by_key(report)["a"] + assert action.action == "updated" + assert action.version == 2 + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == new_content + assert _read_manifest(root)["entries"]["a/SKILL.md"]["version"] == 2 + + async def test_local_tampering_is_overwritten(self, root: Path) -> None: + target = _place_managed(root, "a", SKILL_BODY) + target.write_text("locally tampered\n", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert _actions_by_key(report)["a"].action == "updated" + assert target.read_text(encoding="utf-8") == SKILL_BODY + + async def test_prune_removes_formerly_managed_skill(self, root: Path) -> None: + _place_managed(root, "gone", SKILL_BODY) + + report = await write_skills([], root) + + assert _actions_by_key(report)["gone"].action == "removed" + assert not (root / "gone" / "SKILL.md").exists() + assert not (root / "gone").exists() + assert _read_manifest(root)["entries"] == {} + + async def test_star_does_not_report_an_error_for_a_key_that_wrote( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """A malformed object beside a good version of the same key. + + The good version resolves and materializes, so the run succeeded for + that key. Reporting the malformed sibling as well would flip + ``report.ok`` to ``False`` for a skill that is correctly on disk, and + say the copy there "was left alone" when this run had just written it. + """ + store.put(make_raw_skill(key="a", version=1)) + store.put(make_raw_skill(key="a", version="not-a-version")) + + report = await write_skills("*", root) + + assert report.ok is True + assert [(a.key, a.action) for a in report.actions] == [("a", "written")] + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_star_still_reports_a_key_nothing_could_resolve( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """The converse, and the reason the malformed object is kept at all. + + No version of ``b`` is usable, so it stays in the requested set: the + failure is reported, and prune leaves any copy already on disk alone + rather than reading the key as revoked. + """ + existing = _place_managed(root, "b", SKILL_BODY) + store.put(make_raw_skill(key="a", version=1)) + store.put(make_raw_skill(key="b", version="not-a-version")) + + report = await write_skills("*", root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "written" + assert _actions_by_key(report)["b"].action == "error" + assert [a for a in report.actions if a.action == "removed"] == [] + assert existing.read_text(encoding="utf-8") == SKILL_BODY + + async def test_prune_false_keeps_the_file(self, root: Path) -> None: + target = _place_managed(root, "gone", SKILL_BODY) + + report = await write_skills([], root, prune=False) + + assert target.exists() + assert [a for a in report.actions if a.action == "removed"] == [] + assert "gone/SKILL.md" in _read_manifest(root)["entries"] + + async def test_prune_does_not_touch_unmanaged_files(self, root: Path) -> None: + _place_managed(root, "gone", SKILL_BODY) + bystander = root / "user-notes.md" + bystander.write_text("mine\n", encoding="utf-8") + user_dir_file = root / "user-skill" / "SKILL.md" + user_dir_file.parent.mkdir() + user_dir_file.write_text("hand written\n", encoding="utf-8") + + await write_skills([], root) + + assert bystander.read_text(encoding="utf-8") == "mine\n" + assert user_dir_file.read_text(encoding="utf-8") == "hand written\n" + + async def test_prune_refusal_for_unownable_path_reports_the_version( + self, root: Path + ) -> None: + """A prune refusal carries the manifest's version. + + A manifest entry whose path is not one this SDK could have written is + refused rather than removed. The entry is in hand at that point, so the + error action must carry its version — otherwise a prune *failure* is + strictly less informative than a prune *success*, which does report it. + """ + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + # Right key, wrong filename — not a path this SDK could own. + "orphan/NOTES.md": _entry("orphan", 7, SKILL_BODY), + }, + }, + ) + + report = await write_skills([], root) + + action = _actions_by_key(report)["orphan"] + assert action.action == "error" + assert action.version == 7 + + async def test_prune_refusal_for_symlinked_target_reports_the_version( + self, root: Path + ) -> None: + """Same contract on the symlink refusal path (prune side).""" + if not hasattr(os, "symlink"): + pytest.skip("platform has no symlink support") + (root / "a").mkdir() + outside_file = root.parent / "victim.md" + outside_file.write_text("victim content\n", encoding="utf-8") + (root / "a" / "SKILL.md").symlink_to(outside_file) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 4, "victim content\n")}, + }, + ) + + report = await write_skills([], root) + + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.version == 4 + + async def test_unresolvable_request_still_reports_no_version( + self, root: Path + ) -> None: + """The other half of the contract: do not invent a version. + + A reference that could not be retrieved has neither a manifest entry + nor a ``Skill``, so there is no version to report and ``version`` stays + ``None``. Without this, "always populate version" would be satisfied by + fabricating one. + """ + report = await write_skills([SkillReference(key="ghost", version=3)], root) + + action = _actions_by_key(report)["ghost"] + assert action.action == "error" + assert action.version is None + + async def test_prune_keeps_directory_when_not_empty(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY) + extra = root / "a" / "user-file.txt" + extra.write_text("keep\n", encoding="utf-8") + + report = await write_skills([], root) + + assert _actions_by_key(report)["a"].action == "removed" + assert not (root / "a" / "SKILL.md").exists() + assert extra.exists() + + +class TestRootHandling: + """Root resolution.""" + + async def test_absent_leaf_root_is_created(self, tmp_path: Path) -> None: + target_root = tmp_path / "skills" + report = await write_skills([_skill("a")], target_root) + assert report.ok is True + assert (target_root / "a" / "SKILL.md").exists() + + async def test_missing_ancestors_raise(self, tmp_path: Path) -> None: + with pytest.raises(ValueError): + await write_skills([_skill("a")], tmp_path / "a" / "b" / "c") + + async def test_root_that_is_a_file_raises(self, tmp_path: Path) -> None: + file_root = tmp_path / "not-a-dir" + file_root.write_text("x", encoding="utf-8") + with pytest.raises(ValueError): + await write_skills([_skill("a")], file_root) + + async def test_accepts_string_root(self, root: Path) -> None: + report = await write_skills([_skill("a")], str(root)) + assert report.ok is True + + +class TestSkillsArgumentErrors: + """A bare string that is not ``"*"`` raises. + + A ``ValueError``, not a ``TypeError``: a string *is* an accepted argument + type here, since ``"*"`` means "everything the store holds", so this is an + acceptable type carrying an invalid value. The accessors' equivalent guard + is a ``TypeError`` because a string is never a valid argument there. + """ + + async def test_bare_non_star_string_raises_value_error(self, root: Path) -> None: + with pytest.raises(ValueError) as excinfo: + await write_skills("pdf-extraction", root) + + # Naming the accepted forms is the actionable half of the message. + assert '"*"' in str(excinfo.value) + + async def test_star_is_accepted(self, root: Path) -> None: + """Positive control — otherwise the guard above could reject every string.""" + store = InMemorySkillStore() + store.put( + { + "key": "a", + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + ) + skills_module._set_store(store) + + report = await write_skills("*", root) + + assert report.ok is True + assert (root / "a" / "SKILL.md").exists() + + async def test_bare_string_writes_nothing(self, root: Path) -> None: + """The raise precedes any filesystem work. + + Asserting only the raise would also pass for an implementation that + created one directory per character before failing. + """ + with pytest.raises(ValueError): + await write_skills("abc", root) + + assert list(root.iterdir()) == [] + + +class TestUninitializedStore: + """A store that has not received its initial data must not authorize a prune. + + Through ``all_objects`` there is no difference between "this environment + holds no skills" and "delivery has not answered yet": both are an empty + result. ``write_skills("*")`` reads the first as every skill having been + revoked, so without the optional ``is_initialized()`` probe a reconcile + racing a slow boot deletes every managed file and reports success. + """ + + class _Waiting: + """A delivery store whose first payload has not arrived.""" + + def __init__(self, initialized: bool = False) -> None: + self._initialized = initialized + + def is_initialized(self) -> bool: + return self._initialized + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + return None + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + return {} + + async def test_star_does_not_prune_before_the_first_payload( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + existing = _place_managed(root, "a", SKILL_BODY) + spy = _UnlinkSpy().install(monkeypatch) + skills_module._set_store(self._Waiting()) + + report = await write_skills("*", root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert [a for a in report.actions if a.action == "removed"] == [] + assert spy.targets == [] + # The entry survives, so a later reconcile still knows it owns the file. + assert "a/SKILL.md" in _read_manifest(root)["entries"] + assert any("initial data" in m for m in _error_messages(report)) + + async def test_star_raises_in_raise_mode_before_the_first_payload( + self, root: Path + ) -> None: + skills_module._set_store(self._Waiting()) + with pytest.raises(RuntimeError, match=r"initial data"): + await write_skills("*", root, on_unavailable="raise") + + async def test_an_initialized_store_prunes_as_before( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The positive control: the probe must not disable pruning outright.""" + _place_managed(root, "a", SKILL_BODY) + skills_module._set_store(self._Waiting(initialized=True)) + + report = await write_skills("*", root) + + assert report.ok is True + assert _actions_by_key(report)["a"].action == "removed" + assert not (root / "a").exists() + + async def test_a_store_without_the_probe_prunes_as_before( + self, root: Path, store: InMemorySkillStore + ) -> None: + """``is_initialized()`` is optional; absent means initialized. + + A hand-populated store is never waiting for anything, so requiring the + probe would break every ``InMemorySkillStore`` caller. + """ + assert not hasattr(store, "is_initialized") + _place_managed(root, "a", SKILL_BODY) + + report = await write_skills("*", root) + + assert report.ok is True + assert _actions_by_key(report)["a"].action == "removed" + + async def test_a_probe_that_raises_counts_as_uninitialized( + self, root: Path + ) -> None: + """A store that cannot say whether it is ready does not get to delete.""" + + class Exploding(TestUninitializedStore._Waiting): + def is_initialized(self) -> bool: + raise RuntimeError("cannot tell") + + existing = _place_managed(root, "a", SKILL_BODY) + skills_module._set_store(Exploding()) + + report = await write_skills("*", root) + + assert report.ok is False + assert existing.exists() + assert [a for a in report.actions if a.action == "removed"] == [] + + +class TestResilience: + """Unavailable retrieval and timeout.""" + + async def test_keep_is_the_default_and_does_not_raise(self, root: Path) -> None: + existing = _place_managed(root, "a", SKILL_BODY) + + report = await write_skills([SkillReference(key="a", version=1)], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert existing.read_text(encoding="utf-8") == SKILL_BODY + + async def test_raise_mode_propagates(self, root: Path) -> None: + with pytest.raises(RuntimeError, match=r"(?i)(unavailable|skill store)"): + await write_skills( + [SkillReference(key="a", version=1)], root, on_unavailable="raise" + ) + + async def test_store_error_is_reported_not_raised( + self, root: Path, exploding_store: Any + ) -> None: + report = await write_skills([SkillReference(key="a", version=1)], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + + async def test_exhausted_timeout_behaves_as_unavailable( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a")) + report = await write_skills( + [SkillReference(key="a", version=1)], root, timeout=0 + ) + assert report.ok is False + assert not (root / "a" / "SKILL.md").exists() + + async def test_exhausted_timeout_raises_in_raise_mode( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a")) + with pytest.raises(RuntimeError, match=r"(?i)(unavailable|timeout|timed out)"): + await write_skills( + [SkillReference(key="a", version=1)], + root, + timeout=0, + on_unavailable="raise", + ) + + async def test_exhausted_timeout_stops_pruning( + self, root: Path, store: InMemorySkillStore + ) -> None: + """The deadline bounds pruning too, not just retrieval and the writes. + + A run whose writes all land just inside the deadline would otherwise go + on to stat, unlink and rmdir every stale manifest entry unbounded — the + opposite of what a small ``timeout`` asks for. + """ + existing = _place_managed(root, "stale", SKILL_BODY) + + report = await write_skills([], root, timeout=0) + + assert report.ok is False + assert existing.exists(), "prune ran past the exhausted deadline" + assert any("timeout was exhausted" in m for m in _error_messages(report)) + # The entry survives, so the next reconcile picks it up. + assert "stale/SKILL.md" in _read_manifest(root)["entries"] + + async def test_a_verification_failure_never_prunes_the_good_copy( + self, root: Path + ) -> None: + """A store may key ``all_objects`` differently from the object's own key. + + The on-disk copy lives under the object's own key, so a failure recorded + under the *store's* dict key would drop the real key out of the + requested set and let prune delete the last known-good copy. + """ + + class AliasKeyedStore: + """Keys objects by an internal id, not by the skill's own key.""" + + def __init__(self, raw: dict[str, Any]) -> None: + self._raw = raw + + def get_object(self, kind: str, key: str) -> dict[str, Any] | None: + return None + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + return {"internal-uuid-1": self._raw} + + existing = _place_managed(root, "pdf-extraction", SKILL_BODY) + tampered = { + "key": "pdf-extraction", + "version": 1, + "content": "tampered\n", + "contentHash": _hash(SKILL_BODY), # does not match the content + } + skills_module._set_store(AliasKeyedStore(tampered)) + + report = await write_skills("*", root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert [a.action for a in report.actions] == ["error"] + assert _actions_by_key(report)["pdf-extraction"].action == "error" + assert "pdf-extraction/SKILL.md" in _read_manifest(root)["entries"] + + async def test_a_non_mapping_listing_never_prunes(self, root: Path) -> None: + """A store that cannot list is not a store holding nothing. + + A listing collapsed to "no skills" is indistinguishable from every + skill having been revoked, and prune would then delete every managed + file and report a clean run. The listing failure has to reach the + prune gate as an incomplete run. + """ + + class NoListingStore: + """Answers the listing with something that is not a mapping.""" + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return None + + def all_objects(self, kind: str) -> Any: + return None + + existing = _place_managed(root, "pdf-extraction", SKILL_BODY) + skills_module._set_store(NoListingStore()) + + report = await write_skills("*", root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert [a.action for a in report.actions] == ["error"] + assert any("rather than an object" in m for m in _error_messages(report)) + # The entry survives, so the next reconcile picks it up. + assert "pdf-extraction/SKILL.md" in _read_manifest(root)["entries"] + + async def test_an_answer_under_another_key_writes_nothing(self, root: Path) -> None: + """The file is named after the key the object carries, so a store + answering under a different key would write one path and prune another. + + Left unchecked, the run wrote the aliased key, then deleted it in the + same pass because prune keys off the request — and reported ok. The + requested key has to be the one the outcome is reported against. + """ + + class AliasingStore: + """Answers every lookup with an object carrying its own key.""" + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return { + "key": "other-key", + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + skills_module._set_store(AliasingStore()) + + report = await write_skills(["requested-key"], root) + + assert report.ok is False + assert [a.action for a in report.actions] == ["error"] + # Reported against the key that was asked for, not the one served. + assert _actions_by_key(report)["requested-key"].action == "error" + assert not (root / "other-key").exists() + assert _read_manifest(root)["entries"] == {} + + async def test_an_answer_under_another_key_does_not_overwrite_that_key( + self, root: Path + ) -> None: + """The aliased answer must not reach the real key's file. + + Both keys are requested here, so nothing is prunable and the write + itself is what is under test: unchecked, the object served under the + alias is written to the *other* key's path, clobbering the content that + key's own lookup resolved — and the run still reports ok. + """ + aliased = "aliased\n" + + class AliasingStore: + """Answers one key honestly and the other under that same key.""" + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + if key == "other-key": + return { + "key": "other-key", + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + return { + "key": "other-key", + "version": 2, + "content": aliased, + "contentHash": _hash(aliased), + } + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + existing = _place_managed(root, "other-key", SKILL_BODY) + skills_module._set_store(AliasingStore()) + + # The alias is resolved last, so an unchecked write lands on top. + report = await write_skills(["other-key", "requested-key"], root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert _actions_by_key(report)["requested-key"].action == "error" + assert _actions_by_key(report)["other-key"].action == "skipped_current" + + async def test_an_unattributable_failure_never_prunes(self, root: Path) -> None: + """A withholding with no usable key at all still must not prune. + + The sibling case above recovers the object's own key and keeps it in the + requested set. When neither the object's key nor the store's is usable + the failure is run-level, so there is no key to hold the on-disk copy + with — reporting the run incomplete is the only thing left that stops + prune reading an unreadable object as a revocation. + """ + existing = _place_managed(root, "pdf-extraction", SKILL_BODY) + # The shipped store keys ``all_objects`` as ":", which is + # never a valid skill key, so a missing 'key' field leaves no fallback. + skills_module._set_store( + InMemorySkillStore( + { + "pdf-extraction:1": { + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + } + ) + ) + + report = await write_skills("*", root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert [a.action for a in report.actions] == ["error"] + assert "pdf-extraction/SKILL.md" in _read_manifest(root)["entries"] + + async def test_unavailable_run_does_not_corrupt_manifest(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY) + before = _read_manifest(root) + + await write_skills([SkillReference(key="b", version=1)], root) + + assert ( + _read_manifest(root)["entries"]["a/SKILL.md"] + == (before["entries"]["a/SKILL.md"]) + ) + + +class TestVerifyThenWrite: + """Hash re-verified immediately before writing.""" + + async def test_hash_mismatch_aborts_the_write( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + bad = Skill( + key="a", + version=1, + content=SKILL_BODY.encode("utf-8"), + content_hash="0" * 64, + ) + + report = await write_skills([bad], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert not (root / "a" / "SKILL.md").exists() + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_oversize_skill_aborts_the_write( + self, root: Path, oversize_content: tuple[str, str] + ) -> None: + oversize, _ = oversize_content + report = await write_skills([_skill("a", 1, oversize)], root) + assert report.ok is False + assert not (root / "a" / "SKILL.md").exists() + + async def test_mismatch_does_not_disturb_existing_managed_file( + self, root: Path + ) -> None: + target = _place_managed(root, "a", SKILL_BODY) + bad = Skill(key="a", version=2, content=b"new content\n", content_hash="f" * 64) + + await write_skills([bad], root) + + assert target.read_text(encoding="utf-8") == SKILL_BODY + + +class TestAtomicityAndPermissions: + """Atomic writes, no partial files, 0644.""" + + async def test_written_file_is_0644_and_not_executable(self, root: Path) -> None: + await write_skills([_skill("a")], root) + mode = stat.S_IMODE((root / "a" / "SKILL.md").stat().st_mode) + assert mode == 0o644 + assert not mode & stat.S_IXUSR + assert not mode & stat.S_IXGRP + assert not mode & stat.S_IXOTH + + async def test_write_goes_through_a_single_atomic_rename( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Positive control for the interception hook. + + Without this, the ``spy.calls == []`` assertions in the failure tests + below and in the traversal matrix could pass in a suite where the hook + is never reachable at all. + """ + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True + # The temp file is created in the *same* directory + # as the target, so the rename is atomic rather than cross-device. + _assert_atomic_rename_of(spy, root / "a") + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_rename_failure_leaves_prior_content_intact( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + target = _place_managed(root, "a", SKILL_BODY) + spy = _ReplaceSpy(fail=True).install(monkeypatch) + + report = await write_skills([_skill("a", 2, "brand new content\n")], root) + + # The injected failure — not an unrelated rejection, and not an + # implementation that attempted nothing — is what produced the error. + _assert_atomic_rename_of(spy, target.parent) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert _INJECTED in (action.error or "") + + assert target.read_text(encoding="utf-8") == SKILL_BODY + # No temp artifact survives the failed run. + assert sorted(p.name for p in target.parent.iterdir()) == ["SKILL.md"] + + async def test_no_partial_file_at_target_after_failure( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + spy = _ReplaceSpy(fail=True).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + _assert_atomic_rename_of(spy, root / "a") + + assert report.ok is False + assert _INJECTED in (_actions_by_key(report)["a"].error or "") + assert not (root / "a" / "SKILL.md").exists() + # Neither a partial target nor a leaked temp file. + skill_dir = root / "a" + leftovers = ( + sorted(p.name for p in skill_dir.iterdir()) if skill_dir.exists() else [] + ) + assert leftovers == [] + + async def test_manifest_is_valid_json_after_a_run_with_errors( + self, root: Path + ) -> None: + report = await write_skills([_skill("a"), _skill("../evil")], root) + assert report.ok is False + assert isinstance(_read_manifest(root), dict) + + +# --------------------------------------------------------------------------- +# Security abuse matrix +# --------------------------------------------------------------------------- + +HOSTILE_KEYS = [ + "../evil", + "..", + ".", + "", + "/etc/cron.d/x", + "..\\evil", + "c:evil", + "skill:ads", + "sk\0ill", + "-skill", + "Evil", + "a/b", + "x" * 257, + "a/../../b", + "./a", + " leading-space", + "trailing-space ", +] + + +class TestPathTraversal: + """Nothing is ever written outside the root.""" + + @pytest.mark.parametrize("hostile_key", HOSTILE_KEYS) + async def test_hostile_key_is_rejected( + self, tmp_path: Path, hostile_key: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + root = tmp_path / "skills" + root.mkdir() + outside_before = sorted(p.name for p in tmp_path.iterdir()) + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill(hostile_key)], root) + + assert report.ok is False + assert [a.action for a in report.actions if a.key == hostile_key] == ["error"] + + # The SDK's key validation — not the operating system — must be what + # stopped this. An overlong key exceeds NAME_MAX, a null byte raises in + # the path API, and an absolute path outside the root usually fails on + # permissions, so "an error was reported" is not evidence of a defense + # (and the absolute-path verdict would flip on a privileged runner). + # Assert instead that no write was ever attempted. + assert spy.calls == [] + + # Nothing created outside the root, and no skill directory inside it. + assert sorted(p.name for p in tmp_path.iterdir()) == outside_before + assert [p.name for p in root.iterdir() if p.name != MANIFEST_NAME] == [] + assert list(root.rglob("SKILL.md")) == [] + + async def test_interception_hook_fires_for_a_valid_key( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Positive control for the ``spy.calls == []`` assertion above.""" + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("ok-key")], root) + + assert report.ok is True + assert [Path(call.dst).name for call in spy.calls] == ["SKILL.md"] + + async def test_long_but_filesystem_legal_key_is_written(self, root: Path) -> None: + """The ≤ 256 length bound cannot be exercised through ``write_skills``. + + A key becomes a single directory name and NAME_MAX is 255 bytes on Linux + and macOS, so the longest key the data model permits cannot exist on + disk at all. Assert the accepting side at the largest writable length; + the bound itself is covered by the pure layers (config validation and + accessor revalidation). + """ + key = "k" * 255 + report = await write_skills([_skill(key)], root) + + assert report.ok is True + assert (root / key / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_key_at_the_data_model_bound_is_reported_not_raised( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A 256-character key is valid to every pure layer but fits no filesystem. + + Config validation and the accessors must both accept exactly 256 + characters, yet NAME_MAX is 255 + on Linux and macOS, so this key reaches ``write_skills`` legitimately and + cannot become a directory. Every outcome must be visible in + the report, so it must surface as an ``error`` action rather than an + ``OSError`` escaping the call — which would also skip the manifest rewrite + and orphan any file already written in the same run. + """ + spy = _ReplaceSpy().install(monkeypatch) + long_key = "a" * 256 + + report = await write_skills([_skill("good"), _skill(long_key)], root) + + by_key = _actions_by_key(report) + assert by_key[long_key].action == "error" + assert by_key["good"].action == "written" + # The bare-filename ``dst`` of a ``dir_fd``-relative rename carries no + # directory, so "the path does not contain the hostile key" is no longer + # a meaningful check. Assert the stronger thing instead: the only rename + # that happened was into the valid skill's own directory. Through the + # shared helper, so the check holds on the path fallback too — reading + # ``dst_dir_id`` directly would compare ``None`` there and fail a run + # that had in fact renamed correctly. + _assert_atomic_rename_of(spy, root / "good") + # The valid skill is fully reconciled: written AND recorded, not orphaned. + assert (root / "good" / "SKILL.md").exists() + assert "good/SKILL.md" in _read_manifest(root)["entries"] + + async def test_valid_keys_still_write_alongside_rejected_ones( + self, root: Path + ) -> None: + report = await write_skills([_skill("good"), _skill("../evil")], root) + by_key = _actions_by_key(report) + assert by_key["good"].action == "written" + assert by_key["../evil"].action == "error" + assert (root / "good" / "SKILL.md").exists() + + async def test_traversal_key_does_not_create_parent_files( + self, tmp_path: Path + ) -> None: + root = tmp_path / "skills" + root.mkdir() + await write_skills([_skill("../../escaped")], root) + assert not (tmp_path / "escaped").exists() + assert not (tmp_path.parent / "escaped").exists() + + +@pytest.mark.skipif( + not hasattr(os, "symlink"), reason="platform has no symlink support" +) +class TestSymlinkAttacks: + """Never write through a symlink.""" + + async def test_symlinked_root_raises(self, tmp_path: Path) -> None: + real_dir = tmp_path / "real" + real_dir.mkdir() + link_root = tmp_path / "link" + link_root.symlink_to(real_dir, target_is_directory=True) + + with pytest.raises(ValueError): + await write_skills([_skill("a")], link_root) + + assert list(real_dir.iterdir()) == [] + + async def test_symlinked_skill_directory_is_refused(self, tmp_path: Path) -> None: + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (root / "a").symlink_to(outside, target_is_directory=True) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert list(outside.iterdir()) == [] + + async def test_symlinked_target_file_is_refused(self, tmp_path: Path) -> None: + root = tmp_path / "skills" + root.mkdir() + outside_file = tmp_path / "victim.md" + outside_file.write_text("victim content\n", encoding="utf-8") + (root / "a").mkdir() + (root / "a" / "SKILL.md").symlink_to(outside_file) + # Manifest lists the path so clobber protection is not what saves us. + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 1, "victim content\n")}, + }, + ) + + report = await write_skills([_skill("a", 2, "attacker payload\n")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert outside_file.read_text(encoding="utf-8") == "victim content\n" + + async def test_symlinked_target_is_not_pruned(self, tmp_path: Path) -> None: + """A manifest-listed path that is a symlink is refused, not unlinked. + + Asserting only that the victim file survives proves nothing here: + unlinking a symlink never touches its target, so that assertion holds + for an implementation with no symlink check at all. The observable + contract is the refusal itself (prune path). + """ + root = tmp_path / "skills" + root.mkdir() + outside_file = tmp_path / "victim.md" + outside_file.write_text("victim content\n", encoding="utf-8") + (root / "a").mkdir() + link = root / "a" / "SKILL.md" + link.symlink_to(outside_file) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 1, "victim content\n")}, + }, + ) + + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert [a for a in report.actions if a.action == "removed"] == [] + # The symlink itself is left in place and stays managed. + assert link.is_symlink() + assert "a/SKILL.md" in _read_manifest(root)["entries"] + assert outside_file.read_text(encoding="utf-8") == "victim content\n" + + @_needs_dir_fd + async def test_directory_swapped_at_the_rename_cannot_redirect_the_write( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The swap window is closed, not merely narrowed. + + Every check in the world is worthless if the final rename re-resolves + ``/`` from its path: an attacker holding write permission on + the managed root can replace the validated directory with a symlink in + between and redirect the write out of the root. The rename is therefore + performed relative to a descriptor pinned to the directory that was + checked, so it follows the inode rather than the name. + """ + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + race = _SwapDirectoryDuring("replace", root / "a", outside).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert list(outside.iterdir()) == [] + assert (race.moved_to / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + assert report.ok is True + + @_needs_dir_fd + async def test_directory_swapped_at_the_prune_cannot_redirect_the_unlink( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The same window on the destructive side. + + ``unlink`` never follows a *trailing* symlink, but it does resolve the + directory above it, so the swap turns a prune into a delete of an + attacker-chosen outside file. The unlink is descriptor-relative for the + same reason the rename is. + """ + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + victim = outside / "SKILL.md" + victim.write_text("precious\n", encoding="utf-8") + _place_managed(root, "a", SKILL_BODY) + race = _SwapDirectoryDuring("unlink", root / "a", outside).install(monkeypatch) + + report = await write_skills([], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert victim.read_text(encoding="utf-8") == "precious\n" + assert not (race.moved_to / "SKILL.md").exists() + assert [a.action for a in report.actions if a.key == "a"] == ["removed"] + + +@pytest.mark.skipif( + not hasattr(os, "symlink"), reason="platform has no symlink support" +) +class TestRootSwapRaces: + """The swap one level up: the managed *root*, not ``/``. + + ``_resolve_root`` validates the root and returns a path. It used to end + there: each write and each prune then opened ``/`` *by path* with + ``O_NOFOLLOW | O_DIRECTORY`` and pinned that, and ``O_NOFOLLOW`` guards only + the final component — so the root and every ancestor were re-resolved on + every such open, and a root renamed aside and replaced with a symlink after + validation redirected the open, and with it every descriptor-relative step + behind it, into the attacker's directory. The manifest rewrite refused, but + by then the skill file was already outside the root. + + ``write_skills`` now pins the root once, before anything is read or written, + and holds that descriptor for the whole reconcile: the skill directory is + created and opened relative to it, the unlink and the ``rmdir`` run relative + to it, and so does the manifest write. The swap still happens in each of + these — ``race.swapped`` asserts it did — and the descriptor still names the + directory that was validated, so the reconcile carries on inside the real + root, which the rename has moved to ``race.moved_to``. + + ``TestSymlinkAttacks`` proves the same for a swapped ``/``. The + contract here is the root's: nothing lands outside it, no outside file is + overwritten, no outside file is removed. Note what each assertion permits — + either the reconcile refused outright, or it completed against the real + root — because *which* of those happens depends on where in the sequence + the swap lands, and neither is an escape. + """ + + @_needs_dir_fd + async def test_root_swapped_at_the_skill_directory_create_cannot_redirect_the_write( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """First reconcile against a fresh root: the skill directory does not + exist yet, so ``open_or_create_directory`` calls ``os.mkdir(/a)``. + The swap fires there; the ``mkdir`` and the ``O_NOFOLLOW`` open that + follows both resolve through the link, and ``SKILL.md`` is written into + ``/a/``.""" + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + race = _SwapRootDuring("mkdir", root, "a", outside).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert list(outside.iterdir()) == [] + # Either the skill landed in the real root or the run says it did not. + assert report.ok is False or ( + (race.moved_to / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + ) + + @_needs_dir_fd + async def test_root_swapped_at_the_skill_directory_open_cannot_clobber_an_outside_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Update of an already-managed skill: the swap fires at the first + ``O_NOFOLLOW`` open of ``/a`` (the orphan-temp sweep's), and the + write re-opens by path and inherits it. ``/a/SKILL.md`` — a + file the manifest never recorded — is replaced with LaunchDarkly-served + content.""" + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + (outside / "a").mkdir(parents=True) + victim = outside / "a" / "SKILL.md" + victim.write_text("precious\n", encoding="utf-8") + _place_managed(root, "a", SKILL_BODY) + race = _SwapRootDuring("open", root, "a", outside).install(monkeypatch) + + report = await write_skills([_skill("a", 2, "served update\n")], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert victim.read_text(encoding="utf-8") == "precious\n" + assert report.ok is False or ( + (race.moved_to / "a" / "SKILL.md").read_text(encoding="utf-8") + == "served update\n" + ) + + @_needs_dir_fd + async def test_root_swapped_at_the_prune_cannot_redirect_the_unlink( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The destructive side. A prune of a formerly-managed key opens + ``/a`` by path, unlinks ``SKILL.md`` relative to that descriptor, + then ``rmdir``s the directory — all three resolve through the swapped + root, so an outside file and its directory are removed.""" + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + (outside / "a").mkdir(parents=True) + victim = outside / "a" / "SKILL.md" + victim.write_text("precious\n", encoding="utf-8") + _place_managed(root, "a", SKILL_BODY) + race = _SwapRootDuring("open", root, "a", outside).install(monkeypatch) + + report = await write_skills([], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert victim.exists() and victim.read_text(encoding="utf-8") == "precious\n" + assert report.ok is False or not (race.moved_to / "a" / "SKILL.md").exists() + + @_needs_dir_fd + async def test_a_root_swapped_before_the_pin_is_refused_at_the_run_level( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The other side of the window the three above race into. + + Those swap the root *after* it is pinned, and the held descriptor is + what carries the reconcile through to the real directory. This one swaps + it in the only interval left — after ``_resolve_root`` has validated the + root and before ``write_skills`` opens it — so there is no descriptor + yet to fall back on. The ``O_NOFOLLOW`` open of the root is what has to + refuse it, and because the root passed validation a moment earlier this + is a run-level error rather than the ``ValueError`` an unusable root + raises: nothing has been touched, and the report says so. + """ + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + (outside / "a").mkdir(parents=True) + victim = outside / "a" / "SKILL.md" + victim.write_text("precious\n", encoding="utf-8") + moved_to = tmp_path / "skills.real" + + real_resolve_root = skills_fs_module._resolve_root + + def resolve_then_swap(argument: Any) -> Path: + resolved = real_resolve_root(argument) + os.rename(root, moved_to) + os.symlink(outside, root, target_is_directory=True) + return resolved + + monkeypatch.setattr(skills_fs_module, "_resolve_root", resolve_then_swap) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert [action.key for action in report.actions] == [""], _error_messages( + report + ) + assert "could not be pinned" in (report.actions[0].error or "") + # Neither written into nor read as a managed root. + assert victim.read_text(encoding="utf-8") == "precious\n" + assert not (outside / "a" / MANIFEST_NAME).exists() + assert list(moved_to.iterdir()) == [] + + @staticmethod + def _swap_after_the_pin( + monkeypatch: pytest.MonkeyPatch, root: Path, outside: Path, moved_to: Path + ) -> None: + """Swaps the root in the window between the pin and the manifest read. + + The three races above intercept a destructive call. This one intercepts + the pin itself and swaps the root the instant it returns, which is the + earliest point the held descriptor is already in hand — so everything + the reconcile *decides*, not just everything it does, happens with a + hostile root on the path. The manifest is the first thing read in that + window, and it is the only input that says which of the customer's + files the SDK may overwrite and delete. + """ + real_pin = skills_fs_module.open_directory_nofollow + + def pin_then_swap(path: Any) -> int | None: + fd = real_pin(path) + os.rename(root, moved_to) + os.symlink(outside, root, target_is_directory=True) + return fd + + monkeypatch.setattr(skills_fs_module, "open_directory_nofollow", pin_then_swap) + + @_needs_dir_fd + async def test_a_root_swapped_before_the_manifest_read_cannot_supply_the_entries( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The manifest is read through the descriptor, not through the path. + + Read by path, the manifest was the one input to the reconcile that the + pin did not cover: the swap redirected the read into the attacker's + directory, the run adopted whatever entries it found there, and + ``_rewrite_manifest`` then committed them back over the *real* manifest + through the held descriptor. Nothing escaped the root — but the + ownership record inside it was destroyed, and that record is the only + thing standing between the next reconcile and the customer's own files. + Note that this is strictly worse than the behavior it replaced: before + the root was pinned at all, the manifest write was the one operation + that took its own ``O_NOFOLLOW`` descriptor, so a swapped root made the + write *fail* and left the real manifest intact. + + The attacker's manifest here is valid and empty, which is the cheapest + version to plant and the one that does the most damage: every managed + file looks unowned, so the entry recording it is simply dropped on the + rewrite. ``prune`` is off so that what this asserts is the entries the + run read, uncoupled from what a prune driven by them would then remove + — which is the next test. + + The write of the requested skill is refused either way, by the path + checks: they resolve ``/other`` into the attacker's tree and see + it land outside the managed root. That refusal is what makes this test + narrow rather than weaker — with no write and no prune, the manifest + rewrite is the only thing left in the run, so the surviving entry can + only have come from reading the real manifest. It is also the two + layers doing the jobs they are each documented to do: the path checks + turn a hostile layout into a reported refusal, and the descriptor is + what makes the refusal unnecessary for correctness. + """ + root = tmp_path / "skills" + root.mkdir() + _place_managed(root, "keep", SKILL_BODY) + outside = tmp_path / "outside" + outside.mkdir() + _write_manifest(outside, {"manifestVersion": 1, "entries": {}}) + moved_to = tmp_path / "skills.real" + + self._swap_after_the_pin(monkeypatch, root, outside, moved_to) + + report = await write_skills([_skill("other")], root, prune=False) + + # The real root is where the rename left it, and its manifest still + # records the skill it owned before the run. + assert "keep/SKILL.md" in _read_manifest(moved_to)["entries"] + # The attacker's manifest is neither the one that was read nor the one + # that was written. + assert _read_manifest(outside)["entries"] == {} + # The refusal named above, asserted so that a future change which + # starts writing through the hostile path does not pass this quietly. + assert [a.key for a in report.errors] == ["other"], _error_messages(report) + + @_needs_dir_fd + async def test_a_root_swapped_before_the_manifest_read_cannot_poison_ownership( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The delayed half of the same window, and the damaging one. + + A populated manifest in the attacker's directory does not get to delete + anything *during* the swapped run: the path checks resolve + ``/victim`` into the attacker's tree, see it land outside the + managed root, and refuse the prune. What they cannot refuse is the + record. ``_rewrite_manifest`` commits the entries that were read back + into the real root through the held descriptor, so the swapped run ends + with the real manifest claiming a file the SDK never wrote. + + The deletion then happens on the *next* reconcile — an ordinary one, + with no attacker present and every path check passing, because by then + the entry is in the legitimate manifest, the key is well formed, and the + path really is inside the real root. That is what makes reading the + manifest by path worth fixing rather than noting: the blast radius is + not the swapped run, it is every run after it, and the report for the + run that did the damage shows only a refusal. + + Both phases run here for that reason. Asserting only that the entry is + absent after phase one would leave the consequence implicit, and the + consequence is a customer's own file. + """ + root = tmp_path / "skills" + root.mkdir() + victim = root / "victim" / "SKILL.md" + victim.parent.mkdir() + victim.write_text("the customer's own file\n", encoding="utf-8") + outside = tmp_path / "outside" + (outside / "victim").mkdir(parents=True) + # Present so that a path-based existence probe, were one to return, + # would be satisfied; without it a run whose probe resolved through the + # swapped root would skip the unlink for a reason that has nothing to + # do with the defense under test. + (outside / "victim" / "SKILL.md").write_text("bait\n", encoding="utf-8") + _write_manifest( + outside, + { + "manifestVersion": 1, + "entries": {"victim/SKILL.md": _entry("victim", 1, SKILL_BODY)}, + }, + ) + moved_to = tmp_path / "skills.real" + + self._swap_after_the_pin(monkeypatch, root, outside, moved_to) + await write_skills([_skill("a")], root) + monkeypatch.undo() + + # The attacker withdraws, restoring the root exactly as it was found. + os.unlink(root) + os.rename(moved_to, root) + assert "victim/SKILL.md" not in _read_manifest(root)["entries"] + + # An ordinary reconcile, which is where the planted entry would cash in. + report = await write_skills([], root) + + assert [a for a in report.actions if a.action == "removed"] == [] + assert victim.read_text(encoding="utf-8") == "the customer's own file\n" + + @_needs_dir_fd + async def test_a_root_swapped_before_the_prune_probe_still_removes_the_real_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The existence probe before a prune is answered from the pinned + directory, not from the path. + + Answered from the path, a swap landing between the pin and the probe + made the probe look into the attacker's tree, find nothing, and skip + the unlink — while the prune still dropped the manifest entry and still + reported ``removed``. The revoked skill stayed on disk, now unmanaged, + under a report that said it was gone: the one outcome a revocation must + never have. The attacker's directory is empty here for exactly that + reason — it is the "nothing to remove" answer, planted. + + Pinned, the probe sees the real file, the unlink removes it, and the + report is true. + """ + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + (outside / "a").mkdir(parents=True) + _place_managed(root, "a", SKILL_BODY) + race = _SwapRootBefore( + "stat", root, outside, _names_the_skill_file(root / "a") + ).install(monkeypatch) + + report = await write_skills([], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert [a.action for a in report.actions if a.key == "a"] == ["removed"] + # ``removed`` is true: the real file is gone, and so is its entry. + assert not (race.moved_to / "a" / "SKILL.md").exists() + assert "a/SKILL.md" not in _read_manifest(race.moved_to)["entries"] + # And nothing happened in the attacker's tree. + assert list((outside / "a").iterdir()) == [] + + @_needs_dir_fd + async def test_a_root_swapped_before_the_compare_read_cannot_adopt_an_outside_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Adoption is decided from the bytes in the pinned directory. + + The real root holds a customer-authored file at the managed path with + no manifest entry; the attacker's tree holds a byte-identical copy of + the resolved content at the same relative path. Read by path, the + compare read landed on the attacker's copy, matched, and adopted — and + the entry it recorded was then written into the *real* manifest through + the held descriptor, claiming the customer's file for the next reconcile + to overwrite or delete. Read through the pin, the bytes are the + customer's, they differ, and the write is refused for the reason the + real root warrants. + """ + root = tmp_path / "skills" + root.mkdir() + target = _place_unmanaged(root, "a", "user authored\n") + outside = tmp_path / "outside" + _place_unmanaged(outside, "a", SKILL_BODY) + race = _SwapRootBefore( + "stat", root, outside, _names_the_skill_file(root / "a") + ).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert "refusing to overwrite a file this SDK did not write" in ( + action.error or "" + ) + real = race.moved_to / "a" / "SKILL.md" + assert real == Path(str(target).replace(str(root), str(race.moved_to))) + assert real.read_text(encoding="utf-8") == "user authored\n" + assert "a/SKILL.md" not in _read_manifest(race.moved_to)["entries"] + assert (outside / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + @_needs_dir_fd + async def test_a_root_swapped_before_the_compare_read_cannot_refuse_over_an_outside_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The mirror image: the real root's file *is* the resolved content, + left unrecorded by a reconcile killed before its manifest rewrite, and + the attacker's tree holds something else at the same path. Read by + path, the compare read saw the attacker's bytes, they differed, and the + SDK refused to adopt its own file — wedging that skill on every later + run, on the strength of a file that was never inside the root. Read + through the pin, the file is adopted as ``skipped_current``, which is + what the real root's contents call for. + """ + root = tmp_path / "skills" + root.mkdir() + _place_unmanaged(root, "a", SKILL_BODY) + outside = tmp_path / "outside" + _place_unmanaged(outside, "a", "user authored\n") + race = _SwapRootBefore( + "stat", root, outside, _names_the_skill_file(root / "a") + ).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert _actions_by_key(report)["a"].action == "skipped_current", ( + _error_messages(report) + ) + assert "a/SKILL.md" in _read_manifest(race.moved_to)["entries"] + assert (race.moved_to / "a" / "SKILL.md").read_text( + encoding="utf-8" + ) == SKILL_BODY + assert (outside / "a" / "SKILL.md").read_text( + encoding="utf-8" + ) == "user authored\n" + assert not (outside / MANIFEST_NAME).exists() + + @_needs_dir_fd + async def test_a_root_swapped_before_the_orphan_listing_cannot_reach_outside( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The orphan sweep lists the pinned directory, not the path. + + The removals were already descriptor-relative, so a listing redirected + into the attacker's tree could never have unlinked anything there; what + it could do is come back with the wrong names — the attacker's, which + the pinned unlink then fails to find — so the real orphan is never + swept and keeps its directory pinned forever. Listed off the descriptor, + the real orphan is found and removed, and the attacker's temp file is + neither listed nor touched. + """ + root = tmp_path / "skills" + root.mkdir() + _place_managed(root, "a", SKILL_BODY) + real_orphan = root / "a" / _temp_name("0123456789abcdef") + real_orphan.write_text("half-written body", encoding="utf-8") + outside = tmp_path / "outside" + (outside / "a").mkdir(parents=True) + planted = outside / "a" / _temp_name("fedcba9876543210") + planted.write_text("bait", encoding="utf-8") + race = _SwapRootBefore( + "listdir", root, outside, _lists_the_skill_directory(root / "a") + ).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert report.ok is True, _error_messages(report) + assert not (race.moved_to / "a" / real_orphan.name).exists() + assert planted.read_text(encoding="utf-8") == "bait" + assert sorted(p.name for p in (outside / "a").iterdir()) == [planted.name] + + @_needs_dir_fd + async def test_every_destructive_call_runs_relative_to_a_descriptor( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The invariant behind the three races, asserted directly. + + Each test above plants one swap and checks the blast radius, so each + covers the one sequence it races. This covers the property they are + each an instance of: across a reconcile that creates a directory, + writes a file, renames a temp over it, writes the manifest, unlinks and + removes the directory, *no* destructive call names an absolute path. + Every one passes a bare component and a descriptor to resolve it + against, which is what makes the swap unable to redirect any of them. + + Written as an audit rather than another race because the failure mode is + a new call site, not a new attack: one operation added on the full path + would reopen the window for that operation alone, and no swap test aimed + at the existing sequences would notice. + + The reads that *decide* those calls are held to the same bar. The + existence probe and the compare read name ``SKILL.md``, the manifest + read names the manifest, and each must pass the bare name and a + descriptor; the orphan listing must be handed the descriptor itself. + ``_unsafe_path_reason`` is stubbed out for the run: its checks stat + absolute paths on purpose, as defense in depth ahead of the pin, and + they are not what this audit is about. With them out of the way, every + remaining read that names the skill file or the manifest is a decision + read, and must be pinned. + """ + monkeypatch.setattr( + skills_fs_module, "_unsafe_path_reason", lambda *args, **kwargs: None + ) + destructive = ("mkdir", "rmdir", "unlink", "replace") + deciding = ("stat", "open", "listdir") + recorded: list[tuple[str, Any, bool]] = [] + + def recorder(name: str) -> Any: + real = getattr(os, name) + + def wrapper(first: Any, *args: Any, **kwargs: Any) -> Any: + recorded.append( + ( + name, + os.fspath(first) + if isinstance(first, (str, os.PathLike)) + else first, + # replace takes src_dir_fd/dst_dir_fd rather than dir_fd. + any("dir_fd" in keyword for keyword in kwargs), + ) + ) + return real(first, *args, **kwargs) + + return wrapper + + for name in destructive + deciding: + monkeypatch.setattr(safe_fs_module.os, name, recorder(name)) + + written = await write_skills([_skill("a")], root) + pruned = await write_skills([], root) + + assert written.ok is True, _error_messages(written) + assert pruned.ok is True, _error_messages(pruned) + # mkdir, replace (the skill file), replace (the manifest), unlink, + # rmdir, replace (the manifest again) — the sequence must have run, or + # the audit below is vacuous. + assert {entry[0] for entry in recorded} >= set(destructive), recorded + assert [ + entry + for entry in recorded + if entry[0] in destructive + and ( + not isinstance(entry[1], str) or os.path.isabs(entry[1]) or not entry[2] + ) + ] == [] + + # The decision reads. The prune run's probe found the file and the + # write run's found none, so both answers were given through a + # descriptor; the sweep listed on both runs. + about_a_file = [ + entry + for entry in recorded + if entry[0] in ("stat", "open") + and isinstance(entry[1], str) + and entry[1].endswith(("SKILL.md", MANIFEST_NAME)) + ] + assert about_a_file != [], recorded + assert [ + entry for entry in about_a_file if os.path.isabs(entry[1]) or not entry[2] + ] == [] + listings = [entry for entry in recorded if entry[0] == "listdir"] + assert listings != [], recorded + assert [entry for entry in listings if not isinstance(entry[1], int)] == [] + + +class TestWithoutDirFd: + """The full-path fallback for platforms with no ``*at()`` family. + + On Windows ``os.open`` cannot open a directory at all, so acquiring the + descriptor must not even be attempted there — a fallback reached only after + a descriptor open would leave every write, prune and manifest rewrite + failing rather than falling back. These tests force the flag off so the + fallback is exercised on POSIX too. + """ + + @pytest.fixture(autouse=True) + def _no_dir_fd(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Models Windows: no ``*at()`` family, and directories cannot be opened. + + Forcing the flag off alone would not reproduce the platform, because + ``os.open`` on a directory succeeds on POSIX — the fallback would be + reached either way. Making that call raise the ``PermissionError`` + Windows raises is what proves the descriptor open is never attempted. + """ + monkeypatch.setattr(safe_fs_module, "SUPPORTS_DIR_FD", False) + real_open = os.open + + def no_directory_open(path: Any, *args: Any, **kwargs: Any) -> int: + if os.path.isdir(path): + raise PermissionError(13, "Permission denied") + return real_open(path, *args, **kwargs) + + monkeypatch.setattr(safe_fs_module.os, "open", no_directory_open) + + async def test_write_prune_and_manifest_all_succeed(self, root: Path) -> None: + first = await write_skills([_skill("a"), _skill("b")], root) + assert first.ok is True, _error_messages(first) + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + assert _manifest_path(root).exists() + assert stat.S_IMODE((root / "a" / "SKILL.md").stat().st_mode) == 0o644 + + second = await write_skills([_skill("a")], root) + + assert second.ok is True, _error_messages(second) + assert not (root / "b" / "SKILL.md").exists() + assert "b/SKILL.md" not in _read_manifest(root)["entries"] + + async def test_a_symlinked_skill_directory_is_still_refused( + self, root: Path, tmp_path: Path + ) -> None: + """The fallback keeps the ``lstat`` floor: no writing through a link.""" + outside = tmp_path / "outside" + outside.mkdir() + (root / "a").symlink_to(outside, target_is_directory=True) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert list(outside.iterdir()) == [] + + +class TestNonRegularFiles: + """A managed path that is not a regular file is refused, never read.""" + + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="no FIFOs on this platform") + async def test_a_fifo_at_the_managed_path_does_not_block(self, root: Path) -> None: + """Reading a FIFO with no writer blocks forever. + + Same attacker capability the symlink checks defend against: swapping a + managed ``SKILL.md`` for a FIFO would otherwise hang the whole reconcile + — and the caller's event loop with it — well past any ``timeout``, since + the deadline is only consulted between steps. + """ + skill_dir = root / "a" + skill_dir.mkdir() + os.mkfifo(skill_dir / "SKILL.md") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 1, SKILL_BODY)}, + }, + ) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "regular file" in action.error + assert stat.S_ISFIFO(os.lstat(skill_dir / "SKILL.md").st_mode) + + +class TestClobberProtection: + """Destructive ops only on manifest-listed paths.""" + + async def test_unmanaged_file_is_never_overwritten(self, root: Path) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text("user authored\n", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert target.read_text(encoding="utf-8") == "user authored\n" + + async def test_unmanaged_file_is_never_deleted(self, root: Path) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text("user authored\n", encoding="utf-8") + + await write_skills([], root) + + assert target.read_text(encoding="utf-8") == "user authored\n" + + async def test_manifest_entry_with_mismatched_key_does_not_authorize( + self, root: Path + ) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text("user authored\n", encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "a/SKILL.md": _entry("different-key", 1, "user authored\n") + }, + }, + ) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert target.read_text(encoding="utf-8") == "user authored\n" + + +DIVERGENT_CONTENT = "existing content\n" + + +def _live_entries() -> dict[str, Any]: + """A parseable entries map that really does claim ``a/SKILL.md`` as managed.""" + return {"a/SKILL.md": _entry("a", 1, DIVERGENT_CONTENT)} + + +# The first six variants are unparseable: ``entries`` is missing, the wrong type, +# or the whole document is garbage. That makes "performed no destructive action" +# arithmetic rather than a defense — with no entries to act on, a file at a +# managed path is protected by clobber protection and there is nothing to prune, +# so those cases pass against an implementation that simply treats a corrupt +# manifest as an empty one. +# +# The ``*_live_entries`` variants are the ones that actually test round-tripping: corrupt +# ONLY in ``manifestVersion``, with a valid entries map listing the managed path +# under a matching key. The implementation has everything it needs to overwrite +# and to prune, and must refuse anyway. +CORRUPT_MANIFESTS: list[tuple[str, Any]] = [ + ("garbage", "{not json at all"), + ("empty", ""), + ("wrong_types", {"manifestVersion": 1, "entries": ["a/SKILL.md"]}), + ("entries_missing", {"manifestVersion": 1}), + ("future_version", {"manifestVersion": 2, "entries": {}}), + ("version_not_int", {"manifestVersion": "1", "entries": {}}), + ("future_version_live_entries", {"manifestVersion": 2, "entries": _live_entries()}), + ( + "version_not_int_live_entries", + {"manifestVersion": "1", "entries": _live_entries()}, + ), + # Bounded below as well as above: 1 is the first version ever written, so a + # manifest declaring 0 or a negative is not one this SDK produced. + ("version_zero", {"manifestVersion": 0, "entries": {}}), + ( + "version_negative_live_entries", + {"manifestVersion": -1, "entries": _live_entries()}, + ), +] + +LIVE_ENTRY_MANIFESTS: list[tuple[str, Any]] = [ + case for case in CORRUPT_MANIFESTS if case[0].endswith("_live_entries") +] + + +class TestOversizeManifest: + """A manifest past the read cap is corruption, not a reason to allocate. + + Every other read in the reconcile is bounded. The manifest is a plain file + in a directory the SDK does not own exclusively, so an unbounded read of it + is a way to have a reconcile exhaust the process. + """ + + async def test_a_manifest_over_the_cap_is_refused_non_destructively( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + target = _place_managed(root, "a", SKILL_BODY) + # Derived from the cap rather than written as a literal, so raising the + # bound cannot leave this test passing against a read it no longer bounds. + filler = "x" * (skills_fs_module._MAX_MANIFEST_BYTES + 1) + _write_manifest(root, {"manifestVersion": 1, "entries": {}, "pad": filler}) + spy = _UnlinkSpy().install(monkeypatch) + + report = await write_skills([], root) + + assert report.ok is False + assert any("cap" in m for m in _error_messages(report)) + # Non-destructive on both counts: nothing removed, and the manifest the + # SDK could not read is left for an operator rather than overwritten. + assert spy.targets == [] + assert target.read_text(encoding="utf-8") == SKILL_BODY + assert "pad" in json.loads(_manifest_path(root).read_text(encoding="utf-8")) + + async def test_a_manifest_at_the_cap_is_still_read(self, root: Path) -> None: + """The positive control — the bound is a cap, not an off-by-one refusal.""" + manifest: dict[str, Any] = {"manifestVersion": 1, "entries": {}} + # Pad to exactly the cap, accounting for the rest of the document. + overhead = len(json.dumps({**manifest, "pad": ""}).encode("utf-8")) + manifest["pad"] = "x" * (skills_fs_module._MAX_MANIFEST_BYTES - overhead) + _write_manifest(root, manifest) + assert ( + _manifest_path(root).stat().st_size == skills_fs_module._MAX_MANIFEST_BYTES + ) + + report = await write_skills([], root) + + assert report.ok is True + # Read, so the unknown field round-trips as any other future field does. + assert "pad" in json.loads(_manifest_path(root).read_text(encoding="utf-8")) + + +class TestCorruptManifest: + """Corrupt manifest fails closed, non-destructively.""" + + @pytest.mark.parametrize( + "raw", + [case[1] for case in CORRUPT_MANIFESTS], + ids=[case[0] for case in CORRUPT_MANIFESTS], + ) + async def test_no_destructive_action_and_error_reported( + self, root: Path, raw: Any + ) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text(DIVERGENT_CONTENT, encoding="utf-8") + _write_manifest(root, raw) + + report = await write_skills([_skill("a", 2, "new content\n")], root) + + assert report.ok is False + # The error must name the manifest. For the unparseable variants the file + # at the managed path is also unmanaged, so a bare "some error happened" + # assertion is satisfied by clobber protection alone and says nothing + # about whether the manifest state was detected at all. + errors = _error_messages(report) + assert any("manifest" in e.lower() for e in errors), errors + assert target.read_text(encoding="utf-8") == DIVERGENT_CONTENT + + async def test_run_level_error_carries_the_empty_key_sentinel( + self, root: Path + ) -> None: + """A run-level error has no skill key to hang off. + + The empty string is public API surface: a caller grouping the report by + key has to know the sentinel exists. Asserted here rather than in the + parametrized cases above so it is a statement about the manifest error + specifically, not about whichever error happens to come first. + """ + _write_manifest(root, "{not json at all") + + report = await write_skills([_skill("a")], root) + + manifest_errors = [ + action + for action in report.errors + if "manifest" in (action.error or "").lower() + ] + assert manifest_errors, _error_messages(report) + assert all(action.key == "" for action in manifest_errors) + # A per-skill error in the same report still carries its real key, so the + # sentinel is not simply "every error action has an empty key". + assert all( + action.key != "" + for action in report.errors + if action not in manifest_errors + ) + + @pytest.mark.parametrize( + "raw", + [case[1] for case in LIVE_ENTRY_MANIFESTS], + ids=[case[0] for case in LIVE_ENTRY_MANIFESTS], + ) + async def test_managed_file_is_not_pruned_when_only_the_version_is_corrupt( + self, root: Path, raw: Any + ) -> None: + """The prune counterpart of the live-entries cases. + + Here the implementation can read the entries map and knows exactly which + file it owns, so refusing to remove it is a real decision rather than an + absence of information. + """ + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text(DIVERGENT_CONTENT, encoding="utf-8") + _write_manifest(root, raw) + + report = await write_skills([], root) + + assert report.ok is False + assert target.read_text(encoding="utf-8") == DIVERGENT_CONTENT + assert [a for a in report.actions if a.action == "removed"] == [] + errors = _error_messages(report) + assert any("manifest" in e.lower() for e in errors), errors + + async def test_nothing_is_pruned_under_a_corrupt_manifest(self, root: Path) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text(DIVERGENT_CONTENT, encoding="utf-8") + _write_manifest(root, "{not json at all") + + report = await write_skills([], root) + + assert report.ok is False + assert target.exists() + assert [a for a in report.actions if a.action == "removed"] == [] + errors = _error_messages(report) + assert any("manifest" in e.lower() for e in errors), errors + + async def test_brand_new_paths_may_still_be_written(self, root: Path) -> None: + _write_manifest(root, "{not json at all") + + report = await write_skills([_skill("fresh")], root) + + actions = _actions_by_key(report) + assert actions["fresh"].action == "written" + assert (root / "fresh" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_corrupt_manifest_file_is_not_destroyed(self, root: Path) -> None: + _write_manifest(root, "{not json at all") + + await write_skills([], root) + + assert _manifest_path(root).exists() + assert _manifest_path(root).read_text(encoding="utf-8") == "{not json at all" + + +# The three literal cases the security review names for the prune path. +# +# The distinction from ``TestCorruptManifest`` above is the whole point: a +# corrupt manifest suppresses every destructive action wholesale, so those tests +# say nothing about these. Each manifest here is *well-formed* — parseable, +# a ``manifestVersion`` this release understands, a real ``entries`` map, and an +# entry whose ``key`` is a perfectly valid skill key that is genuinely absent +# from the requested set. The implementation has every input it needs to prune +# and must refuse anyway, because the recorded *path* is not one this SDK could +# have written. +HOSTILE_RECORDED_PATHS: list[str] = [ + # Absolute: the classic. A recorded path read back and unlinked as-is is a + # delete of an attacker-chosen file with the reconcile's privileges. + "/etc/passwd", + # Traversing: the same attack for an implementation that rejects a leading + # slash and then joins the rest onto the root. + "../../../etc/passwd", +] + + +class _UnlinkSpy: + """Records every ``os.unlink`` while delegating to the real one. + + Asserting only that ``/etc/passwd`` still exists proves nothing: the test + process cannot delete it anyway, so that assertion passes against an + implementation with no path check at all — permissions would be doing the + work. What has teeth is that the removal is never *attempted*: the refusal + happens above the syscall, on a path the SDK recomputes rather than trusts. + """ + + def __init__(self) -> None: + self.targets: list[str] = [] + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _UnlinkSpy: + real = os.unlink + + def spy(path: Any, *args: Any, **kwargs: Any) -> None: + self.targets.append(os.fsdecode(path)) + real(path, *args, **kwargs) + + # ``safe_fs_module.os`` *is* the ``os`` module, so this covers both the + # descriptor-relative ``os.unlink(name, dir_fd=...)`` and the + # ``Path.unlink`` used on the no-``*at()`` floor. + monkeypatch.setattr(safe_fs_module.os, "unlink", spy) + return self + + +class TestHostileManifestPrune: + """A well-formed manifest naming a path this SDK could not have written. + + The manifest is untrusted input. It is a plain file on the customer's disk + that anything with write access to the managed root can edit, and ``prune`` + is the one code path in the SDK that deletes. So a recorded path never + authorizes its own removal: it must match ``/SKILL.md`` for a + re-validated key, and the target is recomputed from the *current* managed + root instead of being read back out of the entry. + """ + + @pytest.mark.parametrize("recorded", HOSTILE_RECORDED_PATHS) + async def test_recorded_path_outside_the_root_is_refused( + self, root: Path, recorded: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + spy = _UnlinkSpy().install(monkeypatch) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {recorded: _entry("a", 1, SKILL_BODY)}, + }, + ) + + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + # The refusal is about ownership of the path, not about the file's state. + assert "could own" in action.error + assert [a for a in report.actions if a.action == "removed"] == [] + # Nothing was even attempted, let alone completed. + assert spy.targets == [] + assert Path("/etc/passwd").exists() + # Left in place rather than tidied away: dropping the entry would let a + # single hostile edit erase the SDK's own record of what it manages. + assert recorded in _read_manifest(root)["entries"] + + async def test_entry_under_a_since_symlinked_parent_is_refused( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The recorded path is the SDK's own, and is still not enough. + + Here the entry is exactly what a legitimate reconcile writes — + ``a/SKILL.md`` under key ``a`` — so the shape check that catches the two + cases above passes. What changed is the disk underneath it: ``/a`` + is now a symlink to somewhere else. This is the case a validate-then-act + implementation fails, because the manifest and the entry are both + entirely legitimate; only the current state of the parent is not. + """ + root = tmp_path / "skills" + root.mkdir() + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + victim = elsewhere / "SKILL.md" + victim.write_text("victim content\n", encoding="utf-8") + + # Managed legitimately first, so the manifest entry is one this SDK + # really did write... + managed = _place_managed(root, "a", SKILL_BODY) + # ...then the parent directory is swapped for a link out of the root. + managed.unlink() + (root / "a").rmdir() + (root / "a").symlink_to(elsewhere, target_is_directory=True) + + spy = _UnlinkSpy().install(monkeypatch) + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "symlink" in action.error + assert [a for a in report.actions if a.action == "removed"] == [] + assert spy.targets == [] + # The file the symlink pointed at is untouched, and so is the link. + assert victim.read_text(encoding="utf-8") == "victim content\n" + assert (root / "a").is_symlink() + assert "a/SKILL.md" in _read_manifest(root)["entries"] + + +class TestWriteSkillsTelemetry: + """Materialized / revoked signals from write_skills.""" + + async def test_materialized_signal_per_action( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + _place_managed(root, "same", SKILL_BODY) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "same/SKILL.md": _entry("same", 1, SKILL_BODY), + "stale/SKILL.md": _entry("stale", 1, "old\n"), + }, + }, + ) + (root / "stale").mkdir() + (root / "stale" / "SKILL.md").write_text("old\n", encoding="utf-8") + + await write_skills( + [_skill("same"), _skill("stale", 2, "fresh\n"), _skill("brand-new")], + root, + ) + + signals = recording_emitter.signals(MATERIALIZED_SIGNAL) + by_key = {s["skill_key"]: s for s in signals} + assert len(signals) == 3 + assert by_key["same"]["reconcile_action"] == "skipped_current" + assert by_key["stale"]["reconcile_action"] == "updated" + assert by_key["brand-new"]["reconcile_action"] == "written" + + async def test_materialized_signal_properties( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + + await write_skills([_skill("a")], root) + + props = recording_emitter.signals(MATERIALIZED_SIGNAL)[0] + assert props["skill_key"] == "a" + assert props["content_bytes"] == len(SKILL_BODY.encode("utf-8")) + assert props["content_hash"] == _hash(SKILL_BODY) + assert props["reconcile_action"] == "written" + assert props["language"] == "python" + + async def test_no_filesystem_paths_in_telemetry( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + + await write_skills([_skill("a")], root) + + for _signal, props in recording_emitter.records: + assert "target_path" not in props + for value in props.values(): + assert str(root) not in str(value) + + async def test_no_skill_body_in_telemetry( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + + await write_skills([_skill("a")], root) + + for _signal, props in recording_emitter.records: + for value in props.values(): + assert "Do the thing." not in str(value) + + async def test_revoked_signal_on_prune( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + _place_managed(root, "gone", SKILL_BODY, version=4) + + await write_skills([], root) + + revoked = recording_emitter.signals(REVOKED_SIGNAL) + assert len(revoked) == 1 + assert revoked[0]["skill_key"] == "gone" + assert revoked[0]["version"] == 4 + assert revoked[0]["removed_from_disk"] is True + assert revoked[0]["language"] == "python" + + async def test_revoked_signal_redacts_an_untrusted_manifest_version( + self, root: Path, recording_emitter: Any + ) -> None: + """The manifest is untrusted, so its version is shape-checked first. + + Anything with write access to the managed root can plant an arbitrary + string here; echoing it verbatim would publish attacker-controlled + content — a skill body, or PII — as a signal property. + """ + skills_module._set_emitter_for_testing(recording_emitter) + target = root / "gone" / "SKILL.md" + target.parent.mkdir(parents=True) + target.write_text(SKILL_BODY, encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "gone/SKILL.md": { + "key": "gone", + "version": "Do the thing. " * 8, + "sha256": _hash(SKILL_BODY), + } + }, + }, + ) + + await write_skills([], root) + + revoked = recording_emitter.signals(REVOKED_SIGNAL) + assert len(revoked) == 1 + assert "version" not in revoked[0] + assert revoked[0]["skill_key"] == "gone" + for value in revoked[0].values(): + assert "Do the thing." not in str(value) + + async def test_no_revoked_signal_when_prune_disabled( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + _place_managed(root, "gone", SKILL_BODY) + + await write_skills([], root, prune=False) + + assert recording_emitter.signals(REVOKED_SIGNAL) == [] + + async def test_write_skills_records_no_signal_outside_the_approved_set( + self, root: Path, recording_emitter: Any + ) -> None: + """Allowlist sweep over a run that exercises all four actions. + + The accessor-side half of this sweep is + ``test_accessors_record_no_signal_outside_the_approved_set`` in + test_skills.py. Asserted over recorded strings, so no module-level + signal-name constant is required of the implementation. + """ + skills_module._set_emitter_for_testing(recording_emitter) + for key, content in (("same", SKILL_BODY), ("stale", "old\n"), ("gone", "g\n")): + (root / key).mkdir() + (root / key / "SKILL.md").write_text(content, encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "same/SKILL.md": _entry("same", 1, SKILL_BODY), + "stale/SKILL.md": _entry("stale", 1, "old\n"), + "gone/SKILL.md": _entry("gone", 1, "g\n"), + }, + }, + ) + + report = await write_skills( + [_skill("same"), _skill("stale", 2, "fresh\n"), _skill("brand-new")], + root, + ) + + # Positive control: the subset assertion is vacuous unless the run + # really did produce all four actions and record for them. + assert {a.action for a in report.actions} == { + "skipped_current", + "updated", + "written", + "removed", + } + recorded = {signal for signal, _props in recording_emitter.records} + assert recorded <= APPROVED_SIGNALS, ( + f"unapproved signal(s): {sorted(recorded - APPROVED_SIGNALS)}" + ) + assert not recorded & REMOVED_SIGNALS + assert recorded == {MATERIALIZED_SIGNAL, REVOKED_SIGNAL} + + async def test_no_ld_track_calls_from_write_skills( + self, root: Path, mock_ld_client: Any + ) -> None: + await init_client(client=mock_ld_client) + # init_client flushes $ld:ai:sdk:info on the first client of a process, + # so it may or may not have fired depending on what ran before. This + # test is about write_skills, so start counting from here. + mock_ld_client.track.reset_mock() + + await write_skills([_skill("a"), _skill("../evil")], root) + + mock_ld_client.track.assert_not_called() + + async def test_throwing_emitter_never_breaks_the_reconcile( + self, root: Path, throwing_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(throwing_emitter) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_integrity_signal_property_keys_match_across_layers( + self, root: Path, recording_emitter: Any, oversize_content: tuple[str, str] + ) -> None: + """The same defect, caught at either layer, records + the same property keys. + + Verification runs twice by design: once at the accessor boundary and + again immediately before a write. The signal contract marks ``expected_hash`` + optional, so an implementation that populates it on one path and omits + it on the other passes every other assertion here while making the + signal's shape depend on which internal code path noticed. Oversize + content is the case reachable from both layers with the expected hash in + hand throughout. + """ + skills_module._set_emitter_for_testing(recording_emitter) + oversize, content_hash = oversize_content + + # Layer 1 — the accessor boundary. + store = InMemorySkillStore() + store.put( + { + "key": "big", + "version": 1, + "content": oversize, + "contentHash": content_hash, + } + ) + skills_module._set_store(store) + assert await get_skill("big") is None + + # Layer 2 — verify-then-write, on a directly constructed Skill. + report = await write_skills( + [ + Skill( + key="big", + version=1, + content=oversize.encode("utf-8"), + content_hash=content_hash, + ) + ], + root, + ) + assert report.ok is False + + failures = recording_emitter.signals(INTEGRITY_SIGNAL) + assert len(failures) == 2, failures + accessor_keys, write_keys = (set(props) for props in failures) + assert accessor_keys == write_keys, ( + f"accessor-only keys: {sorted(accessor_keys - write_keys)}; " + f"write-only keys: {sorted(write_keys - accessor_keys)}" + ) + assert "expected_hash" in accessor_keys + + +# --------------------------------------------------------------------------- +# Self-healing partial reconciles +# --------------------------------------------------------------------------- + + +def _place_unmanaged(root: Path, key: str, content: str) -> Path: + """A file at a managed path with **no** manifest entry. + + Exactly the state a reconcile killed between the content writes and the + final manifest rewrite leaves behind — and, indistinguishably on disk, the + state a customer authoring their own file there creates. Which is why the + bytes are the only thing that may decide between them. + """ + target = root / key / "SKILL.md" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return target + + +class TestCrashMidReconcileRecovery: + """A crash between the writes and the manifest rewrite must not wedge a skill.""" + + async def test_byte_identical_unmanaged_file_is_adopted(self, root: Path) -> None: + """The whole point: the second reconcile repairs the first one's crash. + + Without adoption every later reconcile takes the clobber-refusal branch + forever, because the file is at a managed path with no manifest entry. + """ + target = _place_unmanaged(root, "a", SKILL_BODY) + + first = await write_skills([_skill("a")], root) + + assert first.ok is True, _error_messages(first) + action = _actions_by_key(first)["a"] + assert action.action == "skipped_current" + assert action.version == 1 + assert action.path == str(target) + # Adopted, not rewritten, and now recorded. + assert target.read_text(encoding="utf-8") == SKILL_BODY + entry = _read_manifest(root)["entries"]["a/SKILL.md"] + assert entry["key"] == "a" + assert entry["version"] == 1 + assert entry["sha256"] == _hash(SKILL_BODY) + + # And the run after it is an ordinary no-op, through the managed path. + second = await write_skills([_skill("a")], root) + assert second.ok is True, _error_messages(second) + assert _actions_by_key(second)["a"].action == "skipped_current" + + async def test_adoption_writes_nothing( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Adoption is a manifest edit, not a write. Nothing touches the bytes.""" + _place_unmanaged(root, "a", SKILL_BODY) + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert spy.calls == [] + + async def test_adoption_reports_skipped_current_not_a_new_action_kind( + self, root: Path, recording_emitter: Any + ) -> None: + """``skipped_current`` is reused deliberately — no ``adopted`` kind exists.""" + skills_module._set_emitter_for_testing(recording_emitter) + _place_unmanaged(root, "a", SKILL_BODY) + + report = await write_skills([_skill("a")], root) + + assert {a.action for a in report.actions} == {"skipped_current"} + props = recording_emitter.signals(MATERIALIZED_SIGNAL)[0] + assert props["reconcile_action"] == "skipped_current" + assert props["skill_key"] == "a" + + async def test_an_adopted_file_is_prunable_afterwards(self, root: Path) -> None: + """The documented caveat, pinned. + + Adoption records a manifest entry, so a later reconcile may prune the + file. That is correct rather than a weakening: only content byte-identical + to what LaunchDarkly resolved is ever adopted, so the prune removes + content LaunchDarkly delivered — exactly what would have happened had the + crash never occurred. + """ + target = _place_unmanaged(root, "a", SKILL_BODY) + await write_skills([_skill("a")], root) + + report = await write_skills([], root) + + assert report.ok is True, _error_messages(report) + assert _actions_by_key(report)["a"].action == "removed" + assert not target.exists() + + async def test_differing_unmanaged_content_is_still_refused( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The clobber guarantee, restated against the adoption rule. + + Adoption compares bytes, so anything that is not byte-identical to the + resolved content falls through to the same refusal as before. + """ + target = _place_unmanaged(root, "a", "user authored\n") + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "did not write" in action.error + assert target.read_text(encoding="utf-8") == "user authored\n" + assert spy.calls == [] + + async def test_a_longer_file_sharing_the_content_prefix_is_not_adopted( + self, root: Path + ) -> None: + """The read is bounded at ``len(content) + 1``, and that one byte matters. + + A bound of exactly ``len(content)`` would make every file that merely + *begins* with the resolved content hash as current, adopting — and later + pruning — a customer file with the skill body at its head. + """ + longer = SKILL_BODY + "and my own notes below\n" + target = _place_unmanaged(root, "a", longer) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert target.read_text(encoding="utf-8") == longer + + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="no FIFOs on this platform") + async def test_an_unmanaged_fifo_is_refused_and_never_read( + self, root: Path + ) -> None: + """Adoption widened the read to foreign files, so this refusal is load-bearing. + + Opening a FIFO with no writer blocks forever; the descriptor-pinned read + opens ``O_NONBLOCK`` and rejects anything that is not a regular file, so + this returns rather than hanging the reconcile and the event loop with it. + """ + skill_dir = root / "a" + skill_dir.mkdir() + os.mkfifo(skill_dir / "SKILL.md") + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "regular file" in action.error + assert stat.S_ISFIFO(os.lstat(skill_dir / "SKILL.md").st_mode) + + async def test_a_read_failure_on_an_unmanaged_file_refuses( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A failed read proves nothing, so it must never become an overwrite. + + The adoption comparison is what would otherwise authorize the write, and + a file whose bytes could not be read has not been shown to be ours. + """ + target = _place_unmanaged(root, "a", SKILL_BODY) + spy = _ReplaceSpy().install(monkeypatch) + real_open = os.open + + def refuse_the_target(path: Any, *args: Any, **kwargs: Any) -> int: + # Both spellings, for the same reason ``_SwapRootDuring`` matches + # both: the compare read names the file as the bare ``SKILL.md`` + # relative to the pinned skill directory wherever the platform has + # descriptors, and as the full path on the lstat floor. + if isinstance(path, (str, os.PathLike)): + named = os.fspath(path) + if named == str(target) or ( + named == "SKILL.md" and kwargs.get("dir_fd") is not None + ): + raise PermissionError(13, "Permission denied") + return real_open(path, *args, **kwargs) + + monkeypatch.setattr(skills_fs_module.os, "open", refuse_the_target) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + # Distinguishable from the byte-mismatch refusal: this one says the + # comparison could not be made at all. + assert "could not be read to compare" in action.error + assert spy.calls == [] + assert target.read_text(encoding="utf-8") == SKILL_BODY + assert "a/SKILL.md" not in _read_manifest(root)["entries"] + + +# --------------------------------------------------------------------------- +# Orphaned temp files +# --------------------------------------------------------------------------- + + +def _temp_name(token: str = "0123456789abcdef") -> str: + """A name ``atomic_write`` could have created for ``SKILL.md``. + + The prefix comes from ``safe_fs`` itself rather than a copy of its format + string, so a change to the naming breaks this helper instead of silently + making the sweep a no-op. + """ + return f"{safe_fs_module.temp_name_prefix('SKILL.md')}{token}.tmp" + + +class TestOrphanedTempFiles: + """A ``SIGKILL`` mid-write leaves a temp file nothing else records.""" + + async def test_an_orphan_is_swept_on_the_next_write(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY) + orphan = root / "a" / _temp_name() + orphan.write_text("half-written body", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert not orphan.exists() + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_an_orphan_no_longer_blocks_directory_cleanup( + self, root: Path + ) -> None: + """The second-order effect: ``rmdir`` fails on a non-empty directory. + + One orphaned temp file would otherwise pin the skill's directory under + the managed root forever, long after the skill itself was revoked. + """ + _place_managed(root, "a", SKILL_BODY) + orphan = root / "a" / _temp_name() + orphan.write_text("half-written body", encoding="utf-8") + + report = await write_skills([], root) + + assert report.ok is True, _error_messages(report) + assert _actions_by_key(report)["a"].action == "removed" + assert not (root / "a").exists() + + @pytest.mark.parametrize( + "innocent", + [ + "notes.tmp", + "SKILL.md.tmp", + ".SKILL.md.tmp", + ".SKILL.md..tmp", + _temp_name("not-a-token"), + _temp_name("0123456789abcdef") + ".bak", + "x" + _temp_name(), + _temp_name("0123456789abcdefff"), + ], + ) + async def test_a_lookalike_name_is_left_alone( + self, root: Path, innocent: str + ) -> None: + """The recognizer is anchored at both ends, and the sweep deletes files. + + Anything that is not exactly the naming ``safe_fs`` produces belongs to + the customer, whatever it resembles. + """ + _place_managed(root, "a", SKILL_BODY) + bystander = root / "a" / innocent + bystander.write_text("mine\n", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert bystander.read_text(encoding="utf-8") == "mine\n" + + @pytest.mark.skipif( + not hasattr(os, "symlink"), reason="platform has no symlink support" + ) + async def test_a_symlink_wearing_the_temp_name_is_not_removed( + self, root: Path, tmp_path: Path + ) -> None: + """The temp naming must not become a way to have the SDK delete elsewhere. + + Only a regular file is ever swept, and the type comes off the descriptor + rather than a followed path. + """ + outside = tmp_path / "precious.txt" + outside.write_text("do not delete\n", encoding="utf-8") + _place_managed(root, "a", SKILL_BODY) + link = root / "a" / _temp_name() + link.symlink_to(outside) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert outside.read_text(encoding="utf-8") == "do not delete\n" + assert link.is_symlink() + + +# --------------------------------------------------------------------------- +# Windows reserved device names +# --------------------------------------------------------------------------- + +# Spelled out independently of the implementation's own set, so a name dropped +# from that set fails here rather than agreeing with itself. +WINDOWS_RESERVED_KEYS = ( + ["con", "prn", "aux", "nul"] + + [f"com{digit}" for digit in range(1, 10)] + + [f"lpt{digit}" for digit in range(1, 10)] +) + + +class TestWindowsReservedNames: + """Keys Windows cannot hold as directory names, refused on every platform.""" + + def test_the_set_is_exactly_twenty_two_names(self) -> None: + assert len(WINDOWS_RESERVED_KEYS) == len(set(WINDOWS_RESERVED_KEYS)) == 22 + + @pytest.mark.parametrize("reserved", WINDOWS_RESERVED_KEYS) + async def test_a_reserved_key_is_refused_by_write_skills( + self, root: Path, reserved: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill(reserved)], root) + + assert report.ok is False + action = _actions_by_key(report)[reserved] + assert action.action == "error" + assert action.error is not None + assert "Windows reserves" in action.error + assert reserved in action.error + # Rejected before any filesystem call, not by the OS. + assert spy.calls == [] + assert not (root / reserved).exists() + + @pytest.mark.parametrize("reserved", WINDOWS_RESERVED_KEYS) + async def test_a_reserved_key_is_refused_by_the_prune_path( + self, root: Path, reserved: str + ) -> None: + """``_key_rejection_reason`` gates both destructive paths, so both refuse. + + A manifest naming a reserved key is left in place rather than acted on: + the same key check that stops the write stops the delete. + """ + target = _place_managed(root, reserved, SKILL_BODY) + + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)[reserved] + assert action.action == "error" + assert action.error is not None + assert "left in place" in action.error + assert target.read_text(encoding="utf-8") == SKILL_BODY + + @pytest.mark.parametrize("not_reserved", ["com0", "lpt0", "con1", "nul2", "conx"]) + async def test_neighbouring_names_are_not_reserved( + self, root: Path, not_reserved: str + ) -> None: + """``com0`` and ``lpt0`` are not device names, and must still write.""" + report = await write_skills([_skill(not_reserved)], root) + + assert report.ok is True, _error_messages(report) + assert (root / not_reserved / "SKILL.md").exists() + + @pytest.mark.parametrize("reserved", WINDOWS_RESERVED_KEYS) + def test_a_reserved_name_is_still_a_valid_key_to_every_pure_layer( + self, reserved: str + ) -> None: + """The layer choice, asserted — this is the whole point of it. + + The constraint lives in the filesystem layer and must not migrate into + the key grammar. At the grammar level a rejection would fail the *entire* + AI Config — model, provider, instructions, tools — for a Linux customer + over a Windows-only constraint, and would shrink ``skill_refs``, which is + what authorizes a prune: "this skill fails to write on Windows" would + become "this skill gets deleted on Linux". + """ + assert is_valid_skill_key(reserved) is True + + parsed = parse_ai_config( + { + "model": {"name": "claude-3"}, + "provider": {"name": "Anthropic"}, + "instructions": "You are helpful.", + "skills": [{"key": reserved, "version": 1}], + } + ) + assert parsed.success is True + + refs = skill_refs(parsed.data) + assert [ref.key for ref in refs] == [reserved] diff --git a/packages/client/tests/test_skills_watch.py b/packages/client/tests/test_skills_watch.py new file mode 100644 index 00000000..4c30f88b --- /dev/null +++ b/packages/client/tests/test_skills_watch.py @@ -0,0 +1,243 @@ +""" +Tests for ``watch_skills`` / ``SkillWatcher`` — the eager re-reconcile. + +The watcher is wired to the ``SkillStore`` interface, not to any one transport: +it needs a store that implements ``add_listener``, and nothing more. These tests +therefore drive it from ``InMemorySkillStore``, whose ``put`` notifies its +listeners synchronously, and from small hand-written store doubles. The +end-to-end path — a ``delete-object`` arriving over a live FDv2 connection and +pruning a skill's files — is exercised in ``test_skills_fdv2.py``, where the fake +endpoint lives. + +Every test writes only inside pytest's ``tmp_path``. The watcher runs a real +worker thread, so tests wait on observable outcomes rather than on fixed sleeps +wherever the outcome is something that *does* happen; a fixed sleep is used only +to assert that something does *not*. +""" + +from __future__ import annotations + +import time +from typing import Any + +import pytest + +from launchdarkly_ai_server import InMemorySkillStore, init_client, watch_skills +from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND + +pytestmark = pytest.mark.usefixtures("reset_skill_state") + + +def wait_until(predicate: Any, timeout: float = 5.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.01) + return predicate() + + +# --------------------------------------------------------------------------- +# Starting a watch, and what it refuses +# --------------------------------------------------------------------------- + + +class TestWatchSkills: + async def test_the_in_memory_store_can_also_drive_a_watch( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + """The watcher is wired to the ``SkillStore`` interface, not to the FDv2 + store.""" + store = InMemorySkillStore() + store.put(make_raw_skill(key="a", version=1, content="body")) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "a" / "SKILL.md" + assert written.read_text() == "body" + store.put(make_raw_skill(key="a", version=2, content="new body")) + assert wait_until(lambda: written.read_text() == "new body", timeout=10) + finally: + watcher.close() + + async def test_a_burst_of_changes_coalesces_into_few_reconciles( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.1) + try: + for i in range(12): + store.put(make_raw_skill(key=f"skill-{i}")) + time.sleep(0.5) + # Twelve objects put back to back fire twelve listener calls; without + # coalescing that is twelve reconciles of one root. + assert watcher.reconciles <= 2 + finally: + watcher.close() + + async def test_a_store_with_no_listener_support_is_refused_loudly( + self, tmp_path: Any + ) -> None: + class NoListeners: + def get_object(self, *_a: Any, **_k: Any) -> None: + return None + + def all_objects(self, _kind: str) -> dict[str, Any]: + return {} + + await init_client(options={"skillStore": NoListeners()}, client=object()) + with pytest.raises(RuntimeError, match="add_listener"): + await watch_skills("*", tmp_path / "s") + + async def test_no_store_configured_raises(self, tmp_path: Any) -> None: + with pytest.raises(RuntimeError, match="configured skill store"): + await watch_skills("*", tmp_path / "s") + + +# --------------------------------------------------------------------------- +# Changes that land while the initial reconcile is running +# --------------------------------------------------------------------------- + + +class TestChangesDuringTheInitialReconcile: + """The listener attaches before the initial reconcile, so a change delivered + while that reconcile is still running is acted on rather than lost.""" + + async def test_a_revocation_landing_mid_reconcile_is_not_missed( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + class RevokesAfterSnapshot(InMemorySkillStore): + """Revokes everything the moment the reconcile has taken its + snapshot — where a ``delete-object`` lands when it arrives a fraction + of a second into startup, with nothing after it.""" + + def __init__(self) -> None: + super().__init__() + self.snapshots = 0 + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + objects = super().all_objects(kind) + self.snapshots += 1 + if self.snapshots == 1: + self._versions.clear() + self._loose.clear() + for listener in self._listeners.get(SKILL_OBJECT_KIND, []): + listener({"key": "pdf-extraction"}) + return objects + + store = RevokesAfterSnapshot() + store.put(make_raw_skill(key="pdf-extraction", version=1, content="body")) + await init_client(options={"skillStore": store}, client=object()) + + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + # The initial reconcile wrote what its snapshot held, so the file is + # on disk and the revocation that followed it is the only change + # left to act on. + assert written.read_text() == "body" + assert wait_until(lambda: not written.exists(), timeout=10) + finally: + watcher.close() + + async def test_a_failed_initial_reconcile_leaves_no_listener_behind( + self, tmp_path: Any + ) -> None: + """Registering first means a reconcile that raises has to detach: the + caller is handed an exception, not a watcher to close.""" + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + not_a_directory = tmp_path / "file" + not_a_directory.write_text("") + + with pytest.raises(ValueError, match="not a directory"): + await watch_skills("*", not_a_directory) + + assert store._listeners.get(SKILL_OBJECT_KIND, []) == [] + + +# --------------------------------------------------------------------------- +# Closing a watch +# --------------------------------------------------------------------------- + + +class TestWatcherDetachesOnClose: + """``SkillWatcher.close`` unregisters ``notify``, so a closed watcher is + neither called nor kept alive by the store.""" + + @staticmethod + def _skill_listeners(store: Any) -> list[Any]: + return list(store._listeners.get(SKILL_OBJECT_KIND, [])) + + async def test_a_closed_watcher_is_no_longer_notified( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="pdf-extraction", version=1, content="first")) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + assert watcher.notify in self._skill_listeners(store) + + watcher.close() + + assert watcher.notify not in self._skill_listeners(store) + store.put(make_raw_skill(key="pdf-extraction", version=4, content="second")) + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + assert wait_until( + lambda: ( + store.get_object(SKILL_OBJECT_KIND, "pdf-extraction", 4) is not None + ), + timeout=10, + ) + time.sleep(0.3) + assert written.read_text() == "first" + assert watcher.reconciles == 0 + + async def test_close_twice_does_not_raise(self, tmp_path: Any) -> None: + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + watcher.close() + watcher.close() + assert self._skill_listeners(store) == [] + + async def test_repeated_watchers_leave_no_listeners_behind( + self, tmp_path: Any + ) -> None: + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + for _ in range(5): + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + assert len(self._skill_listeners(store)) == 1 + watcher.close() + assert self._skill_listeners(store) == [] + + async def test_a_store_without_remove_listener_still_closes( + self, tmp_path: Any + ) -> None: + """``remove_listener`` is optional: an older store keeps working, at the + cost of the listener staying registered.""" + + class AddOnly: + def __init__(self) -> None: + self.listeners: list[Any] = [] + + def get_object(self, *_a: Any, **_k: Any) -> None: + return None + + def all_objects(self, _kind: str) -> dict[str, Any]: + return {} + + def add_listener(self, _kind: str, fn: Any) -> None: + self.listeners.append(fn) + + store = AddOnly() + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + assert store.listeners == [watcher.notify] + + watcher.close() + watcher.close() + + assert store.listeners == [watcher.notify] diff --git a/uv.lock b/uv.lock index c3e6d327..c1c96777 100644 --- a/uv.lock +++ b/uv.lock @@ -790,7 +790,7 @@ wheels = [ [[package]] name = "launchdarkly-ai-claude-agents" -version = "0.2.1" +version = "0.2.2" source = { editable = "packages/claude-agents" } dependencies = [ { name = "anthropic" }, @@ -809,7 +809,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-claude-messages" -version = "0.2.1" +version = "0.2.2" source = { editable = "packages/claude-messages" } dependencies = [ { name = "anthropic" }, @@ -826,7 +826,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-langchain-agents" -version = "0.2.1" +version = "0.2.2" source = { editable = "packages/langchain-agents" } dependencies = [ { name = "langchain-core" }, @@ -845,7 +845,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-langchain-messages" -version = "0.2.1" +version = "0.2.2" source = { editable = "packages/langchain-messages" } dependencies = [ { name = "langchain-core" }, @@ -862,7 +862,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-openai-agents" -version = "0.2.1" +version = "0.2.2" source = { editable = "packages/openai-agents" } dependencies = [ { name = "launchdarkly-ai-server" }, @@ -881,7 +881,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-openai-messages" -version = "0.2.1" +version = "0.2.2" source = { editable = "packages/openai-messages" } dependencies = [ { name = "launchdarkly-ai-server" }, @@ -898,7 +898,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-python" -version = "0.1.5" +version = "0.1.6" source = { editable = "packages/ai" } dependencies = [ { name = "launchdarkly-ai-server" }, @@ -918,7 +918,7 @@ provides-extras = ["otel"] [[package]] name = "launchdarkly-ai-server" -version = "0.2.1" +version = "0.2.2" source = { editable = "packages/client" } dependencies = [ { name = "opentelemetry-api" },