From f1aad0afb7629d231fcffde5159ada136f586032 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 25 Aug 2026 16:12:17 -0400 Subject: [PATCH 01/16] =?UTF-8?q?test(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20the=20filesystem=20abuse=20matrix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fifth and last of five slices. The adversary. Every filesystem defense the previous two slices introduced now has a test that fails if the defense is removed, plus the materialization telemetry allowlist. - **Path traversal.** A key that escapes the root, a key that is the manifest filename, an over-long key, and a key that resolves outside after `realpath` — each refused before any filesystem call, and the resolved containment check asserted on inode identity rather than on path strings. - **Symlink attacks.** A symlinked skill directory, a symlinked target file, and the `/` directory swapped for a symlink at the exact instant of the rename and of the unlink — the narrowest version of the window the descriptor pinning exists to close, fired from the interception point rather than from implementation internals. - **The no-`*at()` shape.** The path fallback Windows takes for every operation, exercised with the capability probe forced off, so the platform that cannot pin a descriptor is not the untested one. The TOCTOU tests skip off that same flag deliberately: a probe that wrongly reported "unsupported" cannot also silently skip the tests that would have caught it. - **Non-regular files and clobber protection.** A fifo or a directory where `SKILL.md` belongs, and a file at a managed path with no matching manifest entry — reported and left alone, never overwritten and never removed. - **Corrupt manifests.** Unreadable, unparseable, not an object, malformed entries, and a `manifestVersion` this release cannot read: no overwrites, no prunes, an error action naming the manifest, and the manifest itself left as it was found. - **Atomicity.** A crash injected between the write and the rename leaves neither a partial file nor a temp file, and the one recorded rename is proved to have moved `SKILL.md` within the target's own directory — by descriptor identity where the platform has `renameat`, which also rules out the descriptor having been redirected between the check and the rename. - **Telemetry.** The three signal names are asserted as an allowlist rather than a floor: any other name reaching the emitter fails, the two deliberately excluded names are called out by name, no signal carries a filesystem path or the skill body, an emitter that raises never fails the reconcile, and `client.track()` is never reached. Two of these are worth naming, because the obvious test does not reach the guard. The unencodable-content cases pin `contentHash` to the sha256 of the bytes a non-strict encoder would have fabricated, since an arbitrary wrong hash is rejected by the mismatch check first and never exercises the encoder guard. The redaction cases smuggle the body through `contentHash` and through `key`, since a sweep using a well-formed digest under a valid key reaches neither replacement branch. Testing: `uv run pytest` → 1280 passed. `ruff check`, `ruff format --check`, and `mypy packages/*/src` all clean. Co-Authored-By: Claude Fable 5 --- packages/client/tests/test_skills_fs.py | 1071 ++++++++++++++++++++++- 1 file changed, 1064 insertions(+), 7 deletions(-) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 62d01d59..6502aa5b 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -1,13 +1,9 @@ """ -Tests for ``write_skills`` — filesystem materialization and manifest reconcile -semantics. +Tests for ``write_skills`` — filesystem materialization, manifest reconcile +semantics, and the full security abuse matrix. Every test writes only inside pytest's ``tmp_path``. No network, no real LaunchDarkly client, no real skill transport. - -The security abuse matrix — path traversal, symlink attacks, clobber -protection, corrupt manifests, atomicity under an injected crash, and the -materialization telemetry allowlist — is a separate module. """ from __future__ import annotations @@ -15,24 +11,195 @@ import hashlib import json import os +import stat from pathlib import Path -from typing import Any +from typing import Any, NamedTuple import pytest +import launchdarkly_ai_server.safe_fs as safe_fs_module import launchdarkly_ai_server.skills as skills_module +import launchdarkly_ai_server.skills_fs as skills_fs_module from launchdarkly_ai_server import ( InMemorySkillStore, Skill, SkillReference, + get_skill, + init_client, write_skills, ) MANIFEST_NAME = ".launchdarkly-skills.json" SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" + +MATERIALIZED_SIGNAL = "AgentControl Skill Materialized" +REVOKED_SIGNAL = "AgentControl Skill Revoked Received" INTEGRITY_SIGNAL = "AgentControl Skill Integrity Failure" +# The three signal names are an allowlist, not a floor. +APPROVED_SIGNALS = frozenset({MATERIALIZED_SIGNAL, REVOKED_SIGNAL, INTEGRITY_SIGNAL}) + +# Considered and deliberately excluded from SDK emission — named explicitly +# so the regression is unmissable. +REMOVED_SIGNALS = frozenset( + { + "AgentControl Skill SDK Reference Returned", + "AgentControl Skill Content Retrieved", + } +) + + +pytestmark = pytest.mark.usefixtures("reset_skill_state") + + +_INJECTED = "simulated crash between write and rename" + + +def _dir_id(path: Path) -> tuple[int, int]: + """``(st_dev, st_ino)`` — a directory's identity, independent of its name.""" + info = os.stat(path) + return (info.st_dev, info.st_ino) + + +class _RenameCall(NamedTuple): + """One intercepted ``os.replace`` of a ``SKILL.md``. + + ``src``/``dst`` are exactly what the implementation passed. Where the rename + is ``dir_fd``-relative they are bare filenames and the location lives in the + descriptors, so ``*_dir_id`` carries each descriptor's ``(st_dev, st_ino)`` + resolved *at call time* — the implementation closes the descriptors as soon + as the write returns, so they cannot be resolved from the assertions. + """ + + src: str + dst: str + src_dir_fd: int | None + dst_dir_fd: int | None + src_dir_id: tuple[int, int] | None + dst_dir_id: tuple[int, int] | None + + +class _ReplaceSpy: + """Records — and optionally fails — every atomic rename of a ``SKILL.md``. + + Write/rename interception hook: the implementation performs + the final rename through a single ``os.replace`` call site, so patching the + attribute on the ``os`` module observes it. Destinations other than + ``SKILL.md`` (i.e. the manifest's own atomic write) pass straight through — + the filter holds for both call shapes, since the ``dir_fd``-relative form + passes ``"SKILL.md"`` itself as ``dst``. + + Used two ways: to prove an injected failure is what produced an ``error`` + action (atomicity), and to prove no write was *attempted* for a + rejected key — the OS would reject several hostile keys on its + own, so a failed write is not evidence of a defense. + """ + + def __init__(self, fail: bool = False) -> None: + self.calls: list[_RenameCall] = [] + self._fail = fail + self._real = os.replace + + def __call__(self, src: Any, dst: Any, **kwargs: Any) -> None: + if str(dst).endswith("SKILL.md"): + src_dir_fd = kwargs.get("src_dir_fd") + dst_dir_fd = kwargs.get("dst_dir_fd") + self.calls.append( + _RenameCall( + src=str(src), + dst=str(dst), + src_dir_fd=src_dir_fd, + dst_dir_fd=dst_dir_fd, + src_dir_id=None if src_dir_fd is None else _fd_id(src_dir_fd), + dst_dir_id=None if dst_dir_fd is None else _fd_id(dst_dir_fd), + ) + ) + if self._fail: + raise OSError(_INJECTED) + self._real(src, dst, **kwargs) + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _ReplaceSpy: + # The attribute is set on the shared ``os`` module, so the + # single ``os.replace`` call site in safe_fs is intercepted wherever it is + # reached from. Named through the calling module rather than an arbitrary + # one so the hook documents which code it covers. + monkeypatch.setattr(safe_fs_module.os, "replace", self) + return self + + +def _fd_id(fd: int) -> tuple[int, int]: + info = os.fstat(fd) + return (info.st_dev, info.st_ino) + + +def _assert_atomic_rename_of(spy: _ReplaceSpy, skill_dir: Path) -> None: + """Assert the one recorded rename put ``SKILL.md`` into *skill_dir*. + + The temp file must be created in the target's own + directory, so the rename is atomic rather than cross-device. Two call + shapes prove it. Where the platform has ``renameat`` + the rename is ``dir_fd``-relative and the property is asserted by descriptor + identity — one descriptor for both sides, resolving to *skill_dir*'s inode — + which is stronger than comparing path strings, because it also rules out the + descriptor having been redirected between the check and the rename. On the + ``lstat`` floor (Windows) the names are full paths and share a parent. + """ + assert len(spy.calls) == 1 + call = spy.calls[0] + + if safe_fs_module.SUPPORTS_DIR_FD: + assert call.dst == "SKILL.md" + assert call.src != "SKILL.md" + assert call.src_dir_fd is not None + assert call.src_dir_fd == call.dst_dir_fd + assert call.dst_dir_id == _dir_id(skill_dir) + else: + assert Path(call.dst) == skill_dir / "SKILL.md" + assert Path(call.src).parent == skill_dir + assert Path(call.src).name != "SKILL.md" + + +class _SwapDirectoryDuring: + """Fires the directory-swap race at the exact instant of an operation. + + Renames ``/`` aside and leaves a symlink to *outside* in its + place, then lets the intercepted call proceed — the narrowest possible + version of the window an attacker with write access to the managed root + would otherwise have to hit by timing. Both hooks are the + interception points (``os.replace`` for the write, ``os.unlink`` for the + prune), so no implementation internals are touched. + """ + + def __init__(self, attribute: str, skill_dir: Path, outside: Path) -> None: + self.attribute = attribute + self.skill_dir = skill_dir + self.moved_to = skill_dir.parent / f"{skill_dir.name}.real" + self.outside = outside + self.swapped = False + self._real = getattr(os, attribute) + + def __call__(self, first: Any, *args: Any, **kwargs: Any) -> Any: + named = args[0] if args else first + if str(named).endswith("SKILL.md") and not self.swapped: + os.rename(self.skill_dir, self.moved_to) + os.symlink(self.outside, self.skill_dir, target_is_directory=True) + self.swapped = True + return self._real(first, *args, **kwargs) + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _SwapDirectoryDuring: + # ``os.replace`` is called from safe_fs, ``os.unlink`` from skills_fs; both + # resolve to the same module object, so either name reaches both. + module = safe_fs_module if self.attribute == "replace" else skills_fs_module + monkeypatch.setattr(module.os, self.attribute, self) + return self + + +_needs_dir_fd = pytest.mark.skipif( + not safe_fs_module.SUPPORTS_DIR_FD, + reason="no *at() family on this platform; the per-component lstat floor applies", +) + @pytest.fixture def root(tmp_path: Path) -> Path: @@ -580,3 +747,893 @@ async def test_mismatch_does_not_disturb_existing_managed_file( await write_skills([bad], root) assert target.read_text(encoding="utf-8") == SKILL_BODY + + +class TestAtomicityAndPermissions: + """Atomic writes, no partial files, 0644.""" + + async def test_written_file_is_0644_and_not_executable(self, root: Path) -> None: + await write_skills([_skill("a")], root) + mode = stat.S_IMODE((root / "a" / "SKILL.md").stat().st_mode) + assert mode == 0o644 + assert not mode & stat.S_IXUSR + assert not mode & stat.S_IXGRP + assert not mode & stat.S_IXOTH + + async def test_write_goes_through_a_single_atomic_rename( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Positive control for the interception hook. + + Without this, the ``spy.calls == []`` assertions in the failure tests + below and in the traversal matrix could pass in a suite where the hook + is never reachable at all. + """ + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True + # The temp file is created in the *same* directory + # as the target, so the rename is atomic rather than cross-device. + _assert_atomic_rename_of(spy, root / "a") + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_rename_failure_leaves_prior_content_intact( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + target = _place_managed(root, "a", SKILL_BODY) + spy = _ReplaceSpy(fail=True).install(monkeypatch) + + report = await write_skills([_skill("a", 2, "brand new content\n")], root) + + # The injected failure — not an unrelated rejection, and not an + # implementation that attempted nothing — is what produced the error. + _assert_atomic_rename_of(spy, target.parent) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert _INJECTED in (action.error or "") + + assert target.read_text(encoding="utf-8") == SKILL_BODY + # No temp artifact survives the failed run. + assert sorted(p.name for p in target.parent.iterdir()) == ["SKILL.md"] + + async def test_no_partial_file_at_target_after_failure( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + spy = _ReplaceSpy(fail=True).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + _assert_atomic_rename_of(spy, root / "a") + + assert report.ok is False + assert _INJECTED in (_actions_by_key(report)["a"].error or "") + assert not (root / "a" / "SKILL.md").exists() + # Neither a partial target nor a leaked temp file. + skill_dir = root / "a" + leftovers = ( + sorted(p.name for p in skill_dir.iterdir()) if skill_dir.exists() else [] + ) + assert leftovers == [] + + async def test_manifest_is_valid_json_after_a_run_with_errors( + self, root: Path + ) -> None: + report = await write_skills([_skill("a"), _skill("../evil")], root) + assert report.ok is False + assert isinstance(_read_manifest(root), dict) + + +# --------------------------------------------------------------------------- +# Security abuse matrix +# --------------------------------------------------------------------------- + +HOSTILE_KEYS = [ + "../evil", + "..", + ".", + "", + "/etc/cron.d/x", + "..\\evil", + "c:evil", + "skill:ads", + "sk\0ill", + "-skill", + "Evil", + "a/b", + "x" * 257, + "a/../../b", + "./a", + " leading-space", + "trailing-space ", +] + + +class TestPathTraversal: + """Nothing is ever written outside the root.""" + + @pytest.mark.parametrize("hostile_key", HOSTILE_KEYS) + async def test_hostile_key_is_rejected( + self, tmp_path: Path, hostile_key: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + root = tmp_path / "skills" + root.mkdir() + outside_before = sorted(p.name for p in tmp_path.iterdir()) + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill(hostile_key)], root) + + assert report.ok is False + assert [a.action for a in report.actions if a.key == hostile_key] == ["error"] + + # The SDK's key validation — not the operating system — must be what + # stopped this. An overlong key exceeds NAME_MAX, a null byte raises in + # the path API, and an absolute path outside the root usually fails on + # permissions, so "an error was reported" is not evidence of a defense + # (and the absolute-path verdict would flip on a privileged runner). + # Assert instead that no write was ever attempted. + assert spy.calls == [] + + # Nothing created outside the root, and no skill directory inside it. + assert sorted(p.name for p in tmp_path.iterdir()) == outside_before + assert [p.name for p in root.iterdir() if p.name != MANIFEST_NAME] == [] + assert list(root.rglob("SKILL.md")) == [] + + async def test_interception_hook_fires_for_a_valid_key( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Positive control for the ``spy.calls == []`` assertion above.""" + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("ok-key")], root) + + assert report.ok is True + assert [Path(call.dst).name for call in spy.calls] == ["SKILL.md"] + + async def test_long_but_filesystem_legal_key_is_written(self, root: Path) -> None: + """The ≤ 256 length bound cannot be exercised through ``write_skills``. + + A key becomes a single directory name and NAME_MAX is 255 bytes on Linux + and macOS, so the longest key the data model permits cannot exist on + disk at all. Assert the accepting side at the largest writable length; + the bound itself is covered by the pure layers (config validation and + accessor revalidation). + """ + key = "k" * 255 + report = await write_skills([_skill(key)], root) + + assert report.ok is True + assert (root / key / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_key_at_the_data_model_bound_is_reported_not_raised( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A 256-character key is valid to every pure layer but fits no filesystem. + + Config validation and the accessors must both accept exactly 256 + characters, yet NAME_MAX is 255 + on Linux and macOS, so this key reaches ``write_skills`` legitimately and + cannot become a directory. Every outcome must be visible in + the report, so it must surface as an ``error`` action rather than an + ``OSError`` escaping the call — which would also skip the manifest rewrite + and orphan any file already written in the same run. + """ + spy = _ReplaceSpy().install(monkeypatch) + long_key = "a" * 256 + + report = await write_skills([_skill("good"), _skill(long_key)], root) + + by_key = _actions_by_key(report) + assert by_key[long_key].action == "error" + assert by_key["good"].action == "written" + # The bare-filename ``dst`` of a ``dir_fd``-relative rename carries no + # directory, so "the path does not contain the hostile key" is no longer + # a meaningful check. Assert the stronger thing instead: the only rename + # that happened was into the valid skill's own directory. + assert [call.dst_dir_id for call in spy.calls] == [_dir_id(root / "good")] + # The valid skill is fully reconciled: written AND recorded, not orphaned. + assert (root / "good" / "SKILL.md").exists() + assert "good/SKILL.md" in _read_manifest(root)["entries"] + + async def test_valid_keys_still_write_alongside_rejected_ones( + self, root: Path + ) -> None: + report = await write_skills([_skill("good"), _skill("../evil")], root) + by_key = _actions_by_key(report) + assert by_key["good"].action == "written" + assert by_key["../evil"].action == "error" + assert (root / "good" / "SKILL.md").exists() + + async def test_traversal_key_does_not_create_parent_files( + self, tmp_path: Path + ) -> None: + root = tmp_path / "skills" + root.mkdir() + await write_skills([_skill("../../escaped")], root) + assert not (tmp_path / "escaped").exists() + assert not (tmp_path.parent / "escaped").exists() + + +@pytest.mark.skipif( + not hasattr(os, "symlink"), reason="platform has no symlink support" +) +class TestSymlinkAttacks: + """Never write through a symlink.""" + + async def test_symlinked_root_raises(self, tmp_path: Path) -> None: + real_dir = tmp_path / "real" + real_dir.mkdir() + link_root = tmp_path / "link" + link_root.symlink_to(real_dir, target_is_directory=True) + + with pytest.raises(ValueError): + await write_skills([_skill("a")], link_root) + + assert list(real_dir.iterdir()) == [] + + async def test_symlinked_skill_directory_is_refused(self, tmp_path: Path) -> None: + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (root / "a").symlink_to(outside, target_is_directory=True) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert list(outside.iterdir()) == [] + + async def test_symlinked_target_file_is_refused(self, tmp_path: Path) -> None: + root = tmp_path / "skills" + root.mkdir() + outside_file = tmp_path / "victim.md" + outside_file.write_text("victim content\n", encoding="utf-8") + (root / "a").mkdir() + (root / "a" / "SKILL.md").symlink_to(outside_file) + # Manifest lists the path so clobber protection is not what saves us. + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 1, "victim content\n")}, + }, + ) + + report = await write_skills([_skill("a", 2, "attacker payload\n")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert outside_file.read_text(encoding="utf-8") == "victim content\n" + + async def test_symlinked_target_is_not_pruned(self, tmp_path: Path) -> None: + """A manifest-listed path that is a symlink is refused, not unlinked. + + Asserting only that the victim file survives proves nothing here: + unlinking a symlink never touches its target, so that assertion holds + for an implementation with no symlink check at all. The observable + contract is the refusal itself (prune path). + """ + root = tmp_path / "skills" + root.mkdir() + outside_file = tmp_path / "victim.md" + outside_file.write_text("victim content\n", encoding="utf-8") + (root / "a").mkdir() + link = root / "a" / "SKILL.md" + link.symlink_to(outside_file) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 1, "victim content\n")}, + }, + ) + + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert [a for a in report.actions if a.action == "removed"] == [] + # The symlink itself is left in place and stays managed. + assert link.is_symlink() + assert "a/SKILL.md" in _read_manifest(root)["entries"] + assert outside_file.read_text(encoding="utf-8") == "victim content\n" + + @_needs_dir_fd + async def test_directory_swapped_at_the_rename_cannot_redirect_the_write( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The swap window is closed, not merely narrowed. + + Every check in the world is worthless if the final rename re-resolves + ``/`` from its path: an attacker holding write permission on + the managed root can replace the validated directory with a symlink in + between and redirect the write out of the root. The rename is therefore + performed relative to a descriptor pinned to the directory that was + checked, so it follows the inode rather than the name. + """ + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + race = _SwapDirectoryDuring("replace", root / "a", outside).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert list(outside.iterdir()) == [] + assert (race.moved_to / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + assert report.ok is True + + @_needs_dir_fd + async def test_directory_swapped_at_the_prune_cannot_redirect_the_unlink( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The same window on the destructive side. + + ``unlink`` never follows a *trailing* symlink, but it does resolve the + directory above it, so the swap turns a prune into a delete of an + attacker-chosen outside file. The unlink is descriptor-relative for the + same reason the rename is. + """ + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + victim = outside / "SKILL.md" + victim.write_text("precious\n", encoding="utf-8") + _place_managed(root, "a", SKILL_BODY) + race = _SwapDirectoryDuring("unlink", root / "a", outside).install(monkeypatch) + + report = await write_skills([], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert victim.read_text(encoding="utf-8") == "precious\n" + assert not (race.moved_to / "SKILL.md").exists() + assert [a.action for a in report.actions if a.key == "a"] == ["removed"] + + +class TestWithoutDirFd: + """The full-path fallback for platforms with no ``*at()`` family. + + On Windows ``os.open`` cannot open a directory at all, so acquiring the + descriptor must not even be attempted there — a fallback reached only after + a descriptor open would leave every write, prune and manifest rewrite + failing rather than falling back. These tests force the flag off so the + fallback is exercised on POSIX too. + """ + + @pytest.fixture(autouse=True) + def _no_dir_fd(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Models Windows: no ``*at()`` family, and directories cannot be opened. + + Forcing the flag off alone would not reproduce the platform, because + ``os.open`` on a directory succeeds on POSIX — the fallback would be + reached either way. Making that call raise the ``PermissionError`` + Windows raises is what proves the descriptor open is never attempted. + """ + monkeypatch.setattr(safe_fs_module, "SUPPORTS_DIR_FD", False) + real_open = os.open + + def no_directory_open(path: Any, *args: Any, **kwargs: Any) -> int: + if os.path.isdir(path): + raise PermissionError(13, "Permission denied") + return real_open(path, *args, **kwargs) + + monkeypatch.setattr(safe_fs_module.os, "open", no_directory_open) + + async def test_write_prune_and_manifest_all_succeed(self, root: Path) -> None: + first = await write_skills([_skill("a"), _skill("b")], root) + assert first.ok is True, _error_messages(first) + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + assert _manifest_path(root).exists() + assert stat.S_IMODE((root / "a" / "SKILL.md").stat().st_mode) == 0o644 + + second = await write_skills([_skill("a")], root) + + assert second.ok is True, _error_messages(second) + assert not (root / "b" / "SKILL.md").exists() + assert "b/SKILL.md" not in _read_manifest(root)["entries"] + + async def test_a_symlinked_skill_directory_is_still_refused( + self, root: Path, tmp_path: Path + ) -> None: + """The fallback keeps the ``lstat`` floor: no writing through a link.""" + outside = tmp_path / "outside" + outside.mkdir() + (root / "a").symlink_to(outside, target_is_directory=True) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert list(outside.iterdir()) == [] + + +class TestNonRegularFiles: + """A managed path that is not a regular file is refused, never read.""" + + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="no FIFOs on this platform") + async def test_a_fifo_at_the_managed_path_does_not_block(self, root: Path) -> None: + """Reading a FIFO with no writer blocks forever. + + Same attacker capability the symlink checks defend against: swapping a + managed ``SKILL.md`` for a FIFO would otherwise hang the whole reconcile + — and the caller's event loop with it — well past any ``timeout``, since + the deadline is only consulted between steps. + """ + skill_dir = root / "a" + skill_dir.mkdir() + os.mkfifo(skill_dir / "SKILL.md") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 1, SKILL_BODY)}, + }, + ) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "regular file" in action.error + assert stat.S_ISFIFO(os.lstat(skill_dir / "SKILL.md").st_mode) + + +class TestClobberProtection: + """Destructive ops only on manifest-listed paths.""" + + async def test_unmanaged_file_is_never_overwritten(self, root: Path) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text("user authored\n", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert target.read_text(encoding="utf-8") == "user authored\n" + + async def test_unmanaged_file_is_never_deleted(self, root: Path) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text("user authored\n", encoding="utf-8") + + await write_skills([], root) + + assert target.read_text(encoding="utf-8") == "user authored\n" + + async def test_manifest_entry_with_mismatched_key_does_not_authorize( + self, root: Path + ) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text("user authored\n", encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "a/SKILL.md": _entry("different-key", 1, "user authored\n") + }, + }, + ) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert target.read_text(encoding="utf-8") == "user authored\n" + + +DIVERGENT_CONTENT = "existing content\n" + + +def _live_entries() -> dict[str, Any]: + """A parseable entries map that really does claim ``a/SKILL.md`` as managed.""" + return {"a/SKILL.md": _entry("a", 1, DIVERGENT_CONTENT)} + + +# The first six variants are unparseable: ``entries`` is missing, the wrong type, +# or the whole document is garbage. That makes "performed no destructive action" +# arithmetic rather than a defense — with no entries to act on, a file at a +# managed path is protected by clobber protection and there is nothing to prune, +# so those cases pass against an implementation that simply treats a corrupt +# manifest as an empty one. +# +# The ``*_live_entries`` variants are the ones that actually test round-tripping: corrupt +# ONLY in ``manifestVersion``, with a valid entries map listing the managed path +# under a matching key. The implementation has everything it needs to overwrite +# and to prune, and must refuse anyway. +CORRUPT_MANIFESTS: list[tuple[str, Any]] = [ + ("garbage", "{not json at all"), + ("empty", ""), + ("wrong_types", {"manifestVersion": 1, "entries": ["a/SKILL.md"]}), + ("entries_missing", {"manifestVersion": 1}), + ("future_version", {"manifestVersion": 2, "entries": {}}), + ("version_not_int", {"manifestVersion": "1", "entries": {}}), + ("future_version_live_entries", {"manifestVersion": 2, "entries": _live_entries()}), + ( + "version_not_int_live_entries", + {"manifestVersion": "1", "entries": _live_entries()}, + ), +] + +LIVE_ENTRY_MANIFESTS: list[tuple[str, Any]] = [ + case for case in CORRUPT_MANIFESTS if case[0].endswith("_live_entries") +] + + +class TestCorruptManifest: + """Corrupt manifest fails closed, non-destructively.""" + + @pytest.mark.parametrize( + "raw", + [case[1] for case in CORRUPT_MANIFESTS], + ids=[case[0] for case in CORRUPT_MANIFESTS], + ) + async def test_no_destructive_action_and_error_reported( + self, root: Path, raw: Any + ) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text(DIVERGENT_CONTENT, encoding="utf-8") + _write_manifest(root, raw) + + report = await write_skills([_skill("a", 2, "new content\n")], root) + + assert report.ok is False + # The error must name the manifest. For the unparseable variants the file + # at the managed path is also unmanaged, so a bare "some error happened" + # assertion is satisfied by clobber protection alone and says nothing + # about whether the manifest state was detected at all. + errors = _error_messages(report) + assert any("manifest" in e.lower() for e in errors), errors + assert target.read_text(encoding="utf-8") == DIVERGENT_CONTENT + + async def test_run_level_error_carries_the_empty_key_sentinel( + self, root: Path + ) -> None: + """A run-level error has no skill key to hang off. + + The empty string is public API surface: a caller grouping the report by + key has to know the sentinel exists. Asserted here rather than in the + parametrized cases above so it is a statement about the manifest error + specifically, not about whichever error happens to come first. + """ + _write_manifest(root, "{not json at all") + + report = await write_skills([_skill("a")], root) + + manifest_errors = [ + action + for action in report.errors + if "manifest" in (action.error or "").lower() + ] + assert manifest_errors, _error_messages(report) + assert all(action.key == "" for action in manifest_errors) + # A per-skill error in the same report still carries its real key, so the + # sentinel is not simply "every error action has an empty key". + assert all( + action.key != "" + for action in report.errors + if action not in manifest_errors + ) + + @pytest.mark.parametrize( + "raw", + [case[1] for case in LIVE_ENTRY_MANIFESTS], + ids=[case[0] for case in LIVE_ENTRY_MANIFESTS], + ) + async def test_managed_file_is_not_pruned_when_only_the_version_is_corrupt( + self, root: Path, raw: Any + ) -> None: + """The prune counterpart of the live-entries cases. + + Here the implementation can read the entries map and knows exactly which + file it owns, so refusing to remove it is a real decision rather than an + absence of information. + """ + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text(DIVERGENT_CONTENT, encoding="utf-8") + _write_manifest(root, raw) + + report = await write_skills([], root) + + assert report.ok is False + assert target.read_text(encoding="utf-8") == DIVERGENT_CONTENT + assert [a for a in report.actions if a.action == "removed"] == [] + errors = _error_messages(report) + assert any("manifest" in e.lower() for e in errors), errors + + async def test_nothing_is_pruned_under_a_corrupt_manifest(self, root: Path) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text(DIVERGENT_CONTENT, encoding="utf-8") + _write_manifest(root, "{not json at all") + + report = await write_skills([], root) + + assert report.ok is False + assert target.exists() + assert [a for a in report.actions if a.action == "removed"] == [] + errors = _error_messages(report) + assert any("manifest" in e.lower() for e in errors), errors + + async def test_brand_new_paths_may_still_be_written(self, root: Path) -> None: + _write_manifest(root, "{not json at all") + + report = await write_skills([_skill("fresh")], root) + + actions = _actions_by_key(report) + assert actions["fresh"].action == "written" + assert (root / "fresh" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_corrupt_manifest_file_is_not_destroyed(self, root: Path) -> None: + _write_manifest(root, "{not json at all") + + await write_skills([], root) + + assert _manifest_path(root).exists() + assert _manifest_path(root).read_text(encoding="utf-8") == "{not json at all" + + +class TestWriteSkillsTelemetry: + """Materialized / revoked signals from write_skills.""" + + async def test_materialized_signal_per_action( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + _place_managed(root, "same", SKILL_BODY) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "same/SKILL.md": _entry("same", 1, SKILL_BODY), + "stale/SKILL.md": _entry("stale", 1, "old\n"), + }, + }, + ) + (root / "stale").mkdir() + (root / "stale" / "SKILL.md").write_text("old\n", encoding="utf-8") + + await write_skills( + [_skill("same"), _skill("stale", 2, "fresh\n"), _skill("brand-new")], + root, + ) + + signals = recording_emitter.signals(MATERIALIZED_SIGNAL) + by_key = {s["skill_key"]: s for s in signals} + assert len(signals) == 3 + assert by_key["same"]["reconcile_action"] == "skipped_current" + assert by_key["stale"]["reconcile_action"] == "updated" + assert by_key["brand-new"]["reconcile_action"] == "written" + + async def test_materialized_signal_properties( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + + await write_skills([_skill("a")], root) + + props = recording_emitter.signals(MATERIALIZED_SIGNAL)[0] + assert props["skill_key"] == "a" + assert props["content_bytes"] == len(SKILL_BODY.encode("utf-8")) + assert props["content_hash"] == _hash(SKILL_BODY) + assert props["reconcile_action"] == "written" + assert props["language"] == "python" + + async def test_no_filesystem_paths_in_telemetry( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + + await write_skills([_skill("a")], root) + + for _signal, props in recording_emitter.records: + assert "target_path" not in props + for value in props.values(): + assert str(root) not in str(value) + + async def test_no_skill_body_in_telemetry( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + + await write_skills([_skill("a")], root) + + for _signal, props in recording_emitter.records: + for value in props.values(): + assert "Do the thing." not in str(value) + + async def test_revoked_signal_on_prune( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + _place_managed(root, "gone", SKILL_BODY, version=4) + + await write_skills([], root) + + revoked = recording_emitter.signals(REVOKED_SIGNAL) + assert len(revoked) == 1 + assert revoked[0]["skill_key"] == "gone" + assert revoked[0]["version"] == 4 + assert revoked[0]["removed_from_disk"] is True + assert revoked[0]["language"] == "python" + + async def test_revoked_signal_redacts_an_untrusted_manifest_version( + self, root: Path, recording_emitter: Any + ) -> None: + """The manifest is untrusted, so its version is shape-checked first. + + Anything with write access to the managed root can plant an arbitrary + string here; echoing it verbatim would publish attacker-controlled + content — a skill body, or PII — as a signal property. + """ + skills_module._set_emitter_for_testing(recording_emitter) + target = root / "gone" / "SKILL.md" + target.parent.mkdir(parents=True) + target.write_text(SKILL_BODY, encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "gone/SKILL.md": { + "key": "gone", + "version": "Do the thing. " * 8, + "sha256": _hash(SKILL_BODY), + } + }, + }, + ) + + await write_skills([], root) + + revoked = recording_emitter.signals(REVOKED_SIGNAL) + assert len(revoked) == 1 + assert "version" not in revoked[0] + assert revoked[0]["skill_key"] == "gone" + for value in revoked[0].values(): + assert "Do the thing." not in str(value) + + async def test_no_revoked_signal_when_prune_disabled( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + _place_managed(root, "gone", SKILL_BODY) + + await write_skills([], root, prune=False) + + assert recording_emitter.signals(REVOKED_SIGNAL) == [] + + async def test_write_skills_records_no_signal_outside_the_approved_set( + self, root: Path, recording_emitter: Any + ) -> None: + """Allowlist sweep over a run that exercises all four actions. + + The accessor-side half of this sweep is + ``test_accessors_record_no_signal_outside_the_approved_set`` in + test_skills.py. Asserted over recorded strings, so no module-level + signal-name constant is required of the implementation. + """ + skills_module._set_emitter_for_testing(recording_emitter) + for key, content in (("same", SKILL_BODY), ("stale", "old\n"), ("gone", "g\n")): + (root / key).mkdir() + (root / key / "SKILL.md").write_text(content, encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "same/SKILL.md": _entry("same", 1, SKILL_BODY), + "stale/SKILL.md": _entry("stale", 1, "old\n"), + "gone/SKILL.md": _entry("gone", 1, "g\n"), + }, + }, + ) + + report = await write_skills( + [_skill("same"), _skill("stale", 2, "fresh\n"), _skill("brand-new")], + root, + ) + + # Positive control: the subset assertion is vacuous unless the run + # really did produce all four actions and record for them. + assert {a.action for a in report.actions} == { + "skipped_current", + "updated", + "written", + "removed", + } + recorded = {signal for signal, _props in recording_emitter.records} + assert recorded <= APPROVED_SIGNALS, ( + f"unapproved signal(s): {sorted(recorded - APPROVED_SIGNALS)}" + ) + assert not recorded & REMOVED_SIGNALS + assert recorded == {MATERIALIZED_SIGNAL, REVOKED_SIGNAL} + + async def test_no_ld_track_calls_from_write_skills( + self, root: Path, mock_ld_client: Any + ) -> None: + await init_client(client=mock_ld_client) + + await write_skills([_skill("a"), _skill("../evil")], root) + + mock_ld_client.track.assert_not_called() + + async def test_throwing_emitter_never_breaks_the_reconcile( + self, root: Path, throwing_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(throwing_emitter) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_integrity_signal_property_keys_match_across_layers( + self, root: Path, recording_emitter: Any + ) -> None: + """The same defect, caught at either layer, records + the same property keys. + + Verification runs twice by design: once at the accessor boundary and + again immediately before a write. The signal contract marks ``expected_hash`` + optional, so an implementation that populates it on one path and omits + it on the other passes every other assertion here while making the + signal's shape depend on which internal code path noticed. Oversize + content is the case reachable from both layers with the expected hash in + hand throughout. + """ + skills_module._set_emitter_for_testing(recording_emitter) + oversize = "x" * (64 * 1024 + 1) + content_hash = _hash(oversize) + + # Layer 1 — the accessor boundary. + store = InMemorySkillStore() + store.put( + { + "key": "big", + "version": 1, + "content": oversize, + "contentHash": content_hash, + } + ) + skills_module._set_store(store) + assert await get_skill("big") is None + + # Layer 2 — verify-then-write, on a directly constructed Skill. + report = await write_skills( + [ + Skill( + key="big", + version=1, + content=oversize.encode("utf-8"), + content_hash=content_hash, + ) + ], + root, + ) + assert report.ok is False + + failures = recording_emitter.signals(INTEGRITY_SIGNAL) + assert len(failures) == 2, failures + accessor_keys, write_keys = (set(props) for props in failures) + assert accessor_keys == write_keys, ( + f"accessor-only keys: {sorted(accessor_keys - write_keys)}; " + f"write-only keys: {sorted(write_keys - accessor_keys)}" + ) + assert "expected_hash" in accessor_keys From 568729d3945d1448f4f1a17d066155ad4c269970 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 25 Aug 2026 16:10:53 -0400 Subject: [PATCH 02/16] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20materialize=20onto=20disk=20under=20a=20manifest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fourth of five slices. Adds `write_skills`, which writes `//SKILL.md` and reconciles against a manifest recording what the SDK owns, so it overwrites or removes only files it wrote — a file you placed yourself is reported and left untouched. report = await write_skills(refs, ".claude/skills") `skills` accepts `Skill` values, references, bare keys, or the literal `"*"` for everything the store holds. Every outcome is visible in the returned `ReconcileReport`: one `ReconcileAction` per skill, carrying `written`, `updated`, `skipped_current`, `removed` or `error`, plus `.ok` and `.errors`. A failure belonging to the run rather than to one skill — an unreadable manifest, a retrieval that failed before any key was known — carries the empty string as its key. Writes are atomic, at mode 0644, and every destructive step runs against a descriptor pinned to a directory that was already checked, so a path swapped after the check cannot redirect a write or an unlink out of the managed root. Where the platform has no `*at()` family the identical sequence runs against full paths. The defenses, all of them deliberate and all of them tested: - The key is re-validated here regardless of upstream validation, before any filesystem call, because a key becomes a directory name. The data model allows 256 characters and `NAME_MAX` is 255 bytes, so an over-long key is refused too. - Never write or unlink through a symlink, on the write path or the prune path. - Destruction only on manifest-listed paths whose key matches. - A corrupt manifest fails closed: no overwrites and no prunes, brand-new paths may still be written, an error action names the manifest, and the manifest is not rewritten. - An incomplete retrieval suppresses pruning, so a transport outage cannot read as "everything was revoked". - Content is re-verified immediately before the write, because a `Skill` can also be constructed directly by a caller. Pruning removes formerly-managed skills that are no longer referenced, which is how revocation takes effect. `timeout` bounds retrieval, the writes and the pruning; only the final manifest rewrite runs past it, so files already written are never orphaned. `write_skills` performs synchronous filesystem I/O and does not yield — it is `async` for signature parity with the other accessors. Reconcile one root at a time: a run is atomic against the rest of the loop today, so wrapping it to run concurrently makes two runs against one root race on the manifest. The `"*"` form collapses to one object per key at its newest version, since `//SKILL.md` is a single path and writing it twice in one run is a bug rather than a policy, and it reports a withholding count at WARN for the same reason the accessors do. The security abuse matrix — path traversal, symlink attacks, clobber protection, corrupt manifests, atomicity under an injected crash, and the materialization telemetry allowlist — is the next slice. The guards it exercises are all here; what lands next is the adversary that proves each one fails without them. Testing: `uv run pytest` → 1216 passed. `ruff check`, `ruff format --check`, and `mypy packages/*/src` all clean. Co-Authored-By: Claude Fable 5 --- packages/client/README.md | 63 +- packages/client/agents.md | 114 ++- .../src/launchdarkly_ai_server/__init__.py | 21 + .../src/launchdarkly_ai_server/skills.py | 4 +- .../src/launchdarkly_ai_server/skills_fs.py | 960 ++++++++++++++++++ .../src/launchdarkly_ai_server/types.py | 48 + packages/client/tests/test_skills.py | 123 ++- packages/client/tests/test_skills_fs.py | 582 +++++++++++ 8 files changed, 1889 insertions(+), 26 deletions(-) create mode 100644 packages/client/src/launchdarkly_ai_server/skills_fs.py create mode 100644 packages/client/tests/test_skills_fs.py diff --git a/packages/client/README.md b/packages/client/README.md index ae572b3b..d9c1db15 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -223,15 +223,17 @@ asyncio.run(main()) ### Agent Skills Skills are versioned `SKILL.md` documents managed in LaunchDarkly and attached to AI Config -variations by reference. The SDK surfaces which skills a config references and retrieves -their content. Materializing them onto disk, where agent runtimes discover them, follows. +variations by reference. The SDK surfaces which skills a config references, retrieves their +content, and materializes them onto disk where agent runtimes (Claude Agent SDK, and +anything else following the `//SKILL.md` convention) discover them. ```python import asyncio import hashlib +from pathlib import Path from launchdarkly_ai_server import ( - init_client, inspect_config, skill_refs, get_skill, get_skills, + init_client, inspect_config, skill_refs, get_skill, write_skills, InMemorySkillStore, ) @@ -260,13 +262,18 @@ async def main(): if skill is not None: print(skill.content) - # 3. Or resolve the config's references in one call. - for s in await get_skills(refs): - print(s.key, s.version) + # 3. Write them where the agent runtime will look. Only the leaf directory is + # created, so the parent must already exist. + Path(".claude").mkdir(exist_ok=True) + report = await write_skills(refs, ".claude/skills") + for action in report.errors: + print(f"skill {action.key or ''}: {action.error}") asyncio.run(main()) ``` +Pass `"*"` instead of a reference list to materialize every skill the store holds. + **`skills` is now a validated field.** Config parsing fails closed on a `skills` value that is not a list of `{key, version}` objects (key matching `^[a-z0-9][a-z0-9-]*$`, version an integer ≥ 1): the whole variation is rejected, `inspect_config` returns `config: None`, and @@ -333,7 +340,22 @@ treat `expected_hash` / `observed_hash` as the evidence pair. **Versions are selected, not filtered.** A store may hold several versions of one key at once, because a delivery payload does: the newest version of every skill, plus every version a variation currently pins. `get_skill("k", version=1)` asks the store for version -1 and gets it even when a newer one is also held. +1 and gets it even when a newer one is also held. `all_skills()` and `write_skills("*")` +collapse to one skill per key at its newest version, since `//SKILL.md` is a +single path. + +**The root's parent must exist.** `write_skills` creates the root itself but never its +ancestors, so a typo cannot scatter a directory tree across your project. An absent parent, +a root that is an existing file, and a root that is a symlink each raise `ValueError` — +these are caller errors, distinct from the per-skill `error` actions in the report. + +**`write_skills` is deliberately conservative** about your filesystem. It writes only +`//SKILL.md`, tracks what it owns in a manifest at +`/.launchdarkly-skills.json`, and will overwrite or delete **only** paths that +manifest records. A file you placed yourself is reported as an error and left untouched; it +never writes through a symlink; writes are atomic (temp file, `fsync`, rename) at mode +`0644`; and if the manifest is unreadable it performs no destructive action at all. Removing +a skill from a variation is how revocation works — the next reconcile prunes it. | Export | Description | |---|---| @@ -341,11 +363,34 @@ version a variation currently pins. `get_skill("k", version=1)` asks the store f | `get_skill(key, *, version=None)` | One verified skill, or `None`. `version=None` means newest available; a specific `version` matches exactly. Raises only when no store is configured. | | `get_skills(refs)` | Batch form. Accepts `SkillReference` values and bare key strings (string = latest). Results follow input order; missing or unverifiable entries are omitted. | | `all_skills()` | Every verified skill the store holds, one per key at its newest version. | +| `write_skills(skills, root, *, prune=True, timeout=10.0, on_unavailable="keep")` | Materialize skills under `root`, returning a `ReconcileReport`. `prune` removes formerly-managed skills no longer requested. `on_unavailable="raise"` raises instead of reporting when content cannot be retrieved. Raises `ValueError` for an unusable root, a negative `timeout`, or an unrecognised `on_unavailable`. **Performs synchronous filesystem I/O — see the note below.** | | `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)`. | | `InMemorySkillStore(objects=None)` | A dict-backed store with `put(raw)`, for local development and testing. Holds several versions of a key. | Configure the store with `init_client(options={"skillStore": store})`. With none configured, -the accessors raise `RuntimeError` explaining what to do. `shutdown()` clears it. +the accessors raise `RuntimeError` explaining what to do and `write_skills` reports the +failure in its report (or raises, with `on_unavailable="raise"`). `shutdown()` clears it. + +`ReconcileReport.actions` holds one `ReconcileAction` per outcome — `written`, `updated`, +`skipped_current`, `removed`, or `error` — each carrying `key`, `version`, the resolved +`path`, and `error`. `report.ok` is `True` when no action is an `error`, and +`report.errors` is just the `error` actions, so you rarely need to filter `actions` +yourself. A failure that belongs to the whole run rather than to one skill — an unreadable +manifest, for instance — carries the empty string as its `key`. + +The fixed on-disk values are exported too, so you do not have to hardcode them: +`MANIFEST_FILENAME` (`.launchdarkly-skills.json`, handy for a `.gitignore`), +`SKILL_FILENAME`, and `MANIFEST_VERSION`. So are the two closed-set types, for annotating +your own helpers: `ReconcileActionKind` (`written` / `updated` / `skipped_current` / +`removed` / `error`) and `OnUnavailable` (`keep` / `raise`). + +**`write_skills` blocks.** It is `async` for parity with the other accessors and with the +TypeScript SDK, but it awaits nothing: every read, write, `fsync` and rename runs inline, +so a large reconcile holds the event loop for its duration. Wrap it in +`asyncio.to_thread` if that matters. For the same reason `timeout` is checked between +steps rather than interrupting one already in progress. Reconcile one root at a time, +though: because nothing yields today, a run is atomic against the rest of your loop, and +wrapping it to run concurrently makes two runs against the same root race on the manifest. `all_objects` returns one entry per `(key, version)` under keys that are **opaque** to the SDK — identity is read from each object's own `key` and `version` fields, so a store is free @@ -389,3 +434,5 @@ All types are exported from this package. Handler packages import them from here | `GraphTopology` | The parsed graph flag shape (`root` + `edges`) | | `Skill` | A frozen skill document: `.key`, `.version`, `.content` (verified verbatim `bytes`), `.content_hash`, `.name?`, `.description?` | | `SkillReference` | A frozen version-pinned pointer to a skill: `.key`, `.version` | +| `ReconcileAction` | One `write_skills` outcome: `.key`, `.action`, `.version?`, `.path?`, `.error?` | +| `ReconcileReport` | The `write_skills` result: `.actions`, `.ok`, and `.errors` | diff --git a/packages/client/agents.md b/packages/client/agents.md index 152f5771..feb2e348 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,6 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | +| `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | | `src/launchdarkly_ai_server/registry.py` | `Registry`, `global_registry`, `compose`, `resolve_handlers`, `resolve_tools` | @@ -56,7 +57,7 @@ from launchdarkly_ai_server import ( TrackData, UsageDict, HandlerResult, HandlerStreamEvent, StreamEvent, StreamChunkEvent, StreamDoneEvent, ExecuteStreamEvent, ExecuteStreamDoneEvent, VariationMeta, InitClientOptions, JudgeResult, ParseResult, ParseSuccess, ParseFailure, - Skill, SkillReference, + Skill, SkillReference, ReconcileAction, ReconcileReport, ) # Utilities @@ -76,8 +77,10 @@ from launchdarkly_ai_server import config, graph, resolve_graph # Agent Skills from launchdarkly_ai_server import ( - skill_refs, get_skill, get_skills, all_skills, + skill_refs, get_skill, get_skills, all_skills, write_skills, SkillStore, InMemorySkillStore, + SKILL_FILENAME, MANIFEST_FILENAME, MANIFEST_VERSION, + ReconcileActionKind, OnUnavailable, # the two closed-set unions ) ``` @@ -179,7 +182,7 @@ This is an OTel context value, not W3C baggage, so the id does not leak onto out Versioned `SKILL.md` documents attached to AI Config variations by reference, retrieved through an injectable store, and materialized onto disk for agent runtimes to discover. -Three layers, in increasing order of blast radius. Only the first is implemented here: +Three layers, in increasing order of blast radius: 1. **Reference discovery** — `skill_refs(config)` projects the config's `skills` array into typed `SkillReference` values. Pure: no network, no client, no store, no telemetry. @@ -190,7 +193,8 @@ Three layers, in increasing order of blast radius. Only the first is implemented `init_client(options={"skillStore": store})`; with none configured the accessors raise an actionable `RuntimeError`. A delivery transport can be added behind the seam without touching the public API. -3. **Materialization** — writing skills onto disk under a manifest. +3. **Materialization** — `write_skills(skills, root)` writes `//SKILL.md` and + reconciles against a manifest at `/.launchdarkly-skills.json`. ### The store seam, and why version is part of the lookup @@ -241,6 +245,30 @@ Store data is **untrusted input**; the transport is not part of the trust bounda - **Attacker-controlled strings are never echoed into telemetry.** `contentHash` and `key` come off the wire, so a store could put the skill body in either; both are shape-checked and redacted before they reach a signal or a log line. +- **The key is re-validated inside `write_skills`**, regardless of upstream validation — a + key becomes a directory name. Rejection happens before any filesystem call. +- **Never write through a symlink**, in either the skill directory or the target file, on + the write path *and* the prune path. +- **Destructive operations only on manifest-listed paths whose `key` matches.** A file at a + managed path with no matching manifest entry is reported as `error` and left alone. +- **A corrupt manifest fails closed**: unreadable, unparseable, not an object, malformed + `entries`, or a `manifestVersion` this release cannot read means no overwrites and no + prunes, brand-new paths may still be written, an `error` action names the manifest, and + the manifest file itself is not rewritten. +- **An incomplete retrieval suppresses pruning.** Otherwise a transport outage would read + as "everything was revoked" and delete the customer's managed files. +- **Writes are atomic**: temp file created exclusively in the target's *own* directory, + mode `0644` set explicitly (never inherited from the umask, never executable), write, + fsync, `os.replace`, fsync the directory. `os.replace` is the single rename call site + and must not be swapped for `os.rename`. +- **Every operation under the root goes through a pinned descriptor, not a path.** See + "Descriptor-pinned filesystem access" below. Re-resolving `/` from its path at + write or unlink time reopens a swap window that the checks above cannot cover. +- **A key valid to the data model may still be unrepresentable on disk.** The model allows + 256 characters; `NAME_MAX` is 255 bytes. `write_skills` rejects an over-long key before + any filesystem call, and every per-skill filesystem failure is caught at the loop so it + becomes an `error` action — aborting the loop would skip the manifest rewrite and orphan + files already written in that run. - **A key is untrusted input everywhere it appears.** `skill_key_rejection_reason` is the single canonical explanation, so the config parser and the reference projection reject a key for the same stated reason — and so does every layer added later. A silently @@ -262,9 +290,6 @@ Exactly three signals exist, and the list is an **allowlist, not a floor**: | `AgentControl Skill Materialized` | each `written` / `updated` / `skipped_current` | `skill_key`, `content_bytes`, `content_hash`, `reconcile_action`, `language` | | `AgentControl Skill Revoked Received` | prune removes a formerly managed skill | `skill_key`, `version`, `removed_from_disk`, `language` | -The last two belong to the materialization layer and have no caller yet; they live here -with the first so the allowlist is one section of one file rather than three sites to audit. - ### The integrity-failure log record The signal above is product telemetry; the **log record** beside it is the customer-owned @@ -321,11 +346,12 @@ where they cannot see it. `AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` were considered and **deliberately excluded from SDK emission** — both are observable server-side. Do not add them. The skill body never appears in a signal, a log line, or -an error message, and no signal carries a filesystem path. An emitter that raises is caught -and logged; it never fails the operation. +an error message, and no signal carries a filesystem path (paths belong in the returned +`ReconcileReport`, which is user-facing API). An emitter that raises is caught and logged; +it never fails the operation. -Module state lives in `skills_core.py`, so there is exactly one store and one emitter -however the feature is entered. All three signals are emitted from the `record_*` functions +Module state lives in `skills_core.py`, the module `skills.py` and `skills_fs.py` share, +so there is exactly one store and one emitter however the feature is entered. All three signals are emitted from the `record_*` functions next to the seam there — nothing outside that module calls `emit`, so the allowlist is enforced in one place. @@ -366,7 +392,14 @@ primitives live in `safe_fs.py`, which knows nothing about skills: *trailing* symlink, but it does resolve the directory above it, so the same swap turns a removal into a delete of an attacker-chosen file. A symlink found where this SDK expects its own file raises `SymlinkRefused` rather than being tidied away: the state on disk is - not what the caller believes, and that is the caller's to report. + not what the caller believes, and that is the caller's to report. `_prune_one` goes + through it; `rmdir` stays path-based and is safe that way, since it fails `ENOTDIR` on a + symlink and only ever succeeds on an empty directory. + +Every `lstat`, `realpath` and containment check on the skills side lives in one shared +`_unsafe_path_reason`, so the write and prune paths cannot drift apart on what counts as +unsafe. `skills_fs._prune_one` spells its symlink check `os.stat(..., follow_symlinks=False)` +rather than `os.lstat`, matching the name the capability probe advertises. `safe_fs.SUPPORTS_DIR_FD` gates all of it, and the probe is not the obvious one. `os.supports_dir_fd` is populated per underlying syscall, and CPython registers `renameat` @@ -379,7 +412,43 @@ and silently turns the defense off, so the probe names the advertised twins (Windows) `open_directory_nofollow` returns `None` after an `lstat` check instead of attempting the descriptor open — `os.open` cannot open a directory there — and every caller falls back to the identical full-path sequence, the per-component `lstat` floor. The -residual window on those platforms is documented rather than closed. +residual window on those platforms is documented rather than closed; the TOCTOU tests skip +off this same flag, deliberately, so a probe that wrongly reports "unsupported" cannot also +silently skip the tests that would have caught it. + +Both call shapes are admitted by the test seam. `os.replace` remains the single +interceptable rename call site; under the descriptor-relative shape `dst` is the bare string +`"SKILL.md"`, so an `endswith("SKILL.md")` spy filter still matches, and the +same-directory requirement is proved by descriptor identity (`src_dir_fd == dst_dir_fd`, +resolving to the skill directory's `(st_dev, st_ino)`) instead of by comparing path strings. +A spy must `fstat` the descriptor **inside** the intercepted call — the implementation closes +it as soon as the write returns. + +### Deferred: bounded retries + +`timeout` is implemented — a monotonic deadline, checked before each retrieval, before +each write, and before each prune; only the final manifest rewrite runs past it, so files +already written are never orphaned. Bounded retries inside that deadline are **not** +implemented, and belong to the delivery transport, not to this layer. Three structural +reasons, all of which the transport changes: + +1. **There is nothing transient to retry.** `SkillStore.get_object` is a synchronous + in-process read against already-delivered data, modelled on the LaunchDarkly + data-store API. `InMemorySkillStore` reads a dict. A retry re-invokes customer code and + returns the same answer. +2. **The seam cannot classify a failure.** All it surfaces is "this raised". Retrying a + `PermissionError` or a malformed payload spends the caller's `timeout` on a certainty. + The transient/permanent taxonomy a retry policy needs is the transport's to define. +3. **Backoff has nowhere to sleep.** The retrieval path (`_resolve_requests`, + `_resolve_reference`, `_resolve_all`) is synchronous, called from an async + `write_skills`. Backoff would mean either `time.sleep` — blocking the event loop of every + caller — or async-ifying the whole path for a store that cannot benefit. + +Picking a bound and a backoff now would fix numbers in a cross-language contract with no +transport to calibrate them against, so there is **no** retry test and no assumable attempt +count. When the transport lands it owns the policy; keep both languages retry-free until +then, since the number of times a throwing store is invoked is observable and the two would +otherwise diverge. --- @@ -491,7 +560,7 @@ Install with `pip install "launchdarkly-ai-server[otel]"`; see [OTel Setup](#ote |---|---| | `launchdarkly-server-sdk>=9.0`, and the `otel` extra mirrored (`opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-http`) | Each dynamically-resolved or optional package is repeated in the dev group so the test suite can import it. Something that is *only* optional would not be installed in this workspace and the tests covering its present-and-working path could not run. | | `pytest>=8`, `pytest-asyncio>=0.24` | Test runner and the async support the whole suite relies on. `asyncio_mode = "auto"` is set at the workspace root, which is why no test in this package carries an `@pytest.mark.asyncio`. | -| `mypy>=1.10` (`strict`), `ruff>=0.15` | Type checker and linter/formatter. | +| `mypy>=1.10` (`strict`), `ruff>=0.15` | Type checker and linter/formatter. `mypy` strict mode is the only thing enforcing the `Literal[...]` closed set on `ReconcileAction.action` — unlike `write_skills`'s `on_unavailable`, which is also checked at runtime because the value can arrive from untyped code. | --- @@ -513,6 +582,22 @@ convenience accessor that reads meaning into it — no YAML/frontmatter parsing, verified verbatim byte buffer and nothing more; a consumer who wants structure parses it on their side of the boundary. +### 4. Assuming `write_skills` prunes on every run + +Pruning is suppressed when the manifest is corrupt or any retrieval was incomplete — both +mean the SDK cannot tell what it owns or what is still current, and deleting under that +uncertainty is data loss. A run whose report contains a manifest `error` will not have +pruned anything, so do not read "no `removed` actions" as "nothing is stale". + +### 5. Treating "absent from the resolved set" as always meaning revoked + +Revocation is pruning, but only for a skill the store genuinely no longer serves. An object +that is *present and unverifiable* is a different thing, and `_resolve_all` must emit a +failed `_PendingWrite` for it rather than filtering it out: dropping it silently leaves its +key out of the requested set, so prune deletes the last known-good copy on disk and reports +a routine `removed` with `report.ok` still true. Tampered content must never be able to +trigger deletion. + --- ## Adding a New Export @@ -532,4 +617,5 @@ on their side of the boundary. - Do not route skills telemetry through `client.track()`, and do not introduce an LD context anywhere in the skills path. Signals go through the `skills_core.py` emitter seam, whose default is a no-op, and only via its `record_*` functions. - Do not add a signal name outside the three in the Agent Skills table above — the list is an allowlist. `AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` were considered and deliberately excluded from SDK emission. - Do not rename `ld.skills.integrity_failure`, and do not add a ninth `reason_code` in one language only — both are documented compatibility surfaces. See "The integrity-failure log record" above. +- Do not relax any of the `write_skills` filesystem defenses (local key re-validation, symlink refusal, manifest-authorized destruction, corrupt-manifest fail-closed, atomic `0644` writes). Each is a deliberate security property with abuse-case tests attached. - Do not make `SkillStore` lookups key-only. Version is part of the lookup identity because a payload holds several versions of one key; a key-only seam cannot express a version-pinned reference. diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 02c858f2..37b1570a 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -45,6 +45,13 @@ skill_refs, ) from .skills_core import SkillStore +from .skills_fs import ( + MANIFEST_FILENAME, + MANIFEST_VERSION, + SKILL_FILENAME, + OnUnavailable, + write_skills, +) from .tracking import execute_and_stream, execute_and_track, wrap_tool_handlers from .types import ( NATIVE_TOOL_KEY, @@ -74,6 +81,9 @@ ProviderGraphResponse, ProviderHandler, ProviderResponse, + ReconcileAction, + ReconcileActionKind, + ReconcileReport, Skill, SkillReference, StreamChunkEvent, @@ -136,6 +146,9 @@ "ProviderGraphResponse", "ProviderHandler", "ProviderResponse", + "ReconcileAction", + "ReconcileActionKind", + "ReconcileReport", "Skill", "SkillReference", "StreamChunkEvent", @@ -216,6 +229,14 @@ "get_skill", "get_skills", "all_skills", + "write_skills", "SkillStore", "InMemorySkillStore", + # skills — the two closed-set unions a typed consumer needs to name + "ReconcileActionKind", + "OnUnavailable", + # skills — on-disk constants, identical across languages + "SKILL_FILENAME", + "MANIFEST_FILENAME", + "MANIFEST_VERSION", ] diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py index eb3946c2..9d9b3627 100644 --- a/packages/client/src/launchdarkly_ai_server/skills.py +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -183,8 +183,8 @@ def skill_refs(config: AiConfigRep | None) -> list[SkillReference]: A config that came through ``parse_ai_config`` never contains an invalid entry — parsing fails closed on one. A hand-built dict can, and a silently - shortened projection would leave a caller materializing a skill set it - believes is complete, so every dropped entry is logged. + shortened projection would let ``write_skills`` prune the dropped skill's + on-disk copy, so every dropped entry is logged. """ if not isinstance(config, dict): return [] diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py new file mode 100644 index 00000000..feb8fbc9 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -0,0 +1,960 @@ +""" +Agent Skills — filesystem materialization. + +The highest-blast-radius layer of the feature: this is the part that writes to a +customer's disk. Split out of ``skills.py`` on that boundary — everything here +takes already-verified content and reconciles it against a managed root, while +``skills.py`` owns retrieval and verification and knows nothing about the +filesystem. The dependency runs one way only, and the descriptor-pinned +primitives every destructive step goes through live in ``safe_fs.py``. + +The reconcile is manifest-driven and fails closed: destructive operations only +ever touch paths ``/.launchdarkly-skills.json`` records under a matching +key, a corrupt manifest suppresses every destructive action, and an incomplete +retrieval suppresses pruning. Content is re-verified immediately before the +write, because a ``Skill`` can also be constructed directly by a caller. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import stat +import time +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Literal + +from .safe_fs import ( + SymlinkRefused, + atomic_write, + atomic_write_in, + pinned_directory, + unlink_file, +) +from .skills_core import ( + NO_STORE_MESSAGE, + Resolution, + SkillStore, + VerificationFailure, + get_store, + list_raw_objects, + log_withholding_summary, + newest_by_key, + record_materialized, + record_revoked, + reference_target, + resolve_from_store, + verified_bytes, + verify_raw_skill, +) +from .types import ( + ReconcileAction, + ReconcileActionKind, + ReconcileReport, + Skill, + SkillReference, +) +from .types_validation import ( + is_valid_skill_key, + is_valid_skill_version, + skill_key_rejection_reason, +) + +logger = logging.getLogger(__name__) + +MANIFEST_FILENAME = ".launchdarkly-skills.json" +"""The SDK's record of what it has written under a managed root.""" + +MANIFEST_VERSION = 1 +"""Manifest schema version this release writes, and the highest it can read.""" + +SKILL_FILENAME = "SKILL.md" +"""The single file each skill materializes to, under ``//``.""" + +OnUnavailable = Literal["keep", "raise"] +"""How ``write_skills`` reacts to content it could not retrieve.""" + +_UNAVAILABLE_PREFIX = "skill retrieval unavailable: " +""" +Prefix on every error describing content that could not be retrieved. Callers +assert on it, so it lives in one place. +""" + +_MAX_PATH_COMPONENT_BYTES = 255 +""" +NAME_MAX on Linux and macOS, and the component limit on Windows. A skill key +becomes a single directory name, and the data model permits keys up to 256 +characters — one byte longer than any of those filesystems can represent. Such a +key is rejected before any filesystem call so the caller gets a reported action +rather than an ENAMETOOLONG escaping from a stat deep inside the reconcile. +""" + + +# ------------------------------------------------------------------------- +# The reconcile entry point +# ------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _PendingWrite: + """One skill queued for the reconcile: resolved content, or why there is none.""" + + key: str + skill: Skill | None = None + error: str | None = None + + +async def write_skills( + skills: Sequence[Skill | SkillReference | str] | str, + root: str | os.PathLike[str], + *, + prune: bool = True, + timeout: float = 10.0, + on_unavailable: OnUnavailable = "keep", +) -> ReconcileReport: + """ + Materializes skills under a managed root at ``//SKILL.md``. + + *skills* is a sequence of ``Skill`` / ``SkillReference`` / key strings, or + the literal ``"*"`` meaning everything ``all_skills()`` returns. ``Skill`` + values are used as-is; references and strings resolve through the accessors, + so they need a configured store. + + The reconcile is manifest-driven (``/.launchdarkly-skills.json``): + destructive operations only ever touch paths the manifest records under a + matching key, so a file the SDK did not write is never overwritten or + deleted. ``prune`` removes formerly-managed skills that are no longer in the + requested set — which is also how revocation takes effect. ``timeout`` + bounds retrieval, the writes, and pruning; the final manifest rewrite + always runs, so files already written are never orphaned. ``on_unavailable`` + chooses between reporting a failed retrieval (``"keep"``, leaving existing + managed files alone) and raising (``"raise"``). + + Returns a ``ReconcileReport`` in which every outcome is visible; raises + ``ValueError`` for a caller error such as an unusable root. + + **This call performs synchronous filesystem I/O and does not yield.** It is + ``async`` for signature parity with the other accessors and with the + TypeScript SDK, not because it awaits anything: every read, write, ``fsync`` + and rename runs inline, so a large reconcile blocks the event loop for its + duration. Wrap it in ``asyncio.to_thread`` if that matters on your loop. + ``timeout`` is checked between steps rather than interrupting one in + progress, for the same reason. + + **One root, one reconcile at a time.** Because nothing here yields, a whole + reconcile is atomic against every other task on the loop today. Wrapping it + to run concurrently makes that the caller's problem instead: two runs + against the same root interleave on the manifest, and the loser's entries + are lost — which leaves the files it wrote unmanaged, and a later reconcile + then refuses them as files the SDK did not write. + """ + # Both of these are annotated as closed sets, but the values can still arrive + # from untyped code, so they are checked rather than assumed. + if on_unavailable not in ("keep", "raise"): + raise ValueError( + f'on_unavailable must be "keep" or "raise", got {on_unavailable!r}' + ) + if timeout < 0: + raise ValueError(f"timeout must not be negative, got {timeout!r}") + + deadline = time.monotonic() + timeout + root_path = _resolve_root(root) + manifest, manifest_error = _load_manifest(root_path) + entries: dict[str, Any] = manifest.get("entries", {}) + + actions: list[ReconcileAction] = [] + if manifest_error is not None: + # Run-level failure: there is no single skill key to hang it off. + actions.append(_run_error(manifest_error)) + + requests, incomplete = _resolve_requests(skills, deadline, on_unavailable) + + written, write_timed_out = _write_all(root_path, requests, entries, deadline) + actions.extend(written) + incomplete = incomplete or write_timed_out + + # Pruning is destructive, so it needs a trustworthy picture of both sides: a + # corrupt manifest means we do not know what we own, and an incomplete run — + # a retrieval that failed, or a deadline that expired mid-write — means we do + # not know what is still current. Either way, deleting would be a guess. + if prune and manifest_error is None and not incomplete: + actions.extend( + _prune( + root_path, + entries, + {request.key for request in requests}, + deadline, + ) + ) + + if manifest_error is None: + actions.extend(_rewrite_manifest(root_path, manifest, entries)) + + return ReconcileReport(actions=actions) + + +_RUN_LEVEL_KEY = "" +""" +The documented sentinel for a failure that belongs to no single skill (see +``ReconcileAction``). Spelled once so every path that cannot attribute a +failure to a key agrees with the others. +""" + + +def _run_error(message: str) -> ReconcileAction: + """ + A failure belonging to the run rather than to one skill. + + Uses the run-level sentinel key; it is constructed here so every run-level + error agrees. + """ + return ReconcileAction(key=_RUN_LEVEL_KEY, action="error", error=message) + + +def _write_all( + root: Path, + requests: list[_PendingWrite], + entries: dict[str, Any], + deadline: float, +) -> tuple[list[ReconcileAction], bool]: + """ + Reconciles every pending write. Returns ``(actions, timed out mid-run)``. + + The loop never aborts: a per-skill failure becomes an ``error`` action and the + next skill is attempted, because returning early would skip the caller's + manifest rewrite and orphan every file already written in this run. + """ + actions: list[ReconcileAction] = [] + timed_out = False + + for request in requests: + if request.skill is None: + actions.append( + ReconcileAction( + key=request.key, + action="error", + error=request.error + or f"skill '{request.key}' could not be resolved", + ) + ) + continue + if time.monotonic() >= deadline: + timed_out = True + actions.append( + ReconcileAction( + key=request.key, + action="error", + error=( + "the timeout was exhausted before skill " + f"'{request.key}' could be written" + ), + ) + ) + continue + try: + actions.append(_write_one(root, request.skill, entries)) + except OSError as exc: + # A safety net, not the primary defense. pathlib's stat probes swallow + # only ENOENT/ENOTDIR/EBADF/ELOOP and re-raise every other errno, so an + # unexpected filesystem condition must not abort the loop. + actions.append( + ReconcileAction( + key=request.skill.key, + action="error", + version=request.skill.version, + error=f"skill '{request.skill.key}' could not be reconciled: {exc}", + ) + ) + + return actions, timed_out + + +def _rewrite_manifest( + root: Path, manifest: dict[str, Any], entries: dict[str, Any] +) -> list[ReconcileAction]: + """Writes the updated manifest. Returns an error action, or nothing.""" + manifest["manifestVersion"] = MANIFEST_VERSION + manifest["entries"] = entries + try: + # json.dumps is inside the guard: indent= selects the pure-Python encoder, + # and unknown fields must be round-tripped, so a deeply nested + # planted field can raise RecursionError here — after every skill file is + # already on disk. + serialized = json.dumps(manifest, indent=2, sort_keys=True).encode("utf-8") + atomic_write_in(root, MANIFEST_FILENAME, serialized) + except Exception as exc: + return [_run_error(f"the skills manifest could not be written: {exc}")] + return [] + + +# ------------------------------------------------------------------------- +# Request resolution — content in, or a reason there is none +# ------------------------------------------------------------------------- + + +def _unavailable(reason: str) -> str: + """Wraps *reason* as a retrieval-unavailable message.""" + return f"{_UNAVAILABLE_PREFIX}{reason}" + + +@dataclass(frozen=True) +class _RetrievalBlocked: + """Why retrieval must not be attempted. The reason is caller-facing.""" + + reason: str + + +def _available_store(deadline: float, subject: str) -> SkillStore | _RetrievalBlocked: + """ + The configured store, or why retrieval must not be attempted. + + Written once because this gate is what sets ``unavailable`` and therefore + suppresses pruning. If it were maintained in two places, a condition added + to one and not the other would not merely produce a wrong message — it + would delete the user's files. + """ + if time.monotonic() >= deadline: + return _RetrievalBlocked( + _unavailable( + f"the timeout was exhausted before {subject} could be retrieved" + ) + ) + store = get_store() + if store is None: + return _RetrievalBlocked(_unavailable(NO_STORE_MESSAGE)) + return store + + +def _resolve_requests( + skills: Sequence[Skill | SkillReference | str] | str, + deadline: float, + on_unavailable: OnUnavailable, +) -> tuple[list[_PendingWrite], bool]: + """ + Turns the caller's input into one request per skill. + + Returns the requests plus whether any retrieval was left incomplete — an + absent store, a raising store, or an exhausted timeout. That flag suppresses + pruning: deleting managed files because retrieval failed would turn a + transport outage into data loss. + """ + if isinstance(skills, str): + if skills != "*": + raise ValueError( + 'write_skills takes a sequence of skills or the literal "*"; ' + f"got {skills!r}" + ) + return _resolve_all(deadline, on_unavailable) + + requests: list[_PendingWrite] = [] + incomplete = False + for item in skills: + if isinstance(item, Skill): + requests.append(_PendingWrite(key=item.key, skill=item)) + continue + + key, wanted = reference_target(item) + resolved = _resolve_reference(key, wanted, deadline) + if resolved.unavailable: + incomplete = True + if on_unavailable == "raise": + raise RuntimeError(resolved.error) + requests.append( + _PendingWrite(key=key, skill=resolved.skill, error=resolved.error) + ) + + return requests, incomplete + + +def _resolve_reference( + key: str, wanted_version: int | None, deadline: float +) -> Resolution: + """ + Resolves one reference for the materialization path. + + Same core as the accessors, plus the two conditions only this path treats as + data rather than as an exception: an exhausted deadline and an absent store. + """ + store = _available_store(deadline, f"'{key}'") + if isinstance(store, _RetrievalBlocked): + return Resolution(error=store.reason, unavailable=True) + + resolved = resolve_from_store(store, key, wanted_version) + if resolved.unavailable and resolved.error is not None: + return Resolution(error=_unavailable(resolved.error), unavailable=True) + return resolved + + +def _unavailable_run( + error: str, on_unavailable: OnUnavailable +) -> tuple[list[_PendingWrite], bool]: + """ + One run-level retrieval failure — raised, or reported against the empty key. + + Always reports the run incomplete, which is what suppresses pruning: nothing + was retrieved, so every managed file on disk has to be assumed current. + """ + if on_unavailable == "raise": + raise RuntimeError(error) + return [_PendingWrite(key="", error=error)], True + + +def _pending_for_raw(object_key: str, raw: Any) -> _PendingWrite: + """ + One raw store object as a pending write — verified, or reported as failed. + + Present but unverifiable is NOT the same as revoked. Dropping it silently + would leave the key out of the requested set, so prune would delete the last + known-good copy already on disk and report a routine "removed" with + report.ok still true. A failed request instead gets the same treatment the + reference path already gives (see ``_resolve_reference``): the outcome is + surfaced, and the key stays in the requested set so nothing is pruned. + """ + skill = verify_raw_skill(raw) + if skill is not None: + return _PendingWrite(key=skill.key, skill=skill) + # The on-disk copy lives under the object's *own* key, which a custom store + # may key differently in ``all_objects``. The failure must be recorded under + # the object's key, or the copy written under it on an earlier run would + # fall out of the requested set and be pruned — the very deletion this + # function exists to prevent. + raw_key = raw.get("key") if isinstance(raw, dict) else None + key = raw_key if is_valid_skill_key(raw_key) else object_key + if not is_valid_skill_key(key): + # Neither key is usable, so this failure cannot be attributed to a skill + # — the run-level sentinel is the honest report. + return _PendingWrite( + key=_RUN_LEVEL_KEY, + error="the skill store served an object under an invalid key; " + "it was withheld", + ) + return _PendingWrite( + key=key, + error=f"skill '{key}' failed integrity verification and was " + "withheld; the copy already on disk was left alone", + ) + + +def _resolve_all( + deadline: float, on_unavailable: OnUnavailable +) -> tuple[list[_PendingWrite], bool]: + """Resolves the ``"*"`` form — everything the store currently holds.""" + store = _available_store(deadline, "the skill set") + if isinstance(store, _RetrievalBlocked): + return _unavailable_run(store.reason, on_unavailable) + + # Deliberately not via all_skills(), which reports a raising store as an + # empty result — that would look like "every skill was revoked" and let + # prune delete the lot. + objects, error = list_raw_objects(store) + if error is not None: + return _unavailable_run(_unavailable(error), on_unavailable) + + # One object per key, at its newest version. ``all_objects`` may hold several + # versions of one key, and //SKILL.md is a single path — writing it + # twice in one run is a bug rather than a policy. + candidates = newest_by_key(objects) + requests = [_pending_for_raw(key, raw) for key, raw in candidates] + log_withholding_summary( + "skills held by the store", + len(requests), + sum(1 for request in requests if request.skill is not None), + ) + return requests, False + + +# ------------------------------------------------------------------------- +# The managed root and its manifest +# ------------------------------------------------------------------------- + + +def _resolve_root(root: str | os.PathLike[str]) -> Path: + """ + Resolves the managed root once, up front. + + An unusable root is a caller error rather than a per-skill outcome, so this + raises. Only the leaf directory is ever created — recursively creating + missing ancestors would let a typo scatter a directory tree. + """ + path = Path(os.fspath(root)) + + # pathlib re-raises any errno outside ENOENT/ENOTDIR/EBADF/ELOOP, so an + # unreadable parent would surface as PermissionError where the docs + # promise ValueError. + try: + is_symlink = path.is_symlink() + exists = path.exists() + is_dir = path.is_dir() + except OSError as exc: + raise ValueError(f"the skills root could not be inspected: {exc}") from exc + + if is_symlink: + raise ValueError( + f"the skills root must be a real directory, not a symlink: {path}" + ) + + if exists: + if not is_dir: + raise ValueError(f"the skills root is not a directory: {path}") + else: + parent = path.parent + try: + parent_is_dir = parent.is_dir() + except OSError as exc: + raise ValueError( + f"the parent of the skills root could not be inspected: {exc}" + ) from exc + if not parent_is_dir: + raise ValueError( + f"the parent of the skills root does not exist: {parent}. " + "write_skills creates only the leaf directory." + ) + try: + path.mkdir() + except OSError as exc: + raise ValueError(f"the skills root could not be created: {exc}") from exc + + return Path(os.path.realpath(path)) + + +def _load_manifest(root: Path) -> tuple[dict[str, Any], str | None]: + """ + Loads the manifest. Returns ``(manifest, error)``. + + A manifest that cannot be read, cannot be parsed, is not an object, carries a + ``manifestVersion`` this release does not understand, or has a malformed + ``entries`` map is **corrupt**. The caller then performs no destructive + action and leaves the file itself alone: rewriting it would destroy the only + record of what the SDK owns, and acting on a manifest we cannot read would + mean guessing at which of the customer's files are ours. + + An absent manifest is not corrupt — that is simply a fresh root. + """ + path = root / MANIFEST_FILENAME + fresh: dict[str, Any] = {"manifestVersion": MANIFEST_VERSION, "entries": {}} + + if not path.exists(): + return fresh, None + + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + # UnicodeDecodeError is a ValueError, not an OSError: non-UTF-8 bytes in + # the manifest are corruption, and must fail closed like any other. + return {}, f"the skills manifest {MANIFEST_FILENAME} could not be read: {exc}" + + try: + data = json.loads(text) + except (ValueError, RecursionError) as exc: + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} is not valid JSON ({exc}); " + "refusing every destructive action" + ) + + if not isinstance(data, dict): + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} is not a JSON object; " + "refusing every destructive action" + ) + + version = data.get("manifestVersion") + if ( + not isinstance(version, int) + or isinstance(version, bool) + or version > MANIFEST_VERSION + ): + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} declares manifestVersion " + f"{version!r}, which this SDK cannot read; refusing every destructive " + "action" + ) + + if not isinstance(data.get("entries"), dict): + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} has a malformed 'entries' " + "map; refusing every destructive action" + ) + + return data, None + + +# ------------------------------------------------------------------------- +# Per-skill reconcile +# ------------------------------------------------------------------------- + + +def _unsafe_path_reason( + root: Path, skill_dir: Path, target: Path, key: str, *, require_directory: bool +) -> str | None: + """ + The path defenses, in one place. + + Returns why ``//SKILL.md`` must not be touched, or ``None``. + Shared by the write and prune paths: ``agents.md`` marks these checks + non-relaxable, and maintaining them twice is how they drift. + + *require_directory* is the one genuine difference between the two callers. A + write needs a real directory to write into. A prune only needs to not follow + a link — an entry whose directory has been replaced by a plain file has + already lost the file this SDK owned, so reporting ``removed`` is what lets + the stale manifest entry be dropped rather than pinned forever. + + Note that the containment check is unconditional even though ``skill_dir`` + may not exist yet: ``realpath`` resolves the existing prefix and appends the + rest, so a fresh key under a valid root passes. + """ + if skill_dir.is_symlink(): + return f"{key} is a symlink" + if require_directory and skill_dir.exists() and not skill_dir.is_dir(): + return f"{key} exists and is not a directory" + if target.is_symlink(): + return "the target file is a symlink" + if Path(os.path.realpath(skill_dir)).parent != root: + return f"it resolves outside the managed root {root}" + return None + + +def _key_rejection_reason(key: Any) -> str | None: + """ + Why *key* must not become a directory name under the managed root, or ``None``. + + Re-validated locally whatever any upstream layer already did, and + before any filesystem call, because a key becomes a path component. Shared by + the write and the prune paths so the two cannot disagree about which keys + this SDK could own; ``agents.md`` marks these checks non-relaxable, and + maintaining them twice is how they drift. + + ``key.encode`` is safe here only because it runs *after* the pattern check: + the key grammar admits no surrogate, so there is no unencodable key left to + raise on. Do not reorder these two. + """ + if not is_valid_skill_key(key): + return f"{key!r} is not a valid skill key: it {skill_key_rejection_reason(key)}" + # The data model allows 256 characters; no mainstream filesystem allows a + # 256-byte path component. Catch it here so it is a reported action rather + # than an ENAMETOOLONG raised from the first stat in the caller. + key_bytes = len(key.encode("utf-8")) + if key_bytes > _MAX_PATH_COMPONENT_BYTES: + return ( + f"skill key '{key[:32]}...' is {key_bytes} bytes, over the " + f"{_MAX_PATH_COMPONENT_BYTES}-byte limit for a single directory name" + ) + return None + + +def _write_one(root: Path, skill: Skill, entries: dict[str, Any]) -> ReconcileAction: + """Reconciles one verified skill against the managed root.""" + key = skill.key + + def failed(message: str) -> ReconcileAction: + return ReconcileAction( + key=key, action="error", version=skill.version, error=message + ) + + rejection = _key_rejection_reason(key) + if rejection is not None: + return failed(f"{rejection}; nothing was written") + if not is_valid_skill_version(skill.version): + return failed( + f"skill '{key}' has version {skill.version!r}, which is not an " + "integer >= 1; nothing was written" + ) + + skill_dir = root / key + target = skill_dir / SKILL_FILENAME + relative = f"{key}/{SKILL_FILENAME}" + + unsafe = _unsafe_path_reason(root, skill_dir, target, key, require_directory=True) + if unsafe is not None: + return failed(f"'{relative}' was refused: {unsafe}; nothing was written") + + # Re-verify immediately before writing, through the same core the accessors + # use: a Skill can also be constructed directly by a caller. + verified = verified_bytes(key, skill.content, skill.content_hash, skill.version) + if isinstance(verified, VerificationFailure): + return failed( + f"skill '{key}' failed verification immediately before writing: " + f"{verified.reason}; nothing was written" + ) + encoded, content_hash = verified.encoded, verified.content_hash + + # Overwrite only what the manifest records as ours under this key. + entry = entries.get(relative) + managed = isinstance(entry, dict) and entry.get("key") == key + exists = target.exists() + + if exists and not managed: + return failed( + f"'{relative}' exists but the manifest does not record it as managed " + f"under key '{key}'; refusing to overwrite a file this SDK did not write" + ) + + if exists: + try: + on_disk = _read_regular_file(target) + except OSError as exc: + return failed(f"'{relative}' could not be read: {exc}") + + if hashlib.sha256(on_disk).hexdigest() == content_hash: + _update_entry(entries, relative, skill, content_hash) + record_materialized(key, len(encoded), content_hash, "skipped_current") + return ReconcileAction( + key=key, + action="skipped_current", + version=skill.version, + path=str(target), + ) + # Stale version or local tampering — LD-resolved content wins. + action: ReconcileActionKind = "updated" + else: + action = "written" + + write_error = _write_through_descriptor(skill_dir, encoded, key, relative) + if write_error is not None: + return failed(write_error) + + _update_entry(entries, relative, skill, content_hash) + record_materialized(key, len(encoded), content_hash, action) + return ReconcileAction( + key=key, action=action, version=skill.version, path=str(target) + ) + + +def _read_regular_file(target: Path) -> bytes: + """ + Reads *target*, refusing anything that is not a regular file. + + A plain ``Path.read_bytes`` would ``open()`` by name — and opening a FIFO + with no writer blocks forever, so an attacker who can swap the managed file + for one (the same capability the symlink checks defend against) could hang + the whole reconcile, and the event loop with it. ``O_NONBLOCK`` makes that + open return immediately (it is a no-op for regular files), ``O_NOFOLLOW`` + refuses a trailing symlink, and the ``fstat`` on the descriptor — not the + path — is what the type check trusts. ``O_BINARY`` is what keeps these + bytes the *verbatim* bytes: it is 0 on POSIX, but on Windows a descriptor + without it translates CRLF on read, which would fail the hash comparison + against content that is actually current. + """ + flags = ( + os.O_RDONLY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0) + | getattr(os, "O_BINARY", 0) + ) + fd = os.open(target, flags) + try: + if not stat.S_ISREG(os.fstat(fd).st_mode): + raise OSError("the target file is not a regular file") + chunks: list[bytes] = [] + while True: + chunk = os.read(fd, 65536) + if not chunk: + return b"".join(chunks) + chunks.append(chunk) + finally: + os.close(fd) + + +def _write_through_descriptor( + skill_dir: Path, encoded: bytes, key: str, relative: str +) -> str | None: + """ + Performs the write itself. Returns a failure reason, or ``None`` on success. + + Split out of ``_write_one`` because everything above it decides *whether* to + write and this decides nothing: the directory is pinned to a descriptor and + every remaining step is relative to it, so none of the checks above can be + invalidated by a swap between here and the rename. + """ + try: + with pinned_directory(skill_dir, create=True) as dir_fd: + try: + atomic_write(skill_dir, SKILL_FILENAME, encoded, dir_fd=dir_fd) + except OSError as exc: + return f"'{relative}' could not be written: {exc}" + except OSError as exc: + return f"the directory for skill '{key}' could not be created: {exc}" + except ValueError as exc: + return f"'{relative}' was refused: {exc}" + return None + + +def _update_entry( + entries: dict[str, Any], relative: str, skill: Skill, content_hash: str +) -> None: + """ + Records a managed path in the manifest. + + Merges into any existing entry rather than replacing it, so fields written by + a future SDK release survive this one's rewrite. + + ``sha256`` and ``writtenAt`` are recorded for forensics only: the reconcile + decides currency by hashing the bytes on disk, precisely because the + manifest is untrusted, so neither field is ever read back as a decision + input. + """ + existing = entries.get(relative) + entry = dict(existing) if isinstance(existing, dict) else {} + entry["key"] = skill.key + entry["version"] = skill.version + entry["sha256"] = content_hash + entry["writtenAt"] = _utc_timestamp() + entries[relative] = entry + + +# ------------------------------------------------------------------------- +# Pruning — how revocation takes effect +# ------------------------------------------------------------------------- + + +def _prune_error(key: str, message: str, version: Any = None) -> ReconcileAction: + """ + A prune refusal. Mirrors ``_write_one``'s local ``failed`` helper. + + *version* comes off the manifest, which is untrusted, so it is validated here + rather than at each call site — the same guard the ``removed`` action applies, + so a refusal and a removal report the field identically. + Callers that genuinely do not know a version pass nothing; none of them may + invent one. + """ + return ReconcileAction( + key=key, + action="error", + version=version if is_valid_skill_version(version) else None, + error=message, + ) + + +def _prune( + root: Path, entries: dict[str, Any], requested: set[str], deadline: float +) -> list[ReconcileAction]: + """ + Removes managed skills that are no longer requested. + + This is also how revocation takes effect: a revoked skill is simply absent + from the resolved set, so the next reconcile removes it. There is + deliberately no opt-out. + + The deadline applies here just as it does to the writes: a skill left + unpruned is reported as an error and stays in the manifest, so the next + reconcile picks it up. + """ + actions: list[ReconcileAction] = [] + + for relative, entry in list(entries.items()): + if not isinstance(entry, dict): + continue + key = entry.get("key") + if not isinstance(key, str) or key in requested: + continue + + if time.monotonic() >= deadline: + actions.append( + _prune_error( + key, + f"the timeout was exhausted before '{relative}' could be " + "pruned; it was left in place", + entry.get("version"), + ) + ) + continue + + # Only a manifest path this SDK could have written is removable. + if ( + _key_rejection_reason(key) is not None + or relative != f"{key}/{SKILL_FILENAME}" + ): + actions.append( + _prune_error( + key, + f"manifest entry '{relative}' does not name a path this SDK " + f"could own under key '{key}'; it was left in place", + entry.get("version"), + ) + ) + continue + + try: + actions.append(_prune_one(root, relative, key, entries)) + except OSError as exc: + actions.append( + _prune_error( + key, + f"'{relative}' could not be removed: {exc}", + entry.get("version"), + ) + ) + + return actions + + +def _unlink_through_descriptor(skill_dir: Path, relative: str) -> str | None: + """ + Performs the removal itself. Returns a failure reason, or ``None`` on success. + + The mirror of ``_write_through_descriptor``, and split out for the same + reason: everything above it decides *whether* to remove, and this decides + nothing. The directory is pinned before the unlink because unlink never + follows a trailing symlink but does resolve the directory above it, so a + ``/`` swapped for a symlink between the checks and here would + otherwise delete a file outside the root. + """ + try: + with pinned_directory(skill_dir) as dir_fd: + try: + unlink_file(skill_dir, SKILL_FILENAME, dir_fd=dir_fd) + except SymlinkRefused: + return f"'{relative}' was not removed: the target file is a symlink" + except OSError as exc: + return f"'{relative}' could not be removed: {exc}" + except ValueError as exc: + return f"'{relative}' was not removed: {exc}" + return None + + +def _prune_one( + root: Path, relative: str, key: str, entries: dict[str, Any] +) -> ReconcileAction: + """Removes one managed skill file, and its directory when that empties it.""" + skill_dir = root / key + target = skill_dir / SKILL_FILENAME + version = entries[relative].get("version") + + unsafe = _unsafe_path_reason(root, skill_dir, target, key, require_directory=False) + if unsafe is not None: + return _prune_error(key, f"'{relative}' was not removed: {unsafe}", version) + + removed_from_disk = False + if target.exists(): + failure = _unlink_through_descriptor(skill_dir, relative) + if failure is not None: + return _prune_error(key, failure, version) + removed_from_disk = True + try: + # Path-based, and safe that way: rmdir never follows a trailing + # symlink (it fails ENOTDIR) and only ever succeeds on an empty + # directory. + skill_dir.rmdir() + except OSError: + pass # the customer keeps their own files here too + + entries.pop(relative, None) + + if removed_from_disk: + record_revoked(key, version) + + return ReconcileAction( + key=key, + action="removed", + version=version if is_valid_skill_version(version) else None, + path=str(target), + ) + + +def _utc_timestamp() -> str: + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index 4249d5cb..10c90d8c 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -452,6 +452,54 @@ class Skill: """Description from LaunchDarkly metadata; never parsed from the content.""" +ReconcileActionKind = Literal[ + "written", "updated", "skipped_current", "removed", "error" +] +"""The closed set of outcomes ``write_skills`` reports.""" + + +@dataclass(frozen=True) +class ReconcileAction: + """What ``write_skills`` did — or refused to do — for one skill.""" + + key: str + """ + The skill key, or the **empty string** for a failure that belongs to the run + rather than to one skill — a corrupt manifest, a manifest that could not be + rewritten, a retrieval that failed before any key was known. Callers grouping + a report by key need to expect that sentinel; a report may carry both kinds. + """ + action: ReconcileActionKind + version: int | None = None + path: str | None = None + """Canonical resolved path, when one was determined.""" + error: str | None = None + """Failure detail, set only when ``action == "error"``.""" + + +@dataclass(frozen=True) +class ReconcileReport: + """The result of a ``write_skills`` run — every outcome is visible here.""" + + actions: list[ReconcileAction] = field(default_factory=list) + + @property + def ok(self) -> bool: + """``True`` iff no action is an ``error``.""" + return not self.errors + + @property + def errors(self) -> list[ReconcileAction]: + """ + The ``error`` actions, in ``actions`` order. + + Exposed so callers never re-derive it — filtering ``actions`` is + boilerplate that otherwise reappears in every consumer. ``ok`` is defined + in terms of this, so the two can never disagree. + """ + return [a for a in self.actions if a.action == "error"] + + # --------------------------------------------------------------------------- # Model / graph options # --------------------------------------------------------------------------- diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 203eeb4f..0c656ede 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -17,6 +17,8 @@ import launchdarkly_ai_server.skills as skills_module from launchdarkly_ai_server import ( InMemorySkillStore, + ReconcileAction, + ReconcileReport, Skill, SkillReference, all_skills, @@ -129,7 +131,7 @@ def _fabricated_hash_cases() -> list[Any]: class TestSkillTypes: - """Immutability and optional metadata.""" + """Immutability, optional metadata, and ``ReconcileReport.ok``.""" def test_skill_reference_is_immutable(self) -> None: ref = SkillReference(key="pdf-extraction", version=2) @@ -164,6 +166,70 @@ def test_skill_metadata_defaults_to_none(self) -> None: assert skill.name is None assert skill.description is None + def test_report_ok_true_when_no_error_action(self) -> None: + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="skipped_current", version=2), + ReconcileAction(key="c", action="removed"), + ReconcileAction(key="d", action="updated", version=3), + ] + ) + assert report.ok is True + + def test_report_ok_false_with_error_action(self) -> None: + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="error", error="nope"), + ] + ) + assert report.ok is False + + def test_empty_report_is_ok(self) -> None: + assert ReconcileReport(actions=[]).ok is True + + def test_report_errors_lists_error_actions_in_order(self) -> None: + """The report exposes its error actions itself.""" + first = ReconcileAction(key="b", action="error", error="first") + second = ReconcileAction(key="d", action="error", error="second") + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + first, + ReconcileAction(key="c", action="skipped_current", version=2), + second, + ReconcileAction(key="e", action="removed"), + ] + ) + assert report.errors == [first, second] + + def test_report_errors_empty_when_no_error_action(self) -> None: + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="removed"), + ] + ) + assert report.errors == [] + + def test_empty_report_has_no_errors(self) -> None: + assert ReconcileReport(actions=[]).errors == [] + + def test_report_ok_and_errors_always_agree(self) -> None: + """``ok`` is true iff ``errors`` is empty, on the same objects.""" + clean = ReconcileReport( + actions=[ReconcileAction(key="a", action="written", version=1)] + ) + failed = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="error", error="nope"), + ] + ) + for report in (clean, failed, ReconcileReport(actions=[])): + assert report.ok is (report.errors == []) + class TestSkillRefs: """Pure projection of the config's skills array.""" @@ -267,6 +333,59 @@ def test_object_kind_is_not_public_api(self) -> None: assert "SKILL_OBJECT_KIND" not in package.__all__ assert not hasattr(package, "SKILL_OBJECT_KIND") + def test_constants_are_exported_from_the_package_root(self) -> None: + import launchdarkly_ai_server as package + + assert package.SKILL_FILENAME == "SKILL.md" + assert package.MANIFEST_FILENAME == ".launchdarkly-skills.json" + assert package.MANIFEST_VERSION == 1 + + def test_constants_are_listed_in_dunder_all(self) -> None: + """A name absent from ``__all__`` is not part of the public surface.""" + import launchdarkly_ai_server as package + + expected = { + "SKILL_FILENAME", + "MANIFEST_FILENAME", + "MANIFEST_VERSION", + } + assert expected <= set(package.__all__) + + def test_closed_set_types_are_exported_from_the_package_root(self) -> None: + """The two closed-set unions are public API, not implementation detail. + + ``ReconcileActionKind`` types the ``ReconcileAction.action`` field every + consumer of a report reads and switches on, and ``OnUnavailable`` types + a public keyword argument of ``write_skills``. ``agents.md`` forbids + handler packages from importing sub-path modules, so a name exported + only from the implementation module has no supported import path. + """ + import launchdarkly_ai_server as package + + assert hasattr(package, "ReconcileActionKind") + assert hasattr(package, "OnUnavailable") + assert {"ReconcileActionKind", "OnUnavailable"} <= set(package.__all__) + + def test_exported_action_union_admits_exactly_the_five_actions(self) -> None: + """The union must match the actions a report can actually carry. + + Spelled out rather than imported from the implementation for the same + reason as the constants above: deriving the expectation from the thing + under test would make the assertion circular. + """ + import typing + + import launchdarkly_ai_server as package + + assert set(typing.get_args(package.ReconcileActionKind)) == { + "written", + "updated", + "skipped_current", + "removed", + "error", + } + assert set(typing.get_args(package.OnUnavailable)) == {"keep", "raise"} + def test_retrieval_surface_is_exported_from_the_package_root(self) -> None: """A name absent from ``__all__`` is not part of the public surface.""" import launchdarkly_ai_server as package @@ -811,7 +930,7 @@ class TestWithholdingSummary: """ def _tampered(self, make_raw_skill: Any, key: str = "a") -> dict[str, Any]: - raw = make_raw_skill(key=key) + raw: dict[str, Any] = make_raw_skill(key=key) raw["contentHash"] = "0" * 64 return raw diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py new file mode 100644 index 00000000..62d01d59 --- /dev/null +++ b/packages/client/tests/test_skills_fs.py @@ -0,0 +1,582 @@ +""" +Tests for ``write_skills`` — filesystem materialization and manifest reconcile +semantics. + +Every test writes only inside pytest's ``tmp_path``. No network, no real +LaunchDarkly client, no real skill transport. + +The security abuse matrix — path traversal, symlink attacks, clobber +protection, corrupt manifests, atomicity under an injected crash, and the +materialization telemetry allowlist — is a separate module. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any + +import pytest + +import launchdarkly_ai_server.skills as skills_module +from launchdarkly_ai_server import ( + InMemorySkillStore, + Skill, + SkillReference, + write_skills, +) + +MANIFEST_NAME = ".launchdarkly-skills.json" +SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" + +INTEGRITY_SIGNAL = "AgentControl Skill Integrity Failure" + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + r = tmp_path / "skills" + r.mkdir() + return r + + +pytestmark = pytest.mark.usefixtures("reset_skill_state") +"""Every test in this module runs against freshly cleared module state.""" + + +def _hash(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +def _skill( + key: str = "test-skill", version: int = 1, content: str = SKILL_BODY +) -> Skill: + return Skill( + key=key, + version=version, + content=content.encode("utf-8"), + content_hash=_hash(content), + ) + + +def _manifest_path(root: Path) -> Path: + return root / MANIFEST_NAME + + +def _read_manifest(root: Path) -> dict[str, Any]: + return json.loads(_manifest_path(root).read_text(encoding="utf-8")) + + +def _write_manifest(root: Path, raw: Any) -> None: + root.mkdir(parents=True, exist_ok=True) + _manifest_path(root).write_text( + raw if isinstance(raw, str) else json.dumps(raw), encoding="utf-8" + ) + + +def _entry(key: str, version: int, content: str) -> dict[str, Any]: + return { + "key": key, + "version": version, + "sha256": _hash(content), + "writtenAt": "2026-08-14T19:00:00Z", + } + + +def _place_managed(root: Path, key: str, content: str, version: int = 1) -> Path: + """Pre-create a file AND its manifest entry — i.e. an SDK-managed path.""" + target = root / key / "SKILL.md" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {f"{key}/SKILL.md": _entry(key, version, content)}, + }, + ) + return target + + +def _actions_by_key(report: Any) -> dict[str, Any]: + return {a.key: a for a in report.actions} + + +def _error_messages(report: Any) -> list[str]: + """All ``error`` action messages, regardless of which key they hang off. + + Run-level (manifest) errors have no well-defined ``key`` yet, so assertions + about them scan every error action rather than looking one up by key. + """ + return [a.error or "" for a in report.actions if a.action == "error"] + + +class TestBasicWrites: + """Basic writes and the returned report.""" + + async def test_new_skill_is_written_verbatim(self, root: Path) -> None: + report = await write_skills([_skill("pdf-extraction", 2)], root) + + target = root / "pdf-extraction" / "SKILL.md" + assert target.read_text(encoding="utf-8") == SKILL_BODY + assert report.ok is True + action = _actions_by_key(report)["pdf-extraction"] + assert action.action == "written" + assert action.version == 2 + assert action.path is not None + assert Path(action.path).resolve() == target.resolve() + assert action.error is None + + async def test_skill_inputs_need_no_store(self, root: Path) -> None: + report = await write_skills([_skill("a")], root) + assert report.ok is True + assert (root / "a" / "SKILL.md").exists() + + async def test_reference_inputs_resolve_through_store( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=3)) + report = await write_skills([SkillReference(key="a", version=3)], root) + assert report.ok is True + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_string_inputs_resolve_latest( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=9)) + report = await write_skills(["a"], root) + assert _actions_by_key(report)["a"].version == 9 + + async def test_star_writes_everything_in_the_store( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + for k in ("a", "b", "c"): + store.put(make_raw_skill(key=k)) + report = await write_skills("*", root) + assert report.ok is True + assert len([a for a in report.actions if a.action == "written"]) == 3 + for k in ("a", "b", "c"): + assert (root / k / "SKILL.md").exists() + + async def test_one_action_per_requested_skill(self, root: Path) -> None: + report = await write_skills([_skill("a"), _skill("b")], root) + assert sorted(a.key for a in report.actions) == ["a", "b"] + + async def test_empty_request_on_empty_root_is_ok(self, root: Path) -> None: + report = await write_skills([], root) + assert report.ok is True + assert report.actions == [] + + +class TestManifest: + """Manifest format and forward compatibility.""" + + async def test_manifest_format_is_exact(self, root: Path) -> None: + await write_skills([_skill("pdf-extraction", 2)], root) + + manifest = _read_manifest(root) + assert manifest["manifestVersion"] == 1 + entry = manifest["entries"]["pdf-extraction/SKILL.md"] + assert entry["key"] == "pdf-extraction" + assert entry["version"] == 2 + assert entry["sha256"] == _hash(SKILL_BODY) + assert isinstance(entry["writtenAt"], str) + + async def test_entry_paths_are_forward_slash_relative(self, root: Path) -> None: + await write_skills([_skill("a")], root) + keys = list(_read_manifest(root)["entries"].keys()) + assert keys == ["a/SKILL.md"] + assert "\\" not in keys[0] + assert not keys[0].startswith("/") + + async def test_unknown_fields_are_preserved_on_rewrite(self, root: Path) -> None: + entry = _entry("a", 1, SKILL_BODY) + entry["futureEntryField"] = "keep-me" + _write_manifest( + root, + { + "manifestVersion": 1, + "futureTopLevelField": {"keep": True}, + "entries": {"a/SKILL.md": entry}, + }, + ) + (root / "a").mkdir() + (root / "a" / "SKILL.md").write_text(SKILL_BODY, encoding="utf-8") + + await write_skills([_skill("a", 2, SKILL_BODY + "more\n")], root) + + manifest = _read_manifest(root) + assert manifest["futureTopLevelField"] == {"keep": True} + assert manifest["entries"]["a/SKILL.md"]["futureEntryField"] == "keep-me" + + +class TestReconcileSemantics: + """The reconcile state table.""" + + async def test_unchanged_managed_file_is_skipped_current(self, root: Path) -> None: + target = _place_managed(root, "a", SKILL_BODY) + before = target.stat().st_mtime_ns + + report = await write_skills([_skill("a")], root) + + assert _actions_by_key(report)["a"].action == "skipped_current" + assert target.read_text(encoding="utf-8") == SKILL_BODY + assert target.stat().st_mtime_ns == before + + async def test_new_version_updates(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY, version=1) + new_content = SKILL_BODY + "second version\n" + + report = await write_skills([_skill("a", 2, new_content)], root) + + action = _actions_by_key(report)["a"] + assert action.action == "updated" + assert action.version == 2 + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == new_content + assert _read_manifest(root)["entries"]["a/SKILL.md"]["version"] == 2 + + async def test_local_tampering_is_overwritten(self, root: Path) -> None: + target = _place_managed(root, "a", SKILL_BODY) + target.write_text("locally tampered\n", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert _actions_by_key(report)["a"].action == "updated" + assert target.read_text(encoding="utf-8") == SKILL_BODY + + async def test_prune_removes_formerly_managed_skill(self, root: Path) -> None: + _place_managed(root, "gone", SKILL_BODY) + + report = await write_skills([], root) + + assert _actions_by_key(report)["gone"].action == "removed" + assert not (root / "gone" / "SKILL.md").exists() + assert not (root / "gone").exists() + assert _read_manifest(root)["entries"] == {} + + async def test_prune_false_keeps_the_file(self, root: Path) -> None: + target = _place_managed(root, "gone", SKILL_BODY) + + report = await write_skills([], root, prune=False) + + assert target.exists() + assert [a for a in report.actions if a.action == "removed"] == [] + assert "gone/SKILL.md" in _read_manifest(root)["entries"] + + async def test_prune_does_not_touch_unmanaged_files(self, root: Path) -> None: + _place_managed(root, "gone", SKILL_BODY) + bystander = root / "user-notes.md" + bystander.write_text("mine\n", encoding="utf-8") + user_dir_file = root / "user-skill" / "SKILL.md" + user_dir_file.parent.mkdir() + user_dir_file.write_text("hand written\n", encoding="utf-8") + + await write_skills([], root) + + assert bystander.read_text(encoding="utf-8") == "mine\n" + assert user_dir_file.read_text(encoding="utf-8") == "hand written\n" + + async def test_prune_refusal_for_unownable_path_reports_the_version( + self, root: Path + ) -> None: + """A prune refusal carries the manifest's version. + + A manifest entry whose path is not one this SDK could have written is + refused rather than removed. The entry is in hand at that point, so the + error action must carry its version — otherwise a prune *failure* is + strictly less informative than a prune *success*, which does report it. + """ + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + # Right key, wrong filename — not a path this SDK could own. + "orphan/NOTES.md": _entry("orphan", 7, SKILL_BODY), + }, + }, + ) + + report = await write_skills([], root) + + action = _actions_by_key(report)["orphan"] + assert action.action == "error" + assert action.version == 7 + + async def test_prune_refusal_for_symlinked_target_reports_the_version( + self, root: Path + ) -> None: + """Same contract on the symlink refusal path (prune side).""" + if not hasattr(os, "symlink"): + pytest.skip("platform has no symlink support") + (root / "a").mkdir() + outside_file = root.parent / "victim.md" + outside_file.write_text("victim content\n", encoding="utf-8") + (root / "a" / "SKILL.md").symlink_to(outside_file) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 4, "victim content\n")}, + }, + ) + + report = await write_skills([], root) + + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.version == 4 + + async def test_unresolvable_request_still_reports_no_version( + self, root: Path + ) -> None: + """The other half of the contract: do not invent a version. + + A reference that could not be retrieved has neither a manifest entry + nor a ``Skill``, so there is no version to report and ``version`` stays + ``None``. Without this, "always populate version" would be satisfied by + fabricating one. + """ + report = await write_skills([SkillReference(key="ghost", version=3)], root) + + action = _actions_by_key(report)["ghost"] + assert action.action == "error" + assert action.version is None + + async def test_prune_keeps_directory_when_not_empty(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY) + extra = root / "a" / "user-file.txt" + extra.write_text("keep\n", encoding="utf-8") + + report = await write_skills([], root) + + assert _actions_by_key(report)["a"].action == "removed" + assert not (root / "a" / "SKILL.md").exists() + assert extra.exists() + + +class TestRootHandling: + """Root resolution.""" + + async def test_absent_leaf_root_is_created(self, tmp_path: Path) -> None: + target_root = tmp_path / "skills" + report = await write_skills([_skill("a")], target_root) + assert report.ok is True + assert (target_root / "a" / "SKILL.md").exists() + + async def test_missing_ancestors_raise(self, tmp_path: Path) -> None: + with pytest.raises(ValueError): + await write_skills([_skill("a")], tmp_path / "a" / "b" / "c") + + async def test_root_that_is_a_file_raises(self, tmp_path: Path) -> None: + file_root = tmp_path / "not-a-dir" + file_root.write_text("x", encoding="utf-8") + with pytest.raises(ValueError): + await write_skills([_skill("a")], file_root) + + async def test_accepts_string_root(self, root: Path) -> None: + report = await write_skills([_skill("a")], str(root)) + assert report.ok is True + + +class TestSkillsArgumentErrors: + """A bare string that is not ``"*"`` raises. + + A ``ValueError``, not a ``TypeError``: a string *is* an accepted argument + type here, since ``"*"`` means "everything the store holds", so this is an + acceptable type carrying an invalid value. The accessors' equivalent guard + is a ``TypeError`` because a string is never a valid argument there. + """ + + async def test_bare_non_star_string_raises_value_error(self, root: Path) -> None: + with pytest.raises(ValueError) as excinfo: + await write_skills("pdf-extraction", root) + + # Naming the accepted forms is the actionable half of the message. + assert '"*"' in str(excinfo.value) + + async def test_star_is_accepted(self, root: Path) -> None: + """Positive control — otherwise the guard above could reject every string.""" + store = InMemorySkillStore() + store.put( + { + "key": "a", + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + ) + skills_module._set_store(store) + + report = await write_skills("*", root) + + assert report.ok is True + assert (root / "a" / "SKILL.md").exists() + + async def test_bare_string_writes_nothing(self, root: Path) -> None: + """The raise precedes any filesystem work. + + Asserting only the raise would also pass for an implementation that + created one directory per character before failing. + """ + with pytest.raises(ValueError): + await write_skills("abc", root) + + assert list(root.iterdir()) == [] + + +class TestResilience: + """Unavailable retrieval and timeout.""" + + async def test_keep_is_the_default_and_does_not_raise(self, root: Path) -> None: + existing = _place_managed(root, "a", SKILL_BODY) + + report = await write_skills([SkillReference(key="a", version=1)], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert existing.read_text(encoding="utf-8") == SKILL_BODY + + async def test_raise_mode_propagates(self, root: Path) -> None: + with pytest.raises(RuntimeError, match=r"(?i)(unavailable|skill store)"): + await write_skills( + [SkillReference(key="a", version=1)], root, on_unavailable="raise" + ) + + async def test_store_error_is_reported_not_raised( + self, root: Path, exploding_store: Any + ) -> None: + report = await write_skills([SkillReference(key="a", version=1)], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + + async def test_exhausted_timeout_behaves_as_unavailable( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a")) + report = await write_skills( + [SkillReference(key="a", version=1)], root, timeout=0 + ) + assert report.ok is False + assert not (root / "a" / "SKILL.md").exists() + + async def test_exhausted_timeout_raises_in_raise_mode( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a")) + with pytest.raises(RuntimeError, match=r"(?i)(unavailable|timeout|timed out)"): + await write_skills( + [SkillReference(key="a", version=1)], + root, + timeout=0, + on_unavailable="raise", + ) + + async def test_exhausted_timeout_stops_pruning( + self, root: Path, store: InMemorySkillStore + ) -> None: + """The deadline bounds pruning too, not just retrieval and the writes. + + A run whose writes all land just inside the deadline would otherwise go + on to stat, unlink and rmdir every stale manifest entry unbounded — the + opposite of what a small ``timeout`` asks for. + """ + existing = _place_managed(root, "stale", SKILL_BODY) + + report = await write_skills([], root, timeout=0) + + assert report.ok is False + assert existing.exists(), "prune ran past the exhausted deadline" + assert any("timeout was exhausted" in m for m in _error_messages(report)) + # The entry survives, so the next reconcile picks it up. + assert "stale/SKILL.md" in _read_manifest(root)["entries"] + + async def test_a_verification_failure_never_prunes_the_good_copy( + self, root: Path + ) -> None: + """A store may key ``all_objects`` differently from the object's own key. + + The on-disk copy lives under the object's own key, so a failure recorded + under the *store's* dict key would drop the real key out of the + requested set and let prune delete the last known-good copy. + """ + + class AliasKeyedStore: + """Keys objects by an internal id, not by the skill's own key.""" + + def __init__(self, raw: dict[str, Any]) -> None: + self._raw = raw + + def get_object(self, kind: str, key: str) -> dict[str, Any] | None: + return None + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + return {"internal-uuid-1": self._raw} + + existing = _place_managed(root, "pdf-extraction", SKILL_BODY) + tampered = { + "key": "pdf-extraction", + "version": 1, + "content": "tampered\n", + "contentHash": _hash(SKILL_BODY), # does not match the content + } + skills_module._set_store(AliasKeyedStore(tampered)) + + report = await write_skills("*", root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert [a.action for a in report.actions] == ["error"] + assert _actions_by_key(report)["pdf-extraction"].action == "error" + assert "pdf-extraction/SKILL.md" in _read_manifest(root)["entries"] + + async def test_unavailable_run_does_not_corrupt_manifest(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY) + before = _read_manifest(root) + + await write_skills([SkillReference(key="b", version=1)], root) + + assert ( + _read_manifest(root)["entries"]["a/SKILL.md"] + == (before["entries"]["a/SKILL.md"]) + ) + + +class TestVerifyThenWrite: + """Hash re-verified immediately before writing.""" + + async def test_hash_mismatch_aborts_the_write( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + bad = Skill( + key="a", + version=1, + content=SKILL_BODY.encode("utf-8"), + content_hash="0" * 64, + ) + + report = await write_skills([bad], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert not (root / "a" / "SKILL.md").exists() + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_oversize_skill_aborts_the_write(self, root: Path) -> None: + oversize = "x" * (64 * 1024 + 1) + report = await write_skills([_skill("a", 1, oversize)], root) + assert report.ok is False + assert not (root / "a" / "SKILL.md").exists() + + async def test_mismatch_does_not_disturb_existing_managed_file( + self, root: Path + ) -> None: + target = _place_managed(root, "a", SKILL_BODY) + bad = Skill(key="a", version=2, content=b"new content\n", content_hash="f" * 64) + + await write_skills([bad], root) + + assert target.read_text(encoding="utf-8") == SKILL_BODY From c88d5bf5ce0db1e8ad9f8fa5a5ee782ea768b887 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Wed, 26 Aug 2026 16:26:15 -0400 Subject: [PATCH 03/16] test(client): assert rename containment through the branching helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``test_key_at_the_data_model_bound_is_reported_not_raised`` read ``dst_dir_id`` directly, but that field is only populated when ``os.replace`` is called with ``dir_fd`` kwargs. On the path fallback (``SUPPORTS_DIR_FD`` false — the shape Windows takes) it stays ``None``, so the assertion failed even though the valid skill had been renamed correctly into its own directory. ``_assert_atomic_rename_of`` already branches on both call shapes and asserts the same containment property, plus the single-rename count the list comparison implied. Use it. Verified by forcing the probe off for a whole session: this was the only test in the module that broke under the no-``*at()`` shape, and the helper-based check passes under both. Reported by Cursor Bugbot on #54. Co-Authored-By: Claude Opus 5 --- packages/client/tests/test_skills_fs.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 6502aa5b..5af4d19b 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -932,8 +932,11 @@ async def test_key_at_the_data_model_bound_is_reported_not_raised( # The bare-filename ``dst`` of a ``dir_fd``-relative rename carries no # directory, so "the path does not contain the hostile key" is no longer # a meaningful check. Assert the stronger thing instead: the only rename - # that happened was into the valid skill's own directory. - assert [call.dst_dir_id for call in spy.calls] == [_dir_id(root / "good")] + # that happened was into the valid skill's own directory. Through the + # shared helper, so the check holds on the path fallback too — reading + # ``dst_dir_id`` directly would compare ``None`` there and fail a run + # that had in fact renamed correctly. + _assert_atomic_rename_of(spy, root / "good") # The valid skill is fully reconciled: written AND recorded, not orphaned. assert (root / "good" / "SKILL.md").exists() assert "good/SKILL.md" in _read_manifest(root)["entries"] From 49c2741641051df8b666db78097832b745b2ba6f Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Sun, 30 Aug 2026 04:26:10 -0400 Subject: [PATCH 04/16] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20self-healing=20reconciles,=20and=20keys=20no=20file?= =?UTF-8?q?system=20can=20hold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps from the Agent Skills security design review, both in `skills_fs.py`: AV-3 (partial reconciles are unrecoverable) and DV-2 (the key grammar admits Windows device names). **AV-3 — adopt a file whose bytes already are the resolved content.** `_write_all` reconciles every skill and only then rewrites the manifest, once, last. A process killed in that window leaves a skill file at a managed path with no manifest entry — exactly the condition `_write_one` treats as an unmanaged-file collision, so the skill was wedged permanently: every later reconcile took the same refusal branch. Boot-time execution under a ten-second budget makes the crash window realistic. `_write_one` now reads and hashes first and decides from the bytes. Content byte-identical to what LaunchDarkly resolved is adopted — manifest entry recorded, reported `skipped_current` — and anything else falls through to the same refusal as before. This cannot weaken the clobber guarantee: differing unmanaged bytes are never overwritten, and the existing clobber tests pass unchanged. Three details carry the safety: - A read that fails is a refusal, never an overwrite, with a message distinguishable from the byte-mismatch refusal — it is the comparison that would otherwise authorize the write, and a file that could not be read has not been shown to be ours. - The read stays on `_read_regular_file`. Adoption widens it to genuinely foreign files, so its refusal of FIFOs and other non-regular files is now load-bearing rather than defensive. It gains a `max_bytes` bound of `len(content) + 1` — enough to prove inequality for anything longer, and what keeps a foreign file of arbitrary size out of memory. A bound of exactly `len(content)` would adopt every file that merely begins with the resolved content. - `skipped_current` is reused rather than adding an `adopted` action kind, so `ReconcileActionKind` — public, and owned by an approved PR — does not change. Its documented meaning already fits. Adoption also makes the file prunable later. That is correct rather than a weakening: only byte-identical LaunchDarkly content is ever adopted, so a later prune removes content LaunchDarkly delivered anyway — what would have happened had the crash not occurred. The review also floats a write-intent journal. Assessed as over-engineered; not built. **AV-3, secondary — sweep orphaned temp files.** `atomic_write` unlinks its temp file on any exception but not after a `SIGKILL`, and `_prune` walks manifest entries, which an orphan never has, so nothing would ever notice one. The second-order effect is worse than the disk: `_prune_one`'s `rmdir` only succeeds on an empty directory, so a single orphan pins a skill's directory forever. The sweep runs on both the write and the prune path, and is the one place this SDK removes a file the manifest does not list, so it is bounded on every axis: inside `//` only, for a key that passes `_key_rejection_reason`; only names `safe_fs` itself recognizes, via a new `is_temp_name` beside the naming code rather than a copy of the format string that could drift from it; only regular files, with the type read off the descriptor; unlinked through the pinned descriptor. It never raises and never aborts a run. **DV-2 — reject the 22 Windows reserved device names.** `con`, `prn`, `aux`, `nul`, `com1`–`com9`, `lpt1`–`lpt9` are all valid skill keys and none can be a directory name on Windows. Rejected in `_key_rejection_reason`, which the write and prune paths already share, and *not* in the key grammar: `parse_ai_config` fails closed, so a grammar-level rejection would invalidate an entire AI Config for a Linux customer over a Windows-only constraint, and would silently shrink `skill_refs` — which is what authorizes a prune, turning "fails to write on Windows" into "gets deleted on Linux". The 255-byte component bound is in this layer for the same reason. Unconditional, not platform-gated: a root written from a Linux container is routinely read from a Windows host, and neither repository has a Windows CI runner, so a gated branch would be untestable — the condition that produced the gap. No suffix stripping and no case folding: the grammar admits no `.` and no `$`, so `con.txt` and `CONIN$` are unreachable, and keys are lowercase-only. `com0` and `lpt0` are not reserved and are not included. The trade is real and belongs in the release notes: a customer who legitimately names a skill `aux` now gets a reported `error` action on Linux where it previously worked. Neither gap emits the integrity-failure log record. A key rejection and a clobber refusal are `ReconcileAction` errors, not integrity failures. Tests cover crash-mid-reconcile recovery end to end (adopted, reported, recorded, and the next reconcile an ordinary no-op), byte-differing content still refused untouched, an unmanaged FIFO refused without hanging, a read failure refusing rather than overwriting, the `len + 1` off-by-one, the sweep and the `rmdir` it unblocks, lookalike temp names and a symlink wearing one left alone, all 22 reserved names through both destructive paths, and — the point of the layer choice — each reserved name still valid to `is_valid_skill_key`, `parse_ai_config`, and `skill_refs`. Co-Authored-By: Claude Opus 5 --- packages/client/README.md | 26 ++ packages/client/agents.md | 47 ++- .../src/launchdarkly_ai_server/safe_fs.py | 54 ++- .../src/launchdarkly_ai_server/skills_fs.py | 159 ++++++- packages/client/tests/test_skills_fs.py | 388 ++++++++++++++++++ 5 files changed, 655 insertions(+), 19 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index d9c1db15..711ce5bf 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -357,6 +357,32 @@ never writes through a symlink; writes are atomic (temp file, `fsync`, rename) a `0644`; and if the manifest is unreadable it performs no destructive action at all. Removing a skill from a variation is how revocation works — the next reconcile prunes it. +**One exception, and it is what makes a crashed reconcile recoverable.** A file at a managed +path whose bytes are *already byte-identical* to the content LaunchDarkly resolved is +adopted — recorded in the manifest and reported `skipped_current` — rather than refused. +Without that, a process killed after a skill file lands but before the manifest is rewritten +leaves that file managed-but-unrecorded, which is indistinguishable from a file you wrote +yourself, so every later reconcile would refuse it and the skill would stay wedged until +someone intervened. Adoption cannot weaken the guarantee above, because bytes that differ in +any way are still refused and left untouched. Note that an adopted file becomes prunable +like any other managed file — which is the same outcome the crash pre-empted. + +**A few keys are legal to an AI Config but not to a filesystem.** A key becomes a single +directory name, so `write_skills` applies bounds of its own on top of the key grammar: no +mainstream filesystem allows a 256-byte path component, and Windows reserves 22 MS-DOS +device names (`con`, `prn`, `aux`, `nul`, `com1`–`com9`, `lpt1`–`lpt9`) that cannot be +directory names there. Either one is a reported `error` action for that skill, and the +rejection is unconditional rather than platform-gated — a managed root written from a Linux +container is routinely read from a Windows host, so the on-disk result must not depend on +which OS ran the write. The keys stay valid everywhere else: an AI Config referencing a skill +named `aux` parses, and its other fields are unaffected. If you have a skill named for a +device, rename it. + +**Total path length is yours to bound, not the SDK's.** The 255-byte bound above is per +*component*; the root is your path, so `` + `` + `/SKILL.md` can still exceed +Windows' 260-character `MAX_PATH` with a perfectly legal key. Choose a short managed root on +Windows. + | Export | Description | |---|---| | `skill_refs(config)` | Project a config's `skills` array into `list[SkillReference]`. Pure — no client, store, or network needed. Returns `[]` when absent. | diff --git a/packages/client/agents.md b/packages/client/agents.md index feb2e348..cd99b28f 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -250,7 +250,26 @@ Store data is **untrusted input**; the transport is not part of the trust bounda - **Never write through a symlink**, in either the skill directory or the target file, on the write path *and* the prune path. - **Destructive operations only on manifest-listed paths whose `key` matches.** A file at a - managed path with no matching manifest entry is reported as `error` and left alone. + managed path with no matching manifest entry is reported as `error` and left alone — + *unless its bytes already are the resolved content*, in which case it is adopted (manifest + entry recorded, reported `skipped_current`). That single exception is what makes a + reconcile killed between the content writes and the final manifest rewrite recoverable + instead of permanently wedged, and it cannot be widened: the comparison is over the + verbatim bytes against the resolved `contentHash`, a read that fails is a refusal and + never an overwrite, and the read is bounded at `len(content) + 1` bytes so a file that + merely *begins* with the resolved content is refused too. Do not relax it to a prefix, a + length, an mtime, or the manifest's own recorded `sha256` — that field is untrusted and is + never a decision input. `skipped_current` is reused deliberately rather than adding an + `adopted` action kind; `ReconcileActionKind` is a public closed set. +- **Temp files are swept, within the same bounds as everything else.** `atomic_write` unlinks + its own temp file on any exception, but a `SIGKILL` leaves one behind that no manifest + entry records, and a non-empty directory defeats `_prune_one`'s `rmdir` — so one orphan + pins a skill directory forever. The sweep is the only place this SDK removes a file the + manifest does not list, and it is bounded on every axis: inside `//` only, for a + key that passes `_key_rejection_reason`; only names `safe_fs.is_temp_name` recognizes, + anchored at both ends and asked of `safe_fs` rather than re-spelled (a copy would drift + from the writer); only regular files, with the type read off the descriptor; unlinked + through the pinned descriptor. It never raises and never aborts a run. - **A corrupt manifest fails closed**: unreadable, unparseable, not an object, malformed `entries`, or a `manifestVersion` this release cannot read means no overwrites and no prunes, brand-new paths may still be written, an `error` action names the manifest, and @@ -265,10 +284,28 @@ Store data is **untrusted input**; the transport is not part of the trust bounda "Descriptor-pinned filesystem access" below. Re-resolving `/` from its path at write or unlink time reopens a swap window that the checks above cannot cover. - **A key valid to the data model may still be unrepresentable on disk.** The model allows - 256 characters; `NAME_MAX` is 255 bytes. `write_skills` rejects an over-long key before - any filesystem call, and every per-skill filesystem failure is caught at the loop so it - becomes an `error` action — aborting the loop would skip the manifest rewrite and orphan - files already written in that run. + 256 characters; `NAME_MAX` is 255 bytes. Windows additionally reserves 22 MS-DOS device + names, none of which can be a directory name there: `con`, `prn`, `aux`, `nul`, + `com1`–`com9`, `lpt1`–`lpt9` (`com0` and `lpt0` are *not* reserved; do not add them). + `write_skills` rejects both before any filesystem call, and every per-skill filesystem + failure is caught at the loop so it becomes an `error` action — aborting the loop would + skip the manifest rewrite and orphan files already written in that run. +- **Those two bounds live in `_key_rejection_reason`, not in the key grammar, and must not + move.** `is_valid_skill_key` / `skill_key_rejection_reason` keep admitting an over-long or + reserved key on purpose. `parse_ai_config` fails closed on a bad `skills` entry, so a + grammar-level rejection would invalidate the *entire* AI Config — model, provider, + instructions, tools — for a Linux customer over a Windows-only constraint; and it would + silently shrink `skill_refs`, which is what authorizes a prune, converting "this skill + fails to write on Windows" into "this skill gets deleted on Linux". `_key_rejection_reason` + is shared by the write and prune paths, so one edit covers both destructive paths. + The reserved-name check is unconditional rather than `os.name == "nt"`-gated: a root + written from a Linux container is routinely read from a Windows host, and neither + repository has a Windows CI runner (every matrix job is `ubuntu-latest`), so a gated branch + would be untestable — the exact condition that produced the gap. No suffix stripping and no + case folding are needed, because the grammar admits no `.` and no `$` (so `con.txt` and + `CONIN$` are unreachable) and is lowercase-only. The residual the SDK cannot check is total + path length: the 255-byte bound is per *component*, and the root belongs to the customer, + so `MAX_PATH` overflow is a README note rather than a check. - **A key is untrusted input everywhere it appears.** `skill_key_rejection_reason` is the single canonical explanation, so the config parser and the reference projection reject a key for the same stated reason — and so does every layer added later. A silently diff --git a/packages/client/src/launchdarkly_ai_server/safe_fs.py b/packages/client/src/launchdarkly_ai_server/safe_fs.py index 3605399e..73eef878 100644 --- a/packages/client/src/launchdarkly_ai_server/safe_fs.py +++ b/packages/client/src/launchdarkly_ai_server/safe_fs.py @@ -17,6 +17,7 @@ import errno import os +import re import secrets import stat import tempfile @@ -191,6 +192,53 @@ def unlink_file(directory: Path, name: str, *, dir_fd: int | None) -> None: os.unlink(name, dir_fd=dir_fd) +_TEMP_SUFFIX = ".tmp" +"""Suffix on every temp file this module creates.""" + +_TEMP_TOKEN_BYTES = 8 +"""Bytes of randomness in a temp name, as ``secrets.token_hex`` takes them.""" + +_TEMP_TOKEN_PATTERN = re.compile( + # Two producers, one recognizer. The descriptor path below names its temp + # file with ``secrets.token_hex(_TEMP_TOKEN_BYTES)`` — twice that many + # lowercase hex characters. The fallback path hands naming to + # ``tempfile.mkstemp``, whose sequence is eight characters drawn from + # ``[a-z0-9_]``. Matched with ``fullmatch``, which anchors both branches at + # both ends, so nothing longer or otherwise-shaped is ever recognized. + rf"[0-9a-f]{{{_TEMP_TOKEN_BYTES * 2}}}|[a-z0-9_]{{8}}" +) + + +def temp_name_prefix(name: str) -> str: + """ + The prefix every temp file for *name* is created under. + + Spelled once because two callers need to agree on it: ``atomic_write`` + creates the name, and a caller sweeping orphaned temp files left by a crash + has to recognize it. A copy of the format string in the sweeper would be a + copy that can drift out of step with the writer. + """ + return f".{name}." + + +def is_temp_name(candidate: str, name: str) -> bool: + """ + Whether *candidate* is a name this module could have created for *name*. + + The recognizer for the orphan sweep: ``atomic_write`` unlinks its temp file + on any exception, but a ``SIGKILL`` between the create and the rename leaves + it behind, and nothing else on disk records that it exists. Deliberately + narrow — prefix, random token, and suffix must all match, with nothing + before or after — because the only thing a caller does with a ``True`` here + is delete the file. + """ + prefix = temp_name_prefix(name) + if not candidate.startswith(prefix) or not candidate.endswith(_TEMP_SUFFIX): + return False + token = candidate[len(prefix) : -len(_TEMP_SUFFIX)] + return _TEMP_TOKEN_PATTERN.fullmatch(token) is not None + + def _mkstemp_at(dir_fd: int, prefix: str) -> tuple[int, str]: """ ``tempfile.mkstemp`` for a directory descriptor. @@ -202,7 +250,7 @@ def _mkstemp_at(dir_fd: int, prefix: str) -> tuple[int, str]: """ flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) for _ in range(tempfile.TMP_MAX): - name = f"{prefix}{secrets.token_hex(8)}.tmp" + name = f"{prefix}{secrets.token_hex(_TEMP_TOKEN_BYTES)}{_TEMP_SUFFIX}" try: return os.open(name, flags, 0o600, dir_fd=dir_fd), name except FileExistsError: @@ -234,7 +282,7 @@ def atomic_write( semantics on Windows). """ at_fd = dir_fd if dir_fd is not None and SUPPORTS_DIR_FD else None - prefix = f".{name}." + prefix = temp_name_prefix(name) target: str | Path if at_fd is not None: @@ -243,7 +291,7 @@ def atomic_write( else: # mkstemp opens with O_CREAT|O_EXCL, so an existing temp path is never # reused. - fd, temp = tempfile.mkstemp(dir=directory, prefix=prefix, suffix=".tmp") + fd, temp = tempfile.mkstemp(dir=directory, prefix=prefix, suffix=_TEMP_SUFFIX) target = directory / name try: diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py index feb8fbc9..8a865dd8 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fs.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -33,6 +33,7 @@ SymlinkRefused, atomic_write, atomic_write_in, + is_temp_name, pinned_directory, unlink_file, ) @@ -95,6 +96,27 @@ """ +_WINDOWS_RESERVED_NAMES = frozenset( + {"con", "prn", "aux", "nul"} + | {f"com{digit}" for digit in range(1, 10)} + | {f"lpt{digit}" for digit in range(1, 10)} +) +""" +The 22 MS-DOS device names Windows still reserves, which cannot be directory +names there. The key grammar admits every one of them, so a customer who names a +skill ``con`` gets a working reconcile on Linux and a broken one on Windows — +rejected here instead, on every platform, so the on-disk result never depends on +which OS ran the write. Neither repository has a Windows CI runner, which is the +condition that produced the gap in the first place. + +The bare names are the whole set: no suffix stripping is needed because the key +grammar admits no ``.``, so ``con.txt`` is unreachable, and ``CONIN$`` / +``CONOUT$`` are unreachable for want of a ``$``; no case folding is needed +because the grammar is lowercase-only. ``com0`` and ``lpt0`` are deliberately +absent — those are not reserved. +""" + + # ------------------------------------------------------------------------- # The reconcile entry point # ------------------------------------------------------------------------- @@ -644,6 +666,17 @@ def _key_rejection_reason(key: Any) -> str | None: f"skill key '{key[:32]}...' is {key_bytes} bytes, over the " f"{_MAX_PATH_COMPONENT_BYTES}-byte limit for a single directory name" ) + # Same reasoning as the byte bound above, and it lives at the same layer for + # the same reason: the grammar itself must keep admitting these, because + # rejecting them there would fail the whole AI Config over one skill, and + # would shrink ``skill_refs`` — which is what authorizes a prune, so a + # Windows-only constraint would delete the skill's file on Linux. + if key in _WINDOWS_RESERVED_NAMES: + return ( + f"skill key '{key}' is a name Windows reserves for a device and " + "cannot be a directory name there; it is rejected on every platform " + "so a managed root written on one OS is usable on the other" + ) return None @@ -683,24 +716,45 @@ def failed(message: str) -> ReconcileAction: ) encoded, content_hash = verified.encoded, verified.content_hash + # Sweep before writing rather than after, so a temp file this run is about + # to create can never be a candidate. + _sweep_orphan_temp_files(root, key) + # Overwrite only what the manifest records as ours under this key. entry = entries.get(relative) managed = isinstance(entry, dict) and entry.get("key") == key exists = target.exists() - if exists and not managed: - return failed( - f"'{relative}' exists but the manifest does not record it as managed " - f"under key '{key}'; refusing to overwrite a file this SDK did not write" - ) - if exists: + # Hash first, and decide from the bytes. The manifest check below is what + # protects a customer's own file, but it also refuses the file this SDK + # itself wrote and was killed before recording — the reconcile writes + # every skill and only then rewrites the manifest, so a crash in that + # window leaves a managed path with no entry, and every later reconcile + # takes the refusal branch forever. Comparing the bytes distinguishes the + # two cases without weakening anything: only content byte-identical to + # what LaunchDarkly resolved is ever adopted. try: - on_disk = _read_regular_file(target) + on_disk = _read_regular_file(target, max_bytes=len(encoded)) except OSError as exc: + if not managed: + # A read that failed proves nothing, and must never become an + # overwrite: it is the comparison below that would authorize one. + return failed( + f"'{relative}' exists, the manifest does not record it as " + f"managed under key '{key}', and it could not be read to " + f"compare against the resolved content: {exc}; refusing to " + "overwrite a file this SDK may not have written" + ) return failed(f"'{relative}' could not be read: {exc}") if hashlib.sha256(on_disk).hexdigest() == content_hash: + # ``skipped_current`` covers this deliberately, rather than a new + # action kind: its documented meaning is that the bytes on disk + # already are the resolved content, which is exactly as true for an + # adopted file as for one this SDK wrote and recorded. Adoption does + # add a manifest entry, so the file becomes prunable later — correct, + # because a prune then removes content LaunchDarkly delivered anyway. _update_entry(entries, relative, skill, content_hash) record_materialized(key, len(encoded), content_hash, "skipped_current") return ReconcileAction( @@ -709,6 +763,12 @@ def failed(message: str) -> ReconcileAction: version=skill.version, path=str(target), ) + + if not managed: + return failed( + f"'{relative}' exists but the manifest does not record it as managed " + f"under key '{key}'; refusing to overwrite a file this SDK did not write" + ) # Stale version or local tampering — LD-resolved content wins. action: ReconcileActionKind = "updated" else: @@ -725,7 +785,7 @@ def failed(message: str) -> ReconcileAction: ) -def _read_regular_file(target: Path) -> bytes: +def _read_regular_file(target: Path, *, max_bytes: int) -> bytes: """ Reads *target*, refusing anything that is not a regular file. @@ -739,6 +799,12 @@ def _read_regular_file(target: Path) -> bytes: bytes the *verbatim* bytes: it is 0 on POSIX, but on Windows a descriptor without it translates CRLF on read, which would fail the hash comparison against content that is actually current. + + Reads at most ``max_bytes + 1`` bytes. The only consumer compares a hash, and + anything longer than the resolved content cannot match it, so the one extra + byte is enough to prove inequality — which is what keeps a foreign file of + arbitrary size from being pulled into memory now that adoption reads files + the manifest does not list. """ flags = ( os.O_RDONLY @@ -751,15 +817,82 @@ def _read_regular_file(target: Path) -> bytes: if not stat.S_ISREG(os.fstat(fd).st_mode): raise OSError("the target file is not a regular file") chunks: list[bytes] = [] - while True: - chunk = os.read(fd, 65536) + remaining = max_bytes + 1 + while remaining > 0: + chunk = os.read(fd, min(remaining, 65536)) if not chunk: - return b"".join(chunks) + break chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) finally: os.close(fd) +def _sweep_orphan_temp_files(root: Path, key: str) -> None: + """ + Removes temp files a killed reconcile left behind under ``//``. + + ``atomic_write`` unlinks its own temp file on any exception, but a ``SIGKILL`` + between the create and the rename leaves one on disk, and nothing else + records that it exists: ``_prune`` walks manifest entries, and an orphan + never has one. The second-order effect is what makes this worth doing — + ``_prune_one``'s ``rmdir`` only succeeds on an empty directory, so a single + orphaned temp pins a skill's directory permanently. + + Bounded on every axis, because this is the one place the SDK removes a file + the manifest does not list: only inside a directory named by a key that + passes ``_key_rejection_reason``; only names ``safe_fs`` itself recognizes as + its own temp naming for ``SKILL.md``, anchored at both ends, and asked of + ``safe_fs`` rather than re-spelled here so the recognizer cannot drift from + the writer; only regular files; and every removal relative to a descriptor + pinned with ``O_NOFOLLOW``. It never raises and never aborts the run: the + reconcile itself has succeeded either way, so a sweep that cannot happen is + a warning. + """ + if _key_rejection_reason(key) is not None: + return + skill_dir = root / key + if not skill_dir.is_dir(): + return + + try: + with pinned_directory(skill_dir) as dir_fd: + # Listing by path is safe even though the removals are + # descriptor-relative: a name reaches the unlink only if it matches + # the anchored temp pattern, and the unlink resolves it inside the + # pinned directory, so a listing redirected between the pin and here + # can at worst name a file that is not in it. + for name in sorted(os.listdir(skill_dir)): + if is_temp_name(name, SKILL_FILENAME): + _remove_orphan_temp_file(skill_dir, name, dir_fd) + except (OSError, ValueError) as exc: + logger.warning( + "orphaned temp files under skill '%s' could not be swept: %s", key, exc + ) + + +def _remove_orphan_temp_file(skill_dir: Path, name: str, dir_fd: int | None) -> None: + """ + Removes one recognized orphan. A per-file failure warns and moves on. + + The type check is what keeps the temp naming from being a way to have this + SDK delete something it did not write: a symlink or a FIFO wearing that name + is not a file ``atomic_write`` left behind, so it is not this function's to + remove. It is read off the descriptor, not the path, wherever there is one. + """ + try: + if dir_fd is not None: + mode = os.stat(name, dir_fd=dir_fd, follow_symlinks=False).st_mode + else: + mode = os.lstat(skill_dir / name).st_mode + if not stat.S_ISREG(mode): + return + unlink_file(skill_dir, name, dir_fd=dir_fd) + except (OSError, ValueError) as exc: + logger.warning("an orphaned temp file could not be removed: %s", exc) + + def _write_through_descriptor( skill_dir: Path, encoded: bytes, key: str, relative: str ) -> str | None: @@ -929,6 +1062,10 @@ def _prune_one( if unsafe is not None: return _prune_error(key, f"'{relative}' was not removed: {unsafe}", version) + # Before the removal, so the ``rmdir`` below is not defeated by an orphaned + # temp file that nothing else on disk records. + _sweep_orphan_temp_files(root, key) + removed_from_disk = False if target.exists(): failure = _unlink_through_descriptor(skill_dir, relative) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 5af4d19b..223a3262 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -26,8 +26,11 @@ SkillReference, get_skill, init_client, + parse_ai_config, + skill_refs, write_skills, ) +from launchdarkly_ai_server.types_validation import is_valid_skill_key MANIFEST_NAME = ".launchdarkly-skills.json" SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" @@ -1640,3 +1643,388 @@ async def test_integrity_signal_property_keys_match_across_layers( f"write-only keys: {sorted(write_keys - accessor_keys)}" ) assert "expected_hash" in accessor_keys + + +# --------------------------------------------------------------------------- +# Self-healing partial reconciles +# --------------------------------------------------------------------------- + + +def _place_unmanaged(root: Path, key: str, content: str) -> Path: + """A file at a managed path with **no** manifest entry. + + Exactly the state a reconcile killed between the content writes and the + final manifest rewrite leaves behind — and, indistinguishably on disk, the + state a customer authoring their own file there creates. Which is why the + bytes are the only thing that may decide between them. + """ + target = root / key / "SKILL.md" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return target + + +class TestCrashMidReconcileRecovery: + """A crash between the writes and the manifest rewrite must not wedge a skill.""" + + async def test_byte_identical_unmanaged_file_is_adopted(self, root: Path) -> None: + """The whole point: the second reconcile repairs the first one's crash. + + Without adoption every later reconcile takes the clobber-refusal branch + forever, because the file is at a managed path with no manifest entry. + """ + target = _place_unmanaged(root, "a", SKILL_BODY) + + first = await write_skills([_skill("a")], root) + + assert first.ok is True, _error_messages(first) + action = _actions_by_key(first)["a"] + assert action.action == "skipped_current" + assert action.version == 1 + assert action.path == str(target) + # Adopted, not rewritten, and now recorded. + assert target.read_text(encoding="utf-8") == SKILL_BODY + entry = _read_manifest(root)["entries"]["a/SKILL.md"] + assert entry["key"] == "a" + assert entry["version"] == 1 + assert entry["sha256"] == _hash(SKILL_BODY) + + # And the run after it is an ordinary no-op, through the managed path. + second = await write_skills([_skill("a")], root) + assert second.ok is True, _error_messages(second) + assert _actions_by_key(second)["a"].action == "skipped_current" + + async def test_adoption_writes_nothing( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Adoption is a manifest edit, not a write. Nothing touches the bytes.""" + _place_unmanaged(root, "a", SKILL_BODY) + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert spy.calls == [] + + async def test_adoption_reports_skipped_current_not_a_new_action_kind( + self, root: Path, recording_emitter: Any + ) -> None: + """``skipped_current`` is reused deliberately — no ``adopted`` kind exists.""" + skills_module._set_emitter_for_testing(recording_emitter) + _place_unmanaged(root, "a", SKILL_BODY) + + report = await write_skills([_skill("a")], root) + + assert {a.action for a in report.actions} == {"skipped_current"} + props = recording_emitter.signals(MATERIALIZED_SIGNAL)[0] + assert props["reconcile_action"] == "skipped_current" + assert props["skill_key"] == "a" + + async def test_an_adopted_file_is_prunable_afterwards(self, root: Path) -> None: + """The documented caveat, pinned. + + Adoption records a manifest entry, so a later reconcile may prune the + file. That is correct rather than a weakening: only content byte-identical + to what LaunchDarkly resolved is ever adopted, so the prune removes + content LaunchDarkly delivered — exactly what would have happened had the + crash never occurred. + """ + target = _place_unmanaged(root, "a", SKILL_BODY) + await write_skills([_skill("a")], root) + + report = await write_skills([], root) + + assert report.ok is True, _error_messages(report) + assert _actions_by_key(report)["a"].action == "removed" + assert not target.exists() + + async def test_differing_unmanaged_content_is_still_refused( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The clobber guarantee, restated against the adoption rule. + + Adoption compares bytes, so anything that is not byte-identical to the + resolved content falls through to the same refusal as before. + """ + target = _place_unmanaged(root, "a", "user authored\n") + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "did not write" in action.error + assert target.read_text(encoding="utf-8") == "user authored\n" + assert spy.calls == [] + + async def test_a_longer_file_sharing_the_content_prefix_is_not_adopted( + self, root: Path + ) -> None: + """The read is bounded at ``len(content) + 1``, and that one byte matters. + + A bound of exactly ``len(content)`` would make every file that merely + *begins* with the resolved content hash as current, adopting — and later + pruning — a customer file with the skill body at its head. + """ + longer = SKILL_BODY + "and my own notes below\n" + target = _place_unmanaged(root, "a", longer) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert target.read_text(encoding="utf-8") == longer + + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="no FIFOs on this platform") + async def test_an_unmanaged_fifo_is_refused_and_never_read( + self, root: Path + ) -> None: + """Adoption widened the read to foreign files, so this refusal is load-bearing. + + Opening a FIFO with no writer blocks forever; the descriptor-pinned read + opens ``O_NONBLOCK`` and rejects anything that is not a regular file, so + this returns rather than hanging the reconcile and the event loop with it. + """ + skill_dir = root / "a" + skill_dir.mkdir() + os.mkfifo(skill_dir / "SKILL.md") + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "regular file" in action.error + assert stat.S_ISFIFO(os.lstat(skill_dir / "SKILL.md").st_mode) + + async def test_a_read_failure_on_an_unmanaged_file_refuses( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A failed read proves nothing, so it must never become an overwrite. + + The adoption comparison is what would otherwise authorize the write, and + a file whose bytes could not be read has not been shown to be ours. + """ + target = _place_unmanaged(root, "a", SKILL_BODY) + spy = _ReplaceSpy().install(monkeypatch) + real_open = os.open + + def refuse_the_target(path: Any, *args: Any, **kwargs: Any) -> int: + if isinstance(path, (str, os.PathLike)) and os.fspath(path) == str(target): + raise PermissionError(13, "Permission denied") + return real_open(path, *args, **kwargs) + + monkeypatch.setattr(skills_fs_module.os, "open", refuse_the_target) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + # Distinguishable from the byte-mismatch refusal: this one says the + # comparison could not be made at all. + assert "could not be read to compare" in action.error + assert spy.calls == [] + assert target.read_text(encoding="utf-8") == SKILL_BODY + assert "a/SKILL.md" not in _read_manifest(root)["entries"] + + +# --------------------------------------------------------------------------- +# Orphaned temp files +# --------------------------------------------------------------------------- + + +def _temp_name(token: str = "0123456789abcdef") -> str: + """A name ``atomic_write`` could have created for ``SKILL.md``. + + The prefix comes from ``safe_fs`` itself rather than a copy of its format + string, so a change to the naming breaks this helper instead of silently + making the sweep a no-op. + """ + return f"{safe_fs_module.temp_name_prefix('SKILL.md')}{token}.tmp" + + +class TestOrphanedTempFiles: + """A ``SIGKILL`` mid-write leaves a temp file nothing else records.""" + + async def test_an_orphan_is_swept_on_the_next_write(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY) + orphan = root / "a" / _temp_name() + orphan.write_text("half-written body", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert not orphan.exists() + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_an_orphan_no_longer_blocks_directory_cleanup( + self, root: Path + ) -> None: + """The second-order effect: ``rmdir`` fails on a non-empty directory. + + One orphaned temp file would otherwise pin the skill's directory under + the managed root forever, long after the skill itself was revoked. + """ + _place_managed(root, "a", SKILL_BODY) + orphan = root / "a" / _temp_name() + orphan.write_text("half-written body", encoding="utf-8") + + report = await write_skills([], root) + + assert report.ok is True, _error_messages(report) + assert _actions_by_key(report)["a"].action == "removed" + assert not (root / "a").exists() + + @pytest.mark.parametrize( + "innocent", + [ + "notes.tmp", + "SKILL.md.tmp", + ".SKILL.md.tmp", + ".SKILL.md..tmp", + _temp_name("not-a-token"), + _temp_name("0123456789abcdef") + ".bak", + "x" + _temp_name(), + _temp_name("0123456789abcdefff"), + ], + ) + async def test_a_lookalike_name_is_left_alone( + self, root: Path, innocent: str + ) -> None: + """The recognizer is anchored at both ends, and the sweep deletes files. + + Anything that is not exactly the naming ``safe_fs`` produces belongs to + the customer, whatever it resembles. + """ + _place_managed(root, "a", SKILL_BODY) + bystander = root / "a" / innocent + bystander.write_text("mine\n", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert bystander.read_text(encoding="utf-8") == "mine\n" + + @pytest.mark.skipif( + not hasattr(os, "symlink"), reason="platform has no symlink support" + ) + async def test_a_symlink_wearing_the_temp_name_is_not_removed( + self, root: Path, tmp_path: Path + ) -> None: + """The temp naming must not become a way to have the SDK delete elsewhere. + + Only a regular file is ever swept, and the type comes off the descriptor + rather than a followed path. + """ + outside = tmp_path / "precious.txt" + outside.write_text("do not delete\n", encoding="utf-8") + _place_managed(root, "a", SKILL_BODY) + link = root / "a" / _temp_name() + link.symlink_to(outside) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert outside.read_text(encoding="utf-8") == "do not delete\n" + assert link.is_symlink() + + +# --------------------------------------------------------------------------- +# Windows reserved device names +# --------------------------------------------------------------------------- + +# Spelled out independently of the implementation's own set, so a name dropped +# from that set fails here rather than agreeing with itself. +WINDOWS_RESERVED_KEYS = ( + ["con", "prn", "aux", "nul"] + + [f"com{digit}" for digit in range(1, 10)] + + [f"lpt{digit}" for digit in range(1, 10)] +) + + +class TestWindowsReservedNames: + """Keys Windows cannot hold as directory names, refused on every platform.""" + + def test_the_set_is_exactly_twenty_two_names(self) -> None: + assert len(WINDOWS_RESERVED_KEYS) == len(set(WINDOWS_RESERVED_KEYS)) == 22 + + @pytest.mark.parametrize("reserved", WINDOWS_RESERVED_KEYS) + async def test_a_reserved_key_is_refused_by_write_skills( + self, root: Path, reserved: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill(reserved)], root) + + assert report.ok is False + action = _actions_by_key(report)[reserved] + assert action.action == "error" + assert action.error is not None + assert "Windows reserves" in action.error + assert reserved in action.error + # Rejected before any filesystem call, not by the OS. + assert spy.calls == [] + assert not (root / reserved).exists() + + @pytest.mark.parametrize("reserved", WINDOWS_RESERVED_KEYS) + async def test_a_reserved_key_is_refused_by_the_prune_path( + self, root: Path, reserved: str + ) -> None: + """``_key_rejection_reason`` gates both destructive paths, so both refuse. + + A manifest naming a reserved key is left in place rather than acted on: + the same key check that stops the write stops the delete. + """ + target = _place_managed(root, reserved, SKILL_BODY) + + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)[reserved] + assert action.action == "error" + assert action.error is not None + assert "left in place" in action.error + assert target.read_text(encoding="utf-8") == SKILL_BODY + + @pytest.mark.parametrize("not_reserved", ["com0", "lpt0", "con1", "nul2", "conx"]) + async def test_neighbouring_names_are_not_reserved( + self, root: Path, not_reserved: str + ) -> None: + """``com0`` and ``lpt0`` are not device names, and must still write.""" + report = await write_skills([_skill(not_reserved)], root) + + assert report.ok is True, _error_messages(report) + assert (root / not_reserved / "SKILL.md").exists() + + @pytest.mark.parametrize("reserved", WINDOWS_RESERVED_KEYS) + def test_a_reserved_name_is_still_a_valid_key_to_every_pure_layer( + self, reserved: str + ) -> None: + """The layer choice, asserted — this is the whole point of it. + + The constraint lives in the filesystem layer and must not migrate into + the key grammar. At the grammar level a rejection would fail the *entire* + AI Config — model, provider, instructions, tools — for a Linux customer + over a Windows-only constraint, and would shrink ``skill_refs``, which is + what authorizes a prune: "this skill fails to write on Windows" would + become "this skill gets deleted on Linux". + """ + assert is_valid_skill_key(reserved) is True + + parsed = parse_ai_config( + { + "model": {"name": "claude-3"}, + "provider": {"name": "Anthropic"}, + "instructions": "You are helpful.", + "skills": [{"key": reserved, "version": 1}], + } + ) + assert parsed.success is True + + refs = skill_refs(parsed.data) + assert [ref.key for ref in refs] == [reserved] From eda4ae1f9219a8a3c3b363dcfc05fee566820825 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Mon, 31 Aug 2026 14:04:18 -0400 Subject: [PATCH 05/16] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20a=20distinguishable=20outcome=20for=20integrity=20f?= =?UTF-8?q?ailure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``get_skill`` returns ``None`` for four unrelated outcomes: no such skill, the store raised, the requested version is not the one held, and content that failed hash verification. A caller cannot fail closed on suspected tampering while tolerating a merely-absent skill, so no automated customer-side response is possible — finding LA-2 of the Agent Skills security design review. The information already existed internally, as prose in ``Resolution.error``. This gives it a token: ``Resolution`` grows a typed ``reason``, set explicitly at every construction site and declared without a default so a sixth outcome added later has to choose which public token it maps to. ``get_skill_result`` maps that straight through to a frozen ``SkillOutcome`` (``skill``, ``reason``, ``detail``). Deriving the public reason by matching the error string is the fragility LA-2 is about, so the mapping is readable in one table. ``get_skill`` is untouched — its ``None``-for-every-failure contract is documented in its docstring and in the README, and a test now pins that all four failures still collapse to ``None`` and still never raise. Nothing new is emitted: Gap 1's integrity record already fired inside verification before ``resolve_from_store`` returned, and a test asserts one failed retrieval still produces exactly one record and one signal. Co-Authored-By: Claude Opus 5 --- packages/client/README.md | 56 +++- packages/client/agents.md | 59 +++- .../src/launchdarkly_ai_server/__init__.py | 8 +- .../src/launchdarkly_ai_server/skills.py | 36 ++- .../src/launchdarkly_ai_server/skills_core.py | 39 ++- .../src/launchdarkly_ai_server/skills_fs.py | 10 +- .../src/launchdarkly_ai_server/types.py | 55 ++++ packages/client/tests/test_skills.py | 281 ++++++++++++++++++ 8 files changed, 527 insertions(+), 17 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 711ce5bf..045180b5 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -337,6 +337,55 @@ truncated payload; a mismatch means content was delivered whose bytes are not th LaunchDarkly hashed, which is a possible **active-tampering** signal. Alert on it, and treat `expected_hash` / `observed_hash` as the evidence pair. +#### Failing closed on tampering + +The log record above is the operator's surface. `get_skill_result` is the application's: +same retrieval, same verification, same telemetry as `get_skill`, and it reports which of +five outcomes happened instead of collapsing all of them to `None`. + +```python +from launchdarkly_ai_server import get_skill_result + +outcome = await get_skill_result("pdf-extraction") + +if outcome.reason == "integrity_failure": + # Content was delivered and did not verify. Do not degrade quietly. + raise SystemExit(f"refusing to start: {outcome.detail}") + +if outcome.reason == "store_unavailable": + # The store could not answer at all. Retry, alert, or carry on with what + # you already have — but this is an outage, not a revocation. + print(f"skill retrieval unavailable: {outcome.detail}") +elif outcome.reason in ("absent", "wrong_version"): + # Nothing was tampered with — this skill is simply not available to you. + print(f"continuing without a skill: {outcome.detail}") +elif outcome.skill is not None: + print(outcome.skill.content) +``` + +| `reason` | Meaning | +|---|---| +| `ok` | A verified skill was returned; `.skill` is set and `.detail` is `None`. | +| `absent` | The store answered, and does not hold that key. | +| `integrity_failure` | Content was delivered and failed verification, so it was withheld. **The one to fail closed on.** | +| `store_unavailable` | The store itself could not answer — it raised. An outage, not a deletion. | +| `wrong_version` | The store answered with a version other than the one asked for, so the answer was withheld. | + +`.detail` is human-readable and safe to log or show an operator — it names the key and the +failure mode, and never carries skill content or a filesystem path. Branch on `.reason`, +not on `.detail`. `.skill` is populated only when `.reason == "ok"`. `SkillOutcome` is +frozen, like every other value type here. + +**`get_skill` is unchanged.** It still returns `None` for all four failures and still never +raises for one, so no existing caller has to move. The two accessors run the same code path +and differ only in what they report — `get_skill_result` adds no second log record and no +second signal for a failure that already emitted one, so a caller can switch to it without +double-counting anything. + +`get_skills` and `all_skills` have no reported form: they still omit entries that could not +be resolved, and a run that omitted anything logs a count at WARN. Retrieve individually +with `get_skill_result` when you need the reason per key. + **Versions are selected, not filtered.** A store may hold several versions of one key at once, because a delivery payload does: the newest version of every skill, plus every version a variation currently pins. `get_skill("k", version=1)` asks the store for version @@ -387,6 +436,7 @@ Windows. |---|---| | `skill_refs(config)` | Project a config's `skills` array into `list[SkillReference]`. Pure — no client, store, or network needed. Returns `[]` when absent. | | `get_skill(key, *, version=None)` | One verified skill, or `None`. `version=None` means newest available; a specific `version` matches exactly. Raises only when no store is configured. | +| `get_skill_result(key, *, version=None)` | The same retrieval, reporting **why**: a frozen `SkillOutcome` with `.skill`, `.reason` (`ok` / `absent` / `integrity_failure` / `store_unavailable` / `wrong_version`), and `.detail`. Use it to fail closed on tampering — see *Failing closed on tampering* above. Raises only when no store is configured. | | `get_skills(refs)` | Batch form. Accepts `SkillReference` values and bare key strings (string = latest). Results follow input order; missing or unverifiable entries are omitted. | | `all_skills()` | Every verified skill the store holds, one per key at its newest version. | | `write_skills(skills, root, *, prune=True, timeout=10.0, on_unavailable="keep")` | Materialize skills under `root`, returning a `ReconcileReport`. `prune` removes formerly-managed skills no longer requested. `on_unavailable="raise"` raises instead of reporting when content cannot be retrieved. Raises `ValueError` for an unusable root, a negative `timeout`, or an unrecognised `on_unavailable`. **Performs synchronous filesystem I/O — see the note below.** | @@ -406,9 +456,10 @@ manifest, for instance — carries the empty string as its `key`. The fixed on-disk values are exported too, so you do not have to hardcode them: `MANIFEST_FILENAME` (`.launchdarkly-skills.json`, handy for a `.gitignore`), -`SKILL_FILENAME`, and `MANIFEST_VERSION`. So are the two closed-set types, for annotating +`SKILL_FILENAME`, and `MANIFEST_VERSION`. So are the three closed-set types, for annotating your own helpers: `ReconcileActionKind` (`written` / `updated` / `skipped_current` / -`removed` / `error`) and `OnUnavailable` (`keep` / `raise`). +`removed` / `error`), `OnUnavailable` (`keep` / `raise`), and `SkillOutcomeReason` +(`absent` / `integrity_failure` / `ok` / `store_unavailable` / `wrong_version`). **`write_skills` blocks.** It is `async` for parity with the other accessors and with the TypeScript SDK, but it awaits nothing: every read, write, `fsync` and rename runs inline, @@ -460,5 +511,6 @@ All types are exported from this package. Handler packages import them from here | `GraphTopology` | The parsed graph flag shape (`root` + `edges`) | | `Skill` | A frozen skill document: `.key`, `.version`, `.content` (verified verbatim `bytes`), `.content_hash`, `.name?`, `.description?` | | `SkillReference` | A frozen version-pinned pointer to a skill: `.key`, `.version` | +| `SkillOutcome` | A frozen retrieval outcome: `.skill`, `.reason` (`SkillOutcomeReason`), `.detail` | | `ReconcileAction` | One `write_skills` outcome: `.key`, `.action`, `.version?`, `.path?`, `.error?` | | `ReconcileReport` | The `write_skills` result: `.actions`, `.ok`, and `.errors` | diff --git a/packages/client/agents.md b/packages/client/agents.md index cd99b28f..a7b4ce1b 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -77,10 +77,10 @@ from launchdarkly_ai_server import config, graph, resolve_graph # Agent Skills from launchdarkly_ai_server import ( - skill_refs, get_skill, get_skills, all_skills, write_skills, - SkillStore, InMemorySkillStore, + skill_refs, get_skill, get_skill_result, get_skills, all_skills, write_skills, + SkillStore, InMemorySkillStore, SkillOutcome, SKILL_FILENAME, MANIFEST_FILENAME, MANIFEST_VERSION, - ReconcileActionKind, OnUnavailable, # the two closed-set unions + ReconcileActionKind, OnUnavailable, SkillOutcomeReason, # the three closed-set unions ) ``` @@ -188,8 +188,8 @@ Three layers, in increasing order of blast radius: typed `SkillReference` values. Pure: no network, no client, no store, no telemetry. Validation of the array itself lives in `parse_ai_config` and is **fail closed** — one malformed reference fails the whole config parse. -2. **Content accessors** — `get_skill`, `get_skills`, `all_skills` read through the - `SkillStore` seam. Configure a store with +2. **Content accessors** — `get_skill`, `get_skill_result`, `get_skills`, `all_skills` read + through the `SkillStore` seam. Configure a store with `init_client(options={"skillStore": store})`; with none configured the accessors raise an actionable `RuntimeError`. A delivery transport can be added behind the seam without touching the public API. @@ -217,6 +217,55 @@ one place that collapses the result to one object per key, because both whole-st consumers need it — `all_skills`, since a list holding two versions of one key is not a set of skills, and the `"*"` reconcile, since `//SKILL.md` is a single path. +### The reported outcome vocabulary, and the `Resolution` mapping + +`get_skill` returns `Skill | None`; `get_skill_result` returns a frozen `SkillOutcome` +(`skill`, `reason`, `detail`) naming *which* outcome happened. Both are +`resolve_from_store` — one retrieval, one verification, one telemetry pass — and they differ +only in what they report. `get_skill`'s contract is load-bearing and **frozen**: `None` for +every failure, never raises for one, documented in its docstring and in the README. Change +it and every caller that treats `None` as "no skill" breaks silently. + +`SkillOutcomeReason` is five tokens, listed alphabetically for the same reason +`IntegrityReasonCode` is — so the vocabulary reads identically in the Python and TypeScript +SDKs, where the type name, the accessor name, and the tokens are all deliberately the same. +Do not rename one on one side. + +Internal `Resolution.reason` maps 1:1 onto it, set explicitly at every construction site: + +| `resolve_from_store` outcome | `reason` | +|---|---| +| the store raised (`unavailable=True`) | `store_unavailable` | +| `raw` is not a dict | `absent` | +| `verify_raw_skill` returned `None` | `integrity_failure` | +| `skill.version != wanted_version` | `wrong_version` | +| success | `ok` | + +**Adding a sixth internal outcome means choosing which public token it maps to.** +`Resolution.reason` has no default, so the compiler asks the question; answer it rather than +defaulting to `absent`, which claims the store does not hold the skill. If the new outcome +is genuinely neither of the five, the token set grows — on both sides, in the same commit. + +Two things the reason is deliberately *not*: + +- **Not derived from `Resolution.error`.** That string is prose for a human; recovering a + decision a caller fails closed on by matching it is the fragility the typed token exists + to remove. `detail` *is* that string, passed straight through — safe to surface (key and + failure mode only, never content, never a path), and not for matching on. +- **Not `Resolution.unavailable`.** The flag answers "may prune run?" and the token answers + "what does the caller learn?". They agree by construction — `unavailable` is `True` in + exactly the `store_unavailable` case — and both exist because `store_unavailable` must + stay distinct from `absent`: only a raising store suppresses pruning, since deleting + managed files after a failed lookup turns an outage into data loss. + +`get_skill_result` emits nothing of its own. The integrity log record and signal already +fired inside verification before `resolve_from_store` returned; recording anything here +would double-count one failure in a SIEM and in the product counter. + +There is no `get_skills_result` or `all_skills_result`. The batch accessors keep omitting +unresolved entries and keep logging the run-level WARN count, and a second accessor per +batch form would double the surface for a case nobody has asked for. + ### Security posture — do not relax any of this Store data is **untrusted input**; the transport is not part of the trust boundary. diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 37b1570a..753e1e95 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -41,6 +41,7 @@ InMemorySkillStore, all_skills, get_skill, + get_skill_result, get_skills, skill_refs, ) @@ -85,6 +86,8 @@ ReconcileActionKind, ReconcileReport, Skill, + SkillOutcome, + SkillOutcomeReason, SkillReference, StreamChunkEvent, StreamDoneEvent, @@ -150,6 +153,7 @@ "ReconcileActionKind", "ReconcileReport", "Skill", + "SkillOutcome", "SkillReference", "StreamChunkEvent", "StreamDoneEvent", @@ -227,14 +231,16 @@ # skills "skill_refs", "get_skill", + "get_skill_result", "get_skills", "all_skills", "write_skills", "SkillStore", "InMemorySkillStore", - # skills — the two closed-set unions a typed consumer needs to name + # skills — the three closed-set unions a typed consumer needs to name "ReconcileActionKind", "OnUnavailable", + "SkillOutcomeReason", # skills — on-disk constants, identical across languages "SKILL_FILENAME", "MANIFEST_FILENAME", diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py index 9d9b3627..cb21986b 100644 --- a/packages/client/src/launchdarkly_ai_server/skills.py +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -37,7 +37,7 @@ resolve_from_store, verify_raw_skill, ) -from .types import AiConfigRep, Skill, SkillReference +from .types import AiConfigRep, Skill, SkillOutcome, SkillReference from .types_validation import ( is_valid_skill_key, is_valid_skill_version, @@ -247,6 +247,40 @@ async def get_skill(key: str, *, version: int | None = None) -> Skill | None: return resolve_from_store(require_store(), key, version).skill +async def get_skill_result(key: str, *, version: int | None = None) -> SkillOutcome: + """ + Retrieves one verified skill, reporting *why* when there is none. + + Same retrieval, same verification, same telemetry as ``get_skill`` — the two + differ only in what they report. ``get_skill`` collapses "no such skill", + "the store raised", "that is not the version held", and "the content failed + integrity verification" to one ``None``; this returns a ``SkillOutcome`` + whose ``reason`` names which of them happened, so a caller can fail closed on + suspected tampering while tolerating a merely-absent skill: + + ```python + outcome = await get_skill_result("pdf-extraction") + if outcome.reason == "integrity_failure": + raise SystemExit(f"refusing to run: {outcome.detail}") + if outcome.skill is not None: + print(outcome.skill.content) + ``` + + ``detail`` is human-readable and safe to surface — it names the key and the + failure mode, never any skill content or filesystem path. Branch on + ``reason``, not on ``detail``. + + Emits nothing of its own: an integrity failure has already recorded its log + record and its signal inside verification, and recording a second here would + double-count one failure. Raises ``RuntimeError`` only when no skill store is + configured, exactly as ``get_skill`` does. + """ + resolved = resolve_from_store(require_store(), key, version) + return SkillOutcome( + skill=resolved.skill, reason=resolved.reason, detail=resolved.error + ) + + async def get_skills(refs: Sequence[SkillReference | str]) -> list[Skill]: """ Retrieves a batch of verified skills. diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index 1e517e4e..74cb706b 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -44,7 +44,7 @@ from dataclasses import dataclass from typing import Any, Literal, Protocol, get_args -from .types import Skill, SkillReference +from .types import Skill, SkillOutcomeReason, SkillReference from .types_validation import is_valid_skill_key, is_valid_skill_version logger = logging.getLogger(__name__) @@ -672,6 +672,26 @@ def newest_by_key(objects: dict[str, dict[str, Any]]) -> list[tuple[str, Any]]: class Resolution: """One key resolved against a store: the skill, or why there is none.""" + reason: SkillOutcomeReason + """ + Which of the five public outcomes this resolution is. + + Declared first and **without a default**, so every construction site has to + state it. A default would be the wrong shape twice over: a contributor + adding a sixth internal outcome would inherit whichever token happened to be + the default rather than deciding which public token it maps to, and if that + default were ``"ok"`` a failure would publish ``ok`` with no skill attached. + + Carried as a token rather than derived from ``error`` on the way out: + ``get_skill_result`` publishes this value, and pattern-matching prose to + recover a decision a caller fails closed on is exactly the fragility the + typed outcome exists to remove. A reviewer can read the mapping here. + + Distinct from ``unavailable`` on purpose — that flag answers one question + (may prune run?) and this token answers a different one (what does the + caller learn?) — but the two can only disagree by a bug: ``unavailable`` is + ``True`` in exactly the ``store_unavailable`` case. + """ skill: Skill | None = None error: str | None = None unavailable: bool = False @@ -702,26 +722,33 @@ def resolve_from_store( raw = store.get_object(SKILL_OBJECT_KIND, key, wanted_version) except Exception as exc: logger.error("Skill store raised while retrieving '%s'", key, exc_info=True) - return Resolution(error=store_raised(exc), unavailable=True) + return Resolution( + reason="store_unavailable", + error=store_raised(exc), + unavailable=True, + ) if not isinstance(raw, dict): return Resolution( - error=f"skill '{key}' is not available from the configured skill store" + reason="absent", + error=f"skill '{key}' is not available from the configured skill store", ) skill = verify_raw_skill(raw) if skill is None: return Resolution( - error=f"skill '{key}' failed integrity verification and was withheld" + reason="integrity_failure", + error=f"skill '{key}' failed integrity verification and was withheld", ) if wanted_version is not None and skill.version != wanted_version: return Resolution( + reason="wrong_version", error=( f"skill '{key}' version {wanted_version} is not available " f"(the store holds version {skill.version})" - ) + ), ) - return Resolution(skill=skill) + return Resolution(reason="ok", skill=skill) def reference_target(item: SkillReference | str) -> tuple[str, int | None]: diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py index 8a865dd8..fb163796 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fs.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -404,11 +404,17 @@ def _resolve_reference( """ store = _available_store(deadline, f"'{key}'") if isinstance(store, _RetrievalBlocked): - return Resolution(error=store.reason, unavailable=True) + return Resolution( + reason="store_unavailable", error=store.reason, unavailable=True + ) resolved = resolve_from_store(store, key, wanted_version) if resolved.unavailable and resolved.error is not None: - return Resolution(error=_unavailable(resolved.error), unavailable=True) + return Resolution( + reason="store_unavailable", + error=_unavailable(resolved.error), + unavailable=True, + ) return resolved diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index 10c90d8c..65e4850a 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -452,6 +452,61 @@ class Skill: """Description from LaunchDarkly metadata; never parsed from the content.""" +SkillOutcomeReason = Literal[ + "absent", "integrity_failure", "ok", "store_unavailable", "wrong_version" +] +""" +The closed set of outcomes ``get_skill_result`` reports. + +Alphabetical, as ``reason_code`` is in the integrity log record, so the token +list reads identically in every LaunchDarkly AI SDK. Each token is a distinct +*decision* a caller can make, which is the point of the type: ``absent`` is a +skill the store does not hold, ``integrity_failure`` is content that was +delivered and did not verify, and a caller that wants to fail closed on +suspected tampering while tolerating a merely-absent skill needs the two to be +told apart. + +- ``ok`` — a verified skill was returned. +- ``absent`` — the store answered, and does not hold the key. +- ``integrity_failure`` — content was delivered and failed verification; it was + withheld. The one token worth failing closed on. +- ``store_unavailable`` — the store itself could not answer: it raised. + Deliberately distinct from ``absent``, because an outage is not a deletion. +- ``wrong_version`` — the store answered with a version other than the one + asked for, so the answer was withheld. +""" + + +@dataclass(frozen=True) +class SkillOutcome: + """ + Why one retrieval returned what it did — the reported form of ``get_skill``. + + ``get_skill`` collapses every failure to ``None``, which is the right shape + for a caller that only wants content and cannot act on the difference. This + is the shape for a caller that can: ``reason`` names which of the five + outcomes happened, so an integrity failure is distinguishable from a skill + that simply is not configured. The two accessors differ only in what they + report — the retrieval, the verification, and the telemetry are the same + code path, run once. + + Instances are immutable. + """ + + skill: Skill | None + """The verified skill, and only ever populated when ``reason == "ok"``.""" + reason: SkillOutcomeReason + """Which outcome happened. A closed set — see ``SkillOutcomeReason``.""" + detail: str | None + """ + Human-readable detail, set for every reason except ``ok``. + + Safe to log or surface to an operator: it carries the skill key and the + failure mode, and never any skill content or filesystem path. Intended for a + human, not for matching on — branch on ``reason``. + """ + + ReconcileActionKind = Literal[ "written", "updated", "skipped_current", "removed", "error" ] diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 0c656ede..6470f0c0 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -20,10 +20,12 @@ ReconcileAction, ReconcileReport, Skill, + SkillOutcome, SkillReference, all_skills, get_client, get_skill, + get_skill_result, get_skills, init_client, shutdown, @@ -143,6 +145,16 @@ def test_skill_is_immutable(self) -> None: with pytest.raises(dataclasses.FrozenInstanceError): skill.content = b"tampered" # type: ignore[misc] + def test_skill_outcome_is_immutable(self) -> None: + """A reported outcome is a value, like every other public skills type. + + Matters more here than for the others: a caller that fails closed on + ``reason`` must not be handed something a later layer can rewrite. + """ + outcome = SkillOutcome(skill=None, reason="integrity_failure", detail="nope") + with pytest.raises(dataclasses.FrozenInstanceError): + outcome.reason = "ok" # type: ignore[misc] + def test_skill_content_is_bytes(self) -> None: """Content is the verified verbatim bytes — opaque, never text.""" skill = _skill() @@ -385,6 +397,13 @@ def test_exported_action_union_admits_exactly_the_five_actions(self) -> None: "error", } assert set(typing.get_args(package.OnUnavailable)) == {"keep", "raise"} + assert set(typing.get_args(package.SkillOutcomeReason)) == { + "absent", + "integrity_failure", + "ok", + "store_unavailable", + "wrong_version", + } def test_retrieval_surface_is_exported_from_the_package_root(self) -> None: """A name absent from ``__all__`` is not part of the public surface.""" @@ -393,11 +412,14 @@ def test_retrieval_surface_is_exported_from_the_package_root(self) -> None: expected = { "skill_refs", "get_skill", + "get_skill_result", "get_skills", "all_skills", "SkillStore", "InMemorySkillStore", "Skill", + "SkillOutcome", + "SkillOutcomeReason", "SkillReference", } assert expected <= set(package.__all__) @@ -743,6 +765,265 @@ async def test_multibyte_content_verifies( assert skill.content == content.encode("utf-8") +class _RaisingStore: + """A store whose reads raise — the "the transport is down" case. + + Declared with the full ``get_object`` signature on purpose. A double missing + the ``version`` parameter would also produce a raise here, but a + ``TypeError`` from the call itself rather than from the store, and the test + would then pass without the store ever having been consulted. + """ + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + raise RuntimeError("transport failure") + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + raise RuntimeError("transport failure") + + +class _WrongVersionAnsweringStore: + """A store that answers a pinned lookup with some other version.""" + + def __init__(self, make_raw_skill: Any, answered_version: int = 99) -> None: + self._make = make_raw_skill + self._answered_version = answered_version + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + answer: dict[str, Any] = self._make(key=key, version=self._answered_version) + return answer + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + return {} + + +class TestGetSkillResult: + """ + The reported accessor — one token per outcome a retrieval can have. + + ``get_skill`` collapses four distinct failures to ``None``, which leaves a + caller unable to fail closed on suspected tampering while tolerating a skill + that is merely not configured. These tests pin that the five outcomes are + told apart, and that reporting them changed nothing about ``get_skill``. + """ + + def _tampered(self, make_raw_skill: Any, key: str = "a") -> dict[str, Any]: + raw: dict[str, Any] = make_raw_skill(key=key) + raw["contentHash"] = "0" * 64 + return raw + + async def test_ok_carries_the_skill_and_no_detail( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="pdf-extraction", version=2)) + + outcome = await get_skill_result("pdf-extraction") + + assert outcome.reason == "ok" + assert outcome.detail is None + assert outcome.skill is not None + assert outcome.skill.key == "pdf-extraction" + assert outcome.skill.version == 2 + assert outcome.skill.content == SKILL_BODY.encode("utf-8") + + async def test_absent_when_the_store_does_not_hold_the_key( + self, store: InMemorySkillStore + ) -> None: + outcome = await get_skill_result("nope") + + assert outcome.reason == "absent" + assert outcome.skill is None + assert outcome.detail + assert "'nope'" in outcome.detail + + async def test_integrity_failure_when_content_does_not_verify( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """The one outcome a caller is expected to fail closed on.""" + store.put(self._tampered(make_raw_skill, key="a")) + + outcome = await get_skill_result("a") + + assert outcome.reason == "integrity_failure" + assert outcome.skill is None + assert outcome.detail + # The quoted form, so the assertion is about the key and not about a + # letter that appears in half the words in the message. + assert "'a'" in outcome.detail + + async def test_wrong_version_when_the_store_answers_with_another( + self, make_raw_skill: Any + ) -> None: + skills_module._set_store(_WrongVersionAnsweringStore(make_raw_skill)) + + outcome = await get_skill_result("a", version=1) + + assert outcome.reason == "wrong_version" + assert outcome.skill is None + assert outcome.detail + # The detail is what makes this actionable rather than merely negative: + # it names both the version asked for and the version held. + assert "version 1" in outcome.detail + assert "version 99" in outcome.detail + + async def test_store_unavailable_when_the_store_raises( + self, caplog: pytest.LogCaptureFixture + ) -> None: + skills_module._set_store(_RaisingStore()) + + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + outcome = await get_skill_result("a") + + assert outcome.reason == "store_unavailable" + assert outcome.skill is None + assert outcome.detail + assert "RuntimeError" in outcome.detail + + async def test_store_unavailable_is_distinct_from_absent( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """An outage is not a deletion, and the two must not read alike. + + This is the distinction ``write_skills`` already depends on to decide + whether pruning may run — only a raising store suppresses it — so + collapsing the two tokens here would put the public vocabulary at odds + with a policy the SDK already enforces internally. + """ + skills_module._set_store(_RaisingStore()) + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + raised = await get_skill_result("a") + + skills_module._set_store(InMemorySkillStore()) + empty = await get_skill_result("a") + + # Asserted as two named tokens rather than as an inequality: the type + # checker can already see that these two literals differ, so an + # inequality here would be dead weight. + assert raised.reason == "store_unavailable" + assert empty.reason == "absent" + + async def test_every_non_ok_outcome_carries_a_detail( + self, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """A reason with no detail leaves an operator nothing to act on. + + Swept over all four failures in one test rather than asserted per case + only, so a fifth failure path added later without a message is caught by + a test whose name says what it is about. + """ + stores: list[tuple[str, Any]] = [ + ("absent", InMemorySkillStore()), + ( + "integrity_failure", + InMemorySkillStore({"a": self._tampered(make_raw_skill)}), + ), + ("wrong_version", _WrongVersionAnsweringStore(make_raw_skill)), + ("store_unavailable", _RaisingStore()), + ] + + outcomes: list[SkillOutcome] = [] + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + for _expected, store_double in stores: + skills_module._set_store(store_double) + outcomes.append(await get_skill_result("a", version=1)) + + assert [o.reason for o in outcomes] == [expected for expected, _ in stores] + assert all(o.skill is None for o in outcomes) + assert all(o.detail for o in outcomes) + + async def test_detail_never_carries_the_skill_content( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """``detail`` is safe to log, so the body must not travel in it. + + Same rule the integrity log record follows, asserted separately here + because this string reaches the caller through a different surface. + """ + secret = "---\nname: Secret\n---\nSSN 000-00-0000 and an API key.\n" + store.put(make_raw_skill(key="a", content=secret, contentHash="0" * 64)) + + outcome = await get_skill_result("a") + + assert outcome.reason == "integrity_failure" + assert outcome.detail is not None + assert secret not in outcome.detail + assert "SSN" not in outcome.detail + assert "API key" not in outcome.detail + + async def test_raises_the_same_way_as_get_skill_with_no_store(self) -> None: + """Identical failure mode, down to the message. + + The two accessors differ only in what they report about a retrieval; a + missing store is a configuration error in both, so a caller cannot need + to handle it twice. + """ + with pytest.raises(RuntimeError, match="skill store") as reported: + await get_skill_result("a") + with pytest.raises(RuntimeError, match="skill store") as collapsed: + await get_skill("a") + + assert str(reported.value) == str(collapsed.value) + + async def test_records_no_second_integrity_signal( + self, + store: InMemorySkillStore, + make_raw_skill: Any, + recording_emitter: Any, + caplog: pytest.LogCaptureFixture, + ) -> None: + """One failed retrieval is one failure, on both surfaces. + + Verification already recorded the log record and the signal before + ``resolve_from_store`` returned, so reporting the reason must add + nothing: a second record would double-count one event in a SIEM and + inflate the product counter. ``_integrity_records`` is the shared parser + used by the log-record tests further down this module. + """ + skills_module._set_emitter_for_testing(recording_emitter) + store.put(self._tampered(make_raw_skill, key="a")) + + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + outcome = await get_skill_result("a") + + assert outcome.reason == "integrity_failure" + assert len(_integrity_records(caplog)) == 1 + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_get_skill_still_returns_none_for_every_failure( + self, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """The no-behaviour-change guarantee. + + ``get_skill``'s contract — ``None`` for every failure, and it never + raises for one — is documented in its docstring and in the README, and + every existing caller treats ``None`` as "no skill". Adding a reported + accessor beside it must not move that line, so the four failures are + driven through both accessors in one test: the reason is distinguishable + *and* the collapsed form still collapses. + """ + cases: list[tuple[str, Any]] = [ + ("absent", InMemorySkillStore()), + ( + "integrity_failure", + InMemorySkillStore({"a": self._tampered(make_raw_skill)}), + ), + ("wrong_version", _WrongVersionAnsweringStore(make_raw_skill)), + ("store_unavailable", _RaisingStore()), + ] + + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + for expected_reason, store_double in cases: + skills_module._set_store(store_double) + reported = await get_skill_result("a", version=1) + assert reported.reason == expected_reason + # No pytest.raises wrapper: an escaping exception fails the test + # here, which is the "never raises" half of the contract. + assert await get_skill("a", version=1) is None + + class TestGetSkills: """Batch accessor.""" From 6a0e6aa50f6b528b10174d5834f013090817e962 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 4 Sep 2026 10:19:46 -0400 Subject: [PATCH 06/16] =?UTF-8?q?docs(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20close=20out=20the=20security=20review's=20SDK-side?= =?UTF-8?q?=20remainder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four items left open in the response to the Agent Skills security design review. Three are documentation, one is tests; no behavior changes, and the code halves of rows 9 and 2 are deliberately untouched. **Privilege separation** (row 9 docs half, and the agreed counter-proposal for row 26). The recommended deployment runs the reconcile as a different identity than the agent, which is the whole reason the ``0644``/``0755`` modes deny anything: the agent reads its instructions and cannot rewrite them, or the manifest. That is the mitigation for AZ-1, a prompt-injected agent editing its own skills. Write access to the manifest is the worse half — it is what tells the *next* reconcile which paths the SDK may delete — which is why ``_prune`` re-validates every entry from scratch rather than trusting it. The README documents the pattern and hands the operator the check to run, because the SDK cannot run it: it knows only its own identity, which trivially has write access, having just written there. So ``ReconcileReport`` grows no writability field — the review asked for one and we declined, since any check the SDK could make would answer a different question than the one asked and manufacture false confidence exactly where caution is wanted. ``agents.md`` records that reasoning so the field is not added later by someone reading its absence as an oversight. **Three hostile-manifest prune tests** (row 12 remainder): a well-formed manifest listing ``/etc/passwd``, ``../../../etc/passwd``, and a path under a parent that has since become a symlink. ``_prune`` already refuses all three, so these turn asserted into verified. Two things make them worth more than their line count. They are deliberately *well-formed* — the corrupt-manifest suite above them proves nothing here, because a corrupt manifest suppresses every destructive action wholesale, whereas these manifests give the implementation everything it needs to prune. And "deleted nothing" is asserted through an unlink spy rather than by checking that ``/etc/passwd`` still exists: the test process cannot delete that file anyway, so the obvious assertion would pass against an implementation with no path check at all. **One sentence on** ``"*"`` (row 16 remainder). It materializes the whole project library, so every skill's ``description`` enters the agent's context — including skills no AI Config references and skills belonging to other teams. **The Windows platform bound is now explicit** (row 2 residual), in ``safe_fs.py``, ``agents.md`` and the README. Reparse-point checks (``GetFileAttributesW`` / ``FILE_FLAG_OPEN_REPARSE_POINT``) are not implemented, by decision: Windows is not a supported or tested platform for this release, neither repository has a Windows CI runner so the checks would ship unverified, and the TypeScript SDK could not match them in any case because Node exposes no ``*at()`` family on *any* platform. Implementing them in Python alone would break cross-language parity and trade a documented bound for an unverified one. Two consequences are recorded rather than left to be rediscovered: on Windows, write permission on the managed root is the only boundary, which is what makes privilege separation the mitigation and not merely advice; and this retroactively lowers the priority of the row 25 reserved-device-name work, noted where that code lives. Co-Authored-By: Claude Opus 5 --- packages/client/README.md | 68 ++++++++- packages/client/agents.md | 25 ++++ .../src/launchdarkly_ai_server/safe_fs.py | 23 ++++ packages/client/tests/test_skills_fs.py | 130 ++++++++++++++++++ 4 files changed, 245 insertions(+), 1 deletion(-) diff --git a/packages/client/README.md b/packages/client/README.md index 045180b5..bc8df592 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -272,7 +272,12 @@ async def main(): asyncio.run(main()) ``` -Pass `"*"` instead of a reference list to materialize every skill the store holds. +Pass `"*"` instead of a reference list to materialize every skill the store holds — but know +what you are asking for. `"*"` materializes the **whole project library**, which puts every +skill's `description` into the agent's context, including skills no AI Config references and +skills belonging to other teams. `write_skills(skill_refs(...), root)` is the form used above +because it materializes only what the resolved variation actually asked for; reach for `"*"` +when you genuinely want the whole library on disk. **`skills` is now a validated field.** Config parsing fails closed on a `skills` value that is not a list of `{key, version}` objects (key matching `^[a-z0-9][a-z0-9-]*$`, version an @@ -406,6 +411,19 @@ never writes through a symlink; writes are atomic (temp file, `fsync`, rename) a `0644`; and if the manifest is unreadable it performs no destructive action at all. Removing a skill from a variation is how revocation works — the next reconcile prunes it. +**Platform bound: the descriptor-pinned guarantee is POSIX-only.** On POSIX every destructive +step — the open, the rename, the unlink — runs relative to a directory descriptor opened +`O_RDONLY|O_DIRECTORY|O_NOFOLLOW` and held for the whole reconcile, so a directory swapped for +a symlink *after* its checks cannot redirect a write or a delete: the descriptor names the +inode that was checked, which closes the swap window rather than narrowing it. Windows has no +`*at()` syscall family, so there `write_skills` falls back to a per-component `lstat` check +taken immediately before each step. That floor is a check-then-use race rather than a closed +window: an attacker who already holds **write permission on the managed root** can still win +it. Windows reparse-point checks (`GetFileAttributesW` / `FILE_FLAG_OPEN_REPARSE_POINT`) are +deliberately not implemented in this release, and Windows is not a tested platform for it — +neither SDK repository has a Windows CI runner. Treat write permission on the managed root as +the security boundary on every platform, and on Windows as the *only* one. + **One exception, and it is what makes a crashed reconcile recoverable.** A file at a managed path whose bytes are *already byte-identical* to the content LaunchDarkly resolved is adopted — recorded in the manifest and reported `skipped_current` — rather than refused. @@ -477,6 +495,54 @@ to key its own map however the transport underneath does. > what was hashed. The SDK never parses or interprets them; if you want the frontmatter, > decode and parse the content on your side. +#### Privilege separation: the agent must not be able to rewrite its own skills + +**The recommended deployment runs `write_skills` as a different identity than the agent.** +Reconcile as one user, run the agent as another. Everything the reconcile puts on disk is +owner-write-only, and set explicitly rather than inherited from your umask: skill files and +the manifest at `0644` (via `fchmod` on the descriptor, so it cannot be redirected), the +per-skill `//` directories at `0755`, and the execute bit never set on anything. +Those modes are only a defense if the two identities actually differ — under a single identity +they describe a directory the agent can freely rewrite. + +**What to verify, as the identity that will run the agent.** The SDK cannot check this for you +(see below), so make it a deployment step: confirm the agent's identity has no write access to + +- the managed root itself, +- the per-skill directories `//` and the files `//SKILL.md`, +- the manifest at `/.launchdarkly-skills.json`. + +```bash +# Run as the agent's user. Every line should print DENIED. +root=.claude/skills +for target in "$root" "$root/.launchdarkly-skills.json" "$root"/*/ "$root"/*/SKILL.md; do + [ -e "$target" ] || continue + if [ -w "$target" ]; then echo "WRITABLE — fix this: $target"; else echo "DENIED: $target"; fi +done +``` + +Note that the managed root's own mode is **yours, not the SDK's**: `write_skills` creates only +that one leaf directory and does so with your umask, precisely because the root is a path you +chose. Own it — `chown reconcile-user:agent-group` and `chmod 0755` on the root is the shape +that makes the rest of the tree's modes mean something. + +**Why this is the mitigation that matters.** A `SKILL.md` is agent *instructions*. An agent +that can write its own skills directory can rewrite its own instructions, and an agent +processing untrusted input is exactly the thing that might be induced to do so. Write access +to the manifest is worse than write access to a skill, because the manifest is what tells the +*next* reconcile which paths the SDK owns and may delete: an agent that can edit it can keep a +skill LaunchDarkly has revoked, or aim the SDK's own delete path at something it should not +touch. `write_skills` re-validates every manifest entry from scratch for exactly that reason — +it treats that file as untrusted input, never as authorization — but an agent that cannot edit +it at all is the stronger position, and only your deployment can provide that. + +**The SDK deliberately does not report whether the root is writable.** There is no such field +on `ReconcileReport`, and its absence is a decision rather than an oversight. The SDK knows +only its own identity, which trivially has write access — it just wrote there. It cannot know +which identity will later run the agent, so any check it could make would answer a different +question than the one that matters, and would read as reassurance exactly where caution is +wanted. You know both identities; the SDK knows one. + --- ### Utility Helpers diff --git a/packages/client/agents.md b/packages/client/agents.md index a7b4ce1b..2558f9c3 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -510,6 +510,31 @@ resolving to the skill directory's `(st_dev, st_ino)`) instead of by comparing p A spy must `fstat` the descriptor **inside** the intercepted call — the implementation closes it as soon as the write returns. +**The platform bound is POSIX-only, and that is a decision — do not quietly "fix" it.** +Windows reparse-point checks (`GetFileAttributesW`, `FILE_FLAG_OPEN_REPARSE_POINT`) are not +implemented because Windows is not a supported or tested platform for this release: there is +no Windows CI runner in either repository, so the checks would ship unverified, and the +TypeScript SDK could not match them at all — Node exposes no `*at()` family on *any* +platform, so its racy floor is universal rather than Windows-only. Implementing them in +Python alone would break cross-language parity and trade a documented bound for an unverified +one. Two follow-on facts: on Windows write permission on the managed root is the only +boundary, which is why the privilege-separated deployment is documented as the mitigation +rather than as advice; and this bound retroactively lowers the priority of the reserved-device-name +work above — keep that code, but do not read it as evidence that Windows is hardened. If +Windows becomes a supported platform, revisit both together, and add the CI runner first. + +**Privilege separation is the deployment-side half of this, and `ReconcileReport` must not +grow a writability field.** The recommended deployment runs the reconcile as a different +identity than the agent, so the `0644`/`0755` modes above actually deny something: the agent +reads its instructions and cannot rewrite them or the manifest. That is the mitigation for a +prompt-injected agent editing its own skills. The security review asked for the report to +surface whether the managed root is writable; we declined, and the reasoning is load-bearing +rather than a preference. The SDK knows only its *own* identity, which trivially has write +access — it just wrote there — and cannot know which identity will later run the agent. Any +check it could perform would answer a different question than the one asked and would create +false confidence exactly where caution is wanted. The operator's verification steps live in +the README instead. Do not add the field. + ### Deferred: bounded retries `timeout` is implemented — a monotonic deadline, checked before each retrieval, before diff --git a/packages/client/src/launchdarkly_ai_server/safe_fs.py b/packages/client/src/launchdarkly_ai_server/safe_fs.py index 73eef878..43c21e58 100644 --- a/packages/client/src/launchdarkly_ai_server/safe_fs.py +++ b/packages/client/src/launchdarkly_ai_server/safe_fs.py @@ -11,6 +11,29 @@ re-resolving a name — which is what closes the swap window rather than merely narrowing it. Where the platform has no ``*at()`` syscall family (Windows) the identical sequence runs against full paths, the per-component ``lstat`` floor. + +**Platform bound — this guarantee is POSIX-only, deliberately.** On POSIX the +descriptor walk closes the swap window. On Windows it does not exist: there is no +``*at()`` family, so the ``lstat`` floor is all that runs, and a floor is a +check-then-use race rather than a closed window. The remedy would be +reparse-point checks (``GetFileAttributesW``, or opening with +``FILE_FLAG_OPEN_REPARSE_POINT``) and it is **not implemented, by decision rather +than by oversight**: Windows is not a supported or tested platform for this +release, and neither SDK repository has a Windows CI runner, so the checks would +ship untested — and the TypeScript SDK could not match them in any case, because +Node exposes no ``*at()`` family on *any* platform. Shipping them in Python alone +would break the cross-language parity the two SDKs are held to and would trade a +documented bound for an unverified one. + +Two consequences worth stating plainly rather than discovering later. First, on +Windows write permission on the managed root is the *only* boundary, so the +privilege-separated deployment the README documents is not advice there but the +mitigation. Second, this bound retroactively lowers the priority of the Windows +reserved-device-name work in ``skills_fs.py`` (``_WINDOWS_RESERVED_NAMES``): that +code stays, because it is cheap and it keeps a managed root written on Linux +usable when read from Windows, but it should not be read as evidence that Windows +is a hardened target. It is not. Revisit both together if Windows becomes +supported. """ from __future__ import annotations diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 223a3262..6d5ab2db 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -1394,6 +1394,136 @@ async def test_corrupt_manifest_file_is_not_destroyed(self, root: Path) -> None: assert _manifest_path(root).read_text(encoding="utf-8") == "{not json at all" +# The three literal cases the security review names for the prune path. +# +# The distinction from ``TestCorruptManifest`` above is the whole point: a +# corrupt manifest suppresses every destructive action wholesale, so those tests +# say nothing about these. Each manifest here is *well-formed* — parseable, +# a ``manifestVersion`` this release understands, a real ``entries`` map, and an +# entry whose ``key`` is a perfectly valid skill key that is genuinely absent +# from the requested set. The implementation has every input it needs to prune +# and must refuse anyway, because the recorded *path* is not one this SDK could +# have written. +HOSTILE_RECORDED_PATHS: list[str] = [ + # Absolute: the classic. A recorded path read back and unlinked as-is is a + # delete of an attacker-chosen file with the reconcile's privileges. + "/etc/passwd", + # Traversing: the same attack for an implementation that rejects a leading + # slash and then joins the rest onto the root. + "../../../etc/passwd", +] + + +class _UnlinkSpy: + """Records every ``os.unlink`` while delegating to the real one. + + Asserting only that ``/etc/passwd`` still exists proves nothing: the test + process cannot delete it anyway, so that assertion passes against an + implementation with no path check at all — permissions would be doing the + work. What has teeth is that the removal is never *attempted*: the refusal + happens above the syscall, on a path the SDK recomputes rather than trusts. + """ + + def __init__(self) -> None: + self.targets: list[str] = [] + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _UnlinkSpy: + real = os.unlink + + def spy(path: Any, *args: Any, **kwargs: Any) -> None: + self.targets.append(os.fsdecode(path)) + real(path, *args, **kwargs) + + # ``safe_fs_module.os`` *is* the ``os`` module, so this covers both the + # descriptor-relative ``os.unlink(name, dir_fd=...)`` and the + # ``Path.unlink`` used on the no-``*at()`` floor. + monkeypatch.setattr(safe_fs_module.os, "unlink", spy) + return self + + +class TestHostileManifestPrune: + """A well-formed manifest naming a path this SDK could not have written. + + The manifest is untrusted input. It is a plain file on the customer's disk + that anything with write access to the managed root can edit, and ``prune`` + is the one code path in the SDK that deletes. So a recorded path never + authorizes its own removal: it must match ``/SKILL.md`` for a + re-validated key, and the target is recomputed from the *current* managed + root instead of being read back out of the entry. + """ + + @pytest.mark.parametrize("recorded", HOSTILE_RECORDED_PATHS) + async def test_recorded_path_outside_the_root_is_refused( + self, root: Path, recorded: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + spy = _UnlinkSpy().install(monkeypatch) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {recorded: _entry("a", 1, SKILL_BODY)}, + }, + ) + + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + # The refusal is about ownership of the path, not about the file's state. + assert "could own" in action.error + assert [a for a in report.actions if a.action == "removed"] == [] + # Nothing was even attempted, let alone completed. + assert spy.targets == [] + assert Path("/etc/passwd").exists() + # Left in place rather than tidied away: dropping the entry would let a + # single hostile edit erase the SDK's own record of what it manages. + assert recorded in _read_manifest(root)["entries"] + + async def test_entry_under_a_since_symlinked_parent_is_refused( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The recorded path is the SDK's own, and is still not enough. + + Here the entry is exactly what a legitimate reconcile writes — + ``a/SKILL.md`` under key ``a`` — so the shape check that catches the two + cases above passes. What changed is the disk underneath it: ``/a`` + is now a symlink to somewhere else. This is the case a validate-then-act + implementation fails, because the manifest and the entry are both + entirely legitimate; only the current state of the parent is not. + """ + root = tmp_path / "skills" + root.mkdir() + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + victim = elsewhere / "SKILL.md" + victim.write_text("victim content\n", encoding="utf-8") + + # Managed legitimately first, so the manifest entry is one this SDK + # really did write... + managed = _place_managed(root, "a", SKILL_BODY) + # ...then the parent directory is swapped for a link out of the root. + managed.unlink() + (root / "a").rmdir() + (root / "a").symlink_to(elsewhere, target_is_directory=True) + + spy = _UnlinkSpy().install(monkeypatch) + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "symlink" in action.error + assert [a for a in report.actions if a.action == "removed"] == [] + assert spy.targets == [] + # The file the symlink pointed at is untouched, and so is the link. + assert victim.read_text(encoding="utf-8") == "victim content\n" + assert (root / "a").is_symlink() + assert "a/SKILL.md" in _read_manifest(root)["entries"] + + class TestWriteSkillsTelemetry: """Materialized / revoked signals from write_skills.""" From dd2d55cd680fe5cd56bc937804797cb6943ecdd7 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Thu, 10 Sep 2026 16:02:00 -0400 Subject: [PATCH 07/16] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20re-reconcile=20on=20delivery=20with=20watch=5Fskill?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit write_skills is a one-shot reconcile, so a revocation takes effect at the next process restart. watch_skills runs that reconcile now and again whenever the configured store reports a change, so a revoked skill's files leave the disk within a debounce interval of the store learning about it. It is wired to the SkillStore interface, not to any one transport: it needs a store that implements add_listener and nothing more, and refuses loudly when the store does not, since a watcher that silently never fires looks exactly like one whose skills never changed. Above the interface, remove_listener joins add_listener as the optional second half of change notification, on the SkillStore contract and on InMemorySkillStore. SkillWatcher.close needs it to detach; without it a store held every watcher ever created for the rest of its life. The watcher probes for it, so a store without it keeps working. Co-Authored-By: Claude Fable 5.1 --- packages/client/README.md | 3 +- packages/client/agents.md | 11 +- .../src/launchdarkly_ai_server/__init__.py | 4 + .../src/launchdarkly_ai_server/skills.py | 16 + .../src/launchdarkly_ai_server/skills_core.py | 16 +- .../launchdarkly_ai_server/skills_watch.py | 292 ++++++++++++++++++ packages/client/tests/test_skills.py | 29 ++ packages/client/tests/test_skills_watch.py | 178 +++++++++++ 8 files changed, 538 insertions(+), 11 deletions(-) create mode 100644 packages/client/src/launchdarkly_ai_server/skills_watch.py create mode 100644 packages/client/tests/test_skills_watch.py diff --git a/packages/client/README.md b/packages/client/README.md index bc8df592..41d5c6b3 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -458,8 +458,9 @@ Windows. | `get_skills(refs)` | Batch form. Accepts `SkillReference` values and bare key strings (string = latest). Results follow input order; missing or unverifiable entries are omitted. | | `all_skills()` | Every verified skill the store holds, one per key at its newest version. | | `write_skills(skills, root, *, prune=True, timeout=10.0, on_unavailable="keep")` | Materialize skills under `root`, returning a `ReconcileReport`. `prune` removes formerly-managed skills no longer requested. `on_unavailable="raise"` raises instead of reporting when content cannot be retrieved. Raises `ValueError` for an unusable root, a negative `timeout`, or an unrecognised `on_unavailable`. **Performs synchronous filesystem I/O — see the note below.** | -| `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)`. | +| `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)` / `remove_listener(kind, fn)`. | | `InMemorySkillStore(objects=None)` | A dict-backed store with `put(raw)`, for local development and testing. Holds several versions of a key. | +| `watch_skills(skills, root, …)` | `write_skills` plus a re-reconcile on every delivery change. Returns `(initial report, SkillWatcher)`; close the watcher when done. Revocation then takes effect within `debounce` of arriving rather than at the next restart. | Configure the store with `init_client(options={"skillStore": store})`. With none configured, the accessors raise `RuntimeError` explaining what to do and `write_skills` reports the diff --git a/packages/client/agents.md b/packages/client/agents.md index 2558f9c3..ca56b80e 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,6 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | +| `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | @@ -198,11 +199,11 @@ Three layers, in increasing order of blast radius: ### The store seam, and why version is part of the lookup -`SkillStore` is `get_object(kind, key, version=None)`, `all_objects(kind)`, and an optional -`add_listener(kind, fn)`. Version is part of the **lookup identity**, not a filter applied -to the answer, and that is load-bearing: a delivery payload carries the newest version of -every skill *plus* every version any variation currently pins, so two versions of one key -coexist routinely. A seam keyed by key alone would answer a pinned reference with the newest +`SkillStore` is `get_object(kind, key, version=None)`, `all_objects(kind)`, and the optional +pair `add_listener(kind, fn)` / `remove_listener(kind, fn)`. Version is part of the **lookup +identity**, not a filter applied to the answer, and that is load-bearing: a delivery payload +carries the newest version of every skill *plus* every version any variation currently pins, +so two versions of one key coexist routinely. A store keyed by key alone would answer a pinned reference with the newest object, and the caller would then have to reject it — turning the primary use case, a version-pinned attachment, into a missing skill. `version=None` asks for the newest held. diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 753e1e95..f863d847 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -53,6 +53,7 @@ OnUnavailable, write_skills, ) +from .skills_watch import SkillWatcher, watch_skills from .tracking import execute_and_stream, execute_and_track, wrap_tool_handlers from .types import ( NATIVE_TOOL_KEY, @@ -237,6 +238,9 @@ "write_skills", "SkillStore", "InMemorySkillStore", + # skills — the eager re-reconcile + "watch_skills", + "SkillWatcher", # skills — the three closed-set unions a typed consumer needs to name "ReconcileActionKind", "OnUnavailable", diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py index cb21986b..9016ab9f 100644 --- a/packages/client/src/launchdarkly_ai_server/skills.py +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -167,6 +167,22 @@ def add_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: """ self._listeners.setdefault(kind, []).append(fn) + def remove_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Unregisters *fn* from *kind*, so a subsequent ``put`` no longer calls it. + + Removes one occurrence: a callable registered twice must be removed twice. + Removing a callable that is not registered is a no-op, not an error, so a + consumer that detaches on close can do so unconditionally. + """ + listeners = self._listeners.get(kind) + if listeners is None: + return + try: + listeners.remove(fn) + except ValueError: + return + # --------------------------------------------------------------------------- # Reference discovery diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index 74cb706b..22166799 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -151,11 +151,17 @@ class SkillStore(Protocol): Duck-typed on purpose, mirroring how the LaunchDarkly client interface works in this package: pass any object carrying these methods. - ``add_listener(kind, fn)`` is part of the seam but - **optional**, which is why it is deliberately not declared here: a Protocol - member is required for structural compatibility, so declaring it would reject - every store that does not implement it. Nothing in this module calls it — it - exists for the delivery transport to push updates through. + ``add_listener(kind, fn)`` and ``remove_listener(kind, fn)`` are part of the + interface but **optional**, which is why they are deliberately not declared + here: a Protocol member is required for structural compatibility, so declaring + them would reject every store that does not implement them. Nothing in this + module calls either — they exist for the delivery transport to push updates + through, and for a consumer such as ``watch_skills`` to stop receiving them. + A store that implements ``add_listener`` should implement ``remove_listener`` + too; consumers probe for it and skip detaching when it is absent, so an + older store keeps working at the cost of a listener that lives as long as + the store does. ``remove_listener`` removes one occurrence of *fn* under + *kind* and is a no-op when *fn* is not registered. The raw objects a store serves are wire-shaped, with camelCase field names identical across language implementations:: diff --git a/packages/client/src/launchdarkly_ai_server/skills_watch.py b/packages/client/src/launchdarkly_ai_server/skills_watch.py new file mode 100644 index 00000000..05263057 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_watch.py @@ -0,0 +1,292 @@ +""" +Agent Skills — re-reconcile on delivery, so revocation does not wait for a restart. + +``write_skills`` is a one-shot reconcile: it materializes what the store holds +now. With a hand-populated store that is sufficient, and a revocation takes +effect at the next process restart. + +A streaming FDv2 connection changes the premise. A ``delete-object`` reaches a +live connection in **seconds**, and the store publishes a change listener, so +wiring the two together collapses the gap between "LaunchDarkly revoked this +skill" and "its ``SKILL.md`` is off the agent's disk" from a process lifetime to +a debounce interval. + +``on_unavailable="keep"`` stays the default: an outage must not read as +"everything was revoked". A watcher that pruned on a failed retrieval would +convert every transport failure into deletion of a customer's skill files. + +Layering: this module sits *above* ``skills_fs`` and calls ``write_skills`` +without modifying it. Nothing in the reconcile, the accessors, or verification +knows this file exists. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import threading +from collections.abc import Callable, Sequence +from typing import Any + +from .skills_core import SKILL_OBJECT_KIND, get_store +from .skills_fs import OnUnavailable, write_skills +from .types import ReconcileReport, Skill, SkillReference + +logger = logging.getLogger(__name__) + +DEFAULT_DEBOUNCE_SECONDS = 0.5 +""" +How long a change waits for its neighbours before a reconcile runs. + +A full payload transfer commits many objects at once and the listener fires per +object, so without coalescing a payload of forty skills would run forty +reconciles against one root. Half a second is far below the seconds-scale +latency this feature is trying to achieve and far above the microseconds a +commit's listener calls take. +""" + + +class SkillWatcher: + """ + A running re-reconcile. Returned by ``watch_skills``; stop it with ``close``. + + One watcher owns one root. **Do not point two watchers at the same root**, + and do not run ``write_skills`` against a watched root concurrently: the + reconcile's own contract is one root, one reconcile at a time, because two + interleaved runs lose the loser's manifest entries and leave the files it + wrote unmanaged. This class enforces that for its *own* reconciles — they run + on a single worker thread, serialised — and cannot enforce it against a + caller who reconciles the same root by hand. + + The watcher owns its registration on *store*: it registers ``notify`` when + constructed and unregisters it in ``close``, so a closed watcher is no longer + reachable from the store and can be collected. *store* must implement + ``add_listener``; ``remove_listener`` is probed for and, when the store does + not offer it, the listener stays registered for the store's lifetime. + """ + + def __init__( + self, + request: Sequence[Skill | SkillReference | str] | str, + root: str | os.PathLike[str], + store: Any, + *, + prune: bool, + timeout: float, + on_unavailable: OnUnavailable, + debounce: float, + on_reconcile: Callable[[ReconcileReport], Any] | None, + ) -> None: + self._request = request + self._root = root + self._prune = prune + self._timeout = timeout + self._on_unavailable = on_unavailable + self._debounce = debounce + self._on_reconcile = on_reconcile + + self._wake = threading.Event() + self._stop = threading.Event() + self._reconciles = 0 + self._lock = threading.Lock() + self._thread = threading.Thread( + target=self._run, name="ld-ai-skills-reconcile", daemon=True + ) + + # Register before starting the worker: ``notify`` only sets an event, so a + # change that lands in between is picked up as soon as the worker runs, + # and a store whose ``add_listener`` raises leaves no thread behind. + self._store = store + self._registered = False + store.add_listener(SKILL_OBJECT_KIND, self.notify) + self._registered = True + self._thread.start() + + # -- the listener the store calls ------------------------------------- + + def notify(self, _raw: Any = None) -> None: + """ + The store's change listener. Records that something changed; runs nothing. + + Deliberately trivial. It is called on the delivery thread, where a + reconcile — which does synchronous filesystem I/O, an fsync per file, and + a manifest rewrite — would stall event processing for the duration and, + on a stream, let the connection's read buffer back up behind a disk write. + The argument is ignored: a put's raw object and a revocation's tombstone + both mean the same thing here, which is "the store is not what it was". + """ + self._wake.set() + + # -- the worker -------------------------------------------------------- + + def _run(self) -> None: + while not self._stop.is_set(): + if not self._wake.wait(timeout=0.5): + continue + if self._stop.is_set(): + return + # Coalesce the rest of the burst. Clearing *before* the sleep rather + # than after is what makes a change arriving mid-debounce trigger the + # next pass instead of being swallowed by this one. + self._wake.clear() + if self._stop.wait(self._debounce): + return + self._reconcile_once() + + def _reconcile_once(self) -> None: + try: + report = asyncio.run( + write_skills( + self._request, + self._root, + prune=self._prune, + timeout=self._timeout, + on_unavailable=self._on_unavailable, + ) + ) + except Exception: + # A watcher that died on one bad reconcile would silently stop + # tracking revocations, which is worse than a noisy one. + logger.error( + "A skill re-reconcile raised; the watcher continues", exc_info=True + ) + return + + with self._lock: + self._reconciles += 1 + changed = [ + action + for action in report.actions + if action.action in ("written", "updated", "removed", "error") + ] + if changed: + logger.info( + "Re-reconciled skills after a delivery change: %d action(s) of note", + len(changed), + ) + if self._on_reconcile is not None: + try: + self._on_reconcile(report) + except Exception: + logger.error("A watch_skills callback raised", exc_info=True) + + # -- lifecycle --------------------------------------------------------- + + @property + def reconciles(self) -> int: + """How many re-reconciles have completed since the watcher started. + + Excludes the initial reconcile ``watch_skills`` awaits, which is the + caller's own result.""" + with self._lock: + return self._reconciles + + def close(self, timeout: float = 15.0) -> None: + """ + Stops watching. Idempotent. Does not undo anything already on disk. + + Waits out an in-flight reconcile rather than interrupting one, because a + reconcile killed between its content writes and its manifest rewrite is + the one case the manifest format has to recover from — worth avoiding when + we control the timing. + + Detaches ``notify`` from the store first, so no further change reaches a + watcher that is shutting down and the store no longer holds a reference to + it. A store without the optional ``remove_listener`` is left as it is + rather than failing the close. + """ + self._detach() + self._stop.set() + self._wake.set() + if self._thread.is_alive() and self._thread is not threading.current_thread(): + self._thread.join(timeout=timeout) + + def _detach(self) -> None: + with self._lock: + if not self._registered: + return + self._registered = False + remove_listener = getattr(self._store, "remove_listener", None) + if callable(remove_listener): + remove_listener(SKILL_OBJECT_KIND, self.notify) + + def __enter__(self) -> SkillWatcher: + return self + + def __exit__(self, *_exc: Any) -> None: + self.close() + + +async def watch_skills( + skills: Sequence[Skill | SkillReference | str] | str, + root: str | os.PathLike[str], + *, + prune: bool = True, + timeout: float = 10.0, + on_unavailable: OnUnavailable = "keep", + debounce: float = DEFAULT_DEBOUNCE_SECONDS, + on_reconcile: Callable[[ReconcileReport], Any] | None = None, +) -> tuple[ReconcileReport, SkillWatcher]: + """ + Reconciles now, then re-reconciles whenever delivery changes. + + Every argument that ``write_skills`` takes means the same thing here and is + passed straight through; the reconcile's semantics are untouched. Returns the + initial reconcile's report — so a caller can fail fast on a bad root or a + corrupt manifest exactly as they would with ``write_skills`` — paired with a + ``SkillWatcher`` to close when the process is done:: + + report, watcher = await watch_skills("*", "/etc/agent/skills") + try: + ... + finally: + watcher.close() + + A revocation delivered over a streaming connection then prunes the skill's + files within ``debounce`` of arriving, rather than at the next restart. + + Requires a store that implements the optional ``add_listener`` half of the + ``SkillStore`` interface. Raises ``RuntimeError`` when no store is configured, + and when the configured store has no ``add_listener`` — the second case + failing loudly rather than degrading to a one-shot reconcile, because a + watcher that silently never fires looks exactly like a watcher whose skills + never changed. The optional ``remove_listener`` lets ``SkillWatcher.close`` + detach from the store; a store without it still works, but each closed + watcher then stays registered for the store's lifetime. + """ + store = get_store() + if store is None: + raise RuntimeError( + "watch_skills needs a configured skill store. Configure one with " + 'init_client(options={"skillStore": store}).' + ) + add_listener = getattr(store, "add_listener", None) + if not callable(add_listener): + raise RuntimeError( + "watch_skills needs a skill store that implements add_listener(kind, " + "fn); the configured store does not, so delivery changes cannot be " + "observed. Use write_skills for a one-shot reconcile, or configure a " + "store with a delivery transport." + ) + if debounce < 0: + raise ValueError(f"debounce must not be negative, got {debounce!r}") + + # The initial reconcile runs first and on the caller's thread, so its report + # is the caller's to inspect and a bad root raises out of `watch_skills` + # rather than into a worker thread's log. + report = await write_skills( + skills, root, prune=prune, timeout=timeout, on_unavailable=on_unavailable + ) + + watcher = SkillWatcher( + skills, + root, + store, + prune=prune, + timeout=timeout, + on_unavailable=on_unavailable, + debounce=debounce, + on_reconcile=on_reconcile, + ) + return report, watcher diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 6470f0c0..d3f83095 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -548,6 +548,35 @@ def test_put_does_not_notify_other_kind_listeners( assert seen == [] + def test_remove_listener_stops_put_notifying_it(self, make_raw_skill: Any) -> None: + s = InMemorySkillStore() + seen: list[dict[str, Any]] = [] + s.add_listener("skill", seen.append) + s.remove_listener("skill", seen.append) + + s.put(make_raw_skill(key="a")) + + assert seen == [] + + def test_remove_listener_removes_one_occurrence(self, make_raw_skill: Any) -> None: + s = InMemorySkillStore() + seen: list[dict[str, Any]] = [] + s.add_listener("skill", seen.append) + s.add_listener("skill", seen.append) + s.remove_listener("skill", seen.append) + + s.put(make_raw_skill(key="a")) + + assert len(seen) == 1 + + def test_remove_listener_of_an_unregistered_callable_is_a_no_op(self) -> None: + s = InMemorySkillStore() + s.remove_listener("skill", print) + s.add_listener("skill", print) + s.remove_listener("flag", print) + s.remove_listener("skill", print) + s.remove_listener("skill", print) + class TestStoreConfiguration: """Store wiring on the lifecycle layer.""" diff --git a/packages/client/tests/test_skills_watch.py b/packages/client/tests/test_skills_watch.py new file mode 100644 index 00000000..48a95f07 --- /dev/null +++ b/packages/client/tests/test_skills_watch.py @@ -0,0 +1,178 @@ +""" +Tests for ``watch_skills`` / ``SkillWatcher`` — the eager re-reconcile. + +The watcher is wired to the ``SkillStore`` interface, not to any one transport: +it needs a store that implements ``add_listener``, and nothing more. These tests +therefore drive it from ``InMemorySkillStore``, whose ``put`` notifies its +listeners synchronously, and from small hand-written store doubles. + +Every test writes only inside pytest's ``tmp_path``. The watcher runs a real +worker thread, so tests wait on observable outcomes rather than on fixed sleeps +wherever the outcome is something that *does* happen; a fixed sleep is used only +to assert that something does *not*. +""" + +from __future__ import annotations + +import time +from typing import Any + +import pytest + +from launchdarkly_ai_server import InMemorySkillStore, init_client, watch_skills +from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND + +pytestmark = pytest.mark.usefixtures("reset_skill_state") + + +def wait_until(predicate: Any, timeout: float = 5.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.01) + return predicate() + + +# --------------------------------------------------------------------------- +# Starting a watch, and what it refuses +# --------------------------------------------------------------------------- + + +class TestWatchSkills: + async def test_the_in_memory_store_can_also_drive_a_watch( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + """The watcher is wired to the ``SkillStore`` interface, not to the FDv2 + store.""" + store = InMemorySkillStore() + store.put(make_raw_skill(key="a", version=1, content="body")) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "a" / "SKILL.md" + assert written.read_text() == "body" + store.put(make_raw_skill(key="a", version=2, content="new body")) + assert wait_until(lambda: written.read_text() == "new body", timeout=10) + finally: + watcher.close() + + async def test_a_burst_of_changes_coalesces_into_few_reconciles( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.1) + try: + for i in range(12): + store.put(make_raw_skill(key=f"skill-{i}")) + time.sleep(0.5) + # Twelve objects put back to back fire twelve listener calls; without + # coalescing that is twelve reconciles of one root. + assert watcher.reconciles <= 2 + finally: + watcher.close() + + async def test_a_store_with_no_listener_support_is_refused_loudly( + self, tmp_path: Any + ) -> None: + class NoListeners: + def get_object(self, *_a: Any, **_k: Any) -> None: + return None + + def all_objects(self, _kind: str) -> dict[str, Any]: + return {} + + await init_client(options={"skillStore": NoListeners()}, client=object()) + with pytest.raises(RuntimeError, match="add_listener"): + await watch_skills("*", tmp_path / "s") + + async def test_no_store_configured_raises(self, tmp_path: Any) -> None: + with pytest.raises(RuntimeError, match="configured skill store"): + await watch_skills("*", tmp_path / "s") + + +# --------------------------------------------------------------------------- +# Closing a watch +# --------------------------------------------------------------------------- + + +class TestWatcherDetachesOnClose: + """``SkillWatcher.close`` unregisters ``notify``, so a closed watcher is + neither called nor kept alive by the store.""" + + @staticmethod + def _skill_listeners(store: Any) -> list[Any]: + return list(store._listeners.get(SKILL_OBJECT_KIND, [])) + + async def test_a_closed_watcher_is_no_longer_notified( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="pdf-extraction", version=1, content="first")) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + assert watcher.notify in self._skill_listeners(store) + + watcher.close() + + assert watcher.notify not in self._skill_listeners(store) + store.put(make_raw_skill(key="pdf-extraction", version=4, content="second")) + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + assert wait_until( + lambda: ( + store.get_object(SKILL_OBJECT_KIND, "pdf-extraction", 4) is not None + ), + timeout=10, + ) + time.sleep(0.3) + assert written.read_text() == "first" + assert watcher.reconciles == 0 + + async def test_close_twice_does_not_raise(self, tmp_path: Any) -> None: + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + watcher.close() + watcher.close() + assert self._skill_listeners(store) == [] + + async def test_repeated_watchers_leave_no_listeners_behind( + self, tmp_path: Any + ) -> None: + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + for _ in range(5): + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + assert len(self._skill_listeners(store)) == 1 + watcher.close() + assert self._skill_listeners(store) == [] + + async def test_a_store_without_remove_listener_still_closes( + self, tmp_path: Any + ) -> None: + """``remove_listener`` is optional: an older store keeps working, at the + cost of the listener staying registered.""" + + class AddOnly: + def __init__(self) -> None: + self.listeners: list[Any] = [] + + def get_object(self, *_a: Any, **_k: Any) -> None: + return None + + def all_objects(self, _kind: str) -> dict[str, Any]: + return {} + + def add_listener(self, _kind: str, fn: Any) -> None: + self.listeners.append(fn) + + store = AddOnly() + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + assert store.listeners == [watcher.notify] + + watcher.close() + watcher.close() + + assert store.listeners == [watcher.notify] From 0f797c0b167adf8be462ae8e8b2431705d75dfd7 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Thu, 10 Sep 2026 16:03:21 -0400 Subject: [PATCH 08/16] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20the=20FDv2=20delivery=20protocol,=20without=20the?= =?UTF-8?q?=20network?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The half of the delivery transport that has no I/O: identifying a skill object on the wire by kind inline-resource plus category skill, translating it into the raw object shape the SkillStore interface defines, holding it by (key, objectVersion), and applying a payload's events as one commit at payload-transferred. The store that puts a connection underneath this follows separately, so the three decisions that matter most can be reviewed on their own: - objectVersion is the skill's version; version is the payload's. The translation happens in one place and TestVersionTranslation asserts it in both directions, because confusing them fails silently. - Changes commit at payload-transferred, not per object. A half-applied full transfer would briefly empty the store, which with pruning on is the difference between a reconcile and deleting a customer's files. - A hashless object is held, not dropped, so verification withholds it with a reason code rather than the transport reporting it absent. Flag and segment objects share the connection and are skipped and counted, not rejected. Nothing here is exported yet; the store exports it. Co-Authored-By: Claude Fable 5.1 --- packages/client/agents.md | 46 ++ .../src/launchdarkly_ai_server/skills_fdv2.py | 601 +++++++++++++++++ packages/client/tests/test_skills_fdv2.py | 605 ++++++++++++++++++ 3 files changed, 1252 insertions(+) create mode 100644 packages/client/src/launchdarkly_ai_server/skills_fdv2.py create mode 100644 packages/client/tests/test_skills_fdv2.py diff --git a/packages/client/agents.md b/packages/client/agents.md index ca56b80e..80ec0538 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,6 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | +| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the `objectVersion`/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | | `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | @@ -218,6 +219,51 @@ one place that collapses the result to one object per key, because both whole-st consumers need it — `all_skills`, since a list holding two versions of one key is not a set of skills, and the `"*"` reconcile, since `//SKILL.md` is a single path. +### The delivery transport, and the one field that will bite you + +`skills_fdv2.py` translates LaunchDarkly's FDv2 delivery protocol into raw objects in the +shape `skills_core.SkillStore` documents. It lives below the store interface; **nothing above +that interface knows it exists**. If a transport change ever seems to require editing an +accessor, verification, or `write_skills`, the adapter boundary is wrong. + +**`objectVersion` is the skill's version. `version` is the payload's.** On the wire a skill +`put-object` carries both, and they are not interchangeable: + +```json +{"key":"pdf-extraction","kind":"inline-resource","category":"skill", + "objectVersion":3,"version":42, + "object":{"contentType":"text/markdown","content":"…","contentHash":"…","name":"…"}} +``` + +`objectVersion` (3) is what a `{key, version}` reference pins and what becomes the stored +`version`. `version` (42) is the version of the *payload* the object arrived in — it moves +when anything in the environment moves, including a flag with nothing to do with skills. +Reading it as the skill's version fails **silently**: the object verifies, the hash matches, +and the caller gets content under a version number that means nothing. Flags and segments +carry only `version` and omit both `category` and `objectVersion`, which is exactly why the +two fields look interchangeable. `_store_object_from_put` is the only place the translation +happens, and `TestVersionTranslation` asserts it in both directions. + +**Skills are identified by `kind == "inline-resource" && category == "skill"`; everything else +is ignored, not rejected.** An environment's payload assignment carries its flag payload +alongside its agent-skill payload, so flag and segment objects arrive as a matter of course. +Erroring on an unrecognised kind would turn a normal payload into a permanent reconnect +loop — a flag-delivery outage caused by a skills rollout. + +**Changes commit at `payload-transferred`, not as objects arrive.** A payload version is the +unit of consistency: a half-applied full transfer would publish a state the server never +described, and would briefly empty the store — which, with pruning on, is the difference +between a reconcile and deleting a customer's skill files. An interrupted transfer therefore +leaves last known good intact, and listeners fire once per commit. + +**A hashless object is held, not dropped.** Verification withholds it with +`missing_content_hash`; the transport's job is to make that loud (an error per object, a +summary per wholly-hashless payload, `diagnostics.hashless_objects`) rather than to work +around it. Dropping it at the transport would report `absent` — indistinguishable from "no +such skill" — and would let a prune delete the last known-good copy on disk. Never synthesize +a hash from the delivered content: that certifies the content against itself and verifies +nothing. + ### The reported outcome vocabulary, and the `Resolution` mapping `get_skill` returns `Skill | None`; `get_skill_result` returns a frozen `SkillOutcome` diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py new file mode 100644 index 00000000..80ef1224 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -0,0 +1,601 @@ +""" +Agent Skills — the FDv2 delivery protocol. + +The half of the delivery transport that has no I/O: identifying skill objects +on the wire, translating them into the raw object shape the ``SkillStore`` +interface defines, holding them by ``(key, objectVersion)``, and applying a +payload's events as one consistent commit. ``FDv2SkillStore``, the store that +puts a network connection underneath this, follows in a separate change. + +It sits *below* the ``SkillStore`` interface, and everything above — the +accessors, integrity verification, the ``Skill`` dataclass, materialization — +is unaware of it. + +Layering:: + + launchdarkly_ai_server + └─ SkillStore protocol (skills_core) ── the interface accessors call + └─ FDv2SkillStore (this module) ── deserialise, hold, serve + └─ LaunchDarkly's SDK-facing FDv2 channel + GET /sdk/poll, GET /sdk/stream, authenticated with the + environment's server-side SDK key + +Dependencies run one way: this module imports nothing from the feature beyond +the version validator in ``types_validation``, and nothing in the feature +imports it. It uses only the standard library, so it adds no dependency +to a package whose sole runtime dependency is ``opentelemetry-api``. + +Three things this layer does *not* do, on purpose: + +- **It does not verify content.** Verification lives at the accessor boundary in + ``skills_core`` so that it applies to every store equally, including a + customer's own. +- **It does not work around a missing ``contentHash``.** A hashless object is + held verbatim and *withheld* by verification with ``missing_content_hash``. + This module's job is to make that outcome loud — see ``StoreDiagnostics``. +- **It does not evaluate anything.** Flag and segment objects that share the + connection are skipped and counted, nothing more. + +The design rationale — why ``objectVersion`` is not ``version``, why changes +commit at ``payload-transferred`` — is in ``agents.md`` under *The delivery +transport*. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass, field +from typing import Any + +from .types_validation import is_valid_skill_version + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# The wire contract +# --------------------------------------------------------------------------- + +FDV2_OBJECT_KIND = "inline-resource" +""" +The FDv2 ``kind`` skills are delivered under. Together with +``FDV2_OBJECT_CATEGORY`` it maps onto the single interface value +``skills_core.SKILL_OBJECT_KIND``; that translation is this adapter's job. +""" + +FDV2_OBJECT_CATEGORY = "skill" +"""The ``category`` that narrows ``inline-resource`` to an agent skill.""" + +SDK_DATA_MODEL_VERSION = 1 +""" +The ``mv`` request parameter. The one request parameter whose value could not be +confirmed against a live server, so treat the default as provisional and +override it through the store's ``data_model_version`` if needed. +""" + +_EVENT_SERVER_INTENT = "server-intent" +_EVENT_PUT_OBJECT = "put-object" +_EVENT_DELETE_OBJECT = "delete-object" +_EVENT_PAYLOAD_TRANSFERRED = "payload-transferred" +_EVENT_HEARTBEAT = "heart-beat" +_EVENT_GOODBYE = "goodbye" +_EVENT_ERROR = "error" + +_INTENT_TRANSFER_FULL = "xfer-full" +_INTENT_TRANSFER_CHANGES = "xfer-changes" +_INTENT_TRANSFER_NONE = "none" + +_ENVELOPE_FIELDS = ("contentType", "content", "contentHash", "name", "description") +""" +The skill object envelope's fields, copied through verbatim. Nothing is coerced +or defaulted: a transport that filled in a missing field would be forging the +very thing verification exists to check. +""" + +_MOBILE_KEY_PREFIX = "mob-" +_SERVER_KEY_PREFIX = "sdk-" +_CLIENT_SIDE_ID = re.compile(r"\A[0-9a-f]{20,}\Z") +"""A client-side environment ID: bare lowercase hex. Server-side and mobile keys +both carry a prefix, so this shape is unambiguous rather than heuristic.""" + + +# --------------------------------------------------------------------------- +# Server-side only +# --------------------------------------------------------------------------- + + +def _require_server_side_credential(sdk_key: str) -> None: + """ + Refuses a mobile key or a client-side environment ID. + + Skill content is customer-confidential, and payload assignment is shared + across credential types, so a client-side credential may well *succeed* + against these endpoints. Raises rather than logs: a store built on the wrong + credential should not exist. + """ + if not isinstance(sdk_key, str) or not sdk_key.strip(): + raise ValueError( + "FDv2SkillStore requires a LaunchDarkly server-side SDK key " + "(sdk-...); none was given." + ) + key = sdk_key.strip() + if key.startswith(_MOBILE_KEY_PREFIX): + raise ValueError( + "FDv2SkillStore was given a mobile key (mob-...). Agent Skills are a " + "server-side feature: skill content is customer-confidential and is " + "never delivered to a mobile or client-side process. Use the " + "environment's server-side SDK key (sdk-...)." + ) + if _CLIENT_SIDE_ID.match(key): + raise ValueError( + "FDv2SkillStore was given what looks like a client-side environment " + "ID. Agent Skills are a server-side feature: skill content is " + "customer-confidential and is never delivered to a client-side " + "process. Use the environment's server-side SDK key (sdk-...)." + ) + if not key.startswith(_SERVER_KEY_PREFIX): + # Not rejected: private instances and test doubles issue credentials + # without the public prefix. Only the two unambiguous shapes above are. + logger.warning( + "The credential given to FDv2SkillStore does not look like a " + "LaunchDarkly server-side SDK key (sdk-...). Skills are delivered " + "only to server-side credentials; if this is a client-side or mobile " + "credential the connection will be rejected or will deliver nothing." + ) + + +# --------------------------------------------------------------------------- +# Diagnostics +# --------------------------------------------------------------------------- + + +@dataclass +class StoreDiagnostics: + """ + What the transport has seen. Read-only from a caller's perspective. + + Not part of the ``SkillStore`` interface. It exists because "this environment + has no skills" and "every skill was withheld" are easy to mistake for each + other, and a counter is easier to assert on than a log line. + """ + + payloads_transferred: int = 0 + """Completed ``payload-transferred`` commits since the store started.""" + skill_objects_received: int = 0 + """``put-object`` events identified as skills, across all payloads.""" + objects_ignored: int = 0 + """Objects skipped because they were not skills: flags, segments, and any + future kind. Skipping is the contract, not a failure.""" + objects_revoked: int = 0 + """``delete-object`` events applied to skills.""" + hashless_objects: int = 0 + """ + Skill objects whose envelope carried no ``contentHash``. + + **Nonzero means skills are being withheld**: verification withholds every one + of these with ``missing_content_hash``. + """ + connection_failures: int = 0 + """Recoverable transport failures since the last successful transfer.""" + last_error: str | None = None + """The most recent transport error, if any. Human-readable; do not parse.""" + + +# --------------------------------------------------------------------------- +# Deserialisation — where objectVersion is not version +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _Tombstone: + """A ``delete-object`` narrowed to the identity it revokes.""" + + key: str + object_version: int | None + + +def _is_skill_event(data: Any) -> bool: + """ + Whether one ``put-object`` / ``delete-object`` payload is a skill. + + Both halves are required: ``inline-resource`` may carry other categories, + and flags and segments omit ``category`` entirely. Every other kind is + ignored, not rejected, because flag and segment objects share the connection + and erroring on them would turn a normal payload into a reconnect loop. + """ + if not isinstance(data, dict): + return False + return ( + data.get("kind") == FDV2_OBJECT_KIND + and data.get("category") == FDV2_OBJECT_CATEGORY + ) + + +def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: + """ + Translates one FDv2 skill ``put-object`` into the raw object shape the + ``SkillStore`` interface defines. + + **The one translation this adapter must get right:** + + wire ``objectVersion`` → stored ``version`` (the skill's own version) + wire ``version`` → dropped (the *payload* version) + + ``objectVersion`` is what a ``{key, version}`` reference pins; ``version`` + moves whenever anything in the environment moves. Confusing them fails + silently: the object verifies and the caller gets content under a version + number that means nothing. + + Returns ``None`` only when ``key`` is not a string, since a keyless object + has no identity to store it under. Every other defect is carried through + verbatim so that verification withholds it with a reason code rather than + the transport dropping it into indistinguishable absence. + """ + key = data.get("key") + if not isinstance(key, str) or not key: + logger.warning( + "An FDv2 skill put-object carried no string 'key' and could not be " + "stored under any identity; it was dropped." + ) + return None + + raw: dict[str, Any] = {"key": key} + + # A membership test rather than a `.get` default, so an explicitly-null + # objectVersion stays null and reaches verification as `invalid_version`. + if "objectVersion" in data: + raw["version"] = data["objectVersion"] + + envelope = data.get("object") + if isinstance(envelope, dict): + for wire_field in _ENVELOPE_FIELDS: + if wire_field in envelope: + raw[wire_field] = envelope[wire_field] + return raw + + +def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: + """ + Narrows one FDv2 skill ``delete-object`` to the identity it revokes, with + the same ``objectVersion`` translation as a put. + + An ``object_version`` of ``None`` means the delete named no usable version + and is read as "revoke every version of this key". That is the safe + direction: the alternative is continuing to serve content LaunchDarkly has + withdrawn. + """ + key = data.get("key") + if not isinstance(key, str) or not key: + logger.warning( + "An FDv2 skill delete-object carried no string 'key'; it was ignored." + ) + return None + object_version = data.get("objectVersion") + return _Tombstone( + key=key, + object_version=object_version + if is_valid_skill_version(object_version) + else None, + ) + + +# --------------------------------------------------------------------------- +# The held object set +# --------------------------------------------------------------------------- + + +class _SkillObjectSet: + """ + Raw skill objects held in memory, keyed by ``(key, objectVersion)``. + + Lookup semantics are identical to ``InMemorySkillStore``'s, down to the + fall-through to a version-less entry, so that the store a caller configures + cannot change how a pinned reference resolves; ``TestInterfaceParity`` + asserts it. Reimplemented rather than inherited because the transport needs + ``delete`` and the atomic ``replace_with`` a full transfer requires. + + An object too malformed to carry a usable version is still held, under its + key alone, so verification withholds it with a signal rather than the + transport dropping it. + """ + + def __init__(self) -> None: + self._versions: dict[str, dict[int, dict[str, Any]]] = {} + self._loose: dict[str, dict[str, Any]] = {} + + def put(self, raw: dict[str, Any]) -> None: + key = raw["key"] + version = raw.get("version") + if is_valid_skill_version(version): + self._versions.setdefault(key, {})[version] = raw + else: + self._loose[key] = raw + + def delete(self, tombstone: _Tombstone) -> list[dict[str, Any]]: + """Removes what *tombstone* revokes; returns the raw objects that went away.""" + removed: list[dict[str, Any]] = [] + if tombstone.object_version is None: + held = self._versions.pop(tombstone.key, {}) + removed.extend(held.values()) + loose = self._loose.pop(tombstone.key, None) + if loose is not None: + removed.append(loose) + return removed + + held = self._versions.get(tombstone.key, {}) + gone = held.pop(tombstone.object_version, None) + if gone is not None: + removed.append(gone) + if not held: + self._versions.pop(tombstone.key, None) + return removed + + def get(self, key: str, version: int | None) -> dict[str, Any] | None: + held = self._versions.get(key, {}) + if version is not None: + # Fall through to the version-less entry so a malformed object + # reaches verification rather than reading as simply absent. + return held.get(version) or self._loose.get(key) + if held: + return held[max(held)] + return self._loose.get(key) + + def snapshot(self) -> dict[str, dict[str, Any]]: + """One entry per ``(key, version)``, under keys opaque to the SDK.""" + out: dict[str, dict[str, Any]] = { + f"{key}:{version}": raw + for key, versions in self._versions.items() + for version, raw in versions.items() + } + out.update(self._loose) + return out + + def all_raw(self) -> list[dict[str, Any]]: + return list(self.snapshot().values()) + + def replace_with(self, other: _SkillObjectSet) -> None: + """Adopts *other*'s contents wholesale — how a full transfer commits.""" + self._versions = other._versions + self._loose = other._loose + + def copy(self) -> _SkillObjectSet: + clone = _SkillObjectSet() + clone._versions = {key: dict(v) for key, v in self._versions.items()} + clone._loose = dict(self._loose) + return clone + + def __len__(self) -> int: + return sum(len(v) for v in self._versions.values()) + len(self._loose) + + +# --------------------------------------------------------------------------- +# The protocol state machine — pure, no I/O +# --------------------------------------------------------------------------- + + +@dataclass +class _TransferOutcome: + """What one event did. Aggregated by the caller; nothing here does I/O.""" + + committed: bool = False + changes: list[dict[str, Any]] = field(default_factory=list) + basis: str | None = None + fatal: str | None = None + disconnect: str | None = None + + +class _ProtocolReader: + """ + Applies FDv2 events to an object set. Pure — no sockets, no threads, no clock — + so every wire case is testable without a server. + + **Changes are buffered and committed at ``payload-transferred``.** A payload + version is the unit of consistency: applying half of one would publish a + state the server never described, and on a full transfer would briefly empty + the store. Listeners therefore fire once per commit, not once per object. + """ + + def __init__(self, committed: _SkillObjectSet) -> None: + self._committed = committed + self._intent: str | None = None + self._pending: _SkillObjectSet | None = None + self._changes: list[dict[str, Any]] = [] + self.diagnostics = StoreDiagnostics() + # Identities already reported by ``_warn_hashless``. Per reader, so a + # recreated store reports again and two stores never quieten each other. + # No lock: ``handle`` runs only on its owner's single delivery thread. + self._warned_hashless: set[tuple[str, Any]] = set() + + # -- events ------------------------------------------------------------ + + def handle(self, name: str, data: Any) -> _TransferOutcome: + """Routes one event. Unknown event names are ignored, by contract.""" + if name == _EVENT_SERVER_INTENT: + return self._server_intent(data) + if name == _EVENT_PUT_OBJECT: + return self._put_object(data) + if name == _EVENT_DELETE_OBJECT: + return self._delete_object(data) + if name == _EVENT_PAYLOAD_TRANSFERRED: + return self._payload_transferred(data) + if name == _EVENT_ERROR: + return self._error(data) + if name == _EVENT_GOODBYE: + return self._goodbye(data) + if name == _EVENT_HEARTBEAT: + return _TransferOutcome() + logger.debug("Ignoring unknown FDv2 event '%s'", name) + return _TransferOutcome() + + def _server_intent(self, data: Any) -> _TransferOutcome: + payloads = data.get("payloads") if isinstance(data, dict) else None + if not isinstance(payloads, list) or not payloads: + return _TransferOutcome( + disconnect="server-intent carried no payload description" + ) + first = payloads[0] + intent = first.get("intentCode") if isinstance(first, dict) else None + self._intent = intent + self._changes = [] + if intent == _INTENT_TRANSFER_FULL: + # Built alongside the live set rather than in place, so an + # interrupted transfer leaves last known good intact. + self._pending = _SkillObjectSet() + elif intent == _INTENT_TRANSFER_CHANGES: + self._pending = self._committed.copy() + else: + if intent != _INTENT_TRANSFER_NONE: + logger.debug("Ignoring FDv2 server-intent with intentCode %r", intent) + self._pending = None + return _TransferOutcome() + + def _target_for(self, data: Any) -> _SkillObjectSet | None: + """ + The pending set a skill object event applies to, or ``None`` when the + event is not a skill or the current intent carries no objects. + + An object arriving with no ``server-intent`` at all is treated as a + delta against what is held rather than dropped. + """ + if not _is_skill_event(data): + self.diagnostics.objects_ignored += 1 + return None + if self._pending is None: + if self._intent is None: + self._intent = _INTENT_TRANSFER_CHANGES + if self._intent not in (_INTENT_TRANSFER_FULL, _INTENT_TRANSFER_CHANGES): + return None + self._pending = self._committed.copy() + return self._pending + + def _put_object(self, data: Any) -> _TransferOutcome: + target = self._target_for(data) + if target is None: + return _TransferOutcome() + raw = _store_object_from_put(data) + if raw is None: + return _TransferOutcome() + target.put(raw) + self._changes.append(raw) + self.diagnostics.skill_objects_received += 1 + if not isinstance(raw.get("contentHash"), str): + self.diagnostics.hashless_objects += 1 + self._warn_hashless(raw) + return _TransferOutcome() + + def _delete_object(self, data: Any) -> _TransferOutcome: + target = self._target_for(data) + if target is None: + return _TransferOutcome() + tombstone = _tombstone_from_delete(data) + if tombstone is None: + return _TransferOutcome() + target.delete(tombstone) + self.diagnostics.objects_revoked += 1 + # A tombstone carries identity and no content, so a listener that reads + # content must check for ``content`` rather than assume it. + self._changes.append( + {"key": tombstone.key, "version": tombstone.object_version} + ) + return _TransferOutcome() + + def _payload_transferred(self, data: Any) -> _TransferOutcome: + state = data.get("state") if isinstance(data, dict) else None + version = data.get("version") if isinstance(data, dict) else None + if self._pending is not None: + self._committed.replace_with(self._pending) + _warn_if_nothing_can_verify(self._committed) + self._pending = None + self._intent = None + changes = self._changes + self._changes = [] + self.diagnostics.payloads_transferred += 1 + logger.debug( + "FDv2 payload transferred: payload version %s, %d skill object(s) held", + version, + len(self._committed), + ) + return _TransferOutcome( + committed=True, + changes=changes, + basis=state if isinstance(state, str) and state else None, + ) + + def _abandon_in_flight(self) -> None: + """Drops the in-flight payload and keeps what is committed.""" + self._pending = None + self._intent = None + self._changes = [] + + def _error(self, data: Any) -> _TransferOutcome: + reason = data.get("reason") if isinstance(data, dict) else None + self._abandon_in_flight() + return _TransferOutcome(disconnect=f"server sent error: {reason}") + + def _goodbye(self, data: Any) -> _TransferOutcome: + reason = data.get("reason") if isinstance(data, dict) else None + catastrophe = bool(data.get("catastrophe")) if isinstance(data, dict) else False + silent = bool(data.get("silent")) if isinstance(data, dict) else False + self._abandon_in_flight() + if not silent: + logger.info("FDv2 connection closing: %s", reason) + if catastrophe: + return _TransferOutcome( + fatal=f"server sent a catastrophic goodbye: {reason}" + ) + return _TransferOutcome(disconnect=f"server said goodbye: {reason}") + + # -- diagnostics --------------------------------------------------------- + + def _warn_hashless(self, raw: dict[str, Any]) -> None: + """ + One ERROR per ``(key, version)`` whose envelope had no ``contentHash``. + + ERROR rather than WARN because an empty accessor result is otherwise + indistinguishable from an environment that has no skills. + """ + identity = (raw["key"], raw.get("version")) + if identity in self._warned_hashless: + return + self._warned_hashless.add(identity) + logger.error( + "Skill '%s' version %s arrived without a contentHash and will be withheld. %s", + raw["key"], + raw.get("version"), + _HASHLESS_ADVICE, + extra={"ld_skill_key": raw["key"], "ld_skill_version": raw.get("version")}, + ) + + +_HASHLESS_ADVICE = ( + "The delivered skill object carries no 'contentHash', so integrity " + "verification withholds it with reason_code 'missing_content_hash' and its " + "content will not resolve. The SDK cannot work around this: verification " + "hashes the delivered bytes and compares them against the envelope's " + "'contentHash', and there is nothing to compare against. 'contentHash' is a " + "sha256 over the verbatim UTF-8 content. Contact LaunchDarkly support if " + "skills in your environment arrive without one." +) + + +def _warn_if_nothing_can_verify(committed: _SkillObjectSet) -> None: + """ + One ERROR per committed payload in which *nothing* held can possibly verify. + + ``log_withholding_summary`` reports the same condition at the accessor + boundary, but only once a caller asks. This fires at delivery time, so it is + visible in a process that boots, materializes nothing, and exits. + """ + held = committed.all_raw() + if not held: + return + hashless = [raw for raw in held if not isinstance(raw.get("contentHash"), str)] + if len(hashless) != len(held): + return + logger.error( + "All %d skill object(s) in the delivered payload arrived without a " + "contentHash. No skill content will resolve from this store. %s", + len(held), + _HASHLESS_ADVICE, + ) diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py new file mode 100644 index 00000000..ff5720bc --- /dev/null +++ b/packages/client/tests/test_skills_fdv2.py @@ -0,0 +1,605 @@ +""" +Tests for the FDv2 skill delivery protocol. + +Wire semantics — which objects are skills, ``objectVersion`` versus ``version``, +revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted +against ``_ProtocolReader``, which has no I/O, so each case reads as the contract +it is rather than as a server script. +""" + +from __future__ import annotations + +import hashlib +from typing import Any, ClassVar + +import pytest + +from launchdarkly_ai_server import InMemorySkillStore +from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND +from launchdarkly_ai_server.skills_fdv2 import ( + FDV2_OBJECT_CATEGORY, + FDV2_OBJECT_KIND, + _is_skill_event, + _ProtocolReader, + _SkillObjectSet, + _store_object_from_put, + _tombstone_from_delete, +) + +pytestmark = pytest.mark.usefixtures("reset_skill_state") + +SKILL_BODY = "---\nname: PDF Extraction\n---\nExtract text from PDFs.\n" + + +def _hash(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +# --------------------------------------------------------------------------- +# Wire builders — one place that knows the shape, so a contract change is one edit +# --------------------------------------------------------------------------- + + +def put_skill( + key: str = "pdf-extraction", + *, + object_version: Any = 3, + payload_version: int = 42, + content: str = SKILL_BODY, + content_hash: Any = None, + omit_hash: bool = False, + name: str = "PDF Extraction", +) -> dict[str, Any]: + """One skill ``put-object`` event's data, in the shape the wire delivers it.""" + envelope: dict[str, Any] = { + "contentType": "text/markdown", + "content": content, + "name": name, + "description": "Extracts text", + } + if not omit_hash: + envelope["contentHash"] = ( + content_hash if content_hash is not None else _hash(content) + ) + return { + "key": key, + "kind": FDV2_OBJECT_KIND, + "category": FDV2_OBJECT_CATEGORY, + "objectVersion": object_version, + "version": payload_version, + "object": envelope, + } + + +def delete_skill( + key: str = "pdf-extraction", *, object_version: Any = 3, payload_version: int = 43 +) -> dict[str, Any]: + return { + "key": key, + "kind": FDV2_OBJECT_KIND, + "category": FDV2_OBJECT_CATEGORY, + "objectVersion": object_version, + "version": payload_version, + } + + +def put_flag(key: str = "my-flag", version: int = 17) -> dict[str, Any]: + """A flag ``put-object``: no ``category``, no ``objectVersion``.""" + return { + "key": key, + "kind": "flag", + "version": version, + "object": { + "key": key, + "version": version, + "on": True, + "variations": [True, False], + }, + } + + +def put_segment(key: str = "beta-users", version: int = 4) -> dict[str, Any]: + return { + "key": key, + "kind": "segment", + "version": version, + "object": {"key": key, "version": version, "included": []}, + } + + +def server_intent( + code: str = "xfer-full", payload_id: str = "agent-skill" +) -> dict[str, Any]: + return { + "payloads": [ + {"id": payload_id, "target": 1, "intentCode": code, "reason": "test"} + ] + } + + +def transferred(state: str = "basis-1", version: int = 42) -> dict[str, Any]: + return {"state": state, "version": version} + + +def events(*pairs: tuple[str, Any]) -> list[dict[str, Any]]: + return [{"event": name, "data": data} for name, data in pairs] + + +def full_payload( + *object_events: tuple[str, Any], state: str = "basis-1" +) -> list[dict[str, Any]]: + return events( + ("server-intent", server_intent("xfer-full")), + *object_events, + ("payload-transferred", transferred(state)), + ) + + +# --------------------------------------------------------------------------- +# Identifying skill objects, and ignoring everything else +# --------------------------------------------------------------------------- + + +class TestObjectIdentification: + def test_kind_and_category_together_identify_a_skill(self) -> None: + assert _is_skill_event(put_skill()) is True + + def test_a_flag_is_not_a_skill(self) -> None: + assert _is_skill_event(put_flag()) is False + + def test_a_segment_is_not_a_skill(self) -> None: + assert _is_skill_event(put_segment()) is False + + def test_inline_resource_of_another_category_is_not_a_skill(self) -> None: + """``inline-resource`` is a broad kind, so the category is required too.""" + other = put_skill() + other["category"] = "prompt-template" + assert _is_skill_event(other) is False + + def test_skill_category_under_another_kind_is_not_a_skill(self) -> None: + other = put_skill() + other["kind"] = "some-future-kind" + assert _is_skill_event(other) is False + + def test_a_flag_shaped_object_with_no_category_is_not_a_skill(self) -> None: + """Flags and segments omit ``category`` entirely — the documented shape.""" + assert "category" not in put_flag() + assert "objectVersion" not in put_flag() + + @pytest.mark.parametrize("value", [None, "skill", 3, [], ()]) + def test_non_dict_events_are_not_skills(self, value: Any) -> None: + assert _is_skill_event(value) is False + + +# --------------------------------------------------------------------------- +# objectVersion is not version +# --------------------------------------------------------------------------- + + +class TestVersionTranslation: + def test_object_version_becomes_the_seam_version(self) -> None: + raw = _store_object_from_put(put_skill(object_version=3, payload_version=42)) + assert raw is not None + assert raw["version"] == 3 + + def test_the_payload_version_never_reaches_the_seam(self) -> None: + """ + The failure this asserts against is silent: a store that read ``version`` + would serve verifiable content under a version number that means nothing, + and every pinned reference would resolve to the wrong thing with no error. + """ + raw = _store_object_from_put(put_skill(object_version=3, payload_version=42)) + assert raw is not None + assert raw["version"] != 42 + assert 42 not in raw.values() + + def test_the_two_are_distinguished_even_when_the_payload_version_is_lower( + self, + ) -> None: + raw = _store_object_from_put(put_skill(object_version=99, payload_version=1)) + assert raw is not None + assert raw["version"] == 99 + + def test_a_missing_object_version_is_not_defaulted_from_the_payload(self) -> None: + wire = put_skill() + del wire["objectVersion"] + raw = _store_object_from_put(wire) + assert raw is not None + assert "version" not in raw + + def test_an_explicitly_null_object_version_is_carried_through_as_null(self) -> None: + """Carried, not invented: verification reports ``invalid_version``.""" + raw = _store_object_from_put(put_skill(object_version=None)) + assert raw is not None + assert raw["version"] is None + + def test_a_delete_translates_object_version_too(self) -> None: + tombstone = _tombstone_from_delete( + delete_skill(object_version=3, payload_version=43) + ) + assert tombstone is not None + assert tombstone.object_version == 3 + + def test_a_delete_with_no_usable_object_version_revokes_every_version(self) -> None: + tombstone = _tombstone_from_delete(delete_skill(object_version=None)) + assert tombstone is not None + assert tombstone.object_version is None + + def test_a_keyless_put_is_dropped_because_it_has_no_identity(self) -> None: + wire = put_skill() + del wire["key"] + assert _store_object_from_put(wire) is None + + def test_the_envelope_is_copied_verbatim(self) -> None: + raw = _store_object_from_put(put_skill()) + assert raw is not None + assert raw["content"] == SKILL_BODY + assert raw["contentHash"] == _hash(SKILL_BODY) + assert raw["name"] == "PDF Extraction" + assert raw["contentType"] == "text/markdown" + + def test_an_absent_envelope_field_is_absent_rather_than_defaulted(self) -> None: + wire = put_skill() + del wire["object"]["name"] + raw = _store_object_from_put(wire) + assert raw is not None + assert "name" not in raw + + +# --------------------------------------------------------------------------- +# The protocol reader +# --------------------------------------------------------------------------- + + +def drive(reader: _ProtocolReader, payload_events: list[dict[str, Any]]) -> list[Any]: + return [reader.handle(e["event"], e.get("data")) for e in payload_events] + + +class TestProtocolReader: + def test_a_full_transfer_commits_at_payload_transferred(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + outcomes = drive(reader, full_payload(("put-object", put_skill()))) + assert len(held) == 1 + assert outcomes[-1].committed is True + assert outcomes[-1].basis == "basis-1" + + def test_nothing_is_visible_before_payload_transferred(self) -> None: + """A payload version is the unit of consistency; half of one is not a state.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill()), + ), + ) + assert len(held) == 0 + + def test_an_interrupted_full_transfer_leaves_last_known_good_intact(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill(object_version=1)))) + assert held.get("pdf-extraction", None) is not None + + # A second full transfer starts and never completes. + drive( + reader, + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill(object_version=2)), + ), + ) + still_held = held.get("pdf-extraction", None) + assert still_held is not None + assert still_held["version"] == 1 + + def test_a_full_transfer_replaces_rather_than_merges(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill("first")))) + drive( + reader, full_payload(("put-object", put_skill("second")), state="basis-2") + ) + assert held.get("first", None) is None + assert held.get("second", None) is not None + + def test_a_change_transfer_applies_deltas_over_what_is_held(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill("first")))) + drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("second")), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert held.get("first", None) is not None + assert held.get("second", None) is not None + + def test_a_delete_object_revokes_the_skill(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill(object_version=3)))) + drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill(object_version=3)), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert held.get("pdf-extraction", None) is None + assert reader.diagnostics.objects_revoked == 1 + + def test_a_delete_notifies_with_a_tombstone_carrying_no_content(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill()))) + outcomes = drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ), + ) + (change,) = outcomes[-1].changes + assert change == {"key": "pdf-extraction", "version": 3} + assert "content" not in change + + def test_a_delete_for_one_version_leaves_the_other_held(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + full_payload( + ("put-object", put_skill(object_version=2)), + ("put-object", put_skill(object_version=3)), + ), + ) + drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill(object_version=3)), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert held.get("pdf-extraction", 2) is not None + assert held.get("pdf-extraction", None)["version"] == 2 + + def test_flag_and_segment_objects_are_skipped_cleanly(self) -> None: + """ + The mixed payload is the normal case, not an edge one: an environment's + assignment carries its flag payload alongside its agent-skill payload. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + outcomes = drive( + reader, + full_payload( + ("put-object", put_flag("flag-a")), + ("put-object", put_skill("pdf-extraction")), + ("put-object", put_segment("beta-users")), + ("put-object", put_flag("flag-b")), + ("delete-object", put_flag("flag-c")), + ), + ) + assert len(held) == 1 + assert held.get("pdf-extraction", None) is not None + assert reader.diagnostics.objects_ignored == 4 + assert reader.diagnostics.skill_objects_received == 1 + assert all(o.fatal is None and o.disconnect is None for o in outcomes) + + def test_an_unknown_kind_is_ignored_rather_than_fatal(self) -> None: + """ + Erroring on an unrecognised kind would turn a normal payload into a + permanent reconnect loop — a flag-delivery outage caused by a skills + rollout. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + exotic = { + "key": "x", + "kind": "quantum-widget", + "version": 1, + "object": {"a": 1}, + } + outcomes = drive(reader, full_payload(("put-object", exotic))) + assert len(held) == 0 + assert all(o.fatal is None and o.disconnect is None for o in outcomes) + + def test_an_unknown_event_name_is_ignored(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + outcome = reader.handle("some-future-event", {"anything": True}) + assert outcome.fatal is None + assert outcome.disconnect is None + + def test_a_heartbeat_does_nothing(self) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle("heart-beat", None) + assert outcome == type(outcome)() + + def test_an_error_event_abandons_the_in_flight_payload(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill(object_version=1)))) + outcomes = drive( + reader, + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill(object_version=2)), + ( + "error", + {"payloadId": "agent-skill", "reason": "backend unavailable"}, + ), + ), + ) + assert outcomes[-1].disconnect is not None + assert held.get("pdf-extraction", None)["version"] == 1 + + def test_a_goodbye_asks_for_a_reconnect(self) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle("goodbye", {"reason": "rebalancing", "silent": False}) + assert outcome.disconnect is not None + assert outcome.fatal is None + + def test_a_catastrophic_goodbye_is_fatal(self) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle( + "goodbye", {"reason": "no", "silent": False, "catastrophe": True} + ) + assert outcome.fatal is not None + + def test_transfer_none_holds_everything_and_commits(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill()))) + drive( + reader, + events( + ("server-intent", server_intent("none")), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert len(held) == 1 + + def test_an_object_arriving_with_no_intent_is_treated_as_a_delta(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + events( + ("put-object", put_skill()), + ("payload-transferred", transferred("basis-1")), + ), + ) + assert len(held) == 1 + + +# --------------------------------------------------------------------------- +# Interface parity with InMemorySkillStore +# --------------------------------------------------------------------------- + + +class TestInterfaceParity: + """ + The two stores must resolve identically. ``_SkillObjectSet`` reimplements the + lookup rather than inheriting it — see its docstring for why — so this is the + test that stops the two from drifting. + """ + + RAWS: ClassVar[list[dict[str, Any]]] = [ + {"key": "a", "version": 1, "content": "x", "contentHash": _hash("x")}, + {"key": "a", "version": 4, "content": "y", "contentHash": _hash("y")}, + {"key": "b", "version": 2, "content": "z", "contentHash": _hash("z")}, + {"key": "malformed", "version": "not-a-version", "content": "q"}, + ] + + def _both(self) -> tuple[InMemorySkillStore, _SkillObjectSet]: + memory = InMemorySkillStore() + objects = _SkillObjectSet() + for raw in self.RAWS: + memory.put(dict(raw)) + objects.put(dict(raw)) + return memory, objects + + @pytest.mark.parametrize( + "key,version", + [ + ("a", None), + ("a", 1), + ("a", 4), + ("a", 9), + ("b", 2), + ("b", None), + ("missing", None), + ("missing", 1), + ("malformed", None), + ("malformed", 7), + ], + ) + def test_get_agrees(self, key: str, version: int | None) -> None: + memory, objects = self._both() + assert memory.get_object(SKILL_OBJECT_KIND, key, version) == objects.get( + key, version + ) + + def test_snapshot_agrees(self) -> None: + memory, objects = self._both() + assert memory.all_objects(SKILL_OBJECT_KIND) == objects.snapshot() + + +# --------------------------------------------------------------------------- +# The contentHash gap +# --------------------------------------------------------------------------- + + +def _per_object_hashless_errors(caplog: Any) -> list[Any]: + """The per-object ERROR, as distinct from the whole-payload summary.""" + return [ + r + for r in caplog.records + if r.levelname == "ERROR" + and "arrived without a contentHash" in r.getMessage() + and "No skill content will resolve" not in r.getMessage() + ] + + +class TestMissingContentHash: + """ + A skill delivered without a ``contentHash``, asserted as behaviour. + + An envelope with no ``contentHash`` must produce a *withheld* skill with the + ``missing_content_hash`` reason — loudly, diagnosably, and without a crash. + There is deliberately no fallback that skips verification: a hash the SDK + computed from the content it was handed would certify the content against + itself and verify nothing. + """ + + def test_a_redelivered_hashless_object_logs_once_per_store( + self, caplog: Any + ) -> None: + """Re-delivering the same ``(key, version)`` to one store must not + multiply the ERROR: a polling store sees every object on every poll.""" + reader = _ProtocolReader(_SkillObjectSet()) + payload = full_payload(("put-object", put_skill(omit_hash=True))) + with caplog.at_level("ERROR"): + drive(reader, payload) + drive(reader, payload) + assert len(_per_object_hashless_errors(caplog)) == 1 + + def test_a_recreated_store_reports_the_same_hashless_object_again( + self, caplog: Any + ) -> None: + """ + The dedupe belongs to the store, not the process. A host that rebuilds + its store (reconnect wrapper, config reload, credential rotation) must + get the ERROR again, since it is the loudest signal that a deployment is + broken rather than empty by design. + """ + payload = full_payload(("put-object", put_skill(omit_hash=True))) + with caplog.at_level("ERROR"): + drive(_ProtocolReader(_SkillObjectSet()), payload) + first = len(_per_object_hashless_errors(caplog)) + drive(_ProtocolReader(_SkillObjectSet()), payload) + assert first == 1 + assert len(_per_object_hashless_errors(caplog)) == 2 + + def test_two_live_stores_do_not_suppress_each_other(self, caplog: Any) -> None: + """Two stores in one process (say, two environments) each report.""" + one = _ProtocolReader(_SkillObjectSet()) + two = _ProtocolReader(_SkillObjectSet()) + payload = full_payload(("put-object", put_skill(omit_hash=True))) + with caplog.at_level("ERROR"): + drive(one, payload) + drive(two, payload) + # And each still dedupes its own re-deliveries. + drive(one, payload) + drive(two, payload) + assert len(_per_object_hashless_errors(caplog)) == 2 From 29ad3feda3c50808d0673e4b21171cdf2b250230 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:17:57 -0400 Subject: [PATCH 09/16] fix(client): attach the skill watcher's listener before its first reconcile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit watch_skills awaited write_skills and only then constructed SkillWatcher, which is where the store listener attaches. The reconcile snapshots the store as its first step and then spends the rest of its time on the filesystem, so every write, fsync, prune and manifest rewrite in that first pass ran with nothing listening. A change delivered in that window was never seen, and since nothing re-reconciles on a timer, a revocation that landed there waited for the next unrelated change — on a quiet root, the next restart. Exactly the gap watch_skills exists to close. The watcher now attaches its listener before the initial reconcile and starts its worker after. notify only sets an event, so a change arriving mid-reconcile is recorded and picked up by the worker's first pass, while holding the thread back keeps write_skills's one-root-one-reconcile contract: the worker cannot race the caller's own reconcile over the same manifest. A reconcile that raises detaches the listener on the way out, since the caller is handed an exception rather than a watcher to close. Co-Authored-By: Claude Opus 5 --- .../launchdarkly_ai_server/skills_watch.py | 51 ++++++++++++--- packages/client/tests/test_skills_watch.py | 62 +++++++++++++++++++ 2 files changed, 103 insertions(+), 10 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_watch.py b/packages/client/src/launchdarkly_ai_server/skills_watch.py index 05263057..94b31d46 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_watch.py +++ b/packages/client/src/launchdarkly_ai_server/skills_watch.py @@ -94,13 +94,26 @@ def __init__( target=self._run, name="ld-ai-skills-reconcile", daemon=True ) - # Register before starting the worker: ``notify`` only sets an event, so a - # change that lands in between is picked up as soon as the worker runs, - # and a store whose ``add_listener`` raises leaves no thread behind. + # Register before the initial reconcile, and leave the worker unstarted + # until ``start``. ``notify`` only sets an event, so a change that lands + # while that reconcile is still running is recorded rather than lost, and + # the worker cannot reconcile the root while the caller's own reconcile is + # in flight. A store whose ``add_listener`` raises leaves no thread behind. self._store = store self._registered = False store.add_listener(SKILL_OBJECT_KIND, self.notify) self._registered = True + + def _start(self) -> None: + """ + Starts the worker. ``watch_skills`` calls this once, after the initial + reconcile; it is not part of the caller-facing interface. + + Split from construction so registration and reconciling can be ordered + independently: the listener attaches first, so no change is missed, while + the first re-reconcile waits for the initial one to finish, so a root only + ever has one reconcile running at a time. + """ self._thread.start() # -- the listener the store calls ------------------------------------- @@ -272,13 +285,13 @@ async def watch_skills( if debounce < 0: raise ValueError(f"debounce must not be negative, got {debounce!r}") - # The initial reconcile runs first and on the caller's thread, so its report - # is the caller's to inspect and a bad root raises out of `watch_skills` - # rather than into a worker thread's log. - report = await write_skills( - skills, root, prune=prune, timeout=timeout, on_unavailable=on_unavailable - ) - + # The watcher attaches its listener before the initial reconcile, not after. + # The reconcile snapshots the store as its first step and then spends the + # rest of its time on the filesystem — a write and an fsync per skill, the + # prune, the manifest rewrite — so a change delivered after that snapshot + # needs something already listening to be seen at all. Nothing re-reconciles + # on a timer, so a revocation that landed unobserved would wait for the next + # unrelated change, which on a quiet root means the next restart. watcher = SkillWatcher( skills, root, @@ -289,4 +302,22 @@ async def watch_skills( debounce=debounce, on_reconcile=on_reconcile, ) + try: + # The initial reconcile runs on the caller's thread, so its report is the + # caller's to inspect and a bad root raises out of `watch_skills` rather + # than into a worker thread's log. + report = await write_skills( + skills, root, prune=prune, timeout=timeout, on_unavailable=on_unavailable + ) + except BaseException: + # The listener is already attached, so a reconcile that raises must not + # leave it on the store: the caller has no watcher to close. + watcher.close() + raise + + # Only now start the worker. A change that arrived during the reconcile has + # already set the wake event, so the worker's first pass picks it up; one that + # arrived before the reconcile's snapshot is already on disk, and the + # redundant pass it triggers converges on the same state. + watcher._start() return report, watcher diff --git a/packages/client/tests/test_skills_watch.py b/packages/client/tests/test_skills_watch.py index 48a95f07..dada2015 100644 --- a/packages/client/tests/test_skills_watch.py +++ b/packages/client/tests/test_skills_watch.py @@ -92,6 +92,68 @@ async def test_no_store_configured_raises(self, tmp_path: Any) -> None: await watch_skills("*", tmp_path / "s") +# --------------------------------------------------------------------------- +# Changes that land while the initial reconcile is running +# --------------------------------------------------------------------------- + + +class TestChangesDuringTheInitialReconcile: + """The listener attaches before the initial reconcile, so a change delivered + while that reconcile is still running is acted on rather than lost.""" + + async def test_a_revocation_landing_mid_reconcile_is_not_missed( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + class RevokesAfterSnapshot(InMemorySkillStore): + """Revokes everything the moment the reconcile has taken its + snapshot — where a ``delete-object`` lands when it arrives a fraction + of a second into startup, with nothing after it.""" + + def __init__(self) -> None: + super().__init__() + self.snapshots = 0 + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + objects = super().all_objects(kind) + self.snapshots += 1 + if self.snapshots == 1: + self._versions.clear() + self._loose.clear() + for listener in self._listeners.get(SKILL_OBJECT_KIND, []): + listener({"key": "pdf-extraction"}) + return objects + + store = RevokesAfterSnapshot() + store.put(make_raw_skill(key="pdf-extraction", version=1, content="body")) + await init_client(options={"skillStore": store}, client=object()) + + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + # The initial reconcile wrote what its snapshot held, so the file is + # on disk and the revocation that followed it is the only change + # left to act on. + assert written.read_text() == "body" + assert wait_until(lambda: not written.exists(), timeout=10) + finally: + watcher.close() + + async def test_a_failed_initial_reconcile_leaves_no_listener_behind( + self, tmp_path: Any + ) -> None: + """Registering first means a reconcile that raises has to detach: the + caller is handed an exception, not a watcher to close.""" + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + not_a_directory = tmp_path / "file" + not_a_directory.write_text("") + + with pytest.raises(ValueError, match="not a directory"): + await watch_skills("*", not_a_directory) + + assert store._listeners.get(SKILL_OBJECT_KIND, []) == [] + + # --------------------------------------------------------------------------- # Closing a watch # --------------------------------------------------------------------------- From 4c6d965b066941fa1d836a5b53ec6ddfe5e513e4 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:20:57 -0400 Subject: [PATCH 10/16] test(client): a broken store answer never deletes managed files Covers the reconcile side of the withheld-answer fix: a listing that is not a mapping leaves every managed file alone rather than reading as a full revocation, and an answer served under a different key writes nothing, is reported against the key that was asked for, and does not reach that other key's file. Each one previously deleted a file and reported a clean run. Co-Authored-By: Claude Opus 5 --- packages/client/tests/test_skills_fs.py | 114 ++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 62d01d59..4aa9ed8e 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -532,6 +532,120 @@ def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: assert _actions_by_key(report)["pdf-extraction"].action == "error" assert "pdf-extraction/SKILL.md" in _read_manifest(root)["entries"] + async def test_a_non_mapping_listing_never_prunes(self, root: Path) -> None: + """A store that cannot list is not a store holding nothing. + + A listing collapsed to "no skills" is indistinguishable from every + skill having been revoked, and prune would then delete every managed + file and report a clean run. The listing failure has to reach the + prune gate as an incomplete run. + """ + + class NoListingStore: + """Answers the listing with something that is not a mapping.""" + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return None + + def all_objects(self, kind: str) -> Any: + return None + + existing = _place_managed(root, "pdf-extraction", SKILL_BODY) + skills_module._set_store(NoListingStore()) + + report = await write_skills("*", root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert [a.action for a in report.actions] == ["error"] + assert any("rather than an object" in m for m in _error_messages(report)) + # The entry survives, so the next reconcile picks it up. + assert "pdf-extraction/SKILL.md" in _read_manifest(root)["entries"] + + async def test_an_answer_under_another_key_writes_nothing(self, root: Path) -> None: + """The file is named after the key the object carries, so a store + answering under a different key would write one path and prune another. + + Left unchecked, the run wrote the aliased key, then deleted it in the + same pass because prune keys off the request — and reported ok. The + requested key has to be the one the outcome is reported against. + """ + + class AliasingStore: + """Answers every lookup with an object carrying its own key.""" + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return { + "key": "other-key", + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + skills_module._set_store(AliasingStore()) + + report = await write_skills(["requested-key"], root) + + assert report.ok is False + assert [a.action for a in report.actions] == ["error"] + # Reported against the key that was asked for, not the one served. + assert _actions_by_key(report)["requested-key"].action == "error" + assert not (root / "other-key").exists() + assert _read_manifest(root)["entries"] == {} + + async def test_an_answer_under_another_key_does_not_overwrite_that_key( + self, root: Path + ) -> None: + """The aliased answer must not reach the real key's file. + + Both keys are requested here, so nothing is prunable and the write + itself is what is under test: unchecked, the object served under the + alias is written to the *other* key's path, clobbering the content that + key's own lookup resolved — and the run still reports ok. + """ + aliased = "aliased\n" + + class AliasingStore: + """Answers one key honestly and the other under that same key.""" + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + if key == "other-key": + return { + "key": "other-key", + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + return { + "key": "other-key", + "version": 2, + "content": aliased, + "contentHash": _hash(aliased), + } + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + existing = _place_managed(root, "other-key", SKILL_BODY) + skills_module._set_store(AliasingStore()) + + # The alias is resolved last, so an unchecked write lands on top. + report = await write_skills(["other-key", "requested-key"], root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert _actions_by_key(report)["requested-key"].action == "error" + assert _actions_by_key(report)["other-key"].action == "skipped_current" + async def test_unavailable_run_does_not_corrupt_manifest(self, root: Path) -> None: _place_managed(root, "a", SKILL_BODY) before = _read_manifest(root) From 1cf523566fdfd0a6fc70ecc9eac5220ac52e2a45 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:46:56 -0400 Subject: [PATCH 11/16] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20name=20the=20payload=20a=20transfer=20completed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The protocol reader took payloads[0]'s intentCode and applied it to the skill object set, which is what the delivery protocol requires — one payload per credential, read the first intent, tolerate the rest — but it left the assumption behind that rule undocumented and unguarded. If the one-payload guarantee ever widens, an xfer-full for another payload would start an empty pending set and the next payload-transferred would publish it: every skill reported revoked, and with pruning on, a customer's files deleted. The first payload is still the payload that is read. What is new is that the reader now knows which payload skills actually arrive on — learnt from the intent's id, or from the (p::) selector, since no object or transfer event carries a payload id of its own — and declines to apply a transfer of any other, holding last known good, warning once, and counting it in diagnostics.payloads_ignored. An intent describing more than one payload warns once on its own, because that is the one case the comparison cannot catch: another payload's transfer arriving before any skill has been seen has nothing to be compared against. Behaviour under one-payload delivery is unchanged, and a full transfer of the skill payload still empties it — every skill deleted is a real state the guard must not mask. Co-Authored-By: Claude Opus 5 --- packages/client/agents.md | 13 + .../src/launchdarkly_ai_server/skills_fdv2.py | 132 +++++++++- packages/client/tests/test_skills_fdv2.py | 226 ++++++++++++++++++ 3 files changed, 370 insertions(+), 1 deletion(-) diff --git a/packages/client/agents.md b/packages/client/agents.md index 80ec0538..e73c96d4 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -256,6 +256,19 @@ described, and would briefly empty the store — which, with pruning on, is the between a reconcile and deleting a customer's skill files. An interrupted transfer therefore leaves last known good intact, and listeners fire once per commit. +**The first payload intent is read, and is assumed to be the skill payload.** Delivery +provides one payload per credential and the protocol requires a client to ignore all but the +first payload intent, so `payloads[0]` is both what arrives and what the protocol says to +read. The cost of that assumption is that an `xfer-full` for somebody *else's* payload would +start an empty pending set, and the next `payload-transferred` would publish it — every skill +reported revoked, and with pruning on, a customer's files deleted. `_ProtocolReader` +therefore learns which payload skills arrive on, from the intent's `id` or from the +`(p::)` selector, and declines to apply a transfer of any other: once at +WARNING, counted in `diagnostics.payloads_ignored`, holding last known good. A transfer that +names no payload is applied, since one-payload delivery is the common case. The residual is +the first transfer of a connection — before a skill has arrived there is nothing to compare +against — which is what the separate WARNING on a multi-payload intent is for. + **A hashless object is held, not dropped.** Verification withholds it with `missing_content_hash`; the transport's job is to make that loud (an error per object, a summary per wholly-hashless payload, `diagnostics.hashless_objects`) rather than to work diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 80ef1224..f644a349 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -36,6 +36,12 @@ - **It does not evaluate anything.** Flag and segment objects that share the connection are skipped and counted, nothing more. +One assumption it *does* make, and states: **the payload intent it reads is the +payload skills arrive on.** Delivery sends one payload per credential and the +protocol tells a client to read only the first payload intent, so today those are +the same payload. ``_ProtocolReader`` keeps the pair apart anyway, because the +cost of conflating them is an emptied skill set. + The design rationale — why ``objectVersion`` is not ``version``, why changes commit at ``payload-transferred`` — is in ``agents.md`` under *The delivery transport*. @@ -93,6 +99,16 @@ very thing verification exists to check. """ +_PAYLOAD_SELECTOR = re.compile(r"\(p:([^:()]+):\d+\)") +""" +The payload identity inside a transfer's selector, ``(p::)``. + +The selector is the only place a completed transfer names its own payload: +``put-object``, ``delete-object`` and ``payload-transferred`` carry no payload id +of their own. ``_ProtocolReader`` reads it as a fallback for an intent that named +no ``id``. +""" + _MOBILE_KEY_PREFIX = "mob-" _SERVER_KEY_PREFIX = "sdk-" _CLIENT_SIDE_ID = re.compile(r"\A[0-9a-f]{20,}\Z") @@ -169,6 +185,11 @@ class StoreDiagnostics: future kind. Skipping is the contract, not a failure.""" objects_revoked: int = 0 """``delete-object`` events applied to skills.""" + payloads_ignored: int = 0 + """ + Transfers not applied because they completed a payload other than the one + skills arrive on. Zero while delivery sends one payload per connection. + """ hashless_objects: int = 0 """ Skill objects whose envelope carried no ``contentHash``. @@ -255,6 +276,22 @@ def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: return raw +def _payload_id_of(intent: Any) -> str | None: + """The payload id one payload intent names, when it names a usable one.""" + if not isinstance(intent, dict): + return None + value = intent.get("id") + return value if isinstance(value, str) and value else None + + +def _payload_id_from_selector(state: Any) -> str | None: + """The payload id inside a transfer's selector, when it carries one.""" + if not isinstance(state, str): + return None + match = _PAYLOAD_SELECTOR.search(state) + return match.group(1) if match else None + + def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: """ Narrows one FDv2 skill ``delete-object`` to the identity it revokes, with @@ -394,6 +431,18 @@ class _ProtocolReader: version is the unit of consistency: applying half of one would publish a state the server never described, and on a full transfer would briefly empty the store. Listeners therefore fire once per commit, not once per object. + + **The first payload intent is read, and is assumed to be the skill payload.** + Delivery provides one payload per credential and the protocol requires a + client to ignore all but the first payload intent, so ``payloads[0]`` is both + what arrives and what the protocol says to read. If that ever widens, an + ``xfer-full`` for somebody else's payload would empty the skill set and the + next ``payload-transferred`` would publish it empty — with pruning on, the + difference between a reconcile and deleting a customer's files. This layer + therefore learns which payload skills arrive on and declines to apply a + transfer of any other, once at WARNING and counted. The residual is the first + transfer of a connection: before a skill has arrived there is nothing to + compare a payload against. """ def __init__(self, committed: _SkillObjectSet) -> None: @@ -406,6 +455,14 @@ def __init__(self, committed: _SkillObjectSet) -> None: # recreated store reports again and two stores never quieten each other. # No lock: ``handle`` runs only on its owner's single delivery thread. self._warned_hashless: set[tuple[str, Any]] = set() + # The payload the current intent describes, and the payload skills have + # actually arrived on. One payload per connection makes these the same + # payload; the class docstring says why they are kept apart regardless. + self._intent_payload_id: str | None = None + self._skill_payload_id: str | None = None + self._skills_in_payload = 0 + self._warned_multiple_payloads = False + self._warned_foreign_payload = False # -- events ------------------------------------------------------------ @@ -434,10 +491,15 @@ def _server_intent(self, data: Any) -> _TransferOutcome: return _TransferOutcome( disconnect="server-intent carried no payload description" ) + if len(payloads) > 1: + self._warn_multiple_payloads(payloads) + # The first payload only, as the protocol requires. first = payloads[0] intent = first.get("intentCode") if isinstance(first, dict) else None self._intent = intent + self._intent_payload_id = _payload_id_of(first) self._changes = [] + self._skills_in_payload = 0 if intent == _INTENT_TRANSFER_FULL: # Built alongside the live set rather than in place, so an # interrupted transfer leaves last known good intact. @@ -479,6 +541,7 @@ def _put_object(self, data: Any) -> _TransferOutcome: target.put(raw) self._changes.append(raw) self.diagnostics.skill_objects_received += 1 + self._skills_in_payload += 1 if not isinstance(raw.get("contentHash"), str): self.diagnostics.hashless_objects += 1 self._warn_hashless(raw) @@ -493,6 +556,8 @@ def _delete_object(self, data: Any) -> _TransferOutcome: return _TransferOutcome() target.delete(tombstone) self.diagnostics.objects_revoked += 1 + # A revocation identifies the payload as ours just as a put does. + self._skills_in_payload += 1 # A tombstone carries identity and no content, so a listener that reads # content must check for ``content`` rather than assume it. self._changes.append( @@ -503,11 +568,23 @@ def _delete_object(self, data: Any) -> _TransferOutcome: def _payload_transferred(self, data: Any) -> _TransferOutcome: state = data.get("state") if isinstance(data, dict) else None version = data.get("version") if isinstance(data, dict) else None - if self._pending is not None: + payload_id = self._intent_payload_id or _payload_id_from_selector(state) + if self._pending is not None and self._is_foreign_payload(payload_id): + self._warn_foreign_payload(payload_id) + self.diagnostics.payloads_ignored += 1 + self._changes = [] + elif self._pending is not None: self._committed.replace_with(self._pending) _warn_if_nothing_can_verify(self._committed) + if self._skills_in_payload and payload_id is not None: + # Learnt, not configured: nothing below the interface is told + # which payload is which, so the payload that carried a skill + # put or revocation is the payload skills arrive on. + self._skill_payload_id = payload_id self._pending = None self._intent = None + self._intent_payload_id = None + self._skills_in_payload = 0 changes = self._changes self._changes = [] self.diagnostics.payloads_transferred += 1 @@ -526,6 +603,8 @@ def _abandon_in_flight(self) -> None: """Drops the in-flight payload and keeps what is committed.""" self._pending = None self._intent = None + self._intent_payload_id = None + self._skills_in_payload = 0 self._changes = [] def _error(self, data: Any) -> _TransferOutcome: @@ -546,8 +625,59 @@ def _goodbye(self, data: Any) -> _TransferOutcome: ) return _TransferOutcome(disconnect=f"server said goodbye: {reason}") + # -- payload identity ---------------------------------------------------- + + def _is_foreign_payload(self, payload_id: str | None) -> bool: + """ + Whether a transfer completes a payload other than the one skills arrive on. + + ``False`` unless both payloads are known, so one-payload delivery and the + first transfer of a connection behave exactly as they did before this + check existed. + """ + return ( + self._skill_payload_id is not None + and payload_id is not None + and payload_id != self._skill_payload_id + ) + # -- diagnostics --------------------------------------------------------- + def _warn_multiple_payloads(self, payloads: list[Any]) -> None: + """ + One WARNING per reader for an intent describing more than one payload. + + Not an error: reading only the first is what the protocol asks for. But it + means the first payload is no longer *guaranteed* to be the skill payload, + and an intent for another payload arriving before any skill has been seen + is the one case ``_is_foreign_payload`` cannot catch. + """ + if self._warned_multiple_payloads: + return + self._warned_multiple_payloads = True + logger.warning( + "An FDv2 server-intent described %d payloads (%s). Only the first is " + "read, as the protocol requires, and it is taken to be the payload " + "skills arrive on. If skills stop resolving from this point, that is " + "the assumption that broke; contact LaunchDarkly support.", + len(payloads), + ", ".join(str(_payload_id_of(p)) for p in payloads), + ) + + def _warn_foreign_payload(self, payload_id: str | None) -> None: + """One WARNING per reader for a transfer this layer declined to apply.""" + if self._warned_foreign_payload: + return + self._warned_foreign_payload = True + logger.warning( + "An FDv2 transfer of payload %s was not applied to the skills held, " + "which arrive on payload %s. Applying it would have replaced them " + "with whatever that payload carried — nothing, in the case of a flag " + "payload. The skills held are unchanged.", + payload_id, + self._skill_payload_id, + ) + def _warn_hashless(self, raw: dict[str, Any]) -> None: """ One ERROR per ``(key, version)`` whose envelope had no ``contentHash``. diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index ff5720bc..87596609 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -482,6 +482,232 @@ def test_an_object_arriving_with_no_intent_is_treated_as_a_delta(self) -> None: assert len(held) == 1 +# --------------------------------------------------------------------------- +# Which payload a transfer completed +# --------------------------------------------------------------------------- + + +def _payload_warnings(caplog: Any, fragment: str) -> list[Any]: + return [ + r + for r in caplog.records + if r.levelname == "WARNING" and fragment in r.getMessage() + ] + + +def skill_payload( + *object_events: tuple[str, Any], + payload_id: str = "agent-skill", + code: str = "xfer-full", + state: str = "basis-1", +) -> list[dict[str, Any]]: + """One payload's events, with the payload it belongs to named explicitly.""" + return events( + ("server-intent", server_intent(code, payload_id)), + *object_events, + ("payload-transferred", transferred(state)), + ) + + +class TestPayloadIdentity: + """ + Which payload a transfer completed, and why this layer tracks it at all. + + Delivery provides one payload per credential and the protocol requires a + client to read only the first payload intent, so today the payload read is + the payload skills arrive on. These assert the behaviour that survives if + the first of those stops holding: another payload's ``xfer-full`` must not + publish an empty skill set, because with pruning on that deletes a + customer's materialized files. + """ + + def test_only_the_first_payload_intent_is_read(self) -> None: + """Reading only the first is what the protocol asks for, however many + arrive — the point of the rest of this class is to make that safe.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + events( + ( + "server-intent", + { + "payloads": [ + { + "id": "agent-skill", + "target": 1, + "intentCode": "xfer-full", + }, + {"id": "env-flags", "target": 2, "intentCode": "none"}, + ] + }, + ), + ("put-object", put_skill()), + ("payload-transferred", transferred()), + ), + ) + assert len(held) == 1 + + def test_more_than_one_payload_intent_warns_once(self, caplog: Any) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + intent = { + "payloads": [ + {"id": "env-flags", "target": 1, "intentCode": "xfer-changes"}, + {"id": "agent-skill", "target": 2, "intentCode": "xfer-changes"}, + ] + } + with caplog.at_level("WARNING"): + reader.handle("server-intent", intent) + reader.handle("server-intent", intent) + assert len(_payload_warnings(caplog, "described 2 payloads")) == 1 + + def test_one_payload_intent_warns_about_nothing(self, caplog: Any) -> None: + with caplog.at_level("WARNING"): + drive( + _ProtocolReader(_SkillObjectSet()), + skill_payload(("put-object", put_skill())), + ) + assert _payload_warnings(caplog, "payload") == [] + + def test_another_payloads_full_transfer_does_not_empty_the_skills_held( + self, caplog: Any + ) -> None: + """ + The case this guard exists for. A flag payload's ``xfer-full`` starts an + empty pending set; applying it at ``payload-transferred`` would publish + every skill as revoked, which a reconcile with pruning on reads as + "delete these files". + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, skill_payload(("put-object", put_skill()))) + with caplog.at_level("WARNING"): + outcomes = drive( + reader, + skill_payload( + ("put-object", put_flag()), payload_id="env-flags", state="basis-2" + ), + ) + assert held.get("pdf-extraction", None) is not None + assert reader.diagnostics.payloads_ignored == 1 + assert len(_payload_warnings(caplog, "was not applied")) == 1 + # Nothing changed, so no listener is woken to reconcile against it. + assert outcomes[-1].changes == [] + + def test_a_declined_transfer_warns_once_however_often_it_repeats( + self, caplog: Any + ) -> None: + """A polling connection sees the other payload on every poll.""" + reader = _ProtocolReader(_SkillObjectSet()) + drive(reader, skill_payload(("put-object", put_skill()))) + foreign = skill_payload(("put-object", put_flag()), payload_id="env-flags") + with caplog.at_level("WARNING"): + drive(reader, foreign) + drive(reader, foreign) + assert len(_payload_warnings(caplog, "was not applied")) == 1 + assert reader.diagnostics.payloads_ignored == 2 + + def test_a_full_transfer_of_the_skill_payload_still_empties_it(self) -> None: + """Every skill deleted is a real state, and the guard must not mask it.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, skill_payload(("put-object", put_skill()))) + drive(reader, skill_payload(state="basis-2")) + assert len(held) == 0 + assert reader.diagnostics.payloads_ignored == 0 + + def test_a_revocation_identifies_the_payload_as_the_skill_payload(self) -> None: + """A payload that only revokes is still a payload skills arrive on.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + skill_payload(("delete-object", delete_skill()), code="xfer-changes"), + ) + drive( + reader, skill_payload(("put-object", put_skill()), payload_id="env-flags") + ) + assert reader.diagnostics.payloads_ignored == 1 + + def test_the_payload_is_identified_from_the_selector_when_no_id_is_named( + self, + ) -> None: + """``payload-transferred``'s selector is the only other place a completed + transfer names its payload.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + unnamed = {"payloads": [{"target": 1, "intentCode": "xfer-full"}]} + drive( + reader, + events( + ("server-intent", unnamed), + ("put-object", put_skill()), + ("payload-transferred", transferred("(p:agent-skill:53)")), + ), + ) + drive( + reader, + events( + ("server-intent", unnamed), + ("put-object", put_flag()), + ("payload-transferred", transferred("(p:env-flags:12)")), + ), + ) + assert held.get("pdf-extraction", None) is not None + assert reader.diagnostics.payloads_ignored == 1 + + def test_an_unidentifiable_payload_is_applied_rather_than_withheld(self) -> None: + """ + A transfer naming no payload at all is the store's own, since delivery + sends it one payload. Withholding it would break the common case to + defend against a hypothetical one. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, skill_payload(("put-object", put_skill()))) + drive( + reader, + events( + ("server-intent", {"payloads": [{"intentCode": "xfer-full"}]}), + ("put-object", put_skill(object_version=4)), + ("payload-transferred", {"version": 44}), + ), + ) + assert held.get("pdf-extraction", None)["version"] == 4 + assert reader.diagnostics.payloads_ignored == 0 + + def test_the_first_transfer_of_a_connection_is_the_residual( + self, caplog: Any + ) -> None: + """ + Before a skill has arrived there is nothing to compare a payload + against, so another payload's ``xfer-full`` arriving first cannot be + told apart. The multiple-payload WARNING is the only signal there is, + which is why it exists. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + with caplog.at_level("WARNING"): + drive( + reader, + events( + ( + "server-intent", + { + "payloads": [ + {"id": "env-flags", "intentCode": "xfer-full"}, + {"id": "agent-skill", "intentCode": "xfer-full"}, + ] + }, + ), + ("put-object", put_flag()), + ("payload-transferred", transferred()), + ), + ) + assert len(held) == 0 + assert len(_payload_warnings(caplog, "described 2 payloads")) == 1 + + # --------------------------------------------------------------------------- # Interface parity with InMemorySkillStore # --------------------------------------------------------------------------- From efc4ca77daf50a878b5988a591613cd96b778fcb Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 15:24:45 -0400 Subject: [PATCH 12/16] =?UTF-8?q?fix(client):=20Agent=20Skills=20=E2=80=94?= =?UTF-8?q?=20read=20the=20skill's=20version=20off=20the=20wire=20key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SDK-facing FDv2 channel now delivers skills the way streamer #4681 and gonfalon #70638 spell them: object kinds are open strings, the agent-skill payload is classified `generic`, and every generic object carries only `key`, `kind`, `version` and `object`, exactly like a flag. A skill arrives under kind `skill` with its own version folded into the key as `:`. There is no `category` field and no `objectVersion` field; both came from an earlier streamer draft that never shipped. Identification is now the kind alone. The wire key is split in one place, `_split_wire_key`, and both the put and the delete translation go through it. A key that will not split cleanly is held rather than dropped — version-less, or with the offending text as its version — so verification withholds it with `invalid_version` under a key the caller recognises; only a key with nothing before the delimiter is dropped, since there is no identity to hold it under. `SDK_DATA_MODEL_VERSION` goes with it: the connection's `mv` parameter only accepts flag model versions, and generic payloads ignore it. The transport stops sending it in the following change. Co-Authored-By: Claude Fable 5.1 --- packages/client/agents.md | 46 ++--- .../src/launchdarkly_ai_server/skills_core.py | 6 +- .../src/launchdarkly_ai_server/skills_fdv2.py | 159 ++++++++++++------ packages/client/tests/test_skills_fdv2.py | 130 ++++++++++---- 4 files changed, 233 insertions(+), 108 deletions(-) diff --git a/packages/client/agents.md b/packages/client/agents.md index e73c96d4..978dc8ad 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,7 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | -| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the `objectVersion`/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | +| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the wire-key/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | | `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | @@ -226,29 +226,37 @@ shape `skills_core.SkillStore` documents. It lives below the store interface; ** that interface knows it exists**. If a transport change ever seems to require editing an accessor, verification, or `write_skills`, the adapter boundary is wrong. -**`objectVersion` is the skill's version. `version` is the payload's.** On the wire a skill -`put-object` carries both, and they are not interchangeable: +**The skill's version is in the object's `key`. `version` is the payload's.** Each version +of a skill is its own object on the wire, identified as `:`: ```json -{"key":"pdf-extraction","kind":"inline-resource","category":"skill", - "objectVersion":3,"version":42, +{"key":"pdf-extraction:3","kind":"skill","version":42, "object":{"contentType":"text/markdown","content":"…","contentHash":"…","name":"…"}} ``` -`objectVersion` (3) is what a `{key, version}` reference pins and what becomes the stored -`version`. `version` (42) is the version of the *payload* the object arrived in — it moves -when anything in the environment moves, including a flag with nothing to do with skills. -Reading it as the skill's version fails **silently**: the object verifies, the hash matches, -and the caller gets content under a version number that means nothing. Flags and segments -carry only `version` and omit both `category` and `objectVersion`, which is exactly why the -two fields look interchangeable. `_store_object_from_put` is the only place the translation -happens, and `TestVersionTranslation` asserts it in both directions. - -**Skills are identified by `kind == "inline-resource" && category == "skill"`; everything else -is ignored, not rejected.** An environment's payload assignment carries its flag payload -alongside its agent-skill payload, so flag and segment objects arrive as a matter of course. -Erroring on an unrecognised kind would turn a normal payload into a permanent reconnect -loop — a flag-delivery outage caused by a skills rollout. +The `3` after the delimiter is what a `{key, version}` reference pins and what becomes the +stored `version`, under the stored key `pdf-extraction`. `version` (42) is the version of the +*payload* the object arrived in — it moves when anything in the environment moves, +including a flag with nothing to do with skills. Reading it as the skill's version fails +**silently**: the object verifies, the hash matches, and the caller gets content under a +version number that means nothing. There is no separate field for the skill's version: the +agent-skill payload is a *generic* payload, and generic objects carry only `key`, `kind`, +`version` and `object`, exactly like a flag. `_split_wire_key` is the only place the wire key +is read, `_store_object_from_put` and `_tombstone_from_delete` both go through it, and +`TestVersionTranslation` asserts the translation in both directions. A wire key that will +not split cleanly is *held*, not dropped — version-less, or with the offending text as its +version — so verification withholds it with `invalid_version` under a key the caller +recognises; only a key with nothing before the delimiter is dropped, since there is no +identity to hold it under. + +**Skills are identified by `kind == "skill"`; everything else is ignored, not rejected.** +Object kinds on the SDK-facing channel are open strings, and the agent-skill payload is +classified `generic`, so a skill arrives under the kind its producer registered — the bare +category name — not under a broader wrapper kind with a narrowing field. An environment's +payload assignment carries its flag payload alongside its agent-skill payload, so flag and +segment objects arrive as a matter of course. Erroring on an unrecognised kind would turn a +normal payload into a permanent reconnect loop — a flag-delivery outage caused by a skills +rollout. **Changes commit at `payload-transferred`, not as objects arrive.** A payload version is the unit of consistency: a half-applied full transfer would publish a state the server never diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index 22166799..d2d870c4 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -56,9 +56,9 @@ An **internal seam value**, deliberately not exported from the package root. It is the string ``skills.py`` and ``skills_fs.py`` pass to ``SkillStore.get_object`` and ``SkillStore.all_objects``, and a store adapter is free to map it onto -whatever the transport underneath actually uses — a delivery payload may well -carry skills under a broader kind with a narrower category, in which case -translating that pair to this one value is the adapter's job. +whatever the transport underneath actually uses — the value happens to match +the kind LaunchDarkly's delivery channel uses today, but a transport that spelt +it differently would translate, and that translation is the adapter's job. Exporting it would publish an SDK-side seam string as though it were the wire contract, which is a claim this side cannot make and would be hard to walk back diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index f644a349..d20a0e2f 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -3,7 +3,7 @@ The half of the delivery transport that has no I/O: identifying skill objects on the wire, translating them into the raw object shape the ``SkillStore`` -interface defines, holding them by ``(key, objectVersion)``, and applying a +interface defines, holding them by ``(key, version)``, and applying a payload's events as one consistent commit. ``FDv2SkillStore``, the store that puts a network connection underneath this, follows in a separate change. @@ -42,9 +42,9 @@ the same payload. ``_ProtocolReader`` keeps the pair apart anyway, because the cost of conflating them is an emptied skill set. -The design rationale — why ``objectVersion`` is not ``version``, why changes -commit at ``payload-transferred`` — is in ``agents.md`` under *The delivery -transport*. +The design rationale — why the skill's version is read from the object's +``key`` and never from ``version``, why changes commit at +``payload-transferred`` — is in ``agents.md`` under *The delivery transport*. """ from __future__ import annotations @@ -63,21 +63,27 @@ # The wire contract # --------------------------------------------------------------------------- -FDV2_OBJECT_KIND = "inline-resource" +FDV2_OBJECT_KIND = "skill" """ -The FDv2 ``kind`` skills are delivered under. Together with -``FDV2_OBJECT_CATEGORY`` it maps onto the single interface value -``skills_core.SKILL_OBJECT_KIND``; that translation is this adapter's job. +The FDv2 ``kind`` skills are delivered under. + +Object kinds on the SDK-facing channel are open strings: the agent-skill payload +is classified ``generic`` and every object in it carries the kind its producer +registered, which for skills is the bare category name. Delivery lower-cases the +kind, so an exact comparison is the whole test. The kind happens to equal +``skills_core.SKILL_OBJECT_KIND`` today; they are still separate constants, +because one is a wire value LaunchDarkly owns and the other is an SDK seam. """ -FDV2_OBJECT_CATEGORY = "skill" -"""The ``category`` that narrows ``inline-resource`` to an agent skill.""" - -SDK_DATA_MODEL_VERSION = 1 +FDV2_KEY_DELIMITER = ":" """ -The ``mv`` request parameter. The one request parameter whose value could not be -confirmed against a live server, so treat the default as provisional and -override it through the store's ``data_model_version`` if needed. +What separates a skill's key from its version inside the object's wire ``key``. + +A generic object is identified on the wire as ``:`` — the skill's +own key, one delimiter, the skill's own version — because each version of a +skill is a distinct object in the payload. Delivery forbids the delimiter inside +a registered category and skill keys cannot contain it, so a well-formed wire key +has exactly one. """ _EVENT_SERVER_INTENT = "server-intent" @@ -204,7 +210,7 @@ class StoreDiagnostics: # --------------------------------------------------------------------------- -# Deserialisation — where objectVersion is not version +# Deserialisation — where the skill's version lives in the key, not in version # --------------------------------------------------------------------------- @@ -220,17 +226,58 @@ def _is_skill_event(data: Any) -> bool: """ Whether one ``put-object`` / ``delete-object`` payload is a skill. - Both halves are required: ``inline-resource`` may carry other categories, - and flags and segments omit ``category`` entirely. Every other kind is - ignored, not rejected, because flag and segment objects share the connection - and erroring on them would turn a normal payload into a reconnect loop. + The kind alone decides it. Every other kind is ignored, not rejected, + because flag and segment objects share the connection and erroring on them + would turn a normal payload into a reconnect loop. """ if not isinstance(data, dict): return False - return ( - data.get("kind") == FDV2_OBJECT_KIND - and data.get("category") == FDV2_OBJECT_CATEGORY - ) + return data.get("kind") == FDV2_OBJECT_KIND + + +@dataclass(frozen=True) +class _WireIdentity: + """A skill object's wire ``key``, split into the skill's key and version.""" + + key: str + version: Any + """``int`` when the wire carried one; the offending text when it did not; + absent (``_NO_VERSION``) when the wire key had no delimiter at all.""" + + +_NO_VERSION = object() + + +def _split_wire_key(wire_key: Any) -> _WireIdentity | None: + """ + Reads ``:`` off one object's wire ``key``. + + Lenient where leniency keeps the object diagnosable and strict only where + there is nothing to diagnose: + + - No delimiter: the whole wire key is the skill key and there is no version, + so the object is held version-less and verification reports + ``invalid_version`` under a key the caller can recognise. + - A version that is not a run of digits (``"pdf:latest"``, ``"pdf:"``, + ``"a:1:2"``): the text is carried through *as the version*, for the same + reason — the caller learns that ``pdf`` arrived broken, not that it is + absent. + - An empty key before the delimiter (``":3"``): there is no identity to hold + it under, so ``None``, and the caller drops it. + + Leading zeros are accepted (``"pdf:03"`` is version 3) since ``int`` is the + identity a reference pins, not the spelling. + """ + if not isinstance(wire_key, str) or not wire_key: + return None + key, delimiter, version_text = wire_key.partition(FDV2_KEY_DELIMITER) + if not key: + return None + if not delimiter: + return _WireIdentity(key=key, version=_NO_VERSION) + if version_text.isascii() and version_text.isdigit(): + return _WireIdentity(key=key, version=int(version_text)) + return _WireIdentity(key=key, version=version_text) def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: @@ -240,33 +287,36 @@ def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: **The one translation this adapter must get right:** - wire ``objectVersion`` → stored ``version`` (the skill's own version) - wire ``version`` → dropped (the *payload* version) + wire ``key`` → stored ``key`` and ``version`` (split on ``:``) + wire ``version`` → dropped (the *payload* version) - ``objectVersion`` is what a ``{key, version}`` reference pins; ``version`` - moves whenever anything in the environment moves. Confusing them fails - silently: the object verifies and the caller gets content under a version - number that means nothing. + Each version of a skill is its own object on the wire, identified as + ``:``; that version is what a ``{key, version}`` reference + pins. The event's ``version`` field is the version of the payload the object + arrived in and moves whenever anything in the environment moves. Confusing + them fails silently: the object verifies and the caller gets content under a + version number that means nothing. - Returns ``None`` only when ``key`` is not a string, since a keyless object - has no identity to store it under. Every other defect is carried through - verbatim so that verification withholds it with a reason code rather than - the transport dropping it into indistinguishable absence. + Returns ``None`` only when the wire ``key`` carries no skill key at all, + since such an object has no identity to store it under. Every other defect + is carried through so that verification withholds it with a reason code + rather than the transport dropping it into indistinguishable absence. """ - key = data.get("key") - if not isinstance(key, str) or not key: + identity = _split_wire_key(data.get("key")) + if identity is None: logger.warning( - "An FDv2 skill put-object carried no string 'key' and could not be " - "stored under any identity; it was dropped." + "An FDv2 skill put-object carried no usable 'key' (%r) and could not " + "be stored under any identity; it was dropped.", + data.get("key"), ) return None - raw: dict[str, Any] = {"key": key} + raw: dict[str, Any] = {"key": identity.key} - # A membership test rather than a `.get` default, so an explicitly-null - # objectVersion stays null and reaches verification as `invalid_version`. - if "objectVersion" in data: - raw["version"] = data["objectVersion"] + # Absent stays absent and malformed stays malformed, so verification sees + # what arrived (as `invalid_version`) rather than something invented here. + if identity.version is not _NO_VERSION: + raw["version"] = identity.version envelope = data.get("object") if isinstance(envelope, dict): @@ -294,25 +344,26 @@ def _payload_id_from_selector(state: Any) -> str | None: def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: """ - Narrows one FDv2 skill ``delete-object`` to the identity it revokes, with - the same ``objectVersion`` translation as a put. + Narrows one FDv2 skill ``delete-object`` to the identity it revokes, reading + the wire ``key`` the same way a put does. An ``object_version`` of ``None`` means the delete named no usable version and is read as "revoke every version of this key". That is the safe direction: the alternative is continuing to serve content LaunchDarkly has - withdrawn. + withdrawn. It also removes whatever a malformed put of the same wire key + left held, since that was stored version-less under the same skill key. """ - key = data.get("key") - if not isinstance(key, str) or not key: + identity = _split_wire_key(data.get("key")) + if identity is None: logger.warning( - "An FDv2 skill delete-object carried no string 'key'; it was ignored." + "An FDv2 skill delete-object carried no usable 'key' (%r); it was ignored.", + data.get("key"), ) return None - object_version = data.get("objectVersion") return _Tombstone( - key=key, - object_version=object_version - if is_valid_skill_version(object_version) + key=identity.key, + object_version=identity.version + if is_valid_skill_version(identity.version) else None, ) @@ -324,7 +375,7 @@ def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: class _SkillObjectSet: """ - Raw skill objects held in memory, keyed by ``(key, objectVersion)``. + Raw skill objects held in memory, keyed by ``(key, version)``. Lookup semantics are identical to ``InMemorySkillStore``'s, down to the fall-through to a version-less entry, so that the store a caller configures diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index 87596609..b88d14d9 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -1,8 +1,8 @@ """ Tests for the FDv2 skill delivery protocol. -Wire semantics — which objects are skills, ``objectVersion`` versus ``version``, -revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted +Wire semantics — which objects are skills, the skill's version in the wire ``key`` +versus the payload's in ``version``, revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted against ``_ProtocolReader``, which has no I/O, so each case reads as the contract it is rather than as a server script. """ @@ -17,7 +17,7 @@ from launchdarkly_ai_server import InMemorySkillStore from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND from launchdarkly_ai_server.skills_fdv2 import ( - FDV2_OBJECT_CATEGORY, + FDV2_KEY_DELIMITER, FDV2_OBJECT_KIND, _is_skill_event, _ProtocolReader, @@ -40,6 +40,18 @@ def _hash(content: str) -> str: # --------------------------------------------------------------------------- +def wire_key(key: str, object_version: Any) -> str: + """ + The wire ``key`` of one skill object: ``:``. + + ``None`` builds a key with no version at all, which is how the tests spell a + malformed object; anything else is spelled after the delimiter verbatim. + """ + if object_version is None: + return key + return f"{key}{FDV2_KEY_DELIMITER}{object_version}" + + def put_skill( key: str = "pdf-extraction", *, @@ -62,10 +74,8 @@ def put_skill( content_hash if content_hash is not None else _hash(content) ) return { - "key": key, + "key": wire_key(key, object_version), "kind": FDV2_OBJECT_KIND, - "category": FDV2_OBJECT_CATEGORY, - "objectVersion": object_version, "version": payload_version, "object": envelope, } @@ -75,16 +85,14 @@ def delete_skill( key: str = "pdf-extraction", *, object_version: Any = 3, payload_version: int = 43 ) -> dict[str, Any]: return { - "key": key, + "key": wire_key(key, object_version), "kind": FDV2_OBJECT_KIND, - "category": FDV2_OBJECT_CATEGORY, - "objectVersion": object_version, "version": payload_version, } def put_flag(key: str = "my-flag", version: int = 17) -> dict[str, Any]: - """A flag ``put-object``: no ``category``, no ``objectVersion``.""" + """A flag ``put-object``: the same envelope fields, a different ``kind``.""" return { "key": key, "kind": "flag", @@ -141,30 +149,37 @@ def full_payload( class TestObjectIdentification: - def test_kind_and_category_together_identify_a_skill(self) -> None: + def test_the_kind_alone_identifies_a_skill(self) -> None: assert _is_skill_event(put_skill()) is True + def test_the_kind_is_the_bare_category_name(self) -> None: + """ + Object kinds on the channel are open strings and the agent-skill payload + is ``generic``, so a skill arrives under the kind its producer + registered — ``skill`` — not under a broader wrapper kind. + """ + assert FDV2_OBJECT_KIND == "skill" + def test_a_flag_is_not_a_skill(self) -> None: assert _is_skill_event(put_flag()) is False def test_a_segment_is_not_a_skill(self) -> None: assert _is_skill_event(put_segment()) is False - def test_inline_resource_of_another_category_is_not_a_skill(self) -> None: - """``inline-resource`` is a broad kind, so the category is required too.""" + def test_another_generic_kind_is_not_a_skill(self) -> None: + """A generic payload may carry other registered kinds one day.""" other = put_skill() - other["category"] = "prompt-template" + other["kind"] = "prompt-template" assert _is_skill_event(other) is False - def test_skill_category_under_another_kind_is_not_a_skill(self) -> None: + def test_a_skill_shaped_envelope_under_another_kind_is_not_a_skill(self) -> None: other = put_skill() other["kind"] = "some-future-kind" assert _is_skill_event(other) is False - def test_a_flag_shaped_object_with_no_category_is_not_a_skill(self) -> None: - """Flags and segments omit ``category`` entirely — the documented shape.""" - assert "category" not in put_flag() - assert "objectVersion" not in put_flag() + def test_nothing_but_the_kind_is_consulted(self) -> None: + """No secondary field narrows the kind, and none may be required.""" + assert set(put_skill()) == {"key", "kind", "version", "object"} @pytest.mark.parametrize("value", [None, "skill", 3, [], ()]) def test_non_dict_events_are_not_skills(self, value: Any) -> None: @@ -172,15 +187,27 @@ def test_non_dict_events_are_not_skills(self, value: Any) -> None: # --------------------------------------------------------------------------- -# objectVersion is not version +# The skill's version is in the wire key; `version` is the payload's # --------------------------------------------------------------------------- class TestVersionTranslation: - def test_object_version_becomes_the_seam_version(self) -> None: + def test_the_wire_key_is_key_colon_version(self) -> None: + assert ( + put_skill("pdf-extraction", object_version=3)["key"] == "pdf-extraction:3" + ) + + def test_the_version_after_the_delimiter_becomes_the_seam_version(self) -> None: raw = _store_object_from_put(put_skill(object_version=3, payload_version=42)) assert raw is not None assert raw["version"] == 3 + assert isinstance(raw["version"], int) + + def test_the_key_before_the_delimiter_becomes_the_seam_key(self) -> None: + """A caller asks for ``pdf-extraction``, never for ``pdf-extraction:3``.""" + raw = _store_object_from_put(put_skill("pdf-extraction", object_version=3)) + assert raw is not None + assert raw["key"] == "pdf-extraction" def test_the_payload_version_never_reaches_the_seam(self) -> None: """ @@ -200,36 +227,75 @@ def test_the_two_are_distinguished_even_when_the_payload_version_is_lower( assert raw is not None assert raw["version"] == 99 - def test_a_missing_object_version_is_not_defaulted_from_the_payload(self) -> None: - wire = put_skill() - del wire["objectVersion"] - raw = _store_object_from_put(wire) + def test_a_key_with_no_delimiter_is_held_version_less(self) -> None: + """Not defaulted from the payload version, and not dropped: verification + reports ``invalid_version`` under a key the caller recognises.""" + raw = _store_object_from_put(put_skill(object_version=None)) assert raw is not None + assert raw["key"] == "pdf-extraction" assert "version" not in raw - def test_an_explicitly_null_object_version_is_carried_through_as_null(self) -> None: - """Carried, not invented: verification reports ``invalid_version``.""" - raw = _store_object_from_put(put_skill(object_version=None)) + @pytest.mark.parametrize("spelling", ["latest", "", "3.0", "-1", "1:2", "３"]) + def test_a_version_that_is_not_digits_is_carried_through_as_invalid( + self, spelling: str + ) -> None: + """Carried, not invented: verification reports ``invalid_version`` for + the object rather than the transport reporting it absent.""" + raw = _store_object_from_put(put_skill(object_version=spelling)) + assert raw is not None + assert raw["key"] == "pdf-extraction" + assert raw["version"] == spelling + + def test_leading_zeros_spell_the_same_version(self) -> None: + raw = _store_object_from_put(put_skill(object_version="03")) assert raw is not None - assert raw["version"] is None + assert raw["version"] == 3 - def test_a_delete_translates_object_version_too(self) -> None: + def test_a_delete_reads_the_wire_key_the_same_way(self) -> None: tombstone = _tombstone_from_delete( delete_skill(object_version=3, payload_version=43) ) assert tombstone is not None + assert tombstone.key == "pdf-extraction" assert tombstone.object_version == 3 - def test_a_delete_with_no_usable_object_version_revokes_every_version(self) -> None: - tombstone = _tombstone_from_delete(delete_skill(object_version=None)) + @pytest.mark.parametrize("spelling", [None, "latest", "0"]) + def test_a_delete_with_no_usable_version_revokes_every_version( + self, spelling: Any + ) -> None: + tombstone = _tombstone_from_delete(delete_skill(object_version=spelling)) assert tombstone is not None + assert tombstone.key == "pdf-extraction" assert tombstone.object_version is None + @pytest.mark.parametrize("bad_key", [":3", "", None, 3]) + def test_a_put_with_no_skill_key_is_dropped_because_it_has_no_identity( + self, bad_key: Any + ) -> None: + wire = put_skill() + wire["key"] = bad_key + assert _store_object_from_put(wire) is None + def test_a_keyless_put_is_dropped_because_it_has_no_identity(self) -> None: wire = put_skill() del wire["key"] assert _store_object_from_put(wire) is None + def test_a_delete_with_no_skill_key_is_ignored(self) -> None: + wire = delete_skill() + wire["key"] = ":3" + assert _tombstone_from_delete(wire) is None + + def test_the_stored_identity_round_trips_to_the_wire_key(self) -> None: + """``_SkillObjectSet.snapshot`` spells its opaque keys the way the wire + does, so a held object can be matched back to the event that carried it.""" + held = _SkillObjectSet() + wire = put_skill("pdf-extraction", object_version=3) + raw = _store_object_from_put(wire) + assert raw is not None + held.put(raw) + assert set(held.snapshot()) == {wire["key"]} + def test_the_envelope_is_copied_verbatim(self) -> None: raw = _store_object_from_put(put_skill()) assert raw is not None From b6a25f9f455258dabb8d13b13978123239327376 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Thu, 10 Sep 2026 16:05:04 -0400 Subject: [PATCH 13/16] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20the=20FDv2=20delivery=20transport?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FDv2SkillStore puts LaunchDarkly's SDK-facing FDv2 channel underneath the protocol layer: GET /sdk/poll and GET /sdk/stream, authenticated with the environment's server-side SDK key, streaming by default. It carries basis across requests, sends If-None-Match and treats 304 as a current answer, retries with capped jittered backoff, honours Retry-After only up to max_backoff, gives up after a bounded run of consecutive failures where a committed payload resets the count, and keeps serving last known good through every failure. A mobile key or client-side environment ID is refused in the constructor. Standard library only. close interrupts the socket rather than only setting a flag, because the delivery thread lives in a read no flag can reach; without that every shutdown of a healthy stream waited out the full join timeout. The no-store message now names FDv2SkillStore first, and watch_skills points at it as the store with a delivery transport. Co-Authored-By: Claude Fable 5.1 --- packages/client/README.md | 62 + packages/client/agents.md | 25 +- .../src/launchdarkly_ai_server/__init__.py | 5 +- .../src/launchdarkly_ai_server/skills_core.py | 13 +- .../src/launchdarkly_ai_server/skills_fdv2.py | 748 ++++++++- .../launchdarkly_ai_server/skills_watch.py | 2 +- packages/client/tests/test_skills.py | 12 + packages/client/tests/test_skills_fdv2.py | 1332 ++++++++++++++++- packages/client/tests/test_skills_watch.py | 5 +- 9 files changed, 2162 insertions(+), 42 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 41d5c6b3..42b1f40d 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -445,6 +445,66 @@ which OS ran the write. The keys stay valid everywhere else: an AI Config refere named `aux` parses, and its other fields are unaffected. If you have a skill named for a device, rename it. +#### Receiving skills from LaunchDarkly + +`InMemorySkillStore` is for tests and bring-your-own-content. In production, skill content +arrives through `FDv2SkillStore`, which speaks LaunchDarkly's SDK-facing FDv2 delivery +channel — the same `GET /sdk/poll` and `GET /sdk/stream` endpoints the base SDK's FDv2 data +source uses, authenticated with the environment's server-side SDK key. + +```python +import os + +from launchdarkly_ai_server import FDv2SkillStore, init_client, watch_skills + +store = FDv2SkillStore(os.environ["LD_SDK_KEY"]).start() +store.wait_for_skills(timeout=10) +await init_client(options={"skillStore": store}) + +# Materialize now, and re-materialize whenever delivery changes. +report, watcher = await watch_skills("*", ".claude/skills") +try: + ... +finally: + watcher.close() + store.close() +``` + +**Nothing above the store changes.** The accessors, verification, and `write_skills` see raw +objects through the `SkillStore` interface and cannot tell which store produced them. + +**Server-side only.** Skills are for server-side agent runtimes and skill content is +customer-confidential. A mobile key (`mob-…`) or a client-side environment ID raises from the +constructor. + +**Streaming is the default, and it is what makes revocation fast.** A `delete-object` reaches +a live stream in seconds; with `mode="poll"` it arrives within one `poll_interval`. Paired +with `watch_skills`, a revoked skill's `SKILL.md` leaves the disk without a restart. During an +outage the store keeps serving the last content it received and `write_skills`' default +`on_unavailable="keep"` leaves managed files alone — an outage must not read as "everything +was revoked". + +**One network timeout, and its default depends on the mode.** `read_timeout` bounds every +socket operation of a request, connecting included. In `mode="poll"` it bounds the whole +request and defaults to 10 seconds; in `mode="stream"` it bounds each wait for the next bytes +and defaults to 300 seconds, well beyond LaunchDarkly's heartbeat interval. + +**The connection also carries your flags.** A client cannot request only the skill payload, +so a skills-enabled environment delivers flag and segment objects on the same connection. +They are skipped, not evaluated — this store does no evaluation of any kind — and +`diagnostics.objects_ignored` counts them. + +> **Beta caveats, worth knowing before you deploy.** Payload signing does not exist on this +> channel yet, so delivery is TLS-only and the content hash establishes self-consistency, not +> origin authenticity. The FDv2 protocol is opt-in per account: without it the endpoints +> return HTTP 403, which the store reports as a fatal error explaining what to do. `ld-relay` +> does not speak the FDv2 endpoints, so relay-only deployments cannot receive skills. + +**If every skill comes back empty, check `diagnostics.hashless_objects`.** Verification +withholds any delivered object without a `contentHash`, so a nonzero count means skills are +being withheld rather than that the environment has none. The store also logs an error per +hashless object naming the reason. There is deliberately no fallback that skips verification. + **Total path length is yours to bound, not the SDK's.** The 255-byte bound above is per *component*; the root is your path, so `` + `` + `/SKILL.md` can still exceed Windows' 260-character `MAX_PATH` with a perfectly legal key. Choose a short managed root on @@ -460,7 +520,9 @@ Windows. | `write_skills(skills, root, *, prune=True, timeout=10.0, on_unavailable="keep")` | Materialize skills under `root`, returning a `ReconcileReport`. `prune` removes formerly-managed skills no longer requested. `on_unavailable="raise"` raises instead of reporting when content cannot be retrieved. Raises `ValueError` for an unusable root, a negative `timeout`, or an unrecognised `on_unavailable`. **Performs synchronous filesystem I/O — see the note below.** | | `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)` / `remove_listener(kind, fn)`. | | `InMemorySkillStore(objects=None)` | A dict-backed store with `put(raw)`, for local development and testing. Holds several versions of a key. | +| `FDv2SkillStore(sdk_key, *, base_uri=…, mode="stream", …)` | The delivery transport: a store fed by LaunchDarkly over the SDK-facing FDv2 channel. `start()`, `wait_for_skills(timeout)`, `close()`, `diagnostics`, `failed`; also a context manager. **Server-side only** — a mobile key or client-side environment ID raises. See *Receiving skills from LaunchDarkly* above. | | `watch_skills(skills, root, …)` | `write_skills` plus a re-reconcile on every delivery change. Returns `(initial report, SkillWatcher)`; close the watcher when done. Revocation then takes effect within `debounce` of arriving rather than at the next restart. | +| `StoreDiagnostics` | What the transport has seen: `payloads_transferred`, `skill_objects_received`, `objects_ignored`, `objects_revoked`, `hashless_objects`, `connection_failures`, `last_error`. | Configure the store with `init_client(options={"skillStore": store})`. With none configured, the accessors raise `RuntimeError` explaining what to do and `write_skills` reports the diff --git a/packages/client/agents.md b/packages/client/agents.md index 978dc8ad..2a19c273 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,7 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | -| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the wire-key/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | +| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery transport — the FDv2 protocol, the wire-key/`version` translation, the held object set, and `FDv2SkillStore`. Sits **below** the store interface; imports `skills_core` only, and nothing imports it | | `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | @@ -221,10 +221,12 @@ of skills, and the `"*"` reconcile, since `//SKILL.md` is a single pa ### The delivery transport, and the one field that will bite you -`skills_fdv2.py` translates LaunchDarkly's FDv2 delivery protocol into raw objects in the -shape `skills_core.SkillStore` documents. It lives below the store interface; **nothing above -that interface knows it exists**. If a transport change ever seems to require editing an -accessor, verification, or `write_skills`, the adapter boundary is wrong. +`FDv2SkillStore` speaks LaunchDarkly's SDK-facing FDv2 channel (`GET /sdk/poll`, +`GET /sdk/stream`, server-side SDK key in `Authorization`, `basis` + `mv` params, +`If-None-Match`/304). It lives below the store interface and produces raw objects in the +shape `skills_core.SkillStore` documents; **nothing above that interface knows it exists**. If a transport +change ever seems to require editing an accessor, verification, or `write_skills`, the adapter +boundary is wrong. **The skill's version is in the object's `key`. `version` is the payload's.** Each version of a skill is its own object on the wire, identified as `:`: @@ -285,6 +287,19 @@ such skill" — and would let a prune delete the last known-good copy on disk. N a hash from the delivered content: that certifies the content against itself and verifies nothing. +**There is one network timeout, not two.** `urllib`'s `timeout` is the socket timeout for the +whole operation, so connect, headers and each read share it, and the module cannot bound the +connect separately without a custom connection class it should not carry. `read_timeout` is +therefore the only knob, and its default is per mode (`DEFAULT_POLL_TIMEOUT` for a whole poll +request, `DEFAULT_STREAM_READ_TIMEOUT` for the gap between reads on a stream). Do not add a +parameter that the standard library cannot honour; `TestTimeouts` measures the bound against a +socket that accepts and never answers. + +**`close` interrupts the socket, it does not just set a flag.** The delivery thread spends its +life blocked in a read that no flag can reach, and closing a response from another thread does +not unblock CPython's buffered reader. `_interrupt_read` shuts the socket down underneath it. +Without that, every shutdown of a *healthy* stream blocks for the full join timeout. + ### The reported outcome vocabulary, and the `Resolution` mapping `get_skill` returns `Skill | None`; `get_skill_result` returns a frozen `SkillOutcome` diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index f863d847..0f987117 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -46,6 +46,7 @@ skill_refs, ) from .skills_core import SkillStore +from .skills_fdv2 import FDv2SkillStore, StoreDiagnostics from .skills_fs import ( MANIFEST_FILENAME, MANIFEST_VERSION, @@ -238,7 +239,9 @@ "write_skills", "SkillStore", "InMemorySkillStore", - # skills — the eager re-reconcile + # skills — the FDv2 delivery transport, and the eager re-reconcile it enables + "FDv2SkillStore", + "StoreDiagnostics", "watch_skills", "SkillWatcher", # skills — the three closed-set unions a typed consumer needs to name diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index d2d870c4..b8043012 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -134,9 +134,18 @@ NO_STORE_MESSAGE = ( "No skill store is configured, so skill content cannot be retrieved. Configure " - 'one with init_client(options={"skillStore": store}) — InMemorySkillStore is ' - "available for local development and testing." + 'one with init_client(options={"skillStore": store}) — FDv2SkillStore receives ' + "content from LaunchDarkly, and InMemorySkillStore is available for local " + "development and testing." ) +""" +The first thing a user sees when no store is configured, so it names both stores. + +``FDv2SkillStore`` comes first because it is the answer in production, and a +message that offered only ``InMemorySkillStore`` would point a deployment at the +development store. Callers match on "skill store"; keep that phrase if the +wording changes. +""" # --------------------------------------------------------------------------- diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index d20a0e2f..da84a3b4 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -1,15 +1,10 @@ """ -Agent Skills — the FDv2 delivery protocol. +Agent Skills — the FDv2 delivery transport. -The half of the delivery transport that has no I/O: identifying skill objects -on the wire, translating them into the raw object shape the ``SkillStore`` -interface defines, holding them by ``(key, version)``, and applying a -payload's events as one consistent commit. ``FDv2SkillStore``, the store that -puts a network connection underneath this, follows in a separate change. - -It sits *below* the ``SkillStore`` interface, and everything above — the -accessors, integrity verification, the ``Skill`` dataclass, materialization — -is unaware of it. +The store implementation that talks to LaunchDarkly. It sits *below* the +``SkillStore`` interface: it produces raw wire objects in the shape +``skills_core`` documents, and everything above — the accessors, integrity +verification, the ``Skill`` dataclass, materialization — is unaware of it. Layering:: @@ -20,12 +15,12 @@ GET /sdk/poll, GET /sdk/stream, authenticated with the environment's server-side SDK key -Dependencies run one way: this module imports nothing from the feature beyond -the version validator in ``types_validation``, and nothing in the feature -imports it. It uses only the standard library, so it adds no dependency +Dependencies run one way: this module imports ``skills_core`` for the +interface's kind constant and nothing else from the feature, and nothing in the +feature imports it. It uses only the standard library, so it adds no dependency to a package whose sole runtime dependency is ``opentelemetry-api``. -Three things this layer does *not* do, on purpose: +Three things this module does *not* do, on purpose: - **It does not verify content.** Verification lives at the accessor boundary in ``skills_core`` so that it applies to every store equally, including a @@ -36,24 +31,29 @@ - **It does not evaluate anything.** Flag and segment objects that share the connection are skipped and counted, nothing more. -One assumption it *does* make, and states: **the payload intent it reads is the -payload skills arrive on.** Delivery sends one payload per credential and the -protocol tells a client to read only the first payload intent, so today those are -the same payload. ``_ProtocolReader`` keeps the pair apart anyway, because the -cost of conflating them is an emptied skill set. - The design rationale — why the skill's version is read from the object's ``key`` and never from ``version``, why changes commit at -``payload-transferred`` — is in ``agents.md`` under *The delivery transport*. +``payload-transferred``, why there is one network timeout — is in +``agents.md`` under *The delivery transport*. """ from __future__ import annotations +import json import logging +import math +import random import re +import socket +import threading +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable from dataclasses import dataclass, field -from typing import Any +from typing import Any, Literal +from .skills_core import SKILL_OBJECT_KIND from .types_validation import is_valid_skill_version logger = logging.getLogger(__name__) @@ -86,6 +86,20 @@ has exactly one. """ +DEFAULT_BASE_URI = "https://sdk.launchdarkly.com" +"""Where the SDK-facing FDv2 endpoints live. Overridable for Federal and private +instances.""" + +POLL_PATH = "/sdk/poll" +STREAM_PATH = "/sdk/stream" + +DEFAULT_POLL_TIMEOUT = 10.0 +"""Default ``read_timeout`` in ``"poll"`` mode: the bound on one whole request.""" + +DEFAULT_STREAM_READ_TIMEOUT = 300.0 +"""Default ``read_timeout`` in ``"stream"`` mode: the longest gap tolerated +between two reads. LaunchDarkly's heartbeats arrive well inside this.""" + _EVENT_SERVER_INTENT = "server-intent" _EVENT_PUT_OBJECT = "put-object" _EVENT_DELETE_OBJECT = "delete-object" @@ -115,6 +129,8 @@ no ``id``. """ +Mode = Literal["stream", "poll"] + _MOBILE_KEY_PREFIX = "mob-" _SERVER_KEY_PREFIX = "sdk-" _CLIENT_SIDE_ID = re.compile(r"\A[0-9a-f]{20,}\Z") @@ -504,7 +520,6 @@ def __init__(self, committed: _SkillObjectSet) -> None: self.diagnostics = StoreDiagnostics() # Identities already reported by ``_warn_hashless``. Per reader, so a # recreated store reports again and two stores never quieten each other. - # No lock: ``handle`` runs only on its owner's single delivery thread. self._warned_hashless: set[tuple[str, Any]] = set() # The payload the current intent describes, and the payload skills have # actually arrived on. One payload per connection makes these the same @@ -609,8 +624,8 @@ def _delete_object(self, data: Any) -> _TransferOutcome: self.diagnostics.objects_revoked += 1 # A revocation identifies the payload as ours just as a put does. self._skills_in_payload += 1 - # A tombstone carries identity and no content, so a listener that reads - # content must check for ``content`` rather than assume it. + # A tombstone carries identity and no content; see + # ``FDv2SkillStore.add_listener`` for what listeners should expect. self._changes.append( {"key": tombstone.key, "version": tombstone.object_version} ) @@ -780,3 +795,686 @@ def _warn_if_nothing_can_verify(committed: _SkillObjectSet) -> None: len(held), _HASHLESS_ADVICE, ) + + +# --------------------------------------------------------------------------- +# HTTP +# --------------------------------------------------------------------------- + + +class _FatalTransportError(Exception): + """A failure retrying cannot fix: bad credential, forbidden, wrong URI.""" + + +class _RecoverableTransportError(Exception): + """A failure worth retrying. Carries a server-requested delay when given one.""" + + def __init__(self, message: str, retry_after: float | None = None) -> None: + super().__init__(message) + self.retry_after = retry_after + + +_FORBIDDEN_ADVICE = ( + "The FDv2 protocol is opt-in per LaunchDarkly account and is served as HTTP " + "403 while it is off. Skill delivery needs it enabled; contact LaunchDarkly " + "support to enable it for your account." +) + + +def _retry_after_seconds(headers: Any) -> float | None: + """ + ``Retry-After`` in seconds, when the server sent a usable one. + + The HTTP-date form, and non-finite values such as ``inf`` or ``1e309`` that + ``float`` accepts, fall back to our own backoff: none of them is a delay, + and an infinite one would overflow the wait that honours it. + """ + if headers is None: + return None + try: + raw = headers.get("Retry-After") + except AttributeError: + return None + if raw is None: + return None + try: + seconds: float = float(str(raw).strip()) + except ValueError: + return None + if not math.isfinite(seconds): + return None + return max(0.0, seconds) + + +def _classify_status(status: int, headers: Any) -> Exception: + """Turns an HTTP error status into the right exception type.""" + if status == 401: + return _FatalTransportError( + "LaunchDarkly rejected the SDK key (HTTP 401). Skill delivery cannot " + "start. Check that the key is the environment's server-side SDK key." + ) + if status == 403: + return _FatalTransportError( + f"LaunchDarkly returned HTTP 403. {_FORBIDDEN_ADVICE}" + ) + if status in (400, 405, 406, 414, 501): + return _FatalTransportError( + f"LaunchDarkly returned HTTP {status}, which retrying will not fix. " + "The request this adapter sent was not understood. It carries only " + "the SDK key and, after the first payload, a 'basis' selector, so " + "check the base URI and that the endpoint speaks FDv2." + ) + return _RecoverableTransportError( + f"LaunchDarkly returned HTTP {status}", _retry_after_seconds(headers) + ) + + +def _interrupt_read(response: Any) -> None: + """ + Best-effort interruption of a read blocked on *response*, from another thread. + + Closing the response is not enough: CPython's buffered reader stays parked in + ``readline`` until bytes arrive. Shutting the *socket* down underneath it + unblocks it immediately. Reaching the socket means walking urllib's private + attribute chain, so every step is guarded and failure is silent: the + delivery thread is a daemon and ``close``'s join timeout is the backstop. + """ + for path in (("fp", "raw", "_sock"), ("fp", "_sock"), ("_sock",)): + found: Any = response + for name in path: + found = getattr(found, name, None) + if found is None: + break + if found is not None and hasattr(found, "shutdown"): + try: + found.shutdown(socket.SHUT_RDWR) + except OSError: + pass + return + + +class _StreamConnection: + """ + One open streaming connection: an event iterator plus a way to interrupt it + from another thread, which is what ``FDv2SkillStore.close`` needs. + """ + + def __init__(self, response: Any) -> None: + self._response = response + self.events = _iter_sse(response) + + def close(self) -> None: + """Interrupts the read. Safe to call from any thread, and twice.""" + _interrupt_read(self._response) + try: + self._response.close() + except Exception: + pass + + +@dataclass(frozen=True) +class _PollResult: + not_modified: bool + events: list[tuple[str, Any]] + etag: str | None + + +class _Requester: + """ + The only place this module opens a socket. Standard library only, on purpose. + + *read_timeout* is applied to every socket operation of a request. ``urllib`` + has no separate connect timeout: its ``timeout`` becomes the socket timeout + for the whole operation, so connecting, waiting for headers and each body + read are all bounded by the same value. + """ + + def __init__( + self, + sdk_key: str, + base_uri: str, + *, + read_timeout: float, + opener: Any = None, + ) -> None: + self._sdk_key = sdk_key + self._base_uri = base_uri.rstrip("/") + self._read_timeout = read_timeout + # Injectable so tests can drive a fake endpoint without a socket. + self._opener = opener or urllib.request.build_opener() + + def _url(self, path: str, basis: str | None) -> str: + """ + The request URL: the path, plus ``basis`` once a payload has committed. + + Deliberately no ``mv`` (data model version). That parameter selects the + *flag* data model and the connection rejects any value but the flag + default; the agent-skill payload is generic, is served regardless of it, + and has no model version of its own to ask for. + """ + if not basis: + return f"{self._base_uri}{path}" + return f"{self._base_uri}{path}?{urllib.parse.urlencode({'basis': basis})}" + + def _request( + self, path: str, basis: str | None, headers: dict[str, str] + ) -> urllib.request.Request: + all_headers = {"Authorization": self._sdk_key, **headers} + return urllib.request.Request( + self._url(path, basis), headers=all_headers, method="GET" + ) + + def poll(self, basis: str | None, etag: str | None) -> _PollResult: + """One ``GET /sdk/poll``. A 304 is a first-class outcome, not an error.""" + headers = {"Accept": "application/json"} + if etag: + headers["If-None-Match"] = etag + request = self._request(POLL_PATH, basis, headers) + try: + with self._opener.open(request, timeout=self._read_timeout) as response: + status = getattr(response, "status", None) or response.getcode() + if status == 304: + return _PollResult(not_modified=True, events=[], etag=etag) + body = response.read() + new_etag = response.headers.get("ETag") or etag + except urllib.error.HTTPError as exc: + if exc.code == 304: + # urllib raises on 304 when no redirect handler swallows it. + return _PollResult(not_modified=True, events=[], etag=etag) + raise _classify_status(exc.code, exc.headers) from exc + except Exception as exc: + raise _RecoverableTransportError( + f"polling request failed: {type(exc).__name__}: {exc}" + ) from exc + + return _PollResult( + not_modified=False, events=_decode_poll_body(body), etag=new_etag + ) + + def stream(self, basis: str | None) -> _StreamConnection: + """Opens ``GET /sdk/stream``.""" + request = self._request( + STREAM_PATH, + basis, + {"Accept": "text/event-stream", "Cache-Control": "no-cache"}, + ) + try: + response = self._opener.open(request, timeout=self._read_timeout) + except urllib.error.HTTPError as exc: + raise _classify_status(exc.code, exc.headers) from exc + except Exception as exc: + raise _RecoverableTransportError( + f"streaming request failed: {type(exc).__name__}: {exc}" + ) from exc + return _StreamConnection(response) + + +def _decode_poll_body(body: bytes) -> list[tuple[str, Any]]: + """ + Unwraps ``{"events": [...]}``. Polling and streaming carry identical event + objects, which is why the protocol reader is shared between the two modes. + """ + try: + parsed = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise _RecoverableTransportError( + f"polling response was not valid JSON: {exc}" + ) from exc + if not isinstance(parsed, dict) or not isinstance(parsed.get("events"), list): + raise _RecoverableTransportError("polling response had no 'events' array") + events: list[tuple[str, Any]] = [] + for entry in parsed["events"]: + if not isinstance(entry, dict): + continue + name = entry.get("event") + if isinstance(name, str): + events.append((name, entry.get("data"))) + return events + + +def _iter_sse(response: Any) -> Any: + """ + Decodes an SSE body into ``(event name, data)`` pairs. + + Minimal on purpose: ``event:``/``data:`` fields, multi-line ``data`` joined + with newlines, a blank line dispatching, and ``:`` comments skipped. + """ + try: + name: str | None = None + data_lines: list[str] = [] + for raw_line in response: + line = raw_line.decode("utf-8", errors="replace").rstrip("\r\n") + if line == "": + if name is not None: + payload = "\n".join(data_lines) + try: + parsed = json.loads(payload) if payload else None + except json.JSONDecodeError: + logger.warning( + "Discarding FDv2 '%s' event whose data was not JSON", name + ) + parsed = None + else: + yield name, parsed + name = None + data_lines = [] + continue + if line.startswith(":"): + continue + field_name, _, value = line.partition(":") + value = value[1:] if value.startswith(" ") else value + if field_name == "event": + name = value + elif field_name == "data": + data_lines.append(value) + finally: + try: + response.close() + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Backoff +# --------------------------------------------------------------------------- + + +def _backoff_delay( + attempt: int, *, base: float, maximum: float, jitter: float = 0.5 +) -> float: + """ + Exponential backoff with jitter, capped at *maximum*. + + Jitter is subtractive over the whole range rather than added on top, so the + cap is a real ceiling: a fleet restarted together must not reconnect in + lockstep, and must not exceed the interval the cap promises. + """ + # float(2 ** n): the integer power is untyped to mypy. + ceiling: float = min(maximum, base * float(2 ** max(0, attempt - 1))) + return ceiling * (1.0 - jitter * random.random()) + + +# --------------------------------------------------------------------------- +# The store +# --------------------------------------------------------------------------- + + +class FDv2SkillStore: + """ + A ``SkillStore`` fed by LaunchDarkly's SDK-facing FDv2 delivery channel. + + Constructed with the environment's server-side SDK key, started explicitly, + and passed to ``init_client``:: + + store = FDv2SkillStore(sdk_key=os.environ["LD_SDK_KEY"]) + store.start() + store.wait_for_skills(timeout=10) + await init_client(options={"skillStore": store}) + + skill = await get_skill("pdf-extraction") + ... + store.close() + + It also works as a context manager. + + **Server-side only.** A mobile key or a client-side environment ID is + refused in the constructor. + + **Delivery is in the background; retrieval is not.** A daemon thread owns + the connection and fills memory, and ``get_object`` only ever reads what has + already arrived. A process that calls ``get_skill`` immediately after + ``start()`` may see an empty store; ``wait_for_skills`` orders boot against + the first payload. + + **Last known good survives an outage.** A transport failure never empties + the store and never makes ``get_object`` raise, which is what makes + ``write_skills(on_unavailable="keep")`` correct. ``diagnostics`` and + ``failed`` report the degradation. + + **What arrives is untrusted.** Raw wire objects are held verbatim and + verified at the accessor boundary, not here. In particular an object with no + ``contentHash`` is held and then *withheld*; see + ``StoreDiagnostics.hashless_objects``. + """ + + def __init__( + self, + sdk_key: str, + *, + base_uri: str = DEFAULT_BASE_URI, + mode: Mode = "stream", + poll_interval: float = 30.0, + read_timeout: float | None = None, + initial_backoff: float = 1.0, + max_backoff: float = 30.0, + max_consecutive_failures: int = 10, + _requester: Any = None, + ) -> None: + """ + *mode* is ``"stream"`` by default. Prefer it: a ``delete-object`` reaches + a live stream in seconds. ``"poll"`` exists for environments that cannot + hold a long-lived connection, and revocation there is one + ``poll_interval`` late. + + *read_timeout* is the only network timeout and bounds every socket + operation of a request, so its meaning and default follow the mode: in + ``"poll"`` it bounds the whole request (``DEFAULT_POLL_TIMEOUT``); in + ``"stream"`` it bounds each wait for the next bytes + (``DEFAULT_STREAM_READ_TIMEOUT``). Must be positive when given. + + *max_backoff* caps every delay between retries, including one the server + asks for with ``Retry-After``. + + *max_consecutive_failures* bounds the retry loop. On exceeding it the + transport stops, logs an error, and the store keeps serving last known + good; ``failed`` reports it. Only failures in a row count: a committed + payload resets the count. + """ + _require_server_side_credential(sdk_key) + if mode not in ("stream", "poll"): + raise ValueError(f'mode must be "stream" or "poll", got {mode!r}') + if poll_interval <= 0: + raise ValueError(f"poll_interval must be positive, got {poll_interval!r}") + if read_timeout is None: + read_timeout = ( + DEFAULT_STREAM_READ_TIMEOUT + if mode == "stream" + else DEFAULT_POLL_TIMEOUT + ) + elif not (math.isfinite(read_timeout) and read_timeout > 0): + raise ValueError(f"read_timeout must be positive, got {read_timeout!r}") + + self._mode: Mode = mode + self._poll_interval = poll_interval + self._initial_backoff = initial_backoff + self._max_backoff = max_backoff + self._max_consecutive_failures = max_consecutive_failures + + self._objects = _SkillObjectSet() + self._reader = _ProtocolReader(self._objects) + self._lock = threading.RLock() + self._listeners: dict[str, list[Callable[[dict[str, Any]], Any]]] = {} + + self._basis: str | None = None + self._etag: str | None = None + + self._requester = _requester or _Requester( + sdk_key.strip(), + base_uri, + read_timeout=read_timeout, + ) + + self._stop = threading.Event() + self._first_payload = threading.Event() + self._thread: threading.Thread | None = None + self._failed_reason: str | None = None + # The open streaming connection, so ``close`` can interrupt its read. + self._connection: Any = None + # Recoverable failures since the last committed payload. Reset at the + # commit rather than when a connection returns: a stream only ever ends + # by being dropped, so resetting on return would count every healthy, + # server-recycled connection as a failure. + self._failures = 0 + + # -- lifecycle --------------------------------------------------------- + + def start(self) -> FDv2SkillStore: + """ + Starts the delivery thread. Idempotent; returns ``self`` so it chains. + + Does not block: use ``wait_for_skills`` when boot ordering matters. + """ + with self._lock: + if self._thread is not None and self._thread.is_alive(): + return self + self._stop.clear() + self._thread = threading.Thread( + target=self._run, name="ld-ai-skills-fdv2", daemon=True + ) + self._thread.start() + return self + + def close(self, timeout: float = 5.0) -> None: + """ + Stops delivery. Idempotent, and safe to call from any thread. + + Held content is *not* dropped: a closed store still answers from what it + received. Detaching the store from the accessors is the job of the + package-level ``launchdarkly_ai_server.shutdown()`` coroutine. + """ + self._stop.set() + # The delivery thread is normally blocked in a socket read that no flag + # can reach; without this the join waits out its full timeout. + with self._lock: + connection = self._connection + if connection is not None: + connection.close() + thread = self._thread + if ( + thread is not None + and thread.is_alive() + and thread is not threading.current_thread() + ): + thread.join(timeout=timeout) + + def __enter__(self) -> FDv2SkillStore: + return self.start() + + def __exit__(self, *_exc: Any) -> None: + self.close() + + def wait_for_skills(self, timeout: float = 10.0) -> bool: + """ + Blocks until the first payload has been committed, or *timeout* elapses. + + ``True`` means a payload arrived — not that any skill in it verified, and + not that the environment has any skills. ``diagnostics`` answers the rest. + """ + return self._first_payload.wait(timeout=timeout) + + @property + def failed(self) -> str | None: + """Why delivery stopped for good, or ``None`` while it is running.""" + with self._lock: + return self._failed_reason + + @property + def diagnostics(self) -> StoreDiagnostics: + """A snapshot of what the transport has seen. See ``StoreDiagnostics``.""" + with self._lock: + return StoreDiagnostics(**vars(self._reader.diagnostics)) + + # -- the SkillStore interface ----------------------------------------- + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + if kind != SKILL_OBJECT_KIND: + return None + with self._lock: + return self._objects.get(key, version) + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + if kind != SKILL_OBJECT_KIND: + return {} + with self._lock: + return self._objects.snapshot() + + def add_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Registers *fn* to be called once per changed object, at + ``payload-transferred`` rather than as objects stream in. + + A put notifies with the raw skill object. A revocation notifies with a + ``{"key", "version"}`` tombstone carrying no content, so a listener that + reads content must check for ``content`` rather than assume it. + + *fn* runs on the delivery thread. Keep it cheap and non-blocking. An + exception it raises is logged and swallowed, because a broken listener + must not be able to kill delivery. + """ + with self._lock: + self._listeners.setdefault(kind, []).append(fn) + + def remove_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Unregisters *fn* from *kind*. Safe to call from any thread, including + from inside a listener: a removal during one commit takes effect from + the next. + + Removes one occurrence; removing a callable that is not registered is a + no-op, so ``SkillWatcher.close`` can detach unconditionally. + """ + with self._lock: + listeners = self._listeners.get(kind) + if listeners is None: + return + try: + listeners.remove(fn) + except ValueError: + return + + def _notify(self, changes: list[dict[str, Any]]) -> None: + with self._lock: + listeners = list(self._listeners.get(SKILL_OBJECT_KIND, [])) + for raw in changes: + for listener in listeners: + try: + listener(raw) + except Exception: + logger.error( + "A skill store change listener raised; delivery continues", + exc_info=True, + ) + + # -- the delivery loop ------------------------------------------------- + + def _run(self) -> None: + while not self._stop.is_set(): + try: + if self._mode == "stream": + self._stream_once() + else: + self._poll_once() + # A poll that returned is a current answer even when it committed + # nothing (HTTP 304). A stream never returns normally; its + # successes are counted at each commit in ``_apply``. + self._record_success() + except _FatalTransportError as exc: + self._give_up(str(exc)) + return + except _RecoverableTransportError as exc: + with self._lock: + self._failures += 1 + failures = self._failures + self._reader.diagnostics.connection_failures = failures + self._reader.diagnostics.last_error = str(exc) + if failures > self._max_consecutive_failures: + self._give_up( + f"gave up after {failures} consecutive failures; " + f"last error: {exc}" + ) + return + delay = exc.retry_after + if delay is None or not math.isfinite(delay): + delay = _backoff_delay( + failures, base=self._initial_backoff, maximum=self._max_backoff + ) + # ``Retry-After`` is a request and ``max_backoff`` is a promise. + # The header may come from a proxy rather than LaunchDarkly, and + # a value in the hours would park revocation for that long. + delay = min(delay, self._max_backoff) + logger.warning( + "Skill delivery failed (%s); retrying in %.1fs", exc, delay + ) + if self._stop.wait(delay): + return + continue + except Exception as exc: # pragma: no cover - defensive + self._give_up(f"unexpected error in skill delivery: {exc!r}") + logger.error("Unexpected error in skill delivery", exc_info=True) + return + + if self._mode == "poll" and self._stop.wait(self._poll_interval): + return + + def _record_success(self) -> None: + with self._lock: + self._failures = 0 + self._reader.diagnostics.connection_failures = 0 + + def _give_up(self, reason: str) -> None: + with self._lock: + self._failed_reason = reason + self._reader.diagnostics.last_error = reason + logger.error( + "Skill delivery has stopped and will not retry: %s. The store keeps " + "serving the last content it received; skills will not update until " + "the process restarts with a working connection.", + reason, + ) + # Unblock anyone waiting on a first payload that is never coming. + self._first_payload.set() + + def _apply(self, name: str, data: Any) -> None: + """ + Feeds one event to the reader, publishes a commit, and raises the + transport error the event calls for, if any. + """ + with self._lock: + outcome = self._reader.handle(name, data) + if outcome.committed and outcome.basis is not None: + self._basis = outcome.basis + if outcome.committed: + # A commit breaks the row of consecutive failures. + self._record_success() + self._first_payload.set() + if outcome.changes: + self._notify(outcome.changes) + if outcome.fatal: + raise _FatalTransportError(outcome.fatal) + if outcome.disconnect: + raise _RecoverableTransportError(outcome.disconnect) + + def _poll_once(self) -> None: + with self._lock: + basis, etag = self._basis, self._etag + result = self._requester.poll(basis, etag) + with self._lock: + self._etag = result.etag + if result.not_modified: + logger.debug("Skill payload unchanged (HTTP 304)") + # A 304 counts as a first payload, so a boot that reconnects with a + # cached basis is not blocked on a transfer the server will not send. + self._first_payload.set() + return + for name, data in result.events: + self._apply(name, data) + + def _stream_once(self) -> None: + with self._lock: + basis = self._basis + connection = self._requester.stream(basis) + with self._lock: + self._connection = connection + try: + # ``close`` may have run while the connect was in flight and found + # no connection to interrupt; this is the last chance to notice + # before the read below blocks. + if self._stop.is_set(): + return + for name, data in connection.events: + if self._stop.is_set(): + return + self._apply(name, data) + except Exception: + if self._stop.is_set(): + # ``close`` interrupted the read on purpose. + return + raise + finally: + connection.close() + with self._lock: + self._connection = None + # A stream that ends without a goodbye is a dropped connection. + raise _RecoverableTransportError("the FDv2 stream closed unexpectedly") diff --git a/packages/client/src/launchdarkly_ai_server/skills_watch.py b/packages/client/src/launchdarkly_ai_server/skills_watch.py index 05263057..e421df6a 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_watch.py +++ b/packages/client/src/launchdarkly_ai_server/skills_watch.py @@ -267,7 +267,7 @@ async def watch_skills( "watch_skills needs a skill store that implements add_listener(kind, " "fn); the configured store does not, so delivery changes cannot be " "observed. Use write_skills for a one-shot reconcile, or configure a " - "store with a delivery transport." + "store with a delivery transport (FDv2SkillStore)." ) if debounce < 0: raise ValueError(f"debounce must not be negative, got {debounce!r}") diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index d3f83095..a07ea6ae 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -603,6 +603,18 @@ async def test_all_skills_raises_actionably_when_no_store(self) -> None: with pytest.raises(RuntimeError, match="skill store"): await all_skills() + async def test_the_no_store_message_names_the_delivery_store_first(self) -> None: + """ + A deployment that hits this message must be pointed at the store that + receives content from LaunchDarkly, not only at the development one. + """ + with pytest.raises(RuntimeError) as reported: + await get_skill("a") + message = str(reported.value) + assert "FDv2SkillStore" in message + assert "InMemorySkillStore" in message + assert message.index("FDv2SkillStore") < message.index("InMemorySkillStore") + async def test_shutdown_clears_the_store( self, make_raw_skill: Any, mock_ld_client: Any ) -> None: diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index b88d14d9..0cce2f0c 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -1,26 +1,55 @@ """ -Tests for the FDv2 skill delivery protocol. - -Wire semantics — which objects are skills, the skill's version in the wire ``key`` -versus the payload's in ``version``, revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted -against ``_ProtocolReader``, which has no I/O, so each case reads as the contract -it is rather than as a server script. +Tests for the FDv2 skill delivery transport. + +Two layers, deliberately: + +- **A real fake endpoint.** ``_FakeFDv2Endpoint`` is an in-process + ``ThreadingHTTPServer`` that implements the wire contract — the ``basis`` + query parameter, ``Authorization``, ``If-None-Match``/304, the + ``{"events": [...]}`` polling envelope, and SSE for streaming. The store under + test opens real sockets against it, so request construction and header + handling are exercised rather than mocked. +- **The protocol reader driven directly.** Wire semantics — which objects are + skills, the skill's version in the wire ``key`` versus the payload's in + ``version``, revocation, mixed payloads — are + asserted against ``_ProtocolReader``, which has no I/O, so those cases read as + the contract they are instead of as a server script. """ from __future__ import annotations import hashlib +import json +import socket +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, ClassVar +from urllib.parse import parse_qs, urlparse import pytest -from launchdarkly_ai_server import InMemorySkillStore +from launchdarkly_ai_server import ( + FDv2SkillStore, + InMemorySkillStore, + all_skills, + get_skill, + get_skill_result, + init_client, + watch_skills, +) from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND from launchdarkly_ai_server.skills_fdv2 import ( + DEFAULT_POLL_TIMEOUT, + DEFAULT_STREAM_READ_TIMEOUT, FDV2_KEY_DELIMITER, FDV2_OBJECT_KIND, + _backoff_delay, _is_skill_event, _ProtocolReader, + _RecoverableTransportError, + _Requester, + _retry_after_seconds, _SkillObjectSet, _store_object_from_put, _tombstone_from_delete, @@ -28,6 +57,7 @@ pytestmark = pytest.mark.usefixtures("reset_skill_state") +SDK_KEY = "sdk-00000000-0000-4000-8000-000000000000" SKILL_BODY = "---\nname: PDF Extraction\n---\nExtract text from PDFs.\n" @@ -143,6 +173,182 @@ def full_payload( ) +# --------------------------------------------------------------------------- +# The fake endpoint +# --------------------------------------------------------------------------- + + +class _FakeFDv2Endpoint: + """ + An in-process server implementing the SDK-facing FDv2 contract. + + Scripted per request: ``queue_poll`` appends a response for the next + ``/sdk/poll``, ``queue_stream`` appends a sequence of SSE events for the next + ``/sdk/stream``. Every request's method, path, query and headers are recorded + in ``requests`` so the tests can assert on what the store actually sent — + which is the only way ``basis`` round-tripping and ``If-None-Match`` can be + checked at all. + """ + + def __init__(self) -> None: + self.requests: list[dict[str, Any]] = [] + self._polls: list[dict[str, Any]] = [] + self._streams: list[list[dict[str, Any]]] = [] + self._lock = threading.Lock() + self.hold_stream_open = False + self._release = threading.Event() + + endpoint = self + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args: Any) -> None: + return + + def do_GET(self) -> None: + parsed = urlparse(self.path) + query = {k: v[0] for k, v in parse_qs(parsed.query).items()} + with endpoint._lock: + endpoint.requests.append( + { + "path": parsed.path, + "query": query, + "authorization": self.headers.get("Authorization"), + "if_none_match": self.headers.get("If-None-Match"), + "accept": self.headers.get("Accept"), + } + ) + if parsed.path == "/sdk/poll": + endpoint._serve_poll(self) + elif parsed.path == "/sdk/stream": + endpoint._serve_stream(self) + else: + self.send_response(404) + self.send_header("Content-Length", "0") + self.end_headers() + + class Server(ThreadingHTTPServer): + # Handler threads are not joined on shutdown: a test that ends while + # a stream is deliberately held open should not pay for the hold. + daemon_threads = True + + self._server = Server(("127.0.0.1", 0), Handler) + # A short poll interval so `shutdown` is prompt: the default 0.5s is + # paid at the teardown of every test that touches the endpoint. + self._thread = threading.Thread( + target=lambda: self._server.serve_forever(poll_interval=0.01), daemon=True + ) + self._thread.start() + + @property + def base_uri(self) -> str: + host, port = self._server.server_address[:2] + return f"http://{host}:{port}" + + # -- scripting --------------------------------------------------------- + + def queue_poll( + self, + payload_events: list[dict[str, Any]] | None = None, + *, + status: int = 200, + etag: str | None = None, + retry_after: str | None = None, + ) -> None: + with self._lock: + self._polls.append( + { + "status": status, + "events": payload_events or [], + "etag": etag, + "retry_after": retry_after, + } + ) + + def queue_stream(self, payload_events: list[dict[str, Any]]) -> None: + with self._lock: + self._streams.append(payload_events) + + # -- serving ----------------------------------------------------------- + + def _serve_poll(self, handler: BaseHTTPRequestHandler) -> None: + with self._lock: + response = ( + self._polls.pop(0) if self._polls else {"status": 304, "events": []} + ) + status = response["status"] + handler.send_response(status) + if response.get("etag"): + handler.send_header("ETag", response["etag"]) + if response.get("retry_after"): + handler.send_header("Retry-After", response["retry_after"]) + if status in (200,): + body = json.dumps({"events": response["events"]}).encode("utf-8") + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + return + handler.send_header("Content-Length", "0") + handler.end_headers() + + def _serve_stream(self, handler: BaseHTTPRequestHandler) -> None: + with self._lock: + payload_events = self._streams.pop(0) if self._streams else [] + handler.send_response(200) + handler.send_header("Content-Type", "text/event-stream") + handler.send_header("Cache-Control", "no-cache") + handler.send_header("Transfer-Encoding", "chunked") + handler.end_headers() + for event in payload_events: + chunk = ( + f"event: {event['event']}\ndata: {json.dumps(event.get('data'))}\n\n" + ).encode() + handler.wfile.write(f"{len(chunk):X}\r\n".encode() + chunk + b"\r\n") + handler.wfile.flush() + if self.hold_stream_open: + # Keeps the connection up so a test can assert on the store's state + # without racing the reconnect path. Released on ``close`` so the + # hold costs the suite nothing once the test is done with it. + self._release.wait(timeout=10) + handler.wfile.write(b"0\r\n\r\n") + + def close(self) -> None: + self._release.set() + self._server.shutdown() + self._server.server_close() + + +@pytest.fixture +def endpoint() -> Any: + server = _FakeFDv2Endpoint() + yield server + server.close() + + +def poll_store(endpoint: Any, **kwargs: Any) -> FDv2SkillStore: + return FDv2SkillStore( + SDK_KEY, + base_uri=endpoint.base_uri, + mode="poll", + poll_interval=kwargs.pop("poll_interval", 0.05), + initial_backoff=kwargs.pop("initial_backoff", 0.01), + max_backoff=kwargs.pop("max_backoff", 0.05), + read_timeout=kwargs.pop("read_timeout", 5.0), + **kwargs, + ) + + +def wait_until(predicate: Any, timeout: float = 5.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.01) + return predicate() + + # --------------------------------------------------------------------------- # Identifying skill objects, and ignoring everything else # --------------------------------------------------------------------------- @@ -827,6 +1033,608 @@ def test_snapshot_agrees(self) -> None: assert memory.all_objects(SKILL_OBJECT_KIND) == objects.snapshot() +# --------------------------------------------------------------------------- +# The store against the fake endpoint +# --------------------------------------------------------------------------- + + +class TestPollingAgainstTheEndpoint: + def test_a_polled_skill_becomes_retrievable_through_the_accessors( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + raw = store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") + assert raw is not None + assert raw["version"] == 3 + + def test_the_request_carries_the_sdk_key_and_no_data_model_version( + self, endpoint: Any + ) -> None: + """ + No ``mv``: that parameter selects the *flag* data model, the connection + rejects any value but the flag default, and the generic agent-skill + payload is served regardless of it. Sending ``mv=1`` — the skill + payload's own model version — gets the whole connection refused. + """ + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + first = endpoint.requests[0] + assert first["path"] == "/sdk/poll" + assert first["authorization"] == SDK_KEY + assert "mv" not in first["query"] + + def test_the_first_request_sends_no_basis(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert "basis" not in endpoint.requests[0]["query"] + + def test_the_basis_from_payload_transferred_is_echoed_on_the_next_request( + self, endpoint: Any + ) -> None: + endpoint.queue_poll( + full_payload(("put-object", put_skill()), state="selector-abc") + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert endpoint.requests[1]["query"]["basis"] == "selector-abc" + + def test_the_basis_advances_across_successive_payloads(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()), state="basis-1")) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("second")), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint): + assert wait_until(lambda: len(endpoint.requests) >= 3) + bases = [r["query"].get("basis") for r in endpoint.requests[:3]] + assert bases == [None, "basis-1", "basis-2"] + + def test_an_etag_is_returned_as_if_none_match(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill())), etag='W/"v1"') + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert endpoint.requests[1]["if_none_match"] == 'W/"v1"' + + def test_a_304_keeps_the_held_content(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill())), etag='W/"v1"') + endpoint.queue_poll(status=304) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert wait_until(lambda: len(endpoint.requests) >= 3) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + assert store.diagnostics.payloads_transferred == 1 + assert store.failed is None + + def test_a_304_before_any_payload_still_releases_wait_for_skills( + self, endpoint: Any + ) -> None: + """A reconnect with a cached basis has nothing to transfer; boot must not + block on a payload the server has no reason to send.""" + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + + def test_a_mixed_payload_over_the_wire_yields_only_the_skill( + self, endpoint: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_flag("flag-a")), + ("put-object", put_segment("beta")), + ("put-object", put_skill("pdf-extraction")), + ("put-object", put_flag("flag-b")), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + held = store.all_objects(SKILL_OBJECT_KIND) + assert len(held) == 1 + assert next(iter(held.values()))["key"] == "pdf-extraction" + assert store.diagnostics.objects_ignored == 3 + + def test_a_revocation_over_the_wire_removes_the_skill(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + assert wait_until( + lambda: ( + store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is None + and store.diagnostics.objects_revoked == 1 + ) + ) + + def test_the_store_asks_for_only_the_kind_it_serves(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert store.get_object("flag", "pdf-extraction") is None + assert store.all_objects("flag") == {} + + +class TestStreamingAgainstTheEndpoint: + def test_a_streamed_payload_lands(self, endpoint: Any) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore( + SDK_KEY, base_uri=endpoint.base_uri, mode="stream", initial_backoff=0.01 + ) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + finally: + store.close() + + def test_the_stream_request_advertises_event_stream(self, endpoint: Any) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + try: + store.start() + store.wait_for_skills(timeout=5) + finally: + store.close() + assert endpoint.requests[0]["path"] == "/sdk/stream" + assert endpoint.requests[0]["accept"] == "text/event-stream" + + def test_a_streamed_revocation_arrives_without_a_restart( + self, endpoint: Any + ) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream( + full_payload(("put-object", put_skill())) + + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ) + ) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + try: + store.start() + assert wait_until( + lambda: ( + store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is None + and store.diagnostics.objects_revoked == 1 + ) + ) + finally: + store.close() + + def test_a_dropped_stream_reconnects_with_the_basis_it_reached( + self, endpoint: Any + ) -> None: + endpoint.queue_stream( + full_payload(("put-object", put_skill()), state="basis-1") + ) + endpoint.queue_stream(events(("heart-beat", None))) + store = FDv2SkillStore( + SDK_KEY, + base_uri=endpoint.base_uri, + mode="stream", + initial_backoff=0.01, + max_backoff=0.05, + ) + try: + store.start() + assert wait_until(lambda: len(endpoint.requests) >= 2) + finally: + store.close() + assert endpoint.requests[1]["query"]["basis"] == "basis-1" + + def test_close_returns_promptly_while_a_stream_is_open(self, endpoint: Any) -> None: + """ + The delivery thread is blocked in a socket read that no stop flag can + reach, so ``close`` closes the connection under it. Without that, every + shutdown of a healthy stream waits out the join timeout. + """ + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + store.start() + assert store.wait_for_skills(timeout=5) is True + started = time.monotonic() + store.close(timeout=5.0) + assert time.monotonic() - started < 1.0 + + def test_an_interrupted_stream_is_not_reported_as_a_failure( + self, endpoint: Any + ) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + store.start() + store.wait_for_skills(timeout=5) + store.close() + assert store.failed is None + + def test_content_survives_a_reconnect(self, endpoint: Any) -> None: + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + endpoint.queue_stream(events(("heart-beat", None))) + store = FDv2SkillStore( + SDK_KEY, base_uri=endpoint.base_uri, mode="stream", initial_backoff=0.01 + ) + try: + store.start() + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + finally: + store.close() + + +# --------------------------------------------------------------------------- +# Failure handling +# --------------------------------------------------------------------------- + + +class _ScriptedConnection: + """Stands in for ``_StreamConnection``: an event iterator plus a close.""" + + def __init__(self, payload_events: Any) -> None: + self.events = iter(payload_events) + self.closed = False + + def close(self) -> None: + self.closed = True + + +class _ScriptedRequester: + """Raises a scripted sequence, so backoff is asserted without real sockets.""" + + def __init__(self, *outcomes: Any) -> None: + self.outcomes = list(outcomes) + self.calls: list[tuple[str | None, str | None]] = [] + + def poll(self, basis: str | None, etag: str | None) -> Any: + self.calls.append((basis, etag)) + outcome = ( + self.outcomes.pop(0) if self.outcomes else _RecoverableTransportError("x") + ) + if isinstance(outcome, Exception): + raise outcome + return outcome + + def stream(self, basis: str | None) -> Any: + self.calls.append((basis, None)) + outcome = ( + self.outcomes.pop(0) if self.outcomes else _RecoverableTransportError("x") + ) + if isinstance(outcome, Exception): + raise outcome + return _ScriptedConnection(outcome) + + +class _RecyclingRequester: + """ + A healthy server that recycles connections: every ``stream`` call succeeds, + transfers a full payload, and then ends the connection, as LaunchDarkly and + any proxy in between do to a long-lived stream. + """ + + def __init__(self) -> None: + self.connections = 0 + + def stream(self, basis: str | None) -> Any: + self.connections += 1 + return _ScriptedConnection( + [ + (e["event"], e["data"]) + for e in full_payload( + ("put-object", put_skill()), state=f"basis-{self.connections}" + ) + ] + ) + + +class _BlockingConnection: + """A stream that never produces an event until it is closed.""" + + def __init__(self) -> None: + self._closed = threading.Event() + + @property + def events(self) -> Any: + self._closed.wait() + return iter(()) + + def close(self) -> None: + self._closed.set() + + +class _SlowConnectRequester: + """ + A ``stream`` whose connect does not return until the test releases it, + standing in for a slow TLS handshake, followed by a read that never yields. + """ + + def __init__(self) -> None: + self.entered = threading.Event() + self.release = threading.Event() + + def stream(self, basis: str | None) -> Any: + self.entered.set() + self.release.wait(timeout=10) + return _BlockingConnection() + + +def stream_store(**kwargs: Any) -> FDv2SkillStore: + return FDv2SkillStore( + SDK_KEY, + mode="stream", + initial_backoff=kwargs.pop("initial_backoff", 0.001), + max_backoff=kwargs.pop("max_backoff", 0.002), + **kwargs, + ) + + +class TestFailureHandling: + def test_a_403_stops_delivery_and_explains_why( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll(status=403) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert "403" in store.failed + assert "opt-in" in store.failed + assert any("opt-in" in r.getMessage() for r in caplog.records) + + def test_a_401_stops_delivery(self, endpoint: Any) -> None: + endpoint.queue_poll(status=401) + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert "401" in store.failed + + def test_a_fatal_failure_releases_wait_for_skills_rather_than_hanging( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(status=401) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + assert store.failed is not None + + def test_a_fatal_failure_keeps_last_known_good_servable( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll(status=403) + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + + def test_a_500_is_retried(self, endpoint: Any) -> None: + endpoint.queue_poll(status=500) + endpoint.queue_poll(status=503) + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + assert store.failed is None + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + + def test_a_retry_resets_the_failure_count_on_success(self, endpoint: Any) -> None: + endpoint.queue_poll(status=500) + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) + assert wait_until(lambda: store.diagnostics.connection_failures == 0) + + def test_retries_are_bounded(self) -> None: + store = FDv2SkillStore( + SDK_KEY, + mode="poll", + poll_interval=0.01, + initial_backoff=0.001, + max_backoff=0.002, + max_consecutive_failures=3, + _requester=_ScriptedRequester(), + ) + try: + store.start() + assert wait_until(lambda: store.failed is not None) + # Four, not three: the bound is the number of failures *tolerated*, + # so the run that exceeds it is the one that gives up. + assert "gave up after 4 consecutive failures" in store.failed + finally: + store.close() + + def test_recycled_stream_connections_are_not_failures(self) -> None: + # A streaming connection only ever ends by being dropped, so a loop + # that counted every drop as a failure would give up on a healthy + # server after max_consecutive_failures + 1 recycles, and delivery + # (including revocation) would silently stop for the process lifetime. + requester = _RecyclingRequester() + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert wait_until(lambda: requester.connections >= 8) + assert store.failed is None + assert store.diagnostics.payloads_transferred >= 8 + # A drop is a failure until the next commit clears it, so the count + # may read 1 mid-reconnect. What it must never do is climb. + assert store.diagnostics.connection_failures <= 1 + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + finally: + store.close() + + def test_a_stream_commit_resets_the_failure_count(self) -> None: + payload = [ + (e["event"], e["data"]) for e in full_payload(("put-object", put_skill())) + ] + requester = _ScriptedRequester( + _RecoverableTransportError("x"), + _RecoverableTransportError("x"), + _RecoverableTransportError("x"), + payload, + ) + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=5) + # Three failures reach the bound, then a commit, then the exhausted + # requester fails on every reconnect. The count must start again at + # the commit: the stream's own drop is failure one, and three more + # connects are owed before giving up. Carrying the three over would + # give up on the drop itself, with no further connect at all. + assert wait_until(lambda: store.failed is not None) + assert "gave up after 4 consecutive failures" in store.failed + assert "last error: x" in store.failed + assert len(requester.calls) == 7 + finally: + store.close() + + def test_stream_retries_are_bounded(self) -> None: + store = stream_store( + max_consecutive_failures=3, _requester=_ScriptedRequester() + ) + try: + store.start() + assert wait_until(lambda: store.failed is not None) + assert "gave up after 4 consecutive failures" in store.failed + finally: + store.close() + + def test_a_retry_after_header_is_honoured(self) -> None: + requester = _ScriptedRequester( + _RecoverableTransportError("slow down", retry_after=0.25), + ) + store = FDv2SkillStore( + SDK_KEY, + mode="poll", + poll_interval=10.0, + initial_backoff=5.0, + _requester=requester, + ) + try: + started = time.monotonic() + store.start() + assert wait_until(lambda: len(requester.calls) >= 2, timeout=3) + elapsed = time.monotonic() - started + # The server asked for 0.25s; our own backoff would have been 5s. + assert 0.2 <= elapsed < 3.0 + finally: + store.close() + + def test_a_retry_after_header_is_parsed_off_the_wire(self, endpoint: Any) -> None: + endpoint.queue_poll(status=429, retry_after="0") + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint, initial_backoff=5.0) as store: + # If Retry-After were ignored the 5s backoff would blow the timeout. + assert store.wait_for_skills(timeout=3) is True + + @pytest.mark.parametrize("raw", ["inf", "Infinity", "-inf", "nan", "1e309"]) + def test_a_non_finite_retry_after_is_ignored(self, raw: str) -> None: + assert _retry_after_seconds({"Retry-After": raw}) is None + + def test_retry_after_parsing_keeps_its_edges(self) -> None: + assert _retry_after_seconds({"Retry-After": "0"}) == 0.0 + assert _retry_after_seconds({"Retry-After": "-5"}) == 0.0 + assert ( + _retry_after_seconds({"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}) + is None + ) + assert _retry_after_seconds({"Retry-After": "2.5"}) == 2.5 + + @pytest.mark.parametrize("retry_after", [float("inf"), float("nan"), 86400.0]) + def test_an_unreasonable_retry_after_neither_kills_delivery_nor_parks_it( + self, retry_after: float + ) -> None: + # An infinite wait would overflow inside the retry handler and kill the + # thread with `failed` still None; a day-long one would be honoured to + # the second. Both must fall back to the max_backoff cap and carry on. + requester = _ScriptedRequester( + _RecoverableTransportError("slow down", retry_after=retry_after), + [ + (e["event"], e["data"]) + for e in full_payload(("put-object", put_skill())) + ], + ) + store = stream_store(max_backoff=0.05, _requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=3) is True + assert store.failed is None + assert store._thread is not None and store._thread.is_alive() + finally: + store.close() + + def test_a_non_finite_retry_after_off_the_wire_falls_back_to_backoff( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(status=429, retry_after="inf") + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=3) is True + assert store.failed is None + + def test_backoff_is_exponential_and_capped(self) -> None: + assert _backoff_delay(1, base=1.0, maximum=30.0, jitter=0.0) == 1.0 + assert _backoff_delay(2, base=1.0, maximum=30.0, jitter=0.0) == 2.0 + assert _backoff_delay(3, base=1.0, maximum=30.0, jitter=0.0) == 4.0 + assert _backoff_delay(20, base=1.0, maximum=30.0, jitter=0.0) == 30.0 + + def test_jitter_never_exceeds_the_cap(self) -> None: + for attempt in range(1, 12): + for _ in range(50): + assert 0.0 <= _backoff_delay(attempt, base=1.0, maximum=5.0) <= 5.0 + + def test_a_malformed_polling_envelope_is_recoverable_not_fatal( + self, endpoint: Any + ) -> None: + store = FDv2SkillStore( + SDK_KEY, + mode="poll", + poll_interval=0.01, + initial_backoff=0.001, + _requester=_ScriptedRequester( + _RecoverableTransportError("polling response had no 'events' array") + ), + ) + try: + store.start() + assert wait_until(lambda: store.diagnostics.connection_failures >= 1) + assert store.failed is None + finally: + store.close() + + def test_a_listener_that_raises_does_not_kill_delivery(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill("first")))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("second")), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.add_listener(SKILL_OBJECT_KIND, lambda _raw: 1 / 0) + assert wait_until( + lambda: store.get_object(SKILL_OBJECT_KIND, "second") is not None + ) + assert store.failed is None + + # --------------------------------------------------------------------------- # The contentHash gap # --------------------------------------------------------------------------- @@ -854,6 +1662,63 @@ class TestMissingContentHash: itself and verify nothing. """ + async def test_a_hashless_skill_is_withheld_with_the_right_reason( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill(omit_hash=True)))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + outcome = await get_skill_result("pdf-extraction") + assert outcome.skill is None + assert outcome.reason == "integrity_failure" + assert await get_skill("pdf-extraction") is None + assert await all_skills() == [] + + async def test_the_object_is_still_held_so_the_outcome_is_not_absent( + self, endpoint: Any + ) -> None: + """ + Holding it is what makes the failure diagnosable. Dropping it at the + transport would report ``absent`` — indistinguishable from "no such + skill" — and would additionally let a prune delete the last known-good + copy already on disk. + """ + endpoint.queue_poll(full_payload(("put-object", put_skill(omit_hash=True)))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + raw = store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") + assert raw is not None + assert "contentHash" not in raw + await init_client(options={"skillStore": store}, client=object()) + assert (await get_skill_result("pdf-extraction")).reason != "absent" + + def test_the_store_counts_hashless_objects(self, endpoint: Any) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill("a", omit_hash=True)), + ("put-object", put_skill("b", omit_hash=True)), + ("put-object", put_skill("c")), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert store.diagnostics.hashless_objects == 2 + assert store.diagnostics.skill_objects_received == 3 + + def test_a_hashless_object_logs_an_error_naming_the_reason_code( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill(omit_hash=True)))) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + rendered = "\n".join(r.getMessage() for r in caplog.records) + assert "missing_content_hash" in rendered + assert "pdf-extraction" in rendered + assert "contentHash" in rendered + def test_a_redelivered_hashless_object_logs_once_per_store( self, caplog: Any ) -> None: @@ -895,3 +1760,456 @@ def test_two_live_stores_do_not_suppress_each_other(self, caplog: Any) -> None: drive(one, payload) drive(two, payload) assert len(_per_object_hashless_errors(caplog)) == 2 + + def test_a_wholly_hashless_payload_says_so_once( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill("a", omit_hash=True)), + ("put-object", put_skill("b", omit_hash=True)), + ) + ) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + summaries = [ + r + for r in caplog.records + if "No skill content will resolve" in r.getMessage() + ] + assert len(summaries) == 1 + assert "All 2 skill object(s)" in summaries[0].getMessage() + + def test_a_partly_hashed_payload_does_not_claim_total_failure( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill("a", omit_hash=True)), + ("put-object", put_skill("b")), + ) + ) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + rendered = "\n".join(r.getMessage() for r in caplog.records) + assert "No skill content will resolve" not in rendered + + async def test_a_hash_that_does_not_match_is_a_different_failure( + self, endpoint: Any + ) -> None: + """``missing_content_hash`` and ``hash_mismatch`` must not collapse: one + means the envelope carried no hash, the other means the content did not + match the hash it carried.""" + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(content_hash=_hash("something else"))) + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert store.diagnostics.hashless_objects == 0 + await init_client(options={"skillStore": store}, client=object()) + assert ( + await get_skill_result("pdf-extraction") + ).reason == "integrity_failure" + + async def test_a_hashed_skill_resolves_end_to_end(self, endpoint: Any) -> None: + """The positive control: a well-formed envelope resolves end to end.""" + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + skill = await get_skill("pdf-extraction") + assert skill is not None + assert skill.key == "pdf-extraction" + assert skill.version == 3 + assert skill.content == SKILL_BODY.encode("utf-8") + assert skill.content_hash == _hash(SKILL_BODY) + assert skill.name == "PDF Extraction" + + async def test_a_pinned_reference_resolves_to_the_pinned_object_version( + self, endpoint: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(object_version=2, content="v2 body")), + ("put-object", put_skill(object_version=5, content="v5 body")), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + pinned = await get_skill("pdf-extraction", version=2) + assert pinned is not None + assert pinned.content == b"v2 body" + newest = await get_skill("pdf-extraction") + assert newest is not None + assert newest.version == 5 + + async def test_the_payload_version_is_not_resolvable_as_a_skill_version( + self, endpoint: Any + ) -> None: + """ + The end-to-end form of the wire-key/``version`` assertion. + + Asking for the payload version resolves nothing — reported ``absent``, + because the store answers "I hold no such version" rather than answering + with the wrong one. The version that *does* resolve is the one after the + delimiter in the object's wire ``key``. + """ + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(object_version=3, payload_version=42)) + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + by_payload_version = await get_skill_result("pdf-extraction", version=42) + assert by_payload_version.skill is None + assert by_payload_version.reason == "absent" + assert await get_skill("pdf-extraction", version=3) is not None + + +# --------------------------------------------------------------------------- +# Server-side only +# --------------------------------------------------------------------------- + + +class TestServerSideOnly: + def test_a_mobile_key_is_refused(self) -> None: + with pytest.raises(ValueError, match="mobile key"): + FDv2SkillStore("mob-00000000-0000-4000-8000-000000000000") + + def test_a_client_side_environment_id_is_refused(self) -> None: + with pytest.raises(ValueError, match="client-side"): + FDv2SkillStore("0123456789abcdef01234567") + + def test_an_empty_credential_is_refused(self) -> None: + with pytest.raises(ValueError, match="server-side SDK key"): + FDv2SkillStore(" ") + + def test_a_server_side_key_is_accepted(self) -> None: + assert FDv2SkillStore(SDK_KEY) is not None + + def test_an_unrecognised_credential_shape_warns_but_is_allowed( + self, caplog: Any + ) -> None: + """Private instances and test doubles issue keys without the public prefix.""" + with caplog.at_level("WARNING"): + FDv2SkillStore("my-private-instance-credential") + assert any("server-side SDK key" in r.message for r in caplog.records) + + def test_an_unknown_mode_is_refused(self) -> None: + with pytest.raises(ValueError, match="stream"): + FDv2SkillStore(SDK_KEY, mode="mobile") # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# The eager re-reconcile, end to end over the transport +# --------------------------------------------------------------------------- + + +class TestWatchSkillsOverTheTransport: + """ + ``watch_skills`` against a live ``FDv2SkillStore``. The watcher's own + behaviour — debounce, refusal of a store without ``add_listener``, detaching + on close — is covered in ``test_skills_watch.py`` against the in-memory + store; these are the cases that only mean something with a transport + underneath: a wire-level revocation, a new skill version, and an outage. + """ + + async def test_a_revocation_prunes_without_a_restart( + self, endpoint: Any, tmp_path: Any + ) -> None: + """ + The store's change listener drives the reconcile, so the file goes away + seconds after the ``delete-object`` rather than at the next process start. + """ + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + + with poll_store(endpoint, poll_interval=0.2) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + report, watcher = await watch_skills( + "*", tmp_path / "skills", debounce=0.05 + ) + try: + written = tmp_path / "skills" / "pdf-extraction" / "SKILL.md" + assert written.exists() + assert any(a.action == "written" for a in report.actions) + assert wait_until(lambda: not written.exists(), timeout=10) + finally: + watcher.close() + + async def test_a_new_version_is_rewritten_without_a_restart( + self, endpoint: Any, tmp_path: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill(content="first")))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill(object_version=4, content="second")), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + + with poll_store(endpoint, poll_interval=0.2) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + assert written.read_text() == "first" + assert wait_until(lambda: written.read_text() == "second", timeout=10) + finally: + watcher.close() + + async def test_the_default_keeps_last_known_good_during_an_outage( + self, endpoint: Any, tmp_path: Any + ) -> None: + """``on_unavailable="keep"`` is the default: an outage must not read as + "everything was revoked".""" + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll(status=500) + with poll_store(endpoint, poll_interval=0.05) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + assert written.exists() + # ``last_error`` rather than ``connection_failures``: the counter + # resets on the next successful poll, so asserting on it races + # the retry that is supposed to happen. + assert wait_until( + lambda: store.diagnostics.last_error is not None, timeout=10 + ) + time.sleep(0.3) + assert written.exists() + finally: + watcher.close() + + +# --------------------------------------------------------------------------- +# Listener registration +# --------------------------------------------------------------------------- + + +class TestListenerRegistration: + @staticmethod + def _skill_listeners(store: Any) -> list[Any]: + return list(store._listeners.get(SKILL_OBJECT_KIND, [])) + + def test_fdv2_remove_listener_of_an_unregistered_callable_is_a_no_op( + self, endpoint: Any + ) -> None: + with poll_store(endpoint) as store: + store.remove_listener(SKILL_OBJECT_KIND, print) + store.add_listener(SKILL_OBJECT_KIND, print) + store.remove_listener("flag", print) + store.remove_listener(SKILL_OBJECT_KIND, print) + store.remove_listener(SKILL_OBJECT_KIND, print) + assert self._skill_listeners(store) == [] + + +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + + +class TestLifecycle: + def test_start_is_idempotent(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store = poll_store(endpoint) + try: + assert store.start() is store + assert store.start() is store + assert store.wait_for_skills(timeout=5) + finally: + store.close() + + def test_close_is_idempotent(self, endpoint: Any) -> None: + store = poll_store(endpoint) + store.start() + store.close() + store.close() + + def test_close_during_a_slow_connect_returns_promptly(self) -> None: + # Before the connect returns there is no connection for close() to + # interrupt. If the delivery thread then enters the read anyway, close() + # sits out its whole join timeout on a stream that will never speak. + requester = _SlowConnectRequester() + store = stream_store(_requester=requester) + store.start() + assert requester.entered.wait(timeout=5) + threading.Timer(0.1, requester.release.set).start() + started = time.monotonic() + store.close(timeout=5.0) + elapsed = time.monotonic() - started + assert elapsed < 2.0 + assert store._thread is not None + assert not store._thread.is_alive() + + def test_a_closed_store_still_answers_from_what_it_received( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store = poll_store(endpoint) + store.start() + store.wait_for_skills(timeout=5) + store.close() + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + + def test_wait_for_skills_times_out_rather_than_hanging(self) -> None: + store = FDv2SkillStore( + SDK_KEY, mode="poll", poll_interval=60, _requester=_ScriptedRequester() + ) + try: + assert store.wait_for_skills(timeout=0.05) is False + finally: + store.close() + + def test_the_store_satisfies_the_seam_before_it_starts(self) -> None: + store = FDv2SkillStore(SDK_KEY) + assert store.get_object(SKILL_OBJECT_KIND, "anything") is None + assert store.all_objects(SKILL_OBJECT_KIND) == {} + + +# --------------------------------------------------------------------------- +# Timeouts +# --------------------------------------------------------------------------- + + +class _BlackHole: + """ + A listening socket that accepts connections and never sends a byte. + + This is the host ``read_timeout`` exists for: the TCP handshake completes, so + nothing fails fast, and then no response ever comes. A request against it can + only end by timing out, which makes the elapsed time a direct measurement of + the timeout actually applied. + """ + + def __init__(self) -> None: + self._listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._listener.bind(("127.0.0.1", 0)) + self._listener.listen(8) + self._accepted: list[socket.socket] = [] + self._stop = threading.Event() + self._thread = threading.Thread(target=self._accept_forever, daemon=True) + self._thread.start() + host, port = self._listener.getsockname() + self.base_uri = f"http://{host}:{port}" + + def _accept_forever(self) -> None: + self._listener.settimeout(0.05) + while not self._stop.is_set(): + try: + conn, _ = self._listener.accept() + except OSError: + continue + self._accepted.append(conn) + + def close(self) -> None: + self._stop.set() + self._thread.join(timeout=2) + for conn in self._accepted: + conn.close() + self._listener.close() + + +@pytest.fixture +def black_hole() -> Any: + server = _BlackHole() + yield server + server.close() + + +class TestTimeouts: + """ + ``read_timeout`` is the only network timeout, and every request honours it. + + The bounds asserted here are loose on purpose: the point is that a request + against an unresponsive host fails in roughly ``read_timeout`` rather than in + minutes, and that a regression back to a much longer default fails this + suite quickly instead of hanging it. + """ + + def test_a_poll_against_an_unresponsive_host_fails_within_read_timeout( + self, black_hole: Any + ) -> None: + requester = _Requester(SDK_KEY, black_hole.base_uri, read_timeout=0.3) + started = time.monotonic() + with pytest.raises(_RecoverableTransportError) as excinfo: + requester.poll(None, None) + elapsed = time.monotonic() - started + assert 0.2 <= elapsed < 2.0 + assert "timed out" in str(excinfo.value) + + def test_a_stream_against_an_unresponsive_host_fails_within_read_timeout( + self, black_hole: Any + ) -> None: + requester = _Requester(SDK_KEY, black_hole.base_uri, read_timeout=0.3) + started = time.monotonic() + with pytest.raises(_RecoverableTransportError): + requester.stream(None) + assert time.monotonic() - started < 2.0 + + def test_the_store_reports_the_timeout_and_keeps_going( + self, black_hole: Any + ) -> None: + store = FDv2SkillStore( + SDK_KEY, + base_uri=black_hole.base_uri, + mode="poll", + poll_interval=0.05, + initial_backoff=0.01, + max_backoff=0.05, + read_timeout=0.3, + ) + try: + store.start() + assert wait_until(lambda: store.diagnostics.connection_failures >= 1) + assert store.failed is None + assert "timed out" in (store.diagnostics.last_error or "") + finally: + store.close() + + def test_the_default_bound_depends_on_the_mode(self) -> None: + assert DEFAULT_POLL_TIMEOUT == 10.0 + assert DEFAULT_STREAM_READ_TIMEOUT == 300.0 + polling = FDv2SkillStore(SDK_KEY, mode="poll") + streaming = FDv2SkillStore(SDK_KEY, mode="stream") + assert polling._requester._read_timeout == DEFAULT_POLL_TIMEOUT + assert streaming._requester._read_timeout == DEFAULT_STREAM_READ_TIMEOUT + + @pytest.mark.parametrize("mode", ["poll", "stream"]) + def test_an_explicit_read_timeout_overrides_the_default(self, mode: Any) -> None: + store = FDv2SkillStore(SDK_KEY, mode=mode, read_timeout=42.0) + assert store._requester._read_timeout == 42.0 + + @pytest.mark.parametrize("value", [0.0, -1.0, float("inf"), float("nan")]) + def test_a_non_positive_read_timeout_is_rejected(self, value: float) -> None: + with pytest.raises(ValueError, match="read_timeout"): + FDv2SkillStore(SDK_KEY, read_timeout=value) + + def test_there_is_no_separate_connect_timeout(self) -> None: + # ``urllib`` cannot bound the connect separately from the reads, so the + # constructor does not offer a parameter that would only pretend to. + with pytest.raises(TypeError): + FDv2SkillStore(SDK_KEY, connect_timeout=2.0) # type: ignore[call-arg] diff --git a/packages/client/tests/test_skills_watch.py b/packages/client/tests/test_skills_watch.py index 48a95f07..b44ffea0 100644 --- a/packages/client/tests/test_skills_watch.py +++ b/packages/client/tests/test_skills_watch.py @@ -4,7 +4,10 @@ The watcher is wired to the ``SkillStore`` interface, not to any one transport: it needs a store that implements ``add_listener``, and nothing more. These tests therefore drive it from ``InMemorySkillStore``, whose ``put`` notifies its -listeners synchronously, and from small hand-written store doubles. +listeners synchronously, and from small hand-written store doubles. The +end-to-end path — a ``delete-object`` arriving over a live FDv2 connection and +pruning a skill's files — is exercised in ``test_skills_fdv2.py``, where the fake +endpoint lives. Every test writes only inside pytest's ``tmp_path``. The watcher runs a real worker thread, so tests wait on observable outcomes rather than on fixed sleeps From c285ff533991b3e74a2d0b976a91de1008302596 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:04:26 -0400 Subject: [PATCH 14/16] fix(client): retry an FDv2 stream that dies mid-read _Requester.stream wrapped only the connect as recoverable, so a read timeout, reset or truncated chunk in the body reached the delivery loop as whatever the socket raised. The loop read that as a bug and gave up: delivery stopped for the process lifetime, taking updates and revocations with it, the first time a socket died. read_timeout exists to bound a stream that has gone quiet so the loop can reconnect, and tripping it did the opposite. The body now carries the same promise the connect already did. Wrapping the line source rather than the whole read keeps protocol reader errors out of it: those are raised from the consumer's loop body, where they still surface as the bugs they are. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fdv2.py | 21 ++++++- packages/client/tests/test_skills_fdv2.py | 61 +++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index da84a3b4..18b81842 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -1032,6 +1032,25 @@ def _decode_poll_body(body: bytes) -> list[tuple[str, Any]]: return events +def _iter_stream_lines(response: Any) -> Any: + """ + Yields a streaming body's raw lines, presenting a read failure as retryable. + + A live stream dies mid-body far more often than it refuses to open: a read + timeout on a stream that went quiet, a reset, a truncated chunk. Each of + those arrives as whatever the socket raised, and the delivery loop retries + only the transport errors this module defines — anything else it reads as a + bug and stops for the process lifetime. Connecting is already wrapped in + ``_Requester.stream``; this is the same promise for the body. + """ + try: + yield from response + except Exception as exc: + raise _RecoverableTransportError( + f"reading the FDv2 stream failed: {type(exc).__name__}: {exc}" + ) from exc + + def _iter_sse(response: Any) -> Any: """ Decodes an SSE body into ``(event name, data)`` pairs. @@ -1042,7 +1061,7 @@ def _iter_sse(response: Any) -> Any: try: name: str | None = None data_lines: list[str] = [] - for raw_line in response: + for raw_line in _iter_stream_lines(response): line = raw_line.decode("utf-8", errors="replace").rstrip("\r\n") if line == "": if name is not None: diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index 0cce2f0c..af5d1812 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -23,6 +23,7 @@ import socket import threading import time +from http.client import IncompleteRead from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, ClassVar from urllib.parse import parse_qs, urlparse @@ -52,6 +53,7 @@ _retry_after_seconds, _SkillObjectSet, _store_object_from_put, + _StreamConnection, _tombstone_from_delete, ) @@ -1287,6 +1289,41 @@ def test_content_survives_a_reconnect(self, endpoint: Any) -> None: # --------------------------------------------------------------------------- +class _DyingResponse: + """ + A streaming body that transfers a payload and then fails mid-read. + + This is how a live stream actually ends: not with a clean end of body but + with a read timeout on a stream that went quiet, or a reset from the server + or a proxy in between. + """ + + def __init__(self, exc: BaseException) -> None: + self._exc = exc + + def __iter__(self) -> Any: + for event in full_payload(("put-object", put_skill())): + yield f"event: {event['event']}\n".encode() + yield f"data: {json.dumps(event['data'])}\n".encode() + yield b"\n" + raise self._exc + + def close(self) -> None: + pass + + +class _DyingStreamRequester: + """Every connection transfers a payload, then dies with *exc* mid-read.""" + + def __init__(self, exc: BaseException) -> None: + self.connections = 0 + self._exc = exc + + def stream(self, basis: str | None) -> Any: + self.connections += 1 + return _StreamConnection(_DyingResponse(self._exc)) + + class _ScriptedConnection: """Stands in for ``_StreamConnection``: an event iterator plus a close.""" @@ -1477,6 +1514,30 @@ def test_recycled_stream_connections_are_not_failures(self) -> None: finally: store.close() + @pytest.mark.parametrize( + "exc", + [ + TimeoutError("timed out"), + ConnectionResetError(54, "Connection reset by peer"), + IncompleteRead(b"partial"), + ], + ids=["read timeout", "reset", "truncated body"], + ) + def test_a_stream_that_dies_mid_read_reconnects(self, exc: BaseException) -> None: + # A stream fails in its body far more often than at its connect, and + # ``read_timeout`` exists to bound one that has gone quiet. Treating + # such a failure as unexpected would stop delivery — including + # revocation — for the process lifetime the first time a socket died. + requester = _DyingStreamRequester(exc) + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + assert wait_until(lambda: requester.connections >= 5) + assert store.failed is None + finally: + store.close() + def test_a_stream_commit_resets_the_failure_count(self) -> None: payload = [ (e["event"], e["data"]) for e in full_payload(("put-object", put_skill())) From b7ee71a3477bad38d31123bed63e17a6203f3cbd Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Mon, 14 Sep 2026 14:36:58 -0400 Subject: [PATCH 15/16] fix(client): keep FDv2 delivery alive on an idle stream, and shut down promptly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three faults in the delivery loop, all of which left the store reporting itself healthy while doing less than it claimed. **An up-to-date stream tripped the failure cap.** The consecutive-failure count reset only at a commit, and an environment whose skills are not changing answers every reconnect with `intentCode: "none"` and transfers nothing. A stream only ever ends by being dropped, so each recycle of a perfectly healthy idle connection counted as a failure — announced with a `goodbye` or not — and `max_consecutive_failures + 1` of them stopped delivery for the process lifetime, revocations included. The reset on commit covered only the case where content had changed, which is the case that was easy to test and not the case that runs in production. `_TransferOutcome` now reports `up_to_date`, and a complete answer that transfers nothing breaks the row of failures exactly as a commit does. An intent this module does not recognise is still not an answer. **`close` could not interrupt a poll.** The interrupt reached the streaming connection only, so polling parked in its request with nothing to reach and `close` returned when its join timed out — on a 300s-class request, long after the process meant to exit. `_Requester` now tracks the response of a poll in flight and offers `interrupt`, which `close` calls alongside the stream's own. A request still inside its connect has no response to reach; that one is bounded by `read_timeout`, and `start` no longer leaves the store inert when a join times out around it. An interrupt we asked for is no longer recorded as a delivery failure. **`close` left a waiter parked.** `wait_for_skills` waited on the first payload alone, so a shutdown racing a waiter added the waiter's whole timeout to it. Delivery ending is now its own event: a waiter is released by a payload, a give-up or a close, and reports whether a payload actually arrived rather than merely that it was let go. That also settles what `_give_up` had been quietly asserting — it set the first-payload flag to unblock waiters, which made `wait_for_skills` answer `True` for a store holding nothing. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fdv2.py | 123 ++++++- packages/client/tests/test_skills_fdv2.py | 315 +++++++++++++++++- 2 files changed, 420 insertions(+), 18 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 18b81842..6a84092e 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -487,6 +487,14 @@ class _TransferOutcome: basis: str | None = None fatal: str | None = None disconnect: str | None = None + up_to_date: bool = False + """ + The server said what we hold is current and it has nothing to transfer. + + A complete answer that commits nothing, which is exactly what a 304 is to a + poll. The delivery loop counts it as a healthy connection; see + ``FDv2SkillStore._apply``. + """ class _ProtocolReader: @@ -576,6 +584,10 @@ def _server_intent(self, data: Any) -> _TransferOutcome: if intent != _INTENT_TRANSFER_NONE: logger.debug("Ignoring FDv2 server-intent with intentCode %r", intent) self._pending = None + # ``none`` is a complete answer that carries nothing. An intent this + # module does not recognise is not an answer at all, so only the + # former reports itself up to date. + return _TransferOutcome(up_to_date=intent == _INTENT_TRANSFER_NONE) return _TransferOutcome() def _target_for(self, data: Any) -> _SkillObjectSet | None: @@ -942,6 +954,25 @@ def __init__( self._read_timeout = read_timeout # Injectable so tests can drive a fake endpoint without a socket. self._opener = opener or urllib.request.build_opener() + self._lock = threading.Lock() + # The response of a poll in flight, so ``interrupt`` can reach its + # socket from another thread. Polling only: a stream's response is + # handed straight to the caller as a ``_StreamConnection``, which + # carries an interrupt of its own. + self._in_flight: Any = None + + def interrupt(self) -> None: + """ + Unblocks a poll parked in its body read, from another thread. + + Best effort, and safe to call when nothing is in flight. A request still + inside its connect has no response to reach yet and is bounded only by + ``read_timeout``; ``FDv2SkillStore.start`` covers what that leaves. + """ + with self._lock: + response = self._in_flight + if response is not None: + _interrupt_read(response) def _url(self, path: str, basis: str | None) -> str: """ @@ -972,11 +1003,17 @@ def poll(self, basis: str | None, etag: str | None) -> _PollResult: request = self._request(POLL_PATH, basis, headers) try: with self._opener.open(request, timeout=self._read_timeout) as response: - status = getattr(response, "status", None) or response.getcode() - if status == 304: - return _PollResult(not_modified=True, events=[], etag=etag) - body = response.read() - new_etag = response.headers.get("ETag") or etag + with self._lock: + self._in_flight = response + try: + status = getattr(response, "status", None) or response.getcode() + if status == 304: + return _PollResult(not_modified=True, events=[], etag=etag) + body = response.read() + new_etag = response.headers.get("ETag") or etag + finally: + with self._lock: + self._in_flight = None except urllib.error.HTTPError as exc: if exc.code == 304: # urllib raises on 304 when no redirect handler swallows it. @@ -1225,6 +1262,18 @@ def __init__( self._stop = threading.Event() self._first_payload = threading.Event() + """A payload has committed. The fact ``wait_for_skills`` reports.""" + self._delivery_ended = threading.Event() + """ + Delivery has stopped, by ``close`` or by ``_give_up``. Kept apart from + ``_first_payload`` because it is not one: a waiter has to be let go + either way, but only a payload makes ``wait_for_skills`` true. + """ + self._released = threading.Event() + """ + Either of the two above, and what a waiter actually parks on: an + ``Event`` cannot wait on two, so the setters funnel through here. + """ self._thread: threading.Thread | None = None self._failed_reason: str | None = None # The open streaming connection, so ``close`` can interrupt its read. @@ -1244,7 +1293,13 @@ def start(self) -> FDv2SkillStore: Does not block: use ``wait_for_skills`` when boot ordering matters. """ with self._lock: + self._rearm_waiters() if self._thread is not None and self._thread.is_alive(): + # A ``close`` whose join timed out leaves the previous thread + # running with the stop flag still set. Clearing it lets that + # thread carry on delivering, rather than leaving a store that + # reports itself started and never delivers again. + self._stop.clear() return self self._stop.clear() self._thread = threading.Thread( @@ -1253,6 +1308,16 @@ def start(self) -> FDv2SkillStore: self._thread.start() return self + def _rearm_waiters(self) -> None: + """ + Re-arms ``wait_for_skills`` for a store being started again after a + ``close``. A payload already held stays an answer; an ended delivery + does not, or the next waiter would be released before it began. + """ + self._delivery_ended.clear() + if not self._first_payload.is_set(): + self._released.clear() + def close(self, timeout: float = 5.0) -> None: """ Stops delivery. Idempotent, and safe to call from any thread. @@ -1262,12 +1327,17 @@ def close(self, timeout: float = 5.0) -> None: package-level ``launchdarkly_ai_server.shutdown()`` coroutine. """ self._stop.set() + # A waiter parked in ``wait_for_skills`` is owed an answer now rather + # than at the end of its timeout; delivery is over either way. + self._end_delivery() # The delivery thread is normally blocked in a socket read that no flag - # can reach; without this the join waits out its full timeout. + # can reach; without this the join waits out its full timeout. Streaming + # parks in the connection, polling in the request, so interrupt both. with self._lock: connection = self._connection if connection is not None: connection.close() + self._requester.interrupt() thread = self._thread if ( thread is not None @@ -1288,8 +1358,24 @@ def wait_for_skills(self, timeout: float = 10.0) -> bool: ``True`` means a payload arrived — not that any skill in it verified, and not that the environment has any skills. ``diagnostics`` answers the rest. + + Returns early, ``False``, when delivery ends before any payload does: + a ``close`` from another thread, or a failure delivery cannot retry. + Waiting out the full timeout for an answer that has already arrived + would delay every shutdown that raced a waiter. """ - return self._first_payload.wait(timeout=timeout) + self._released.wait(timeout=timeout) + return self._first_payload.is_set() + + def _publish_first_payload(self) -> None: + """Records the first committed payload and lets any waiter go.""" + self._first_payload.set() + self._released.set() + + def _end_delivery(self) -> None: + """Records that delivery has stopped and lets any waiter go.""" + self._delivery_ended.set() + self._released.set() @property def failed(self) -> str | None: @@ -1383,6 +1469,11 @@ def _run(self) -> None: self._give_up(str(exc)) return except _RecoverableTransportError as exc: + if self._stop.is_set(): + # ``close`` interrupted the request on purpose. Counting it + # would spend a retry from the bounded budget and leave a + # misleading ``last_error`` on a healthy store. + return with self._lock: self._failures += 1 failures = self._failures @@ -1432,8 +1523,8 @@ def _give_up(self, reason: str) -> None: "the process restarts with a working connection.", reason, ) - # Unblock anyone waiting on a first payload that is never coming. - self._first_payload.set() + # Let go of anyone waiting on a first payload that is never coming. + self._end_delivery() def _apply(self, name: str, data: Any) -> None: """ @@ -1444,10 +1535,16 @@ def _apply(self, name: str, data: Any) -> None: outcome = self._reader.handle(name, data) if outcome.committed and outcome.basis is not None: self._basis = outcome.basis - if outcome.committed: - # A commit breaks the row of consecutive failures. + if outcome.committed or outcome.up_to_date: + # Both break the row of consecutive failures: a commit is a payload + # delivered, and ``up_to_date`` is the server confirming we already + # hold it. Counting only the commit would give up on a healthy + # stream serving an environment whose skills are not changing: + # nothing to transfer means no commit, while every recycled + # connection still ends in a drop. self._record_success() - self._first_payload.set() + if outcome.committed: + self._publish_first_payload() if outcome.changes: self._notify(outcome.changes) if outcome.fatal: @@ -1465,7 +1562,7 @@ def _poll_once(self) -> None: logger.debug("Skill payload unchanged (HTTP 304)") # A 304 counts as a first payload, so a boot that reconnects with a # cached basis is not blocked on a transfer the server will not send. - self._first_payload.set() + self._publish_first_payload() return for name, data in result.events: self._apply(name, data) diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index af5d1812..d063e451 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -538,6 +538,17 @@ def test_a_full_transfer_commits_at_payload_transferred(self) -> None: assert outcomes[-1].committed is True assert outcomes[-1].basis == "basis-1" + def test_an_up_to_date_intent_is_reported_as_such(self) -> None: + """``intentCode: "none"`` is the stream's 304: current, nothing to send.""" + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle("server-intent", server_intent("none")) + assert outcome.up_to_date is True + assert outcome.committed is False + assert outcome.disconnect is None + # A transfer intent is a promise of content, not an up-to-date answer. + transfer = reader.handle("server-intent", server_intent("xfer-full")) + assert transfer.up_to_date is False + def test_nothing_is_visible_before_payload_transferred(self) -> None: """A payload version is the unit of consistency; half of one is not a state.""" held = _SkillObjectSet() @@ -1312,7 +1323,17 @@ def close(self) -> None: pass -class _DyingStreamRequester: +class _FakeRequester: + """ + Base for the requester fakes: supplies the ``interrupt`` the store calls on + ``close``, so each fake only scripts the part it is about. + """ + + def interrupt(self) -> None: + """No real socket to reach; these fakes end their own connections.""" + + +class _DyingStreamRequester(_FakeRequester): """Every connection transfers a payload, then dies with *exc* mid-read.""" def __init__(self, exc: BaseException) -> None: @@ -1335,7 +1356,7 @@ def close(self) -> None: self.closed = True -class _ScriptedRequester: +class _ScriptedRequester(_FakeRequester): """Raises a scripted sequence, so backoff is asserted without real sockets.""" def __init__(self, *outcomes: Any) -> None: @@ -1361,7 +1382,7 @@ def stream(self, basis: str | None) -> Any: return _ScriptedConnection(outcome) -class _RecyclingRequester: +class _RecyclingRequester(_FakeRequester): """ A healthy server that recycles connections: every ``stream`` call succeeds, transfers a full payload, and then ends the connection, as LaunchDarkly and @@ -1383,6 +1404,53 @@ def stream(self, basis: str | None) -> Any: ) +class _UpToDateRecyclingRequester(_FakeRequester): + """ + A healthy server with nothing new to say: every connection answers + ``intentCode: "none"`` — the stream's equivalent of a 304 — transfers + nothing, and is then recycled. This is the steady state of an environment + whose skills are not changing, which is most environments most of the time. + """ + + def __init__(self, farewell: bool = False) -> None: + self.connections = 0 + self._farewell = farewell + + def stream(self, basis: str | None) -> Any: + self.connections += 1 + script: list[tuple[str, Any]] = [ + ("server-intent", server_intent("none")), + ("heart-beat", {}), + ] + if self._farewell: + # A recycle is often announced rather than abrupt. + script.append(("goodbye", {"reason": "connection recycled"})) + return _ScriptedConnection(script) + + +class _SlowPollRequester(_FakeRequester): + """ + A poll whose request does not return until the test releases it, standing in + for one blocked where no interrupt can reach: inside its connect. + """ + + def __init__(self) -> None: + self.entered = threading.Event() + self.release = threading.Event() + + def poll(self, basis: str | None, etag: str | None) -> Any: + self.entered.set() + self.release.wait(timeout=10) + raise _RecoverableTransportError("released") + + +class _SilentStreamRequester(_FakeRequester): + """A stream that connects and then delivers nothing until it is closed.""" + + def stream(self, basis: str | None) -> Any: + return _BlockingConnection() + + class _BlockingConnection: """A stream that never produces an event until it is closed.""" @@ -1398,7 +1466,7 @@ def close(self) -> None: self._closed.set() -class _SlowConnectRequester: +class _SlowConnectRequester(_FakeRequester): """ A ``stream`` whose connect does not return until the test releases it, standing in for a slow TLS handshake, followed by a read that never yields. @@ -1447,7 +1515,11 @@ def test_a_fatal_failure_releases_wait_for_skills_rather_than_hanging( ) -> None: endpoint.queue_poll(status=401) with poll_store(endpoint) as store: - assert store.wait_for_skills(timeout=5) is True + started = time.monotonic() + # Released promptly, and ``False``: no payload arrived, and saying + # otherwise would send a caller on to read a store holding nothing. + assert store.wait_for_skills(timeout=5) is False + assert time.monotonic() - started < 2.0 assert store.failed is not None def test_a_fatal_failure_keeps_last_known_good_servable( @@ -1514,6 +1586,28 @@ def test_recycled_stream_connections_are_not_failures(self) -> None: finally: store.close() + @pytest.mark.parametrize("farewell", [False, True], ids=["dropped", "goodbye"]) + def test_an_up_to_date_recycled_stream_is_not_a_failure( + self, farewell: bool + ) -> None: + # Resetting at a commit covers only a connection that carried new + # content. An environment whose skills are not changing answers every + # reconnect with ``intentCode: "none"`` and transfers nothing, so a loop + # that counted those drops would give up on a *healthy* idle stream + # after max_consecutive_failures + 1 recycles — and revocation, the one + # thing streaming exists to deliver promptly, would never arrive again. + requester = _UpToDateRecyclingRequester(farewell=farewell) + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert wait_until(lambda: requester.connections >= 8) + assert store.failed is None + # As with a payload-carrying recycle, the count may read 1 + # mid-reconnect. What it must never do is climb. + assert store.diagnostics.connection_failures <= 1 + finally: + store.close() + @pytest.mark.parametrize( "exc", [ @@ -2194,6 +2288,64 @@ def close(self) -> None: self._listener.close() +class _StalledBody: + """ + A listening socket that answers with headers and then stalls the body. + + Distinct from ``_BlackHole``: here the request succeeds far enough to hand + urllib a response, and the caller then parks in ``read``. That is the state + ``close`` has to interrupt — and, unlike a request still inside its connect, + the state an interrupt can actually reach. + """ + + def __init__(self) -> None: + self._listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._listener.bind(("127.0.0.1", 0)) + self._listener.listen(8) + self._accepted: list[socket.socket] = [] + self.serving = threading.Event() + self._stop = threading.Event() + self._thread = threading.Thread(target=self._serve_forever, daemon=True) + self._thread.start() + host, port = self._listener.getsockname() + self.base_uri = f"http://{host}:{port}" + + def _serve_forever(self) -> None: + self._listener.settimeout(0.05) + while not self._stop.is_set(): + try: + conn, _ = self._listener.accept() + except OSError: + continue + self._accepted.append(conn) + try: + conn.recv(4096) + # A length far longer than the body that follows, so the read + # blocks rather than seeing the end of the message. + conn.sendall( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" + b"Content-Length: 4096\r\n\r\n" + ) + except OSError: + continue + self.serving.set() + + def close(self) -> None: + self._stop.set() + self._thread.join(timeout=2) + for conn in self._accepted: + conn.close() + self._listener.close() + + +@pytest.fixture +def stalled_body() -> Any: + server = _StalledBody() + yield server + server.close() + + @pytest.fixture def black_hole() -> Any: server = _BlackHole() @@ -2274,3 +2426,156 @@ def test_there_is_no_separate_connect_timeout(self) -> None: # constructor does not offer a parameter that would only pretend to. with pytest.raises(TypeError): FDv2SkillStore(SDK_KEY, connect_timeout=2.0) # type: ignore[call-arg] + + +class TestWaitingForSkills: + """ + ``wait_for_skills`` answers with what happened, and never outlives it. + + Its budget is a boot-ordering allowance, not a delay to spend: a store that + already knows no payload is coming owes the caller that answer immediately. + """ + + def test_close_releases_a_waiter_rather_than_leaving_it_parked(self) -> None: + # A shutdown racing a waiter is the ordinary case, not an exotic one: + # ``close`` on the main thread while a worker is still waiting for its + # first payload. Parking that worker for the rest of its timeout adds + # the whole budget to a process that has already decided to stop. + store = stream_store(_requester=_SilentStreamRequester()) + store.start() + answers: list[bool] = [] + waiter = threading.Thread( + target=lambda: answers.append(store.wait_for_skills(timeout=10)), + daemon=True, + ) + waiter.start() + time.sleep(0.2) + started = time.monotonic() + store.close() + waiter.join(timeout=5) + assert not waiter.is_alive() + assert time.monotonic() - started < 2.0 + assert answers == [False] + + def test_a_payload_already_held_still_answers_true_after_close(self) -> None: + # ``close`` does not drop content, so it must not turn the answer about + # that content into a lie either. + requester = _RecyclingRequester() + store = stream_store(_requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + finally: + store.close() + assert store.wait_for_skills(timeout=5) is True + + def test_a_restarted_store_waits_again(self) -> None: + # The released flag is sticky by design, so a store closed before any + # payload and then started again has to re-arm: otherwise the next + # waiter is let go before delivery has had a chance to begin. + store = stream_store(_requester=_SilentStreamRequester()) + store.start() + store.close() + assert store.wait_for_skills(timeout=0.1) is False + store.start() + try: + started = time.monotonic() + assert store.wait_for_skills(timeout=0.5) is False + # Waited, rather than being released by the previous close. + assert time.monotonic() - started >= 0.4 + finally: + store.close() + + +class TestPollShutdown: + """ + ``close`` has to interrupt a poll in flight, as it already does a stream. + + Without it the delivery thread stays parked in its request and ``close`` + returns only when the join times out — on a 300s-class request, long after + the process meant to exit. The bound is loose on purpose: the point is + promptly rather than a particular number of milliseconds. + """ + + def test_interrupt_unblocks_a_poll_stalled_in_its_body( + self, stalled_body: Any + ) -> None: + requester = _Requester(SDK_KEY, stalled_body.base_uri, read_timeout=30.0) + raised: list[BaseException] = [] + + def poll_until_interrupted() -> None: + try: + requester.poll(None, None) + except BaseException as exc: + raised.append(exc) + + thread = threading.Thread(target=poll_until_interrupted, daemon=True) + thread.start() + assert stalled_body.serving.wait(timeout=5) + # The response is in hand; give the read a moment to park in it. + time.sleep(0.2) + started = time.monotonic() + requester.interrupt() + thread.join(timeout=5) + assert not thread.is_alive() + assert time.monotonic() - started < 2.0 + assert raised and isinstance(raised[0], _RecoverableTransportError) + + def test_close_during_a_stalled_poll_returns_promptly( + self, stalled_body: Any + ) -> None: + store = FDv2SkillStore( + SDK_KEY, + base_uri=stalled_body.base_uri, + mode="poll", + poll_interval=0.05, + read_timeout=30.0, + ) + store.start() + assert stalled_body.serving.wait(timeout=5) + time.sleep(0.2) + started = time.monotonic() + store.close(timeout=5.0) + assert time.monotonic() - started < 2.0 + assert store._thread is not None and not store._thread.is_alive() + + def test_a_poll_we_interrupted_is_not_a_delivery_failure( + self, stalled_body: Any + ) -> None: + # Our own shutdown is not an outage: counting it would spend a retry + # from the bounded budget and leave a misleading ``last_error`` behind + # on a store whose content is still perfectly good. + store = FDv2SkillStore( + SDK_KEY, + base_uri=stalled_body.base_uri, + mode="poll", + poll_interval=0.05, + read_timeout=30.0, + ) + store.start() + assert stalled_body.serving.wait(timeout=5) + time.sleep(0.2) + store.close(timeout=5.0) + assert store.diagnostics.connection_failures == 0 + assert store.diagnostics.last_error is None + assert store.failed is None + + def test_a_close_that_timed_out_leaves_the_store_restartable(self) -> None: + # A request blocked inside its connect is beyond any interrupt, so + # ``close`` can still return with the thread alive. ``start`` must not + # then find that thread and return with the stop flag set: the store + # would report itself started and never deliver again. + requester = _SlowPollRequester() + store = FDv2SkillStore( + SDK_KEY, mode="poll", poll_interval=0.01, _requester=requester + ) + try: + store.start() + assert requester.entered.wait(timeout=5) + store.close(timeout=0.2) + assert store._thread is not None and store._thread.is_alive() + store.start() + assert store._stop.is_set() is False + finally: + requester.release.set() + store.close(timeout=2) From 88c225ebc296613713d32fcfa3f7fb474ffe6495 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Mon, 14 Sep 2026 14:47:44 -0400 Subject: [PATCH 16/16] fix(client): log a recycled FDv2 stream at debug rather than warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A stream only ever ends by being dropped, and LaunchDarkly — and any proxy in between — recycles a long-lived one. Every reconnect therefore logged "Skill delivery failed" at WARNING, for as long as the process ran. Until the previous commit that noise was bounded, because an idle stream gave up after eleven recycles and went quiet; now that delivery correctly survives them, it would run forever and describe a healthy store as failing. A connection that got a complete answer before it ended — a committed payload, or an up-to-date intent — delivered everything it was asked for, so its reconnect is now DEBUG and says so. A connection that ended without answering is the case the warning exists for and still gets it: a connect that never landed, or a transfer that died part-way through. Filling a customer's logs with a fault they do not have is not merely untidy; it teaches them that the level which means something can be ignored. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fdv2.py | 27 ++++++++++++++-- packages/client/tests/test_skills_fdv2.py | 32 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 6a84092e..71411c8a 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -1283,6 +1283,10 @@ def __init__( # by being dropped, so resetting on return would count every healthy, # server-recycled connection as a failure. self._failures = 0 + # Whether the current attempt got a complete answer before it ended. + # A stream only ever ends by being dropped, so this is what separates + # a recycled healthy connection from one that failed. + self._attempt_answered = False # -- lifecycle --------------------------------------------------------- @@ -1456,6 +1460,8 @@ def _notify(self, changes: list[dict[str, Any]]) -> None: def _run(self) -> None: while not self._stop.is_set(): + with self._lock: + self._attempt_answered = False try: if self._mode == "stream": self._stream_once() @@ -1477,6 +1483,7 @@ def _run(self) -> None: with self._lock: self._failures += 1 failures = self._failures + answered = self._attempt_answered self._reader.diagnostics.connection_failures = failures self._reader.diagnostics.last_error = str(exc) if failures > self._max_consecutive_failures: @@ -1494,9 +1501,22 @@ def _run(self) -> None: # The header may come from a proxy rather than LaunchDarkly, and # a value in the hours would park revocation for that long. delay = min(delay, self._max_backoff) - logger.warning( - "Skill delivery failed (%s); retrying in %.1fs", exc, delay - ) + if answered: + # LaunchDarkly, and any proxy in between, recycles a + # long-lived stream. A connection that answered before it + # ended delivered everything it was asked for, so the + # reconnect is routine rather than a fault worth warning + # about for as long as the process runs. + logger.debug( + "The FDv2 stream ended after a complete answer (%s); " + "reconnecting in %.1fs", + exc, + delay, + ) + else: + logger.warning( + "Skill delivery failed (%s); retrying in %.1fs", exc, delay + ) if self._stop.wait(delay): return continue @@ -1511,6 +1531,7 @@ def _run(self) -> None: def _record_success(self) -> None: with self._lock: self._failures = 0 + self._attempt_answered = True self._reader.diagnostics.connection_failures = 0 def _give_up(self, reason: str) -> None: diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index d063e451..b0c6aa86 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -1608,6 +1608,38 @@ def test_an_up_to_date_recycled_stream_is_not_a_failure( finally: store.close() + def test_a_recycled_connection_reconnects_quietly(self, caplog: Any) -> None: + # A healthy idle stream reconnects for as long as the process runs, so + # warning on each one would fill a customer's logs with a fault they do + # not have and teach them to ignore the level that means something. + requester = _UpToDateRecyclingRequester() + store = stream_store(_requester=requester) + with caplog.at_level("DEBUG", logger="launchdarkly_ai_server.skills_fdv2"): + try: + store.start() + assert wait_until(lambda: requester.connections >= 5) + finally: + store.close() + assert store.failed is None + assert not [r for r in caplog.records if r.levelname == "WARNING"] + assert [r for r in caplog.records if "reconnecting in" in r.getMessage()] + + def test_a_connection_that_never_answered_still_warns(self, caplog: Any) -> None: + # The quiet path is earned by answering. A connection that failed before + # it told us anything is the case the warning exists for. + store = stream_store( + max_consecutive_failures=10, _requester=_ScriptedRequester() + ) + with caplog.at_level("DEBUG", logger="launchdarkly_ai_server.skills_fdv2"): + try: + store.start() + assert wait_until(lambda: store.diagnostics.connection_failures >= 3) + finally: + store.close() + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert warnings + assert all("Skill delivery failed" in r.getMessage() for r in warnings) + @pytest.mark.parametrize( "exc", [