diff --git a/specs/TRAINING.md b/specs/TRAINING.md index a4e39a7f..687b939a 100644 --- a/specs/TRAINING.md +++ b/specs/TRAINING.md @@ -368,6 +368,462 @@ Mneme stays independent and its ledger MUST NOT be truncated by recording retent Rewind MUST use records when present; otherwise it is best effort and the trial MUST be labelled as such. +## Rewind + +```sh +spawnfile train ./Spawnfile --agent agent:writer --materialize train:ice-shooting --history ./history --out ./moment +``` + +`--materialize ` MUST require the existing `--out` option and follows +the plan's exact-agent selection (inferred only for a single-agent project). +It requires neither `--train` nor `--training-config`. It is mutually exclusive +with `--plan`. `--history` is OPTIONAL and applies only to materialization. The +command resolves the authoring plan and selected example using local files. It +MUST NOT invoke cognition, Paideia, Docker, authentication, network access, or +compiled-artifact emission. `materializeTrainingMoment({plan, project, example, +historyDir?, out})` exposes the same operation to trial preparation. Selection, +input enumeration, and historical cuts are available as pure library functions. + +The example selector is an ID from the agent's plan targets. If that ID occurs +in several dataset/role pairs it MUST be qualified as `train:`, +`validation:`, or `test:`. Repeated references to the same dataset, role, +and example are one candidate. A qualified selector that still matches several +datasets MUST refuse. Ambiguity errors MUST list role, ID, and dataset for every +candidate. + +### Inputs and history + +Inputs are EVERY effective item in the agent's compiled `workspaceResources`, +accessible `memoryAccess` banks deduplicated by bank source, and Moltnet networks +in its resolved surface attachments, including representative attachments. Each +input is identified by `{section, id, declaredBy}`, using the same declaring-node +resolution as authoring-plan overrides. Sections are `resources`, `memory`, and +`networks`. A dataset example's full declaration replaces the effective item +with the same section and ID; the plan guarantees exactly one match. Relative +paths in replacement declarations resolve from the dataset file's directory. +Every input MUST materialize successfully; present-day data MUST NOT substitute +for unavailable history. + +Records and host logs MUST use the merged [Moment record](#moment-record) and +[Host histories and retention](#host-histories-and-retention) contracts. Rewind +uses their shared parsers and lookup helpers. The local history import layout is: + +```text +DIR/ + moments//--.json + deployments/.resource-log.jsonl # project deployment (choose one log) + deployments//resource-log.jsonl # home deployment (alternative) + bundles/.tar + git/// # local repository mirror + feeds///landings.jsonl + feeds///repo/ # local git source mirror + feeds///trees// # preserve .N re-land suffixes + memory///events.jsonl + moltnet///state.json | moltnet.sqlite + recording//wake-moments.jsonl + recording//snapshots/// + recording//snapshots//.manifest.json + recording//snapshots//.complete +``` + +`` MUST be `encodeURIComponent` of the node ID: `agent:writer` +becomes `agent%3Awriter`. `` is the selected compiled node's existing +slug, including the compiler's collision handling. `` is the +`record.volumes` resource ID. IDs and archive names MUST be safe path segments. +Every read MUST remain inside `DIR` lexically and after realpath resolution; +symlink escapes refuse. JSON and every JSONL line are validated by their owning contract; +invalid lines refuse the affected input with file, line number, and reason. + +Import copies deployment evidence verbatim: the deployment state directory's +`moments/` subtree (including agent slug and hashed filenames); exactly one resource +log from the project output or home `deployments/` location above; the +host feed state's `landings.jsonl` beside `landed.json`; retained feed volume +`trees/` directories; deployed bundle tars from `container/workspace-bundles/`; +Daimon's persisted `recording//` rows, snapshot trees, manifests and completion markers; +Mneme's `/memory/events.jsonl`; and the network's consistent Moltnet store +(including SQLite recovery sidecars when needed). The `git/` repositories and +feed `repo/` directories are local mirrors retaining historical commits, not +recorder output. Rewind neither fetches nor changes these mirrors. + +The log retains its original absolute `deployment_id`; that identity is passed to +`deployResourcesAt` without opening the original path. Its name/location MUST mirror +`deployResourceLogPath(deployment_id)`. Multiple logs, mixed deployment identities, +invalid lines, or ambiguous pins refuse; omit the log if it is unavailable. Resource +lookup is scoped to the exact compiled agent and, when present, `declared_by`. +Moment discovery also accepts nested JSONL imports (one shared-format record per line). +Feed history MUST include the required transition boundary; older entries without it +are invalid under the merged contract and refuse. + +Moment records match exact compiled `agent` and parsed `wake.at == example.at`. +For several attempts of that wake, select the EARLIEST `wake.startedAt` (original +turn), even when replay pins differ or append/capture order differs. Conflicting +copies at that same earliest instant (including replays sharing BOTH execution ID +and startedAt but differing in row hash) still refuse when any input's effective +decision differs: usable pin versus fallback, selected pin, or fallback fidelity +given `exact` and input-specific `errors`. The refusal MUST name the input. +Unrelated errors and unusable pin values do not create a conflict. Discovery sorts +paths lexically, so equivalent earliest copies yield a deterministic representative; +row hash, capture order and execution ID MUST NOT resolve differing input decisions. +Validate recorded commit IDs in every earliest-attempt copy for each used input, +including IDs hidden by declared pin precedence; later replays cannot replace +or invalidate the original turn's pins. + +The report retains dataset `at` and adds `effectiveAt = selected wake.startedAt`, +or `effectiveAt = at` if unmatched. Task-clock origin, historical selection, and +leak bounds MUST use `effectiveAt`. Report the matching count, selected execution +ID, selected start instant, and `earliest-startedAt` choice. Record matching and +payload cuts compare parsed instants with RFC3339 nanosecond +precision, never strings or UUIDs. Host landing/deployment selection uses the merged +helpers' millisecond host-clock contract. + +An `exact:false` record does not invalidate healthy pins: a present pin is used +with `exact` fidelity and `record` basis (`record-cursor` for Moltnet), unless its +`errors` names that input. A record error containing `Deployment activation is non-exact` +instead downgrades every git/bundle pin used from that record to `approximate`, +preserving `record` basis and the error in notes; those pins are still used. +This covers inexact activations and rollback intervals even though the marker +does not name individual resources. Errored pins are ignored, absent pins use fallback +rules, and those inputs are labelled `approximate`, with the record's error text +in notes. Explicit section or declaring-node qualifications apply only to that +input; unqualified ID errors conservatively apply to every input with that ID. +A missing fallback still refuses. No matching record uses the normal +rules below; independent immutable git pins and unambiguous landing logs may +still be exact. + +### Per-input reconstruction and fidelity + +Each successful input reports `fidelity: exact | approximate | overridden`, a +`basis`, and the selected source identifiers. Explicit overrides report +`overridden` and `basis: override`. A replacement git/bundle declaration must +still supply locally available historical bytes. A replacement memory bank, +network, or volume starts fresh: an empty bank/volume, or an empty network store +with its declared rooms. A network override writes a fresh `state.json` snapshot +regardless of its declared storage backend; trial preparation consumes or imports +that snapshot. With no `--history`, only locally materializable pins +and explicit overrides can succeed; everything else refuses. + +Git resources select a full 40- or 64-hex declared +commit first (`exact`, `pinned`), otherwise the matching record's resource +commit (`exact`, `record`). With no usable moment pin, consult the imported +deployment log at `effectiveAt` using `deployResourcesAt`; use the selected commit +with `basis: deploy-log`. Fidelity is `exact` only when `activation_exact` is true +and `exact` is not false; inexact activation boundaries and inclusive rollback +intervals MUST remain `approximate`, with the activation/rollback reason in notes. +The merged helper excludes a rolled-back candidate after its interval and restores +the predecessor. A pin that is present but unavailable in the repository refuses. +Only an absent deployment pin falls back to the declared selector's +first-parent history, choosing its newest commit by committer time among commits +whose author AND committer times are `<= effectiveAt` +(`approximate`, `selector-at`); an absent selector means `HEAD`. Commit dates +do not establish when a commit reached a branch or tag, so this fallback cannot +prove historical ref membership. A missing eligible commit refuses. +The selected commit's author AND committer dates MUST be `<= effectiveAt` before any +git checkout or bundle extraction, including pinned, recorded, and overridden +git sources. All paths MUST use the same selected-commit guard. + +Git output MUST be a clean new repository containing only objects reachable +from the selected commit, with one checked-out branch, no remotes, tags, source +packed refs, reflogs, `FETCH_HEAD`, or `ORIG_HEAD`. Git commands use argv without +a shell and sanitized configuration. History MUST NOT be mutated. The leak +check also rejects any reachable ancestor with author or committer time after +`effectiveAt`. Rebuild the checkout index without live stat data; acceptance checks +every index entry's ctime and mtime against `effectiveAt`. Acceptance also compares +the worktree paths, raw bytes, symlink targets, and executable modes against the +selected commit's tree in the output repository, refusing untracked/ignored extras. +Bundle `build.files` extracts `files.root` at the selected commit and honors +`exclude`. `build.dependencies` and `build.generated` retain their source +directory or declared inputs at that commit and report `build: deferred`; +trial preparation owns builds. Bundle output MAY be a plain tree. A moment record +or, when absent, the deployment log pins a bundle by SHA-256 only. +Exact bundle output requires `bundles/.tar` whose actual digest +matches that pin (`record` or `deploy-log` basis). A record or deploy-log digest retains +the activation/rollback fidelity above even when its bytes verify. A present +deployment bundle pin without a matching archive refuses; it MUST NOT silently +select a different source commit. Validate the archive before extraction: escaping paths, +symlinks, hard links, devices, and other special entries refuse. Acceptance +reopens the archive and compares output paths, bytes, and executable modes. +For a moment-record bundle pin, absent or digest-mismatched archives fall back to +source reconstruction where available, always `approximate` even with a pinned source commit; notes explain +that the recorded SHA-256 could not be reproduced. A prebuilt `source` tar has +no historical source fallback and refuses unless overridden. +Source-built bundles record their resolved roots, prefixes and exclusions in +`source.trees`. Acceptance re-exports that selection from the source repository +and byte-compares the result, including when a local pin supplies the repository. +Generated inputs spanning multiple repositories or independently pinned input +refs cannot be represented by this one-repository/one-commit history contract +and MUST refuse. Bundle roots whose historical repository prefix cannot be +established also refuse. Feed and bundle extraction MUST use raw tree/blob objects +(`git ls-tree -r -z` and `git cat-file`, with `GIT_ATTR_NOSYSTEM=1`), without +attribute-driven transforms, `export-subst`, or `export-ignore` semantics. +Symlinks MUST be recreated as symlinks and pass the independent escape check; +gitlink/submodule entries MUST refuse with an explicit unsupported-history error. + +Fed volumes use a matching record pin with `exact`, `record`, including when no +landing log is retained. Otherwise, lookup inside the inclusive +`[switching_at, landed_at]` window selects the incoming entry with `approximate`, +`landing-log` and an ambiguous-transition note. Outside windows, choose the +newest `landed_at <= effectiveAt`, breaking equal timestamps by last append +(`exact`, `landing-log`). Before the first entry, there is no landed state: +a nonempty log without an applicable landing refuses unless a matching record +supplies the state or the feed is explicitly overridden. + +Without a record pin or a nonempty landing log, a plain git feed evaluates its ref template +and paths at `effectiveAt`, including `${date}` in UTC and `${date:}`. +Use the primary ref only if it has an eligible first-parent commit; otherwise +try the fallback. Its newest eligible commit is `approximate`, `feed-rule-at`. +Command refs, directory feeds, prepared/include recipes and freeze/waiting state +cannot be reconstructed this way and refuse. Commit dates do not establish ref +availability or historical freeze holds. + +Before choosing retained bytes versus extraction, the selected feed source +commit MUST be verified as a commit object with author AND committer dates +`<= effectiveAt`. This single selection guard applies equally to plain, retained, +prepared, and included git feeds; an eligible landing time does not make a +future source commit eligible. Directory feeds have no git source commit. +Plain git feed bytes MUST come from re-extraction of the landing's expanded paths +at its selected commit, with revision verification using the existing feed +revision function. When retained `trees//` bytes also exist, they MUST +match that extraction (paths, types, executable bits, symlink targets and bytes); +any mismatch refuses. A missing history repository refuses even when retained +bytes exist. Acceptance MUST independently re-extract from the history repository +at the selected commit and paths and compare the bundle bytes and revision. +`include` or `prepare` requires retained bytes; rewind MUST NOT run them. +Prepared revisions identify source/include/recipe provenance, not output content. +When prepared provenance is retained, its composite revision MUST match, and its +git source revision MUST match the selected commit and paths. Prepared output +MUST always be `approximate`, with the note `prepared output not content-verifiable` +and `source.prepared: true` in the moment input (including record-only pins without +a landing log). The independent acceptance gate enforces that fidelity and note. +All leak-check classes still apply to prepared output. +Raw blob extraction ignores repository-local `info/attributes` as well as committed +archive attributes; later tags cannot change historical file bytes. +Output preserves the agent-visible `trees//`, including re-land suffixes, relative `current` +symlink, and `.spawnfile-feed.json` (`spawnfile.volume-feed.v1`). Its `landed_at` +is the selected landing time, or the selected commit's committer time for an +feed-rule approximation. A recorded pin or ambiguous incoming transition may +have publication metadata after `effectiveAt`; this does not advance the trial +clock or filesystem timestamps. The independent gate reopens its record/window +before allowing such publication metadata. Plain directory feeds are not +refetchable: retained tree bytes MUST recompute to the recorded revision or refuse. +Prepared directory bytes MUST NOT be compared to their provenance revision as +though it were a directory content digest. + +Memory uses Mneme's single append-only ledger. Any recorded `generation`, +including 0, refuses as unsupported: Mneme has no generations or curation +archives. There is no `generations.jsonl` reader. A record's `seq` selects the +first `seq` ledger lines (`exact`, `record`); `seq:0` is an exact empty bank and +requires no ledger. Otherwise selection stops before the FIRST event after +`effectiveAt` (`approximate`, `created-at-prefix`): a subsequent clock regression +never reopens the cut. The prefix MUST end in LF, contain contiguous sequences +from 1, and only nondecreasing event times `<= effectiveAt`. Legacy seq-less +events are allowed only as a leading run. Invalid sequence or time ordering refuses. + +Memory output is `memory///memory/events.jsonl`. It MUST exclude +`index.sqlite`, `causal*`, `authority-receipts.jsonl`, and `authority-receipts/`. +Spawnfile validates the ledger structurally without importing Mneme: every line +MUST be a JSON event with an instant `createdAt`, and duplicate JSON keys refuse. +Sequence, leading legacy run, LF termination, and prefix time ordering are checked +locally. Event checksums are retained but not verified: Mneme 0.1.1 hashes +`JSON.stringify(event)` (`node_modules/@noopolis/mneme/dist/identity/ids.js:15-17`) +before the persisted event normalizes/defaults fields and injects its sequence +(`node_modules/@noopolis/mneme/dist/store/store.js:249-274`); the original input +and its property order cannot be recovered from the ledger alone. + +In installed Mneme 0.1.1, the index is rebuilt from the ledger before every index +read: embedding recall at `node_modules/@noopolis/mneme/dist/runtime/runtime.js:86-87`, +`memory.search` at `node_modules/@noopolis/mneme/dist/kernel/kernel.js:152-153`, +and `memory.locate` at `node_modules/@noopolis/mneme/dist/kernel/kernel.js:207-208`. +These sites read the store then call `this.index.rebuildFromEvents(allEvents)`. +Opening an index alone does NOT rebuild it. A materialized bank without +`index.sqlite` is complete because this index is derived state; trial preparation +MUST preserve those rebuild-before-read consumers. Rewind requires no Mneme runtime +dependency and does not create even a temporary index. +Missing bank history refuses unless the bank is overridden or recorded at `seq:0`. + +Moltnet JSON and SQLite stores remove messages, rooms, and attachment-delivery +events after `effectiveAt`. Empty threads/DMs and threads/DMs whose root is after the cut +are dropped. Thread/DM message counts and last-message times are recomputed; +attachment cursors are clamped/remapped to retained events. Room removals after +`effectiveAt` are undone when removal time and restorable room metadata exist; a missing +historical removal time retains current removal state. Restoration MUST refuse +when its only available room metadata comes from a future event. +Agents registered after `effectiveAt` and their memberships and delivery cursors +are removed; retained agents' `updated_at` and SQLite migrations' `applied_at` +are clamped to `effectiveAt`. Both store inspectors reject future registration +timestamps; the SQLite inspector also rejects future migration timestamps. +Other registration metadata and memberships have no historical log, explicitly +disclosed in input `notes`: even a cursor-exact content cut cannot prove historical membership. +SQLite changes apply only to a copy through `node:sqlite` on the supported +Node >=22.19 line; if that API is unavailable, SQLite rewind refuses. RFC3339Nano +fractions with omitted trailing zeros MUST be compared as parsed instants. +A record cursor is an instant, MUST equal `wake.startedAt`, and cuts messages at +`created_at <= cursor` (`exact`, `record-cursor`). When `messageIds` is present, +every listed wake message MUST survive the cut; otherwise refuse. With no pin, +cut at `effectiveAt` (`approximate`, `created-at-cut`). Missing stores refuse +unless overridden; UUIDs never order messages. + +Unfed volumes consume the Daimon wake-moment contract required by +[Moment record](#moment-record), including its nonempty deliveries, snapshot name +constraints, and `.complete` publication marker. Read rows with the shared recording +parser. Trees are full point-in-time copies; time selection MUST use row `started_at`, +not the snapshot name. A record names its exact snapshot (`exact`, `record`); +otherwise use the newest complete snapshot row `<= effectiveAt` +(`approximate`, `snapshot-at`). Both the tree and manifest MUST have a published +sibling `.complete` marker. `.partial` directories never count. +Missing named history or no complete eligible snapshot refuses. Independent +acceptance rechecks the marker, manifest and matching wake row. + +The sibling manifest contains `{version,id,snapshot,started_at,sourcePath,entries}`. +Entries map relative paths to `{type,mode,size,source,target?}`: type is file, +directory or symlink; size is a decimal string; source preserves recorder +`dev`, `ino`, `size`, `mtimeNs`, `ctimeNs`, and `mode`; target is symlink text. +Copy file bytes into new inodes; do not hard-link into the bundle. Verify paths, +entry types, modes, file/symlink sizes and targets against the manifest; missing +or extra entries refuse. Restore source directory modes (the recorder protects +retained directories as 0700). Directory inode size and copied inode identity +are filesystem metadata, not portable tree invariants. Verify their manifest +structure but not equality to new inodes. Acceptance independently checks the +manifest and compares copied bytes with the retained tree, catching same-size +byte mutations even though recorder manifests contain no content digest. + +All materialized files and directories MUST have atime/mtime `<= effectiveAt`, using the +source commit, landing, event, or snapshot instant capped at `effectiveAt`. Symlink times +MUST be set with `lutimes`; following a symlink to set times is insufficient. + +### Task clock + +`task-clock.json` is strict `spawnfile.task-clock.v1` with `{version, origin}`; +`origin` is `effectiveAt` normalized to ISO-8601 with `Z`. Trial preparation +supplies one safe-integer real launch anchor in milliseconds. All processes MUST +share `originEpochMs + (realNowEpochMs - anchorEpochMs)`; independently started +children MUST NOT reset their clock to the origin. `originEpochMs` uses JavaScript +`Date.parse(origin)` millisecond precision. + +Both dataset `at` and selected `effectiveAt` MUST be within the inclusive range +`1970-01-01T00:00:00Z` .. `9999-12-31T23:59:59.999Z`, compared as instants before +millisecond truncation. Out-of-range trials MUST refuse before any materialization +or output creation. This range is shared with Daimon's ecosystem task-clock +contract. Historical source timestamps may precede 1970; filesystem stamping and +restoration MUST preserve their negative epochs using `Date` or numeric-string +arguments to `utimes`/`lutimes`, never negative numeric seconds. + +Each process MUST have exactly one offset owner. These pure helpers replace +`taskClockEnvironment`; their outputs MUST NOT be merged for a single process: + +| Helper `(origin, anchorEpochMs)` | Recipient | Returned variables | +| --- | --- | --- | +| `harnessTaskClockEnvironment` | Top-level agent harness (Daimon), with an unshifted process clock | Only `NOOPOLIS_TASK_CLOCK`: compact JSON `{"version":"noopolis.task-clock.v1","origin":"","anchorEpochMs":}` | +| `processClockEnvironment` | Child/tool process under libfaketime | Only `FAKETIME`: signed relative seconds `(originEpochMs - anchorEpochMs) / 1000`, and `FAKETIME_DONT_FAKE_MONOTONIC=1` | +| `clockAwareEnvironment` | Clock-aware child (such as Mneme) without libfaketime | Only `MNEME_CLOCK_ORIGIN`: normalized origin, and `MNEME_CLOCK_ANCHOR_MS`: decimal anchor milliseconds | + +Daimon consumes the top-level contract and derives each child's environment itself. +Trial preparation supplies the installed libfaketime path through `LD_PRELOAD` for +the process-clock case. The helper cannot choose an image-specific library path. +That child's environment MUST omit `NOOPOLIS_TASK_CLOCK`, `MNEME_CLOCK_ORIGIN`, +and `MNEME_CLOCK_ANCHOR_MS`, including inherited values. A clock-aware child MUST +instead omit libfaketime from `LD_PRELOAD` and omit `FAKETIME` and +`NOOPOLIS_TASK_CLOCK`. Applying both offsets would shift time twice. The top-level +harness MUST NOT itself run under libfaketime or receive the Mneme offset variables. + +Relative `FAKETIME` MUST include a sign, use seconds without a suffix, and preserve +milliseconds (three fractional digits when nonzero; no fraction for integral +seconds). Integer-second rounding and the `@` start-at form are forbidden. +[Upstream libfaketime documentation](https://github.com/wolfcw/libfaketime#readme), +sections 4d/4e, supports fractional signed offsets since 0.8 and notes that the +decimal delimiter depends on locale. Trial preparation MUST ensure a dot-decimal +numeric locale (for example `LC_ALL=C`) for these values. + +Shared Spawnfile/Daimon conformance vectors: + +| Origin | Anchor epoch ms | `FAKETIME` | +| --- | --- | --- | +| `2001-01-01T00:00:00.000Z` | `1821692800250` | `-843385600.250` | +| `2001-01-01T00:00:00.000Z` | `1821692860250` | `-843385660.250` | +| `2001-01-01T00:00:00.250Z` | `978307200000` | `+0.250` | +| `2001-01-01T00:00:00.000Z` | `978307200001` | `-0.001` | +| `2001-01-01T00:00:00.000Z` | `978307200000` | `+0` | +| `2001-01-01T00:00:00.000Z` | `978307199000` | `+1` | + +The clock offsets reported wall time; real timeouts and monotonic elapsed time +MUST remain real. Rewind neither sleeps on nor advances a synthetic scheduler. +Trial preparation owns the anchor and runtime environment; the immutable bundle +carries only the origin. + +### Independent leak check and output + +The leak checker MUST re-read the bundle from disk independently of materializer +state. Its strict `spawnfile.leak-check.v1` result contains `{version, at, ok, +checked, findings}`; counters cover files, git objects, memory events, Moltnet +messages/events, and snapshots. Each finding carries `{input, class, path, +detail}`; `input` is an input identity or `null` for bundle-level problems. +Metadata is captured before payload reads; the checker restores the captured access +times after inspection, preserving pre-existing future atimes for refusal. +After restoration and rename, filesystem metadata MUST be checked again on the +final published bundle. A failure removes the payload and records the refusal; +the pre-publication check alone cannot establish the published timestamps. +Every git-backed input MUST record its selected full commit ID in +`moment.json`'s `inputs[].source.commit`; git feed identity MUST record the same +ID. Acceptance MUST independently open the source history repository read-only, +verify that ID names a commit object, and inspect its raw author and committer +dates against `effectiveAt`, regardless of retained bytes or normalized filesystem times. +Missing/unreadable provenance and inconsistent feed commit IDs also refuse. +History repositories are resolved by input identity from `--history`. A local +pin or git-backed override resolved without that history entry MUST additionally +record its absolute local repository path in `source.repository`; acceptance +reopens it read-only. That source repository must remain available for subsequent +verification. Directory feeds and digest-pinned or explicitly overridden prebuilt tars have no git +provenance. Recorded bundles are independently checked against their digest-pinned archive. This source check is independent of the materialized git object's +reachable-history check and MUST NOT reuse the producer's date assertion. +Native payload layouts use exact entry allowlists: a feed volume contains only +`current`, `.spawnfile-feed.json`, and `trees/`; a memory bank contains +only `memory/events.jsonl`; a Moltnet directory contains exactly one of +`state.json` or `moltnet.sqlite`. Bundle-level entries are limited to the input +paths and their directory ancestors plus `moment.json`, `leak-check.json`, and +`task-clock.json`. Unexpected entries refuse even with historical file times. +Every finding refuses the moment: + +| Class | Refusal condition | +| --- | --- | +| `bundle-entry` | An entry outside the input paths, their directory ancestors, and the report allowlist is present. | +| `file-time` | A file, directory, or symlink atime or mtime is after `effectiveAt`. | +| `symlink-escape` | An absolute link or a lexically/physically escaping link resolves outside the bundle. | +| `git-object` | Source provenance is missing, inconsistent, not a commit, or has author/committer time after `effectiveAt`; any materialized commit has a future date, any object is unreachable from the selected commit, or refs/remotes/reflogs/fetch metadata exceed the single branch. Any `.git` directory/gitfile or bare repository (`HEAD` + `objects/` + `refs/`) anywhere outside the materialized git resource repository also refuses, including inside feed trees and bundles. | +| `memory-event` | Future ledger events, sequence gaps, invalid prefix ordering, or unexpected bank entries are present. | +| `moltnet-message` / `moltnet-event` / `moltnet-room` | Retained messages, delivery events, rooms, registration timestamps, or migration timestamps are after `effectiveAt`, or the store directory contains unexpected entries. | +| `feed-identity` | Unproved publication time after `effectiveAt`, a directory revision mismatch, `current` outside its selected tree, or unexpected volume entries/retained trees. | +| `dated-path` | A path segment contains a calendar date `YYYY-MM-DD` later than the UTC date of `effectiveAt + 14h`, the latest possible local date. | +| `snapshot` | An unfed volume differs from its selected full snapshot manifest or retained bytes. | + +`OUT` MUST be absent or an empty directory. The materializer builds in a sibling +staging directory and renames into place only after the disk leak check passes: + +```text +OUT/ + moment.json + leak-check.json + task-clock.json + resources///... + memory///memory/events.jsonl + moltnet///... +``` + +`moment.json` is strict `spawnfile.materialized-moment.v1`: status, agent, +`example: {id, role, dataset}`, normalized `at`, `effectiveAt`, sorted inputs, and `record: {matched, attempts?, +selectedStartedAt?, executionId?, selection?}`, +and `leakCheck: {ok, findings}`. Inputs sort by section, declaring node, then ID; +each reports kind, fidelity (or `null` on refusal), basis, source identifiers, +relative output path (or `null`), notes, and refusal reason (or `null`). Deferred +bundle builds additionally report `build: deferred`. + +On any input refusal or leak, staging is removed and `OUT` contains ONLY +`moment.json` with `status: refused`, all refusal reasons, and `leak-check.json` +if the check ran. Partial input data and task-clock files MUST NOT remain usable. +Success prints `moment.json` to stdout as JSON and exits 0. Refusal emits a +one-line stderr summary and exits 1. Invalid flags, projects, or example +selections can fail before a moment exists and need not create a report. +Generated schemas for the three bundle documents appear below; schema-sync +tests MUST keep them identical to the exported runtime contracts. + ## Public handoff The child invocation is `paideia train --spawnfile-context FILE` followed by the @@ -1318,3 +1774,483 @@ The child deadline is Paideia's declared deadline plus five seconds for cleanup. } ``` + +## Materialized moment JSON Schema + + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "spawnfile.materialized-moment.v1" + }, + "status": { + "type": "string", + "enum": [ + "materialized", + "refused" + ] + }, + "agent": { + "type": "string", + "minLength": 1 + }, + "example": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "role": { + "type": "string", + "enum": [ + "train", + "validation", + "test" + ] + }, + "dataset": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "id", + "role", + "dataset" + ], + "additionalProperties": false + }, + "at": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "effectiveAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "inputs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "section": { + "type": "string", + "enum": [ + "resources", + "memory", + "networks" + ] + }, + "id": { + "type": "string", + "minLength": 1 + }, + "declaredBy": { + "type": "string", + "minLength": 1 + }, + "kind": { + "type": "string", + "enum": [ + "git", + "bundle", + "feed", + "volume", + "memory", + "moltnet" + ] + }, + "fidelity": { + "anyOf": [ + { + "type": "string", + "enum": [ + "exact", + "approximate", + "overridden" + ] + }, + { + "type": "null" + } + ] + }, + "basis": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "object", + "properties": { + "commit": { + "type": "string", + "minLength": 1 + }, + "repository": { + "type": "string", + "minLength": 1 + }, + "revision": { + "type": "string", + "minLength": 1 + }, + "seq": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "trees": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "properties": { + "root": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "exclude": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": [ + "root" + ], + "additionalProperties": false + } + }, + "sha256": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + }, + "agentSlug": { + "type": "string", + "minLength": 1 + }, + "snapshot": { + "type": "string", + "minLength": 1 + }, + "cursor": { + "type": "string", + "minLength": 1 + }, + "switchingAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "landedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "prepared": { + "type": "boolean", + "const": true + } + }, + "additionalProperties": false + }, + "path": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "null" + } + ] + }, + "notes": { + "type": "array", + "items": { + "type": "string" + } + }, + "refusal": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "null" + } + ] + }, + "build": { + "type": "string", + "const": "deferred" + } + }, + "required": [ + "section", + "id", + "declaredBy", + "kind", + "fidelity", + "basis", + "source", + "path", + "notes", + "refusal" + ], + "additionalProperties": false + } + }, + "record": { + "type": "object", + "properties": { + "matched": { + "type": "boolean" + }, + "attempts": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "selectedStartedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "executionId": { + "type": "string", + "minLength": 1 + }, + "selection": { + "type": "string", + "const": "earliest-startedAt" + } + }, + "required": [ + "matched" + ], + "additionalProperties": false + }, + "leakCheck": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "findings": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "ok", + "findings" + ], + "additionalProperties": false + } + }, + "required": [ + "version", + "status", + "agent", + "example", + "at", + "effectiveAt", + "inputs", + "record", + "leakCheck" + ], + "additionalProperties": false +} +``` + + +## Leak check JSON Schema + + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "spawnfile.leak-check.v1" + }, + "at": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "ok": { + "type": "boolean" + }, + "checked": { + "type": "object", + "properties": { + "files": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "gitObjects": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "memoryEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "moltnetMessages": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "moltnetEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "snapshots": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "files", + "gitObjects", + "memoryEvents", + "moltnetMessages", + "moltnetEvents", + "snapshots" + ], + "additionalProperties": false + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "input": { + "anyOf": [ + { + "type": "object", + "properties": { + "section": { + "type": "string", + "enum": [ + "resources", + "memory", + "networks" + ] + }, + "id": { + "type": "string", + "minLength": 1 + }, + "declaredBy": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "section", + "id", + "declaredBy" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "class": { + "type": "string", + "enum": [ + "bundle-entry", + "file-time", + "symlink-escape", + "git-object", + "memory-event", + "moltnet-message", + "moltnet-event", + "moltnet-room", + "feed-identity", + "dated-path", + "snapshot" + ] + }, + "path": { + "type": "string", + "minLength": 1 + }, + "detail": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "input", + "class", + "path", + "detail" + ], + "additionalProperties": false + } + } + }, + "required": [ + "version", + "at", + "ok", + "checked", + "findings" + ], + "additionalProperties": false +} +``` + + +## Task clock JSON Schema + + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "spawnfile.task-clock.v1" + }, + "origin": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "version", + "origin" + ], + "additionalProperties": false +} +``` + diff --git a/src/cli/AGENTS.md b/src/cli/AGENTS.md index d280c66c..43e96d65 100644 --- a/src/cli/AGENTS.md +++ b/src/cli/AGENTS.md @@ -13,7 +13,7 @@ src/cli/ ├── composedLifecycleContractSet.ts # Closed machine command/contract inventory ├── evidenceExportHelperCommand.ts # Local helper construction command ├── compileBuildCommands.ts # `compile` and `build` command registration -├── trainCommand.ts # Canonical agent selection and Paideia CLI option forwarding +├── trainCommand.ts # Canonical plan/rewind selection and Paideia CLI option forwarding ├── paideiaDelegation.ts # Private versioned context handoff, child lifecycle and completion receipts ├── paideiaSupervisor.ts # Packaged group leader retaining identity until native child/group cleanup ├── lifecycleCommands.ts # Thin lifecycle/compile/build/run/publish/up/down registration composition diff --git a/src/cli/runCli.ts b/src/cli/runCli.ts index fd97c30e..9f96f3fa 100644 --- a/src/cli/runCli.ts +++ b/src/cli/runCli.ts @@ -24,7 +24,7 @@ import { clearProjectModelFallbacks, compileProject, createTrainingContext, - createTrainingPlan, + createTrainingPlan, loadTrainingProject, resolveTrainingPlan, materializeTrainingMoment, initProject, listInitTemplates, preflightCompilePlan, @@ -77,9 +77,7 @@ import { delegatePaideiaTraining } from "./paideiaDelegation.js"; const packageJsonPath = new URL("../../package.json", import.meta.url); const readPackageVersion = (): string => { - const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8")) as { - version?: unknown; - }; + const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8")) as { version?: unknown }; return typeof packageJson.version === "string" ? packageJson.version : "0.0.0"; }; @@ -105,6 +103,8 @@ const createDefaultRenderEnvironment = (): CliRenderEnvironment => ({ export interface CliHandlers { createTrainingContext: typeof createTrainingContext; createTrainingPlan: typeof createTrainingPlan; + loadTrainingProject: typeof loadTrainingProject; resolveTrainingPlan: typeof resolveTrainingPlan; + materializeTrainingMoment: typeof materializeTrainingMoment; delegatePaideiaTraining: typeof delegatePaideiaTraining; buildCompilePlan: typeof buildCompilePlan; buildOrganizationView: typeof buildOrganizationView; buildProject: typeof buildProject; compileProject: typeof compileProject; @@ -139,7 +139,7 @@ export interface CliHandlers { } const createDefaultHandlers = (): CliHandlers => ({ - createTrainingContext, createTrainingPlan, delegatePaideiaTraining, + createTrainingContext, createTrainingPlan, loadTrainingProject, resolveTrainingPlan, materializeTrainingMoment, delegatePaideiaTraining, buildCompilePlan, buildOrganizationView, buildProject, compileProject, publishProject, addAgentProject, addProjectModelFallback, addProjectSurface, addSubagentProject, addTeamProject, clearProjectModelFallbacks, diff --git a/src/cli/trainCommand.test.ts b/src/cli/trainCommand.test.ts index e402a640..387e1e5d 100644 --- a/src/cli/trainCommand.test.ts +++ b/src/cli/trainCommand.test.ts @@ -5,9 +5,13 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { runCli, type CliHandlers } from "./runCli.js"; import type { DelegatePaideiaTrainingOptions } from "./paideiaDelegation.js"; +import { + agentManifest, cleanTrainingProjects, dataFiles, example, trainingProject, volume +} from "../compiler/training/plan/fixtures.test-helper.js"; const directories: string[] = []; afterEach(async () => { await Promise.all(directories.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); }); +afterEach(cleanTrainingProjects); async function project(): Promise { const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "spawnfile-train-command-"))); directories.push(root); @@ -24,6 +28,120 @@ const emptyPlan = (root: string): Awaited { + it.each([example.id, `train:${example.id}`])("materializes %s locally without delegated training flags", async (selector) => { + const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest() }); + const out = path.join(root, "rewound"), stdout: string[] = [], stderr: string[] = []; + const forbidden = vi.fn(async () => { throw Error("must not execute, authenticate, or compile output"); }); + const code = await runCli(["train", path.join(root, "Spawnfile"), "--agent", "agent:writer", + "--materialize", selector, "--out", out], { + handlers: { createTrainingContext: forbidden, delegatePaideiaTraining: forbidden, compileProject: forbidden, + buildProject: forbidden, runProject: forbidden, upProject: forbidden, publishProject: forbidden, + importClaudeCodeAuth: forbidden, importCodexAuth: forbidden, importGrokAuth: forbidden, + requireAuthProfile: forbidden, provisionCredentials: forbidden }, + streams: { stdout: (value) => stdout.push(value), stderr: (value) => stderr.push(value) } + }); + expect(code).toBe(0); + expect(stderr).toEqual([]); + expect(stdout).toHaveLength(1); + const moment = JSON.parse(stdout[0]!); + expect(moment).toMatchObject({ status: "materialized", agent: "agent:writer", + example: { id: example.id, role: "train", dataset: "training/train.yaml" }, inputs: [], + record: { matched: false }, leakCheck: { ok: true, findings: 0 } }); + expect(JSON.parse(await readFile(path.join(out, "moment.json"), "utf8"))).toEqual(moment); + expect(JSON.parse(await readFile(path.join(out, "task-clock.json"), "utf8"))).toEqual({ + version: "spawnfile.task-clock.v1", origin: "2026-10-09T08:20:31Z" + }); + expect(forbidden).not.toHaveBeenCalled(); + }); + + it("lists ambiguous roles and materializes a qualified selection", async () => { + const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest(), + "training/test.yaml": { examples: [example] } }); + const out = path.join(root, "rewound"), stderr: string[] = []; + const options = { streams: { stdout() {}, stderr: (value: string) => stderr.push(value) } }; + const args = ["train", root, "--agent", "agent:writer", "--out", out, "--materialize"]; + expect(await runCli([...args, example.id], options)).toBe(1); + expect(stderr).toHaveLength(1); + expect(stderr[0]).toContain(`train:${example.id} (training/train.yaml)`); + expect(stderr[0]).toContain(`test:${example.id} (training/test.yaml)`); + expect(await readdir(root)).not.toContain("rewound"); + expect(await runCli([...args, `test:${example.id}`], options)).toBe(0); + expect(JSON.parse(await readFile(path.join(out, "moment.json"), "utf8")).example.role).toBe("test"); + }); + + it("retains the plan's single-agent inference when --agent is omitted", async () => { + const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest() }); + const stdout: string[] = []; + expect(await runCli(["train", root, "--materialize", example.id, "--out", path.join(root, "out")], { + streams: { stdout: (value) => stdout.push(value), stderr() {} } + })).toBe(0); + expect(JSON.parse(stdout[0]!).agent).toBe("agent:writer"); + }); + + it("forwards a loaded project, resolved plan, history and the existing --out option", async () => { + const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest() }); + const materializeTrainingMoment = vi.fn(async ({ plan }) => ({ + version: "spawnfile.materialized-moment.v1", status: "materialized", agent: plan.agent.id, + example: { id: example.id, role: "train", dataset: "training/train.yaml" }, at: example.at, effectiveAt: example.at, + inputs: [], record: { matched: false }, leakCheck: { ok: true, findings: 0 } + })); + const stdout: string[] = []; + expect(await runCli(["train", root, "--agent", "agent:writer", "--materialize", example.id, + "--history", "history with spaces", "--out", "out with spaces"], { + handlers: { materializeTrainingMoment }, streams: { stdout: (value) => stdout.push(value), stderr() {} } + })).toBe(0); + expect(materializeTrainingMoment).toHaveBeenCalledOnce(); + expect(materializeTrainingMoment.mock.calls[0]![0]).toMatchObject({ + project: { root, manifest: path.join(root, "Spawnfile") }, + plan: { agent: { id: "agent:writer" }, producer: { version: packageVersion } }, + example: example.id, historyDir: "history with spaces", out: "out with spaces" + }); + expect(JSON.parse(stdout[0]!).status).toBe("materialized"); + }); + + it("returns one-line stderr and exit 1 for a refused moment with report-only output", async () => { + const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest({ + workspace: { docs: { system: "AGENTS.md" }, resources: [volume] } + }) }); + const out = path.join(root, "refused"), stdout: string[] = [], stderr: string[] = []; + expect(await runCli(["train", root, "--agent", "agent:writer", "--materialize", example.id, "--out", out], { + streams: { stdout: (value) => stdout.push(value), stderr: (value) => stderr.push(value) } + })).toBe(1); + expect(stdout).toEqual([]); + expect(stderr).toHaveLength(1); + expect(stderr[0]).toMatch(/^error: Moment refused: 1 input refusal\(s\), 0 leak finding\(s\)/u); + expect(stderr[0]).not.toContain("\n"); + expect(await readdir(out)).toEqual(["moment.json"]); + expect(JSON.parse(await readFile(path.join(out, "moment.json"), "utf8"))).toMatchObject({ + status: "refused", inputs: [{ id: volume.id, refusal: expect.any(String) }] + }); + expect((await readdir(root)).filter((name) => name.includes("staging"))).toEqual([]); + }); + + it.each([ + ["--materialize", example.id, "--agent", "agent:writer"], + ["--materialize", example.id, "--agent", "agent:writer", "--out", "output", "--plan"], + ["--history", "history", "--plan"] + ])("rejects incomplete or conflicting local mode options %j before reads", async (...flags) => { + const forbidden = vi.fn(async () => { throw Error("must not read project"); }); + expect(await runCli(["train", "/missing", ...flags], { + handlers: { loadTrainingProject: forbidden, createTrainingPlan: forbidden, materializeTrainingMoment: forbidden, + createTrainingContext: forbidden, delegatePaideiaTraining: forbidden }, streams: { stdout() {}, stderr() {} } + })).toBe(2); + expect(forbidden).not.toHaveBeenCalled(); + }); + + it("registers --out exactly once alongside --materialize and --history", async () => { + const stdout: string[] = []; + expect(await runCli(["train", "--help"], { streams: { stdout: (value) => stdout.push(value), stderr() {} } })).toBe(0); + const help = stdout.join("\n"); + expect(help.match(/--out /gu)).toHaveLength(1); + expect(help).toContain("--materialize "); + expect(help).toContain("--history "); + }); +}); + describe("spawnfile train --plan", () => { it.each([false, true])("prints the real authoring plan without execution with dryRun=%s", async (dryRun) => { const flags = dryRun ? ["--dry-run"] : []; diff --git a/src/cli/trainCommand.ts b/src/cli/trainCommand.ts index 7e5fb989..df9cc8b8 100644 --- a/src/cli/trainCommand.ts +++ b/src/cli/trainCommand.ts @@ -17,13 +17,15 @@ export const registerTrainCommand = ( signal?: AbortSignal ): void => { const command = program.command("train") - .description("Train one canonical agent through Paideia's isolated native integration") + .description("Plan, rewind, or train one canonical agent through its declared training inputs") .argument("[path]", "Canonical project directory or Spawnfile path", process.cwd()) .option("--agent ", "Exact canonical agent node id (inferred only for a single-agent project)") .option("--paideia-command ", "Installed Paideia executable; no shell or automatic install", "paideia") .option("--training-image ", "Pinned image containing the complete training environment") .option("--training-config ", "V2 image recipe and declared inputs, or advanced v1 local bindings") .option("--plan", "Print the resolved training authoring plan without executing training") + .option("--materialize ", "Rewind an authored example into --out without running training") + .option("--history ", "Local recorded history for --materialize") .option("--dry-run", "Validate and estimate without compiling, authenticating or starting models") .option("--repair-measurements ", "Fork captured work into a fresh output for authorized measurement repair") .option("--repair-witness ", "Explicit verified image witness for a legacy parent") @@ -31,9 +33,32 @@ export const registerTrainCommand = ( for (const name of forwarded) { const flag = `--${name} `; if (repeated.has(name)) command.option(flag, `Paideia ${name}; repeatable`, (value: string, previous: string[]) => [...previous, value], []); - else command.option(flag, `Paideia ${name}`); + else command.option(flag, name === "out" ? "Output directory for training or moment materialization" : `Paideia ${name}`); } command.action(async (inputPath: string, options: Record) => { + if (typeof options.materialize === "string") { + if (options.plan === true) { + throw new SpawnfileError("validation_error", "--materialize and --plan are mutually exclusive"); + } + if (typeof options.out !== "string") { + throw new SpawnfileError("validation_error", "--materialize requires --out"); + } + const project = await handlers.loadTrainingProject(inputPath); + const plan = handlers.resolveTrainingPlan(project, options.agent as string | undefined, packageVersion); + const moment = await handlers.materializeTrainingMoment({ plan, project, example: options.materialize, + historyDir: options.history as string | undefined, out: options.out }); + if (moment.status === "refused") { + streams.stderr(`error: Moment refused: ${moment.inputs.filter((input) => input.refusal !== null).length} input refusal(s), ${moment.leakCheck.findings} leak finding(s); see moment.json in --out`); + setExitCode(1); + } else { + streams.stdout(JSON.stringify(moment, null, 2)); + setExitCode(0); + } + return; + } + if (options.history !== undefined) { + throw new SpawnfileError("validation_error", "--history requires --materialize"); + } if (options.plan === true) { const plan = await handlers.createTrainingPlan(inputPath, { agent: options.agent as string | undefined, packageVersion diff --git a/src/compiler/training/AGENTS.md b/src/compiler/training/AGENTS.md index d47925d4..6505ecc9 100644 --- a/src/compiler/training/AGENTS.md +++ b/src/compiler/training/AGENTS.md @@ -5,6 +5,7 @@ - `context.ts` resolves the full compiler graph and pins source files without compiling or launching it. - `resolution.ts` merges direct-member training metadata during traversal and preserves declaring sources. - `plan/` owns local authoring-file loading, pure plan resolution, and the public training-plan schema. +- `rewind/` materializes authored moments from contained local history, with explicit fidelity and an independent disk leak check; it never runs trials. - Preserve resolved inheritance and exact agent IDs. Never create a second agent declaration. - Source mappings are project-relative editable files, not runtime-native destinations or flattened prompts. - Do not serialize secret values, arbitrary environments or transport credentials. diff --git a/src/compiler/training/index.ts b/src/compiler/training/index.ts index 5bd21453..e5d9f970 100644 --- a/src/compiler/training/index.ts +++ b/src/compiler/training/index.ts @@ -1,6 +1,7 @@ export * from "./contract.js"; export * from "./context.js"; export * from "./plan/index.js"; +export * from "./rewind/index.js"; export type { ResolvedTraining, ResolvedTrainingGrader } from "./resolution.js"; export { parseTrainingMappedPreparation } from "./preparation/contract.js"; export type { TrainingMappedPreparation } from "./preparation/contract.js"; diff --git a/src/compiler/training/plan/load.test.ts b/src/compiler/training/plan/load.test.ts index 96300475..a72c4b6d 100644 --- a/src/compiler/training/plan/load.test.ts +++ b/src/compiler/training/plan/load.test.ts @@ -5,6 +5,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { buildCompilePlan } from "../../buildCompilePlan.js"; import { loadTrainingProject, createTrainingPlan } from "./load.js"; import { trainingPlanJsonSchema, trainingPlanSchema } from "./contract.js"; +import { materializedMomentJsonSchema, leakCheckJsonSchema, taskClockJsonSchema } from "../rewind/contract.js"; import { agentManifest, cleanTrainingProjects, dataFiles, example, target, teamManifest, training, trainingProject, writeTrainingFile @@ -76,10 +77,16 @@ describe("training authoring loader", () => { expect((await createTrainingPlan(root, options)).targets[0].files[0].path).toBe("C:notes.md"); }); - it("keeps the documented plan JSON Schema identical to the exported contract", async () => { + it.each([ + ["training-plan-schema", trainingPlanJsonSchema], + ["materialized-moment-schema", materializedMomentJsonSchema], + ["leak-check-schema", leakCheckJsonSchema], + ["task-clock-schema", taskClockJsonSchema] + ])("keeps the documented %s identical to the exported contract", async (marker, expected) => { const doc = await readFile(new URL("../../../../specs/TRAINING.md", import.meta.url), "utf8"); - const schema = doc.match(/\n```json\n([\s\S]*?)\n```/u)?.[1]; - expect(JSON.parse(schema!)).toEqual(trainingPlanJsonSchema); + const pattern = "\\n```json\\n([\\s\\S]*?)\\n```"; + const schema = doc.match(new RegExp(pattern, "u"))?.[1]; + expect(JSON.parse(schema!)).toEqual(expected); }); }); diff --git a/src/compiler/training/rewind/AGENTS.md b/src/compiler/training/rewind/AGENTS.md new file mode 100644 index 00000000..03ed56dd --- /dev/null +++ b/src/compiler/training/rewind/AGENTS.md @@ -0,0 +1,29 @@ +# Training Rewind + +- `contract.ts` owns strict materialization, leak and task-clock documents and their generated schemas; the recording barrel owns moment records. +- `history.ts` is the only boundary for reading history files, logs and retained bytes; every path must be lexically and physically contained. Reuse recording wake/moment parsers, volume landing readers, and deployment resource lookup through their barrels. +- `deployedResource.ts` scopes fallback pins to the compiled agent and declaring node. Import one deployment log under `deployments/`, retaining its canonical original identity without reading that external path. Inexact activation/rollback pins remain approximate; a present but unavailable pin never silently becomes selector-at. +- `select.ts` and `inputs.ts` resolve original attempts, examples and compiled input identities; `recordInput.ts` isolates per-input recording failures. Dataset `at` matches wake time; `effectiveAt` is the original startedAt and owns all cuts, clocks and leak bounds. +- `types.ts` defines input identities, declarations and per-kind contexts/results; `index.ts` exports the public materializer and pure selection/cut helpers. +- `git.ts` owns resources; `gitProcess.ts`, `gitHistory.ts`, `gitTree.ts`, and `gitArchive.ts` isolate argv execution, containment, blob trees, and raw feed trees; `recordedCommit.ts` verifies full recorded IDs name commit objects without peeling. +- `feed.ts`, `memory.ts`, and `volumes.ts` own input reconstruction. `snapshotManifest.ts` requires Daimon completion markers and verifies full trees; `memorySchemas.ts` validates ledger events. Mneme generations are unsupported; seq:0 is exact empty. +- `feedContent.ts` verifies plain git revisions and re-extracts for independent byte acceptance; retained copies must match. Prepared revisions prove provenance only: always approximate, never compared to a directory content digest. +- `moltnet.ts` dispatches stores; `moltnetJson.ts`, `moltnetSqlite.ts`, `moltnetSqliteData.ts`, and `moltnetSchemas.ts` own snapshots, copied databases, SQL decoding, and strict native shapes. +- `materialize.ts` stages all inputs and publishes only after the independent disk reader in `leakCheck.ts` accepts them. +- `leakFiles.ts`, `leakGit.ts`, and `leakInputs.ts` inspect disk metadata, every git object, and native payloads independently of materialization decisions. `leakGitPayload.ts` compares raw output-repository blobs to the worktree and checks index stat times; `leakLayout.ts` enforces closed native layouts. `leakProvenance.ts` reopens source repositories read-only using disk commit receipts, checking type and both raw dates without the producer's guard and re-exporting bundle `source.trees` for byte comparison. Discover unexpected `.git` directories/gitfiles and bare repositories throughout every input; only the materialized git resource repository is allowed. +- `instant.ts` compares parsed nanosecond instants; `timestamps.ts` stamps metadata without following symlinks. +- `taskClock.ts` separates top-level harness JSON, libfaketime child offsets, and clock-aware child variables: one offset owner per process. Relative signed seconds preserve milliseconds and the shared anchor; trial preparation supplies LD_PRELOAD and a dot-decimal numeric locale. Timeouts remain real. +- Adjacent tests use local temporary repositories and fixtures. Never use Docker, production hosts or network fetches. +- `fixtures.test-helper.ts`, `snapshotFixtures.test-helper.ts`, and `git.test-helper.ts` build disposable real projects and repositories; package emission excludes them. +- Selector-at git history is approximate: committer dates cannot prove when a commit reached a ref. Feed-rule approximation cannot reconstruct freeze holds. +- Reuse P2 effective input identity resolution. No input may silently use present-day data. +- Feed/bundle extraction reads raw blobs, never `git archive` or attribute-driven transformations. Preserve symlinks for the escape check; refuse gitlinks. Recorded commits must be full lowercase 40/64-hex IDs verified as commit objects, including retained feeds. +- Guard every selected git/bundle/feed commit's author and committer dates before choosing output bytes, including retained/prepared feeds. Record the selected commit in the moment/identity; local pins or git-backed overrides without a history entry also record their repository path for independent acceptance. +- Validate each non-overridden input's recorded commit IDs in every same-agent/same-wake earliest-attempt copy before accepting that input, even when pin precedence omits those fields from selection comparisons. +- Validate memory JSONL structure locally; rewind never imports Mneme or ships an index/audit state. Checksum verification is skipped because persisted normalized events cannot recover the original JSON checksum input (`node_modules/@noopolis/mneme/dist/identity/ids.js:15-17`, `dist/store/store.js:249-274`). +- Mneme 0.1.1's index is rebuilt from the ledger before every index read, not on open: `node_modules/@noopolis/mneme/dist/runtime/runtime.js:86-87` (embedding recall), `node_modules/@noopolis/mneme/dist/kernel/kernel.js:152-153` (search), and `:207-208` (locate). A ledger-only materialized bank is complete. +- Inspect atime and mtime from metadata captured before payload reads, then restore captured access times without hiding existing future atimes. +- Preserve negative source epochs with Date/numeric-string filesystem times; recheck the final renamed bundle. Trial at/effectiveAt must be within 1970 through 9999 before materialization. +- Generated inputs spanning repositories refuse: this history contract carries one commit per resource. Never claim that one repository proves another repository's bytes. +- `recordedBundle.ts` requires digest-matching retained tars for exact bundles and independently checks extracted bytes; source reconstruction is approximate. `leakFeed.ts` proves late publication metadata against record/window history. +- Keep modules under 400 lines, use named/barrel exports, and never import Mneme internals. diff --git a/src/compiler/training/rewind/CLAUDE.md b/src/compiler/training/rewind/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/src/compiler/training/rewind/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/src/compiler/training/rewind/contract.ts b/src/compiler/training/rewind/contract.ts new file mode 100644 index 00000000..6ac94451 --- /dev/null +++ b/src/compiler/training/rewind/contract.ts @@ -0,0 +1,44 @@ +import { z } from "zod"; +import { recordedSha256Schema } from "../../../recording/index.js"; +import { trainingInstantSchema } from "../../../manifest/index.js"; + +const text = z.string().min(1); +const count = z.number().int().nonnegative(); +export const inputIdentitySchema = z.object({ + section: z.enum(["resources", "memory", "networks"]), id: text, declaredBy: text +}).strict(); +export const materializedInputSchema = inputIdentitySchema.extend({ + kind: z.enum(["git", "bundle", "feed", "volume", "memory", "moltnet"]), + fidelity: z.enum(["exact", "approximate", "overridden"]).nullable(), basis: text, + source: z.object({ commit: text.optional(), repository: text.optional(), revision: text.optional(), seq: count.optional(), + trees: z.array(z.object({ root: z.string(), prefix: z.string().optional(), exclude: z.array(text).optional() }).strict()).min(1).optional(), + sha256: recordedSha256Schema.optional(), agentSlug: text.optional(), snapshot: text.optional(), cursor: text.optional(), + switchingAt: trainingInstantSchema.optional(), landedAt: trainingInstantSchema.optional(), prepared: z.literal(true).optional() }).strict(), + path: text.nullable(), notes: z.array(z.string()), refusal: text.nullable(), build: z.literal("deferred").optional() +}).strict(); +export type MaterializedInput = z.infer; +export const materializedMomentSchema = z.object({ + version: z.literal("spawnfile.materialized-moment.v1"), status: z.enum(["materialized", "refused"]), + agent: text, example: z.object({ id: text, role: z.enum(["train", "validation", "test"]), dataset: text }).strict(), + at: trainingInstantSchema, effectiveAt: trainingInstantSchema, inputs: z.array(materializedInputSchema), + record: z.object({ matched: z.boolean(), attempts: count.optional(), selectedStartedAt: trainingInstantSchema.optional(), + executionId: text.optional(), selection: z.literal("earliest-startedAt").optional() }).strict(), + leakCheck: z.object({ ok: z.boolean(), findings: count }).strict() +}).strict(); +export type MaterializedMoment = z.infer; +export const leakFindingSchema = z.object({ + input: inputIdentitySchema.nullable(), + class: z.enum(["bundle-entry", "file-time", "symlink-escape", "git-object", "memory-event", "moltnet-message", + "moltnet-event", "moltnet-room", "feed-identity", "dated-path", "snapshot"]), path: text, detail: text +}).strict(); +export const leakCheckSchema = z.object({ + version: z.literal("spawnfile.leak-check.v1"), at: trainingInstantSchema, ok: z.boolean(), + checked: z.object({ files: count, gitObjects: count, memoryEvents: count, moltnetMessages: count, + moltnetEvents: count, snapshots: count }).strict(), findings: z.array(leakFindingSchema) +}).strict(); +export type LeakCheck = z.infer; +export type LeakFinding = z.infer; +export const taskClockSchema = z.object({ version: z.literal("spawnfile.task-clock.v1"), origin: trainingInstantSchema }).strict(); +export const materializedMomentJsonSchema = z.toJSONSchema(materializedMomentSchema); +export const leakCheckJsonSchema = z.toJSONSchema(leakCheckSchema); +export const taskClockJsonSchema = z.toJSONSchema(taskClockSchema); diff --git a/src/compiler/training/rewind/deployedResource.test.ts b/src/compiler/training/rewind/deployedResource.test.ts new file mode 100644 index 00000000..03f95101 --- /dev/null +++ b/src/compiler/training/rewind/deployedResource.test.ts @@ -0,0 +1,168 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import * as deployment from "../../../deployment/index.js"; +import { BundleTarWriter } from "../../workspaceBundleTar.js"; +import { AT, BEFORE, emptyRecord, jsonl, rewindFixture, testGit } from "./fixtures.test-helper.js"; +import { materializeTrainingMoment } from "./materialize.js"; +import { composeMomentRecord } from "../../../recording/index.js"; + +const roots: string[] = []; +afterEach(async () => { vi.restoreAllMocks(); await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const DIGEST = `sha256:${"a".repeat(64)}`; +const fixture = async (home = false) => { + const f = await rewindFixture(); roots.push(f.root); + testGit(f.gitRepo, ["checkout", "-B", "main", f.commit]); + await writeFile(path.join(f.gitRepo, "news.txt"), "newer eligible selector\n"); + testGit(f.gitRepo, ["add", "."]); testGit(f.gitRepo, ["commit", "--quiet", "-m", "later"], "2026-10-09T08:00:00Z"); + const later = testGit(f.gitRepo, ["rev-parse", "HEAD"]); + const recordPath = home ? "/original/home/deployments/desk/record.json" : "/original/output/deployments/desk.json"; + const logPath = home ? "deployments/desk/resource-log.jsonl" : "deployments/desk.resource-log.jsonl"; + const entry = (commit = f.commit, fields: Partial = {}): deployment.DeployResources => ({ + version: "spawnfile.deploy-resources.v1", deployment: "desk", deployment_id: recordPath, + deployed_at: "2026-10-10T00:00:00Z", activated_at: BEFORE, activation_exact: true, report_sha256: DIGEST, + resources: [{ id: "code", kind: "git", commit, sha256: DIGEST, agent_node_ids: ["agent:a"], declared_by: "team:publication" }], ...fields + }); + const log = (entries: unknown[]) => jsonl(f.historyDir, logPath, entries); + const run = async () => { + const result = await materializeTrainingMoment(f.options); + return { result, input: result.inputs.find((input) => input.id === "code")! }; + }; + return { ...f, later, entry, log, run, logPath, recordPath }; +}; + +describe("merged deployment resource history in rewind", () => { + it.each((["git", "bundle"] as const).flatMap((kind) => ["activation", "rollback"].map((boundary) => ({ kind, boundary })))) + ("downgrades a matched $kind record with non-exact $boundary", async ({ kind, boundary }) => { + const f = await fixture(); + let sha256 = DIGEST; + if (kind === "bundle") { + const archive = path.join(f.root, "saved.tar"), writer = await BundleTarWriter.create(archive); + await writer.begin("tools.txt", 6, 0o644); await writer.data(Buffer.from("tools\n")); await writer.end(); await writer.finish(); + const bytes = await readFile(archive), hex = createHash("sha256").update(bytes).digest("hex"); sha256 = `sha256:${hex}`; + await mkdir(path.join(f.historyDir, "bundles")); await writeFile(path.join(f.historyDir, "bundles", `${hex}.tar`), bytes); + const agent = f.project.compilePlan.nodes.find((node) => node.id === f.plan.agent.id)!.value; + if (agent.kind !== "agent") throw new Error("Expected agent"); + const index = agent.workspaceResources!.findIndex((resource) => resource.id === "code"); + agent.workspaceResources![index] = { scope: agent.workspaceResources![index]!.scope, sharing: "per_agent", id: "code", kind, + mount: "./code", mode: "readonly", source: "never-read.tar" }; + } + const entry = f.entry(f.commit, { activation_exact: boundary !== "activation", activation_note: "Predecessor may still serve code", + ...(boundary === "rollback" ? { exact: false, rolled_back_at: "2026-10-09T08:20:32Z" } : {}), + resources: [{ id: "code", kind, sha256, ...(kind === "git" ? { commit: f.commit } : {}), agent_node_ids: ["agent:a"], declared_by: "team:publication" } as deployment.DeployResources["resources"][number]], + recording_agents: [{ node_id: "agent:a", slug: "a", directory: "/recording/a", keep_ms: 1000, + resources: [{ id: "code", kind }], feeds: [], memory: [], networks: [], snapshots: [] }] }); + const record = composeMomentRecord({ agentNodeId: "agent:a", deploymentRecordPath: f.recordPath, deployResources: [entry], + row: { version: "noopolis.daimon.wake-moment.v1", agent_id: "agent:a", execution_id: "execution-17", started_at: AT, + deliveries: [{ acceptance_id: "accepted", delivery_id: "schedule:edition", kind: "schedule", occurred_at: AT }], snapshots: [] }, + feedLandings: {}, memoryLedgers: {}, recordedAt: AT }); + expect(record.resources).toHaveLength(1); + expect(record.errors).toContainEqual(expect.stringContaining("Deployment activation is non-exact")); + await jsonl(f.historyDir, "moments/pin.jsonl", [record]); + const { result, input } = await f.run(); + expect(result.status, JSON.stringify(result.inputs)).toBe("materialized"); + expect(result.record.matched).toBe(true); + expect(input).toMatchObject({ basis: "record", fidelity: "approximate", notes: record.errors, + source: kind === "git" ? { commit: f.commit } : { sha256 } }); + }); + + it.each([false, true])("uses the deployed commit instead of a newer selector (home=%s)", async (home) => { + const f = await fixture(home), lookup = vi.spyOn(deployment, "deployResourcesAt"); + const entry = f.entry(); await f.log([entry]); + const { result, input } = await f.run(); + expect(result.status, JSON.stringify(result.inputs)).toBe("materialized"); + expect(input).toMatchObject({ fidelity: "exact", basis: "deploy-log", source: { commit: f.commit } }); + expect(await readFile(path.join(f.options.out, input.path!, "news.txt"), "utf8")).toBe("Before the wake\n"); + expect(lookup).toHaveBeenCalledWith([entry], f.recordPath, AT, "agent:a"); + }); + + it.each(["2026-10-09T08:20:30Z", AT, "2026-10-09T08:20:32Z", "2026-10-09T08:20:32.001Z"])("respects inclusive rollback and restoration at %s", async (at) => { + const f = await fixture(); + const activated_at = "2026-10-09T08:20:30Z", rolled_back_at = "2026-10-09T08:20:32Z"; + const candidate = f.entry(f.later, { activated_at, report_sha256: `sha256:${"b".repeat(64)}` }); + await f.log([f.entry(), candidate, { ...candidate, activation_exact: false, exact: false, rolled_back_at }]); + // The dataset wake matches a record; all history must be selected at its effective startedAt. + const record = emptyRecord(); record.wake.startedAt = at; + await jsonl(f.historyDir, "moments/attempt.jsonl", [record]); + const { result, input } = await f.run(), rolledBack = at === "2026-10-09T08:20:32.001Z"; + expect(result.status, JSON.stringify(result.inputs)).toBe("materialized"); + expect(input).toMatchObject({ basis: "deploy-log", fidelity: rolledBack ? "exact" : "approximate", + source: { commit: rolledBack ? f.commit : f.later } }); + if (!rolledBack) expect(input.notes).toContainEqual(expect.stringContaining("rollback interval")); + }); + + it("carries an inexact activation note beside the selected pin", async () => { + const f = await fixture(); await f.log([f.entry(f.commit, { activation_exact: false, activation_note: "Predecessor overlaps launch" })]); + const { result, input } = await f.run(); + expect(result.status).toBe("materialized"); + expect(input).toMatchObject({ basis: "deploy-log", fidelity: "approximate", source: { commit: f.commit }, notes: ["Predecessor overlaps launch"] }); + }); + + it.each(["before activation", "other agent", "other declaring node", "absent resource"])("uses selector-at only when the deploy pin is absent: %s", async (state) => { + const f = await fixture(), entry = f.entry(); + if (state === "before activation") entry.activated_at = "2026-10-09T09:00:00Z"; + if (state === "other agent") entry.resources[0]!.agent_node_ids = ["agent:other"]; + if (state === "other declaring node") entry.resources[0]!.declared_by = "team:other"; + if (state === "absent resource") entry.resources = []; + await f.log([entry]); + expect((await f.run()).input).toMatchObject({ basis: "selector-at", fidelity: "approximate", source: { commit: f.later } }); + }); + + it.each(["moment", "declared"])("preserves %s pin precedence over deployment history", async (kind) => { + const f = await fixture(); await f.log([f.entry(f.later)]); + if (kind === "moment") { + const record = emptyRecord(); record.resources = [{ id: "code", kind: "git", commit: f.commit }]; + await jsonl(f.historyDir, "moments/pin.jsonl", [record]); + } else { + const agent = f.project.compilePlan.nodes.find((node) => node.id === f.plan.agent.id)!.value; + if (agent.kind !== "agent") throw new Error("Expected agent"); + const resource = agent.workspaceResources!.find((item) => item.id === "code")!; + if (resource.kind !== "git") throw new Error("Expected git"); resource.ref = f.commit; + } + expect((await f.run()).input).toMatchObject({ basis: kind === "moment" ? "record" : "pinned", fidelity: "exact", source: { commit: f.commit } }); + }); + + it.each(["malformed", "mixed identities", "multiple logs", "duplicate pin"])("refuses %s instead of guessing selector-at", async (state) => { + const f = await fixture(), entry = f.entry(); + if (state === "duplicate pin") entry.resources.push({ ...entry.resources[0]! }); + await f.log(state === "malformed" ? [entry, {}] : state === "mixed identities" ? [entry, { ...entry, deployment_id: "/other/desk.json" }] : [entry]); + if (state === "multiple logs") await jsonl(f.historyDir, "deployments/other.resource-log.jsonl", [entry]); + const { result, input } = await f.run(); + expect(result.status).toBe("refused"); + expect(input.refusal).toMatch(/Invalid history|deployment.*ambiguous|Ambiguous deployment/u); + expect(await readdir(f.options.out)).toEqual(["moment.json"]); + }); + + it.each([true, false])("restores digest-verified deploy bundle bytes with activation_exact=%s", async (exact) => { + const f = await fixture(), archive = path.join(f.root, "saved.tar"); + const writer = await BundleTarWriter.create(archive); + await writer.begin("tools.txt", 6, 0o644); await writer.data(Buffer.from("tools\n")); await writer.end(); await writer.finish(); + const bytes = await readFile(archive), hex = createHash("sha256").update(bytes).digest("hex"); + await mkdir(path.join(f.historyDir, "bundles")); await writeFile(path.join(f.historyDir, "bundles", `${hex}.tar`), bytes); + const agent = f.project.compilePlan.nodes.find((node) => node.id === f.plan.agent.id)!.value; + if (agent.kind !== "agent") throw new Error("Expected agent"); + const index = agent.workspaceResources!.findIndex((item) => item.id === "code"); + agent.workspaceResources![index] = { scope: agent.workspaceResources![index]!.scope, sharing: "per_agent", id: "code", kind: "bundle", mount: "./code", mode: "readonly", source: "never-read.tar" }; + await f.log([f.entry(f.commit, { activation_exact: exact, resources: [{ id: "code", kind: "bundle", sha256: `sha256:${hex}`, + agent_node_ids: ["agent:a"], declared_by: "team:publication" }] })]); + const { result, input } = await f.run(); + expect(result.status, JSON.stringify(result.inputs)).toBe("materialized"); + expect(input).toMatchObject({ basis: "deploy-log", fidelity: exact ? "exact" : "approximate", source: { sha256: `sha256:${hex}` } }); + expect(await readFile(path.join(f.options.out, input.path!, "tools.txt"), "utf8")).toBe("tools\n"); + }); + + it("refuses an unavailable deployment bundle instead of reconstructing a different selector", async () => { + const f = await fixture(), agent = f.project.compilePlan.nodes.find((node) => node.id === f.plan.agent.id)!.value; + if (agent.kind !== "agent") throw new Error("Expected agent"); + const index = agent.workspaceResources!.findIndex((item) => item.id === "code"), previous = agent.workspaceResources![index]!; + agent.workspaceResources![index] = { scope: previous.scope, sharing: "per_agent", id: "code", kind: "bundle", + mount: "./code", mode: "readonly", build: { files: { root: ".", ref: f.later } } }; + await f.log([f.entry(f.commit, { resources: [{ id: "code", kind: "bundle", sha256: DIGEST, + agent_node_ids: ["agent:a"], declared_by: "team:publication" }] })]); + const { result, input } = await f.run(); + expect(result.status).toBe("refused"); + expect(input.refusal).toContain(`Deployment bundle ${DIGEST} could not be reproduced`); + expect(await readdir(f.options.out)).toEqual(["moment.json"]); + }); +}); diff --git a/src/compiler/training/rewind/deployedResource.ts b/src/compiler/training/rewind/deployedResource.ts new file mode 100644 index 00000000..35b386a0 --- /dev/null +++ b/src/compiler/training/rewind/deployedResource.ts @@ -0,0 +1,18 @@ +import type { InputContext } from "./types.js"; + +/** Deployment IDs remain evidence only: all bytes are read from the contained history mirror. */ +export const deployedResource = async (context: InputContext) => { + const agentId = context.agentId ?? context.record?.agent + ?? (context.input.declaredBy.startsWith("agent:") ? context.input.declaredBy : undefined); + const entry = await context.history?.deploymentResources(context.at, agentId); + if (!entry) return null; + const pins = entry.resources.filter((resource) => resource.id === context.input.id + && (resource.declared_by === undefined || resource.declared_by === context.input.declaredBy)); + if (pins.length > 1) throw new Error(`Ambiguous deployment resource pin for ${context.input.declaredBy}/${context.input.id}`); + const pin = pins[0]; + if (!pin) return null; + const exact = entry.activation_exact && entry.exact !== false; + return { pin, exact, notes: exact ? [] : [entry.activation_note + ?? (entry.rolled_back_at ? "Trial instant falls inside an inexact deployment rollback interval." + : "Deployment activation boundary is inexact; an overlapping predecessor may have served different pins.")] }; +}; diff --git a/src/compiler/training/rewind/feed.test.ts b/src/compiler/training/rewind/feed.test.ts new file mode 100644 index 00000000..282a80dd --- /dev/null +++ b/src/compiler/training/rewind/feed.test.ts @@ -0,0 +1,289 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { mkdir, readFile, readdir, readlink, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import type { TeamWorkspaceResource } from "../../../manifest/index.js"; +import { digestDirectory, feedIdentityFindings, resolveFeedSource, type FeedLanding } from "../../../volume/index.js"; +import { materializeFeed } from "./feed.js"; +import { History } from "./history.js"; +import { commitTestGit, gitTestRoot, initTestGit, testGit, testMomentRecord } from "./git.test-helper.js"; +import type { InputContext } from "./types.js"; +import { stampTree } from "./timestamps.js"; + +const AT = "2026-10-09T08:20:31Z", EARLY = "2026-10-09T08:00:00Z", LATE = "2026-10-09T09:00:00Z"; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const fixture = async () => { + const root = await gitTestRoot(); roots.push(root); + const directory = path.join(root, "history/feeds/agent%3Awriter/news"), repository = path.join(directory, "repo"); + await initTestGit(repository); + const past = await commitTestGit(repository, { "published/news.txt": "past\n", "ignored/private.txt": "not-fed\n" }, EARLY); + const future = await commitTestGit(repository, { "published/news.txt": "future\n" }, LATE); + const revision = resolveFeedSource({ kind: "git", fetch: false, repo: repository, ref: past, paths: ["published"] }).revision; + const landing: FeedLanding = { version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "org-news", + source: { kind: "git", repo: "https://never-fetch.invalid/source" }, revision, tree: `trees/${revision}`, commit: past, ref: "main", paths: ["published"], switching_at: EARLY, landed_at: EARLY }; + const resource: TeamWorkspaceResource = { id: "news", kind: "volume", mode: "mutable", mount: "./news", name: "org-news", + feed: { git: { repo: "https://never-fetch.invalid/source", ref: "main", paths: ["published"], fetch: true } } }; + const context: InputContext = { at: AT, destination: path.join(root, "output"), history: new History(path.join(root, "history")), record: null, + input: { section: "resources", declaredBy: "agent:writer", id: "news", declarationBase: root, overridden: false, declaration: resource } }; + const log = async (entries: FeedLanding[]) => writeFile(path.join(directory, "landings.jsonl"), `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`); + return { root, repository, directory, past, future, revision, landing, context, resource, log }; +}; + +describe("feed rewind", () => { + it.each(["retained", "reconstructed"].flatMap((bytes) => ["author", "committer"].map((role) => [bytes, role]))) + ("refuses a future selected commit before using %s bytes (%s date)", async (bytes, role) => { + const { context, repository, past, directory, landing, log } = await fixture(); + testGit(repository, ["checkout", "--detach", past]); + testGit(repository, ["commit", "--allow-empty", "-m", "future metadata", `--date=${role === "author" ? LATE : EARLY}`], role === "committer" ? LATE : EARLY); + const commit = testGit(repository, ["rev-parse", "HEAD"]); + const revision = resolveFeedSource({ kind: "git", fetch: false, repo: repository, ref: commit, paths: ["published"] }).revision; + await log([{ ...landing, commit, revision, tree: `trees/${revision}`, switching_at: EARLY, landed_at: EARLY }]); + if (bytes === "retained") { + const retained = path.join(directory, "trees", revision); + await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "future.txt"), "future retained bytes"); + await stampTree(retained, EARLY, AT); + } + await expect(materializeFeed(context)).rejects.toThrow(`Selected commit ${commit} has an author or committer date after ${AT}`); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it("selects exact boundary landings and matches offset-equivalent records", async () => { + const { context, landing, log, revision } = await fixture(); + const selected = { ...landing, landed_at: AT }; + await log([selected, { ...landing, revision: "f".repeat(64), tree: `trees/${"f".repeat(64)}`, landed_at: "2026-10-09T08:20:31.001Z" }]); + context.record = { ...testMomentRecord(), feeds: [{ id: "news", revision, tree: `trees/${revision}`, commit: landing.commit!, ref: "main", paths: ["published"], landedAt: "2026-10-09T10:20:31+02:00" }] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: landing.commit, revision }, instant: "2026-10-09T10:20:31+02:00" }); + expect(await readlink(path.join(context.destination, "current"))).toBe(`trees/${revision}`); + expect(await readFile(path.join(context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); + const identity = JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")); + expect(feedIdentityFindings(identity)).toEqual([]); + expect(identity).toMatchObject({ landed_at: "2026-10-09T10:20:31+02:00", files: 1, source: { commit: landing.commit, paths: ["published"], ref: "main" } }); + }); + + it("uses a present pin from an inexact record without a landing log or optional ref", async () => { + const { context, landing } = await fixture(); + context.record = { ...testMomentRecord(), exact: false, errors: ["memory self unavailable"], + feeds: [{ id: "news", revision: landing.revision, tree: landing.tree, commit: landing.commit, + paths: landing.paths, landedAt: EARLY }] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: landing.commit } }); + expect(await readFile(path.join(context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); + expect(JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")).source.ref).toBe(landing.commit); + }); + + it("keeps the original wake pin when another landing precedes the replay", async () => { + const { context, landing, log, repository } = await fixture(); + const commit = await commitTestGit(repository, { "published/news.txt": "later landing\n" }, "2026-10-09T08:10:00Z"); + const revision = resolveFeedSource({ kind: "git", fetch: false, repo: repository, ref: commit, paths: ["published"] }).revision; + await log([landing, { ...landing, commit, revision, tree: `trees/${revision}`, landed_at: "2026-10-09T08:11:00Z" }]); + context.record = { ...testMomentRecord(), feeds: [{ id: "news", revision: landing.revision, tree: landing.tree, + commit: landing.commit, ref: landing.ref, paths: landing.paths, landedAt: EARLY }] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: landing.commit } }); + expect(await readFile(path.join(context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); + }); + + it.each(["2026-10-09T08:20:30Z", AT, "2026-10-09T08:20:32Z"]) + ("uses the incoming transition landing approximately at %s", async (at) => { + const { context, landing, log } = await fixture(); + const incoming = { ...landing, switching_at: "2026-10-09T08:20:30Z", landed_at: "2026-10-09T08:20:32Z" }; + await log([landing, incoming]); context.at = at; + const result = await materializeFeed(context); + expect(result).toMatchObject({ fidelity: "approximate", basis: "landing-log", instant: at, + source: { switchingAt: incoming.switching_at, landedAt: incoming.landed_at } }); + expect(result.notes).toEqual([expect.stringContaining("ambiguous feed transition")]); + expect(JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")).landed_at).toBe(incoming.landed_at); + expect(await readFile(path.join(context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); + }); + + it("falls back to the feed rule at the trial instant with historical committer landing time", async () => { + const { context, past, revision } = await fixture(); + expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", basis: "feed-rule-at", source: { commit: past, revision }, instant: "2026-10-09T08:00:00.000Z" }); + const identity = JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")); + expect(Date.parse(identity.landed_at)).toBeLessThanOrEqual(Date.parse(AT)); + expect(await readdir(path.join(context.destination, "current"))).toEqual(["published"]); + }); + + it("steps back past a future-authored feed commit", async () => { + const { context, resource, repository, past } = await fixture(); + testGit(repository, ["checkout", "--detach", past]); + await commitTestGit(repository, { "published/news.txt": "future-authored\n" }, "2026-10-09T08:10:00Z"); + testGit(repository, ["commit", "--amend", "--no-edit", `--date=${LATE}`], "2026-10-09T08:10:00Z"); + resource.feed!.git!.ref = "HEAD"; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", basis: "feed-rule-at", source: { commit: past } }); + expect(await readFile(path.join(context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); + }); + + it("expands the Berlin date in both ref and paths near UTC midnight", async () => { + const { context, resource, repository } = await fixture(); + const commit = await commitTestGit(repository, { "2026-10-10/news.txt": "Berlin tomorrow\n" }, "2026-10-09T22:00:00Z"); + testGit(repository, ["branch", "daily/2026-10-10", commit]); + resource.feed = { git: { repo: repository, ref: { template: "daily/${date:Europe/Berlin}" }, paths: ["${date:Europe/Berlin}"] } }; + context.at = "2026-10-09T22:30:00Z"; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", source: { commit } }); + expect(await readFile(path.join(context.destination, "current/2026-10-10/news.txt"), "utf8")).toBe("Berlin tomorrow\n"); + expect(JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")).source).toMatchObject({ ref: "daily/2026-10-10", paths: ["2026-10-10"] }); + }); + + it("uses fallback when the primary ref exists but has no commit at or before at", async () => { + const { context, resource, repository, past } = await fixture(); + testGit(repository, ["checkout", "--orphan", "daily/2026-10-09"]); + await commitTestGit(repository, { "published/news.txt": "future orphan\n" }, LATE); + resource.feed!.git!.ref = { template: "daily/${date}", fallback: "main" }; + expect(await materializeFeed(context)).toMatchObject({ source: { commit: past } }); + expect(JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")).source.ref).toBe("main"); + }); + + it("uses retained trees for prepared feeds without executing the recipe or ref command", async () => { + const { context, resource, directory, landing, log } = await fixture(); + resource.feed = { git: { repo: "never-used", ref: { command: ["never-run-ref"] } }, freeze: { timezone: "UTC", after: "08:00" }, + include: [{ from: "not-on-disk", to: "assets" }], prepare: { host: true, command: ["never-run-prepare"] } }; + const prepared = { include: [], prepare: { command: ["never-run-prepare"], image: null, network: null, platform: null }, source_revision: landing.revision }; + const revision = createHash("sha256").update(`prepared\n${JSON.stringify(prepared)}\n`).digest("hex"); + const retained = path.join(directory, "trees", `${revision}.2`); + await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "prepared.txt"), "retained\n"); + await log([{ ...landing, revision, tree: `trees/${revision}.2`, source: { kind: "git", repo: "never-used", prepared } }]); + expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", basis: "landing-log", + notes: ["prepared output not content-verifiable"] }); + expect(await readlink(path.join(context.destination, "current"))).toBe(`trees/${revision}.2`); + expect(await readFile(path.join(context.destination, "current/prepared.txt"), "utf8")).toBe("retained\n"); + expect(JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")).source.prepared.prepare.command).toEqual(["never-run-prepare"]); + }); + + it.each(["unchanged", "changed bytes", "extra file", "missing file", "wrong revision", "missing repository"]) + ("verifies retained plain git feed with %s against commit and paths", async (change) => { + const f = await fixture(); + const revision = change === "wrong revision" ? "0".repeat(64) : f.revision; + const retained = path.join(f.directory, "trees", revision); + await mkdir(path.join(retained, "published"), { recursive: true }); + if (change !== "missing file") await writeFile(path.join(retained, "published/news.txt"), change === "changed bytes" ? "future\n" : "past\n"); + if (change === "extra file") await writeFile(path.join(retained, "extra.txt"), "future\n"); + await f.log([{ ...f.landing, revision, tree: `trees/${revision}` }]); + if (change === "missing repository") await rm(f.repository, { recursive: true }); + if (change === "unchanged") { + expect(await materializeFeed(f.context)).toMatchObject({ fidelity: "exact" }); + expect(await readFile(path.join(f.context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); + expect(await readdir(f.context.destination)).not.toContain(".retained-check"); + } else await expect(materializeFeed(f.context)).rejects.toThrow(change === "wrong revision" ? "Feed revision mismatch" + : change === "missing repository" ? "Missing feed source repository" : "Retained git feed bytes disagree with selected commit and paths"); + }); + + it("uses retained directory feed landings and starts overridden feeds empty", async () => { + const { context, resource, directory, log } = await fixture(); + resource.feed = { directory: "never-read" }; + const retainedTemp = path.join(directory, "retained"); await mkdir(retainedTemp); await writeFile(path.join(retainedTemp, "data.txt"), "directory\n"); + const revision = digestDirectory(retainedTemp), retained = path.join(directory, "trees", revision); + await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "data.txt"), "directory\n"); + await log([{ version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "org-news", + source: { kind: "directory", directory: "never-read" }, revision, tree: `trees/${revision}`, switching_at: EARLY, landed_at: EARLY }]); + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", source: { revision } }); + expect(await readFile(path.join(context.destination, "current/data.txt"), "utf8")).toBe("directory\n"); + await rm(context.destination, { recursive: true }); context.input.overridden = true; context.history = undefined; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "overridden", basis: "override" }); + expect(await readdir(context.destination)).toEqual([]); + }); + + it("uses retained directory record pins without optional git metadata or a log", async () => { + const { context, resource, directory } = await fixture(); + resource.feed = { directory: "never-read" }; + const retained = path.join(directory, "retained"); await mkdir(retained); await writeFile(path.join(retained, "data.txt"), "past\n"); + const revision = digestDirectory(retained), selected = path.join(directory, "trees", `${revision}.2`); + await mkdir(selected, { recursive: true }); await writeFile(path.join(selected, "data.txt"), "past\n"); + context.record = { ...testMomentRecord(), feeds: [{ id: "news", revision, tree: `trees/${revision}.2`, landedAt: EARLY }] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { revision } }); + expect(await readFile(path.join(context.destination, "current/data.txt"), "utf8")).toBe("past\n"); + }); + + it.each(["changed bytes", "extra file", "missing file"])("refuses retained directory feeds with %s", async (change) => { + const { context, directory, landing, log } = await fixture(); + const retained = path.join(directory, "retained"); await mkdir(retained); await writeFile(path.join(retained, "data.txt"), "past\n"); + const revision = digestDirectory(retained), selected = path.join(directory, "trees", `${revision}.3`); + await mkdir(selected, { recursive: true }); + if (change !== "missing file") await writeFile(path.join(selected, "data.txt"), change === "changed bytes" ? "evil\n" : "past\n"); + if (change === "extra file") await writeFile(path.join(selected, "extra.txt"), "extra\n"); + // Historical source kind controls verification even after the declaration switched to git. + const { commit: _commit, ref: _ref, paths: _paths, ...base } = landing; + await log([{ ...base, source: { kind: "directory", directory: "never-read" }, revision, tree: `trees/${revision}.3` }]); + await expect(materializeFeed(context)).rejects.toThrow("Directory feed revision mismatch"); + }); + + it("does not substitute another retained generation for a missing directory tree", async () => { + const { context, directory, landing, log } = await fixture(); + const retained = path.join(directory, "retained"); await mkdir(retained); await writeFile(path.join(retained, "data.txt"), "past\n"); + const revision = digestDirectory(retained), previous = path.join(directory, "trees", revision); + await mkdir(previous, { recursive: true }); await writeFile(path.join(previous, "data.txt"), "past\n"); + const { commit: _commit, ref: _ref, paths: _paths, ...base } = landing; + await log([{ ...base, source: { kind: "directory", directory: "never-read" }, revision, tree: `trees/${revision}.3` }]); + await expect(materializeFeed(context)).rejects.toThrow("Directory/prepared/included feed requires retained landed tree bytes"); + }); + + it("refuses revisions which cannot be reproduced from the selected commit", async () => { + const { context, landing, log } = await fixture(); + await log([{ ...landing, revision: "0".repeat(64), tree: `trees/${"0".repeat(64)}` }]); + await expect(materializeFeed(context)).rejects.toThrow("Feed revision mismatch"); + }); + + it("refuses nonhistorical feed rules and missing retained prepared bytes", async () => { + const { context, resource, landing, log } = await fixture(); + resource.feed = { directory: "never-read" }; + await expect(materializeFeed(context)).rejects.toThrow("Directory feed cannot be rewound"); + resource.feed = { git: { repo: "never-read", ref: { command: ["never-execute"] } } }; + await expect(materializeFeed(context)).rejects.toThrow("command refs cannot be evaluated"); + resource.feed = { git: { repo: "never-read" }, freeze: { after: "08:00", timezone: "UTC" } }; + await expect(materializeFeed(context)).rejects.toThrow("freeze/waiting"); + resource.feed = { git: { repo: "never-read" }, prepare: { host: true, command: ["never-execute"] } }; + const prepared = { include: [], prepare: { command: ["never-execute"], image: null, network: null, platform: null }, source_revision: landing.revision }; + const revision = createHash("sha256").update(`prepared\n${JSON.stringify(prepared)}\n`).digest("hex"); + await log([{ ...landing, revision, tree: `trees/${revision}`, source: { kind: "git", repo: "never-read", prepared } }]); + await expect(materializeFeed(context)).rejects.toThrow("retained landed tree bytes"); + context.history = undefined; + await expect(materializeFeed(context)).rejects.toThrow("--history is required"); + }); + + it("refuses before the first landing in a nonempty log unless overridden", async () => { + const { context, landing, log } = await fixture(); + await log([{ ...landing, switching_at: LATE, landed_at: LATE }]); + await expect(materializeFeed(context)).rejects.toThrow(`Feed news has no landed state at ${AT}; provide an override`); + await expect(readdir(context.destination)).rejects.toThrow(); + context.input.overridden = true; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "overridden", basis: "override" }); + expect(await readdir(context.destination)).toEqual([]); + }); + + it("preserves a matching wake pin before the first retained landing", async () => { + const { context, landing, log } = await fixture(); + await log([{ ...landing, switching_at: LATE, landed_at: LATE }]); + context.record = { ...testMomentRecord(), feeds: [{ id: "news", revision: landing.revision, tree: landing.tree, + commit: landing.commit, ref: landing.ref, paths: landing.paths, landedAt: EARLY }] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: landing.commit } }); + expect(await readFile(path.join(context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); + }); + + it("validates every landing line and falls back for an empty landing log", async () => { + const { context, directory, landing } = await fixture(); + await writeFile(path.join(directory, "landings.jsonl"), `${JSON.stringify(landing)}\n${JSON.stringify({ ...landing, switching_at: undefined })}\n`); + await expect(materializeFeed(context)).rejects.toThrow(/landings.jsonl:2/u); + await writeFile(path.join(directory, "landings.jsonl"), ""); + expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", basis: "feed-rule-at" }); + await rm(context.destination, { recursive: true }); + }); + + it("ignores archive attributes and preserves long names when reconstructing feed bytes", async () => { + const { context, repository, resource } = await fixture(); + const longName = `published/${"x".repeat(150)}.txt`; + const commit = await commitTestGit(repository, { ".gitattributes": "published/hidden.txt export-ignore\npublished/version.txt export-subst\n", + "published/hidden.txt": "omitted\n", "published/version.txt": "$Format:%H$\n", [longName]: "long\n" }, "2026-10-09T08:10:00Z"); + resource.feed!.git!.ref = commit; + await materializeFeed(context); + expect(await readFile(path.join(context.destination, "current/published/version.txt"), "utf8")).toBe("$Format:%H$\n"); + expect(await readFile(path.join(context.destination, "current", longName), "utf8")).toBe("long\n"); + expect(await readFile(path.join(context.destination, "current/published/hidden.txt"), "utf8")).toBe("omitted\n"); + }); + + it("refuses retained history symlinks outside the history root", async () => { + const { context, directory, landing, log } = await fixture(); + const retained = path.join(directory, "trees", landing.revision); + await mkdir(retained, { recursive: true }); await symlink("/etc/passwd", path.join(retained, "escape")); await log([landing]); + await expect(materializeFeed(context)).rejects.toThrow("History symlink escape"); + }); + +}); diff --git a/src/compiler/training/rewind/feed.ts b/src/compiler/training/rewind/feed.ts new file mode 100644 index 00000000..e5e33261 --- /dev/null +++ b/src/compiler/training/rewind/feed.ts @@ -0,0 +1,156 @@ +import { lstat, mkdir, readdir, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { TeamWorkspaceResource } from "../../../manifest/index.js"; +import { digestDirectory, expandFeedPaths, expandFeedRefTemplate, FEED_IDENTITY_FILE, FEED_IDENTITY_VERSION, + feedLandingAt, isFeedLanding, type FeedIdentity, type FeedLanding } from "../../../volume/index.js"; +import { assertPreparedFeedRevision, feedGitRevision } from "./feedContent.js"; +import { resolveRecordedCommit } from "./recordedCommit.js"; +import { exportFeedGitTree } from "./gitArchive.js"; +import { safeCommittedPath } from "./gitTree.js"; +import { assertLocalGitRepository } from "./gitHistory.js"; +import { assertCommitNotFuture, commitAt, type SelectedCommit } from "./gitProcess.js"; +import { historyKey } from "./history.js"; +import { compareInstants } from "./instant.js"; +import type { InputContext, InputResult } from "./types.js"; + +type VolumeFeed = NonNullable["feed"]>; + +const feedPaths = (paths: string[] | undefined): string[] | undefined => paths?.length ? paths : undefined; +const treeName = (tree: string): string => tree.replace(/^trees\//u, ""); + +/** Same template/fallback ordering as chooseFeedRef, with historical eligibility replacing existence. */ +export const selectFeedCommit = async (feed: VolumeFeed, repository: string, at: string): Promise => { + if (!feed.git) throw new Error("Directory feed cannot be rewound without a landing log; provide an override"); + if (feed.freeze) throw new Error("Feed freeze/waiting state cannot be reconstructed without a landing log; provide an override"); + const rule = feed.git.ref; + if (typeof rule === "object" && rule.command) throw new Error("Feed command refs cannot be evaluated historically without a landing log; provide an override"); + const selectors = typeof rule === "object" ? [expandFeedRefTemplate(rule.template!, new Date(at)), rule.fallback] + : [rule ?? "HEAD"]; + for (const selector of selectors) { + if (!selector) continue; + const selected = await commitAt(repository, selector, at); + if (selected) return { ...selected, ref: selector }; + } + throw new Error(`No feed ref has a first-parent commit at or before ${at}`); +}; + +const fileCount = async (directory: string): Promise => { + let count = 0; + for (const name of await readdir(directory)) { + const file = path.join(directory, name), stats = await lstat(file); + count += stats.isDirectory() ? await fileCount(file) : 1; + } + return count; +}; + +export const materializeFeed = async (context: InputContext): Promise => { + const { input, history } = context; + if (input.section !== "resources" || input.declaration.kind !== "volume" || !input.declaration.feed) throw new Error("Feed rewind requires a fed volume"); + const resource = input.declaration, feed = resource.feed!; + if (input.overridden) { + await mkdir(context.destination, { recursive: true }); + return { fidelity: "overridden", basis: "override", source: {}, notes: ["Volume override starts empty; no feed refresh is run."], instant: context.at }; + } + if (!history) throw new Error(`Missing feed history for ${input.id}; --history is required`); + const key = `feeds/${historyKey(input)}`; + const log = await history.feedLandings(key, input.id); + let landing: FeedLanding, repository: string | null = null, basis: string, ambiguous = false; + const openRepository = async (): Promise => { + repository ??= await history.repository(`${key}/repo`); + if (!repository) throw new Error(`Missing feed source repository for ${input.id}`); + await assertLocalGitRepository(repository, history); + return repository; + }; + for (const [index, entry] of (log ?? []).entries()) { + for (const selected of entry.paths ?? []) safeCommittedPath(selected); + if (!entry.commit) continue; + try { await resolveRecordedCommit(await openRepository(), entry.commit); } + catch (error) { throw new Error(`Invalid history ${key}/landings.jsonl:${index + 1}: ${error instanceof Error ? error.message : String(error)}`); } + } + for (const candidate of context.candidateRecords ?? []) { + const entry = candidate.feeds.find((record) => record.id === input.id); + if (entry?.commit !== undefined) await resolveRecordedCommit(await openRepository(), entry.commit); + } + const record = context.record?.feeds.find((entry) => entry.id === input.id); + const recordedCommit = record?.commit === undefined ? undefined : await resolveRecordedCommit(await openRepository(), record.commit); + const matching = record ? log?.findLast((entry) => entry.revision === record.revision && treeName(entry.tree) === treeName(record.tree) + && compareInstants(entry.landed_at, record.landedAt) === 0 && entry.commit === recordedCommit) : undefined; + const selected = record ? null : feedLandingAt(log, context.at); + if (record) { + // A wake pin is authoritative even if a later attempt or landing changed the currently selected log row. + landing = { version: "spawnfile.volume-feed-landing.v1", resource: input.id, volume: matching?.volume ?? resource.name!, + revision: record.revision, tree: `trees/${treeName(record.tree)}`, landed_at: record.landedAt, + switching_at: matching?.switching_at ?? record.landedAt, + source: matching?.source ?? (feed.git ? { kind: "git", repo: feed.git.repo } : { kind: "directory", directory: feed.directory! }), + ...(recordedCommit ? { commit: recordedCommit, ref: record.ref ?? recordedCommit } : {}), + ...(record.paths ? { paths: record.paths } : {}), ...(record.period ? { period: record.period } : {}) }; + basis = "record"; + } else if (selected) { + landing = selected; ambiguous = selected.ambiguous === true; basis = "landing-log"; + } else { + if (log.length) throw new Error(`Feed ${input.id} has no landed state at ${context.at}; provide an override`); + // Refuse unknown historical behavior before touching any declared source or executing any command. + if (!feed.git || feed.freeze || typeof feed.git.ref === "object" && feed.git.ref.command) await selectFeedCommit(feed, "", context.at); + if (feed.include || feed.prepare) throw new Error("Prepared/included feed requires a landing log and retained tree bytes"); + const repo = await openRepository(), commit = await selectFeedCommit(feed, repo, context.at); + const paths = expandFeedPaths(feed.git!.paths, new Date(context.at)); + const revision = feedGitRevision(repo, commit.commit, commit.ref, paths); + landing = { version: "spawnfile.volume-feed-landing.v1", resource: input.id, volume: resource.name!, + revision, tree: `trees/${revision}`, source: { kind: "git", repo: feed.git!.repo }, + commit: commit.commit, ref: commit.ref, ...(paths ? { paths } : {}), switching_at: commit.instant, landed_at: commit.instant }; + basis = "feed-rule-at"; + } + if (!isFeedLanding(landing)) throw new Error(`Invalid feed landing for ${input.id}`); + for (const selected of landing.paths ?? []) safeCommittedPath(selected); + // Selection owns this check: retained/prepared bytes must obey the same cutoff as extraction. + if (landing.source.kind === "git" && (!landing.commit || !landing.ref)) throw new Error("Git feed landing must record its commit and ref"); + if (landing.commit) { + landing.commit = await resolveRecordedCommit(await openRepository(), landing.commit); + await assertCommitNotFuture(await openRepository(), landing.commit, context.at); + } + const retainedName = treeName(landing.tree), tree = path.join(context.destination, "trees", retainedName); + const isPrepared = landing.source.prepared !== undefined + || basis === "record" && !matching && (feed.include !== undefined || feed.prepare !== undefined); + if (landing.source.prepared) { + assertPreparedFeedRevision(landing.revision, landing.source.prepared); + if (landing.commit && feedGitRevision(await openRepository(), landing.commit, landing.ref!, landing.paths) + !== landing.source.prepared.source_revision) throw new Error("Prepared feed source revision disagrees with selected commit and paths"); + } + if (landing.source.kind === "git" && !isPrepared) { + const repo = await openRepository(); + const revision = feedGitRevision(repo, landing.commit!, landing.ref!, landing.paths); + if (revision !== landing.revision) throw new Error(`Feed revision mismatch: recorded ${landing.revision}, reconstructed ${revision}`); + await exportFeedGitTree(repo, landing.commit!, tree, feedPaths(landing.paths)); + if (await history.path(`${key}/trees/${retainedName}`)) { + const retained = path.join(context.destination, ".retained-check"); + try { + await history.copyTree(`${key}/trees/${retainedName}`, retained); + if (digestDirectory(retained) !== digestDirectory(tree)) throw new Error("Retained git feed bytes disagree with selected commit and paths"); + } finally { await rm(retained, { recursive: true, force: true }); } + } + } else { + if (!await history.path(`${key}/trees/${retainedName}`)) throw new Error("Directory/prepared/included feed requires retained landed tree bytes"); + await history.copyTree(`${key}/trees/${retainedName}`, tree); + } + if (landing.source.kind === "directory" && !isPrepared) { + const actual = digestDirectory(tree); + if (actual !== landing.revision) throw new Error(`Directory feed revision mismatch: recorded ${landing.revision}, retained ${actual}`); + } + const files = await fileCount(tree); + if (!files) throw new Error("Retained feed tree is empty; a fed volume never serves nothing"); + const prepared = landing.source.prepared ? { prepared: landing.source.prepared } : {}; + const identity: FeedIdentity = { version: FEED_IDENTITY_VERSION, resource: input.id, volume: landing.volume, revision: landing.revision, + tree: `trees/${retainedName}`, files, landed_at: landing.landed_at, + source: landing.source.kind === "git" ? { kind: "git", commit: landing.commit!, ref: landing.ref!, paths: landing.paths ?? null, ...prepared } + : { kind: "directory", ...prepared } }; + await symlink(identity.tree, path.join(context.destination, "current")); + await writeFile(path.join(context.destination, FEED_IDENTITY_FILE), `${JSON.stringify(identity, null, 2)}\n`); + return { fidelity: isPrepared || basis === "feed-rule-at" || ambiguous ? "approximate" : "exact", basis, + source: { revision: landing.revision, landedAt: landing.landed_at, + ...(basis === "landing-log" || matching ? { switchingAt: landing.switching_at } : {}), + ...(landing.commit ? { commit: landing.commit } : {}), ...(isPrepared ? { prepared: true } : {}) }, + instant: compareInstants(landing.landed_at, context.at) > 0 ? context.at : landing.landed_at, + notes: [...(isPrepared ? ["prepared output not content-verifiable"] : []), + ...(ambiguous ? ["The trial instant falls inside an ambiguous feed transition; the incoming landing is used."] + : basis === "feed-rule-at" ? ["Commit dates do not establish when a ref advanced; historical freeze holds cannot be reconstructed."] : [])] }; +}; diff --git a/src/compiler/training/rewind/feedContent.test.ts b/src/compiler/training/rewind/feedContent.test.ts new file mode 100644 index 00000000..dd1c9959 --- /dev/null +++ b/src/compiler/training/rewind/feedContent.test.ts @@ -0,0 +1,99 @@ +import { mkdir, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { digestDirectory, hostExec } from "../../../volume/index.js"; +import { resolveFeedContent } from "../../../volume/feedPrepare.js"; +import { cleanTrainingProjects } from "../plan/fixtures.test-helper.js"; +import * as feed from "./feed.js"; +import { AT, BEFORE, emptyRecord, jsonl, rewindFixture } from "./fixtures.test-helper.js"; +import { leakCheckTrainingMoment } from "./leakCheck.js"; +import { materializeTrainingMoment, resolveTrainingMoment } from "./materialize.js"; +import { stampTree } from "./timestamps.js"; + +afterEach(cleanTrainingProjects); +afterEach(() => vi.restoreAllMocks()); + +const preparedFixture = async (kind: "git" | "directory", recipe: "include" | "prepare", recordOnly = false) => { + const f = await rewindFixture(), directory = path.dirname(f.feedRepo), sourceDirectory = path.join(f.root, "source"); + await mkdir(sourceDirectory); await writeFile(path.join(sourceDirectory, "source.txt"), "source bytes\n"); + const resolved = resolveFeedContent({ resourceId: "news", volume: directory, volumeName: "news", stateDir: directory, keep: 1, healLimit: 3, + source: kind === "git" ? { kind, repo: f.feedRepo, ref: f.commit, fetch: false } : { kind, directory: sourceDirectory }, + ...(recipe === "include" ? { include: [] } : { prepare: { kind: "host" as const, command: ["never-run-recipe"], timeoutMs: 1 } }) + }, { exec: hostExec }); + const retained = path.join(directory, "trees", resolved.revision); + await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "prepared.txt"), "landed prepared bytes\n"); + expect(digestDirectory(retained)).not.toBe(resolved.revision); + const log = "feeds/team%3Apublication/news/landings.jsonl"; + await jsonl(f.historyDir, log, [{ version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "news", + revision: resolved.revision, tree: `trees/${resolved.revision}`, switching_at: BEFORE, landed_at: BEFORE, + source: { kind, ...(kind === "git" ? { repo: f.feedRepo } : { directory: sourceDirectory }), prepared: resolved.provenance.prepared }, + ...(kind === "git" ? { commit: f.commit, ref: "main" } : {}) }]); + if (recordOnly) { + const input = resolveTrainingMoment(f.plan, f.project, f.options.example).inputs.find((entry) => entry.id === "news" && entry.section === "resources")!; + if (input.section !== "resources" || input.declaration.kind !== "volume") throw new Error("Expected feed"); + input.declaration.feed = { ...(kind === "git" ? { git: { repo: f.feedRepo } } : { directory: sourceDirectory }), include: [] }; + const record = emptyRecord(); record.feeds = [{ id: "news", revision: resolved.revision, tree: `trees/${resolved.revision}`, landedAt: BEFORE, + ...(kind === "git" ? { commit: f.commit, ref: "main" } : {}) }]; + await jsonl(f.historyDir, "moments/prepared.jsonl", [record]); await rm(path.join(f.historyDir, log)); + } + return f; +}; + +describe("independent feed byte verification", () => { + it.each(["changed bytes", "extra file", "wrong revision", "missing repository"]) + ("rejects a broken plain git producer with %s at acceptance", async (change) => { + const f = await rewindFixture(), correct = feed.materializeFeed; + vi.spyOn(feed, "materializeFeed").mockImplementation(async (context) => { + const result = await correct(context); + if (change === "missing repository") await rm(f.feedRepo, { recursive: true }); + else if (change === "wrong revision") { + const file = path.join(context.destination, ".spawnfile-feed.json"), identity = JSON.parse(await readFile(file, "utf8")); + identity.revision = "0".repeat(64); result.source.revision = identity.revision; + await writeFile(file, JSON.stringify(identity)); + } else await writeFile(path.join(context.destination, "current", change === "changed bytes" ? "news.txt" : "extra.txt"), "future bytes\n"); + return result; + }); + const result = await materializeTrainingMoment(f.options); + expect(result.status).toBe("refused"); + const check = JSON.parse(await readFile(path.join(f.options.out, "leak-check.json"), "utf8")); + expect(check.findings).toContainEqual(expect.objectContaining({ class: "feed-identity", path: "resources/team%3Apublication/news", + detail: change === "missing repository" ? "Missing feed source history repository for content verification" + : change === "wrong revision" ? "Feed revision disagrees with selected commit and paths" : "Git feed bytes disagree with selected commit and paths" })); + expect(check.findings.some((finding: { class: string }) => finding.class === "file-time")).toBe(false); + expect(await readdir(f.options.out)).toEqual(["leak-check.json", "moment.json"]); + }); + + it.each((["git", "directory"] as const).flatMap((kind) => (["include", "prepare"] as const).map((recipe) => ({ kind, recipe })))) + ("accepts recording-format $kind/$recipe provenance with approximate prepared bytes", async ({ kind, recipe }) => { + const f = await preparedFixture(kind, recipe), result = await materializeTrainingMoment(f.options); + expect(result, JSON.stringify(result.inputs)).toMatchObject({ status: "materialized", leakCheck: { ok: true } }); + const input = result.inputs.find((entry) => entry.kind === "feed")!; + expect(input).toMatchObject({ fidelity: "approximate", source: { prepared: true }, notes: ["prepared output not content-verifiable"] }); + expect(await readFile(path.join(f.options.out, input.path!, "current/prepared.txt"), "utf8")).toBe("landed prepared bytes\n"); + await stampTree(f.options.out, BEFORE, AT); + expect((await leakCheckTrainingMoment(f.options.out, { historyDir: f.historyDir })).findings).toEqual([]); + result.inputs.find((entry) => entry.kind === "feed")!.fidelity = "exact"; + await writeFile(path.join(f.options.out, "moment.json"), JSON.stringify(result)); await stampTree(f.options.out, BEFORE, AT); + expect((await leakCheckTrainingMoment(f.options.out, { historyDir: f.historyDir })).findings) + .toContainEqual(expect.objectContaining({ class: "feed-identity", detail: "Prepared feed must be approximate: prepared output not content-verifiable" })); + }); + + it.each(["git", "directory"] as const)("keeps record-only prepared %s bytes approximate when the landing log is absent", async (kind) => { + const f = await preparedFixture(kind, "include", true), result = await materializeTrainingMoment(f.options); + expect(result, JSON.stringify(result.inputs)).toMatchObject({ status: "materialized", leakCheck: { ok: true } }); + expect(result.inputs.find((entry) => entry.kind === "feed")).toMatchObject({ fidelity: "approximate", basis: "record", + source: { prepared: true }, notes: ["prepared output not content-verifiable"] }); + }); + + it("still applies filesystem leak classes to prepared output", async () => { + const f = await preparedFixture("directory", "prepare"), result = await materializeTrainingMoment(f.options); + expect(result.status).toBe("materialized"); + const tree = path.join(f.options.out, result.inputs.find((entry) => entry.kind === "feed")!.path!, "current"); + await mkdir(path.join(tree, ".git")); await writeFile(path.join(tree, "2099-01-01.txt"), "future path"); + await symlink("/outside", path.join(tree, "escape")); await stampTree(f.options.out, BEFORE, AT); + await writeFile(path.join(tree, "prepared.txt"), "changed now"); + const check = await leakCheckTrainingMoment(f.options.out, { historyDir: f.historyDir }); + for (const kind of ["file-time", "git-object", "dated-path", "symlink-escape"]) expect(check.findings) + .toContainEqual(expect.objectContaining({ class: kind, path: expect.stringContaining("resources/team%3Apublication/news/trees/") })); + }); +}); diff --git a/src/compiler/training/rewind/feedContent.ts b/src/compiler/training/rewind/feedContent.ts new file mode 100644 index 00000000..e9c873c9 --- /dev/null +++ b/src/compiler/training/rewind/feedContent.ts @@ -0,0 +1,36 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { digestDirectory, resolveFeedSource, type FeedExec, type FeedIdentity } from "../../../volume/index.js"; +import { exportFeedGitTree } from "./gitArchive.js"; +import { rewindGitSync } from "./gitProcess.js"; + +const revisionExec: FeedExec = (command, args) => { + if (command !== "git" || args[0] !== "-C" || !args[1]) throw new Error("Feed revision may only read a local git repository"); + return rewindGitSync(args[1], args.slice(2)); +}; +export const feedGitRevision = (repo: string, commit: string, ref: string, paths?: string[] | null): string => + resolveFeedSource({ kind: "git", fetch: false, repo, ref: commit, label: ref, paths: paths?.length ? paths : undefined }, + { exec: revisionExec }).revision; + +/** A prepared revision proves the recipe/source identity, never its output bytes. */ +export const assertPreparedFeedRevision = (revision: string, prepared: NonNullable): void => { + const provenance = { include: prepared.include, prepare: prepared.prepare, source_revision: prepared.source_revision }; + const actual = createHash("sha256").update(`prepared\n${JSON.stringify(provenance)}\n`).digest("hex"); + if (actual !== revision) throw new Error("Prepared feed revision disagrees with its provenance"); +}; + +/** Acceptance re-extracts from history; it does not consume any producer-computed content digest. */ +export const verifyPlainFeedTree = async (repo: string, identity: FeedIdentity, directory: string): Promise => { + const source = identity.source; + if (source.kind !== "git") throw new Error("Plain git feed verification requires git provenance"); + if (feedGitRevision(repo, source.commit, source.ref, source.paths) !== identity.revision) { + throw new Error("Feed revision disagrees with selected commit and paths"); + } + const scratch = await mkdtemp(path.join(os.tmpdir(), "spawnfile-feed-check-")); + try { + await exportFeedGitTree(repo, source.commit, scratch, source.paths?.length ? source.paths : undefined); + if (digestDirectory(directory) !== digestDirectory(scratch)) throw new Error("Git feed bytes disagree with selected commit and paths"); + } finally { await rm(scratch, { recursive: true, force: true }); } +}; diff --git a/src/compiler/training/rewind/fixtures.test-helper.ts b/src/compiler/training/rewind/fixtures.test-helper.ts new file mode 100644 index 00000000..b94bdaba --- /dev/null +++ b/src/compiler/training/rewind/fixtures.test-helper.ts @@ -0,0 +1,66 @@ +import { execFileSync } from "node:child_process"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { trainingProject, writeTrainingFile, teamFiles, teamManifest, volume, example, network } from "../plan/fixtures.test-helper.js"; +import { loadTrainingProject, resolveTrainingPlan } from "../plan/index.js"; +import { rewindGitEnvironment } from "./gitProcess.js"; +import { resolveFeedSource } from "../../../volume/feedSource.js"; +import { writeRecordedVolumeSnapshot } from "./snapshotFixtures.test-helper.js"; +import type { MomentRecord } from "../../../recording/index.js"; +import type { MemoryLedgerEvent } from "./memorySchemas.js"; + +export const AT = "2026-10-09T08:20:31Z"; +export const BEFORE = "2026-10-09T07:00:00Z"; +export const FUTURE = "2026-10-09T08:20:31.001Z"; +export const memoryEvent = (seq: number, createdAt = BEFORE): MemoryLedgerEvent => ({ + id: `event-${seq}`, seq, createdAt, type: "memory.observed", principal: { agentId: "a", scope: "global" }, + scope: "global", visibility: "global", source: "rewind-test", content: { kind: "text", text: `Fact ${seq}` }, + tags: [], entities: [], sensitivity: "normal", parentEventIds: [], checksum: `checksum-${seq}` +}); +export const emptyRecord = (agent = "agent:a", at = AT): MomentRecord => ({ + version: "spawnfile.moment-record.v1", agent, wake: { at, startedAt: at, executionId: "execution-17", + deliveries: [{ deliveryId: "delivery-17", kind: "message", occurredAt: at }], surface: "moltnet", room: "assignment", from: "brass" }, + recordedAt: FUTURE, exact: true, resources: [], feeds: [], memory: [], moltnet: [], volumes: [] +}); +export const jsonl = async (root: string, relative: string, values: unknown[]): Promise => { + const file = path.join(root, relative); + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, values.map((value) => JSON.stringify(value)).join("\n") + (values.length ? "\n" : "")); +}; +export const testGit = (repo: string, args: string[], date = BEFORE): string => execFileSync("git", ["-C", repo, ...args], { + encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env: { ...rewindGitEnvironment(), GIT_AUTHOR_NAME: "Rewind", GIT_AUTHOR_EMAIL: "rewind@example.invalid", + GIT_COMMITTER_NAME: "Rewind", GIT_COMMITTER_EMAIL: "rewind@example.invalid", GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date } +}).trim(); + +/** Real compiled team with all five storage paths, including a historical git feed. */ +export const rewindFixture = async () => { + const root = await trainingProject(teamFiles()); + const historyDir = path.join(root, "history"), gitRepo = path.join(historyDir, "git/team%3Apublication/code"); + await mkdir(gitRepo, { recursive: true }); + testGit(gitRepo, ["init", "--quiet", "--initial-branch=main"]); + await writeFile(path.join(gitRepo, "news.txt"), "Before the wake\n"); + testGit(gitRepo, ["add", "."]); testGit(gitRepo, ["commit", "--quiet", "-m", "before"]); + const commit = testGit(gitRepo, ["rev-parse", "HEAD"]); + const feedRepo = path.join(historyDir, "feeds/team%3Apublication/news/repo"); + await mkdir(path.dirname(feedRepo), { recursive: true }); + testGit(root, ["clone", "--quiet", "--no-local", gitRepo, feedRepo]); + const feedSource = resolveFeedSource({ kind: "git", repo: feedRepo, ref: commit, fetch: false }); + await jsonl(historyDir, "feeds/team%3Apublication/news/landings.jsonl", [{ version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "news", source: { kind: "git", repo: feedRepo }, revision: feedSource.revision, + tree: `trees/${feedSource.revision}`, commit, ref: "main", switching_at: BEFORE, landed_at: BEFORE }]); + await writeFile(path.join(gitRepo, "future.txt"), "secret after wake\n"); + testGit(gitRepo, ["add", "."]); testGit(gitRepo, ["commit", "--quiet", "-m", "future"], "2026-10-10T00:00:00Z"); + const futureCommit = testGit(gitRepo, ["rev-parse", "HEAD"]); + const resources = [volume, { id: "code", kind: "git", mount: "./code", mode: "readonly", url: gitRepo, branch: "main" }, + { id: "news", kind: "volume", name: "news", mount: "./news", mode: "mutable", feed: { git: { repo: feedRepo } } }]; + const manifest = teamManifest({ memory: [] }); + manifest.shared.workspace.resources = resources as typeof manifest.shared.workspace.resources; + await writeTrainingFile(root, "Spawnfile", manifest); + await writeTrainingFile(root, "training/train.yaml", { examples: [{ ...example, networks: [network] }] }); + await jsonl(historyDir, "memory/agent%3Aa/self/events.jsonl", [memoryEvent(1), memoryEvent(2, FUTURE)]); + const project = await loadTrainingProject(root), plan = resolveTrainingPlan(project, "agent:a", "test"); + const agentSlug = project.compilePlan.nodes.find((node) => node.id === plan.agent.id)!.slug; + await writeRecordedVolumeSnapshot(historyDir, { agentSlug, agentId: plan.agent.id, rootId: "edition-state", + snapshot: "20261009T070000000Z-execution-17", startedAt: BEFORE, files: { "saved.txt": "snapshot bytes\n" } }); + return { root, historyDir, project, plan, gitRepo, futureCommit, commit, feedRepo, + options: { project, plan, example: example.id, historyDir, out: path.join(root, "out") } }; +}; diff --git a/src/compiler/training/rewind/git.test-helper.ts b/src/compiler/training/rewind/git.test-helper.ts new file mode 100644 index 00000000..7fe9df0c --- /dev/null +++ b/src/compiler/training/rewind/git.test-helper.ts @@ -0,0 +1,31 @@ +import { execFileSync } from "node:child_process"; +import { mkdir, mkdtemp, realpath, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { MomentRecord } from "../../../recording/index.js"; +import { rewindGitEnvironment } from "./gitProcess.js"; + +export const gitTestRoot = async (): Promise => realpath(await mkdtemp(path.join(os.tmpdir(), "spawnfile-rewind-git-"))); +export const testGit = (repository: string, args: string[], at = "2026-10-09T08:00:00Z"): string => execFileSync("git", [ + "-c", "core.hooksPath=/dev/null", "-c", "core.autocrlf=false", "-C", repository, ...args +], { encoding: "utf8", env: { ...rewindGitEnvironment(), GIT_AUTHOR_NAME: "Rewind Test", GIT_AUTHOR_EMAIL: "rewind@example.invalid", + GIT_COMMITTER_NAME: "Rewind Test", GIT_COMMITTER_EMAIL: "rewind@example.invalid", GIT_AUTHOR_DATE: at, GIT_COMMITTER_DATE: at }, stdio: ["ignore", "pipe", "pipe"] }).trim(); +export const initTestGit = async (repository: string): Promise => { + await mkdir(repository, { recursive: true }); + testGit(repository, ["init", "--quiet", "--initial-branch=main", "--template="]); +}; +export const commitTestGit = async (repository: string, files: Record, at: string): Promise => { + for (const [name, contents] of Object.entries(files)) { + const file = path.join(repository, name); + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, contents); + } + testGit(repository, ["add", "--all"]); + testGit(repository, ["commit", "--quiet", "--allow-empty", "-m", `at ${at}`], at); + return testGit(repository, ["rev-parse", "HEAD"]); +}; +export const testMomentRecord = (at = "2026-10-09T08:20:31Z"): MomentRecord => ({ + version: "spawnfile.moment-record.v1", agent: "agent:writer", wake: { at, startedAt: at, executionId: "execution-17", + deliveries: [{ deliveryId: "delivery-17", kind: "message", occurredAt: at }], surface: "moltnet", room: "assignment", from: "brass" }, + recordedAt: at, exact: true, resources: [], feeds: [], memory: [], moltnet: [], volumes: [] +}); diff --git a/src/compiler/training/rewind/git.test.ts b/src/compiler/training/rewind/git.test.ts new file mode 100644 index 00000000..57537f2e --- /dev/null +++ b/src/compiler/training/rewind/git.test.ts @@ -0,0 +1,196 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { mkdir, readFile, readdir, readlink, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { TeamWorkspaceResource } from "../../../manifest/index.js"; +import { BundleTarWriter } from "../../workspaceBundleTar.js"; +import { materializeGit, localRepositoryPath, selectCommitAt } from "./git.js"; +import { History } from "./history.js"; +import { commitTestGit, gitTestRoot, initTestGit, testGit, testMomentRecord } from "./git.test-helper.js"; +import type { InputContext } from "./types.js"; + +const AT = "2026-10-09T08:20:31Z", EARLY = "2026-10-09T08:00:00Z", LATE = "2026-10-09T09:00:00Z"; +const roots: string[] = []; +afterEach(async () => { vi.unstubAllEnvs(); await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const fixture = async () => { + const root = await gitTestRoot(); roots.push(root); + const repository = path.join(root, "history/git/agent%3Awriter/archive"); + await initTestGit(repository); + const past = await commitTestGit(repository, { "readme.txt": "past\n", "package/package.json": "{}\n", "package/secret.txt": "omit\n" }, EARLY); + const future = await commitTestGit(repository, { "readme.txt": "future\n", "future.txt": "tomorrow\n" }, LATE); + testGit(repository, ["tag", "future-tag", future]); + testGit(repository, ["remote", "add", "origin", "https://network-must-not-run.invalid/repo.git"]); + const resource: TeamWorkspaceResource = { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: repository }; + const context: InputContext = { at: AT, destination: path.join(root, "output"), history: new History(path.join(root, "history")), record: null, + input: { section: "resources", declaredBy: "agent:writer", id: "archive", declarationBase: repository, overridden: false, declaration: resource } }; + return { root, repository, past, future, context }; +}; + +describe("git rewind", () => { + it.each(["git", "bundle"] as const)("refuses future recorded %s commits before materializing", async (kind) => { + const { repository, past, context } = await fixture(); + context.input.declaration = kind === "git" + ? { id: "archive", kind, mode: "readonly", mount: "./archive", url: repository } + : { id: "archive", kind, mode: "readonly", mount: "./archive", build: { files: { root: "package" } } }; + for (const role of ["author", "committer"]) { + testGit(repository, ["checkout", "--detach", past]); + testGit(repository, ["commit", "--allow-empty", "-m", "future metadata", `--date=${role === "author" ? LATE : EARLY}`], role === "committer" ? LATE : EARLY); + const commit = testGit(repository, ["rev-parse", "HEAD"]); + context.record = { ...testMomentRecord(), resources: kind === "git" ? [{ id: "archive", kind, commit }] : [] }; + if (context.input.declaration.kind === "bundle") context.input.declaration.build!.files!.ref = commit; + await expect(materializeGit(context)).rejects.toThrow(`Selected commit ${commit} has an author or committer date after ${AT}`); + await expect(readdir(context.destination)).rejects.toThrow(); + } + }); + + it("records exactly one commit's reachable closure and removes future objects, refs and configuration", async () => { + const { repository, past, future, context } = await fixture(); + context.record = { ...testMomentRecord(), resources: [{ id: "archive", kind: "git", commit: past }] }; + const result = await materializeGit(context); + expect(result).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: past } }); + expect(await readFile(path.join(context.destination, "readme.txt"), "utf8")).toBe("past\n"); + const objects = testGit(context.destination, ["cat-file", "--batch-all-objects", "--batch-check=%(objectname)"]).split("\n").sort(); + expect(objects).toEqual(testGit(repository, ["rev-list", "--objects", past]).split("\n").map((line) => line.split(" ")[0]).sort()); + expect(objects).not.toContain(future); + expect(testGit(context.destination, ["for-each-ref", "--format=%(refname)"])).toBe("refs/heads/moment"); + expect(testGit(context.destination, ["remote"])).toBe(""); + expect(testGit(context.destination, ["status", "--porcelain"])).toBe(""); + for (const name of ["logs", "FETCH_HEAD", "ORIG_HEAD", "packed-refs"]) await expect(readFile(path.join(context.destination, ".git", name))).rejects.toThrow(); + expect(testGit(repository, ["rev-parse", "HEAD"])).toBe(future); + expect(testGit(repository, ["tag"])).toBe("future-tag"); + }); + + it("can materialize a local full-object pin without history and ignores injected git environment", async () => { + const { repository, past, context } = await fixture(); + context.history = undefined; + context.input.declaration = { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: repository, ref: past.toUpperCase() }; + vi.stubEnv("GIT_DIR", "/does/not/exist"); + vi.stubEnv("GIT_CONFIG_COUNT", "1"); vi.stubEnv("GIT_CONFIG_KEY_0", "core.worktree"); vi.stubEnv("GIT_CONFIG_VALUE_0", "/elsewhere"); + expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "pinned", source: { commit: past } }); + expect(await readFile(path.join(context.destination, "readme.txt"), "utf8")).toBe("past\n"); + }); + + it("uses selector-at first-parent fallback and labels its approximation", async () => { + const { past, context } = await fixture(); + expect(await materializeGit(context)).toMatchObject({ fidelity: "approximate", basis: "selector-at", source: { commit: past }, notes: [expect.stringContaining("dates")] }); + }); + + it("does not visit a later merged side branch when selecting the first-parent date", async () => { + const { repository, past, context } = await fixture(); + testGit(repository, ["checkout", "-b", "side", past]); + await commitTestGit(repository, { "side.txt": "side\n" }, "2026-10-09T08:19:00Z"); + testGit(repository, ["checkout", "main"]); + testGit(repository, ["merge", "--no-ff", "--no-edit", "side"], "2026-10-09T10:00:00Z"); + expect((await materializeGit(context)).source.commit).toBe(past); + }); + + it("uses override declaration content and marks it overridden", async () => { + const { past, repository, context } = await fixture(); + context.history = undefined; context.input.overridden = true; + context.input.declaration = { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: repository, ref: past }; + expect(await materializeGit(context)).toMatchObject({ fidelity: "overridden", basis: "override", source: { commit: past } }); + }); + + it("refuses absent history, prebuilt bundles without records and network-only pins", async () => { + const { past, context } = await fixture(); context.history = undefined; + await expect(materializeGit(context)).rejects.toThrow("unpinned resources require --history"); + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", source: "today.tar" }; + await expect(materializeGit(context)).rejects.toThrow("Prebuilt bundle has no matching recorded archive"); + context.input.declaration = { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: "https://example.invalid/repo", ref: past }; + await expect(materializeGit(context)).rejects.toThrow("network access is forbidden"); + }); + + it("exports files bundle roots and exclusions at their pinned commit", async () => { + const { past, context } = await fixture(); + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { files: { root: "package", ref: past, exclude: ["secret.txt"] } } }; + expect(await materializeGit(context)).toMatchObject({ fidelity: "approximate", basis: "pinned", source: { commit: past } }); + expect(await readdir(context.destination)).toEqual(["package.json"]); + }); + + it("refuses future pinned bundle metadata before exporting a plain tree", async () => { + const { repository, past, future, context } = await fixture(); + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { files: { root: "package", ref: future } } }; + await expect(materializeGit(context)).rejects.toThrow("author or committer date after"); + // The author clock is independently checked: an old committer date alone is insufficient. + testGit(repository, ["checkout", "--detach", past]); + testGit(repository, ["commit", "--amend", "--no-edit", `--date=${LATE}`], EARLY); + context.input.declaration.build!.files!.ref = testGit(repository, ["rev-parse", "HEAD"]); + await expect(materializeGit(context)).rejects.toThrow("author or committer date after"); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it("defers dependency and generated recipes while keeping all declared historical inputs", async () => { + const { context } = await fixture(); + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { dependencies: { directory: "package", image: `node@sha256:${"0".repeat(64)}` } } }; + expect(await materializeGit(context)).toMatchObject({ build: "deferred", fidelity: "approximate" }); + expect(await readdir(context.destination)).toEqual(["package.json", "secret.txt"]); + await rm(context.destination, { recursive: true }); + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { generated: { + command: ["never-run"], cwd: ".", inputs: [{ root: ".", exclude: ["package/secret.txt"] }, { root: "package" }] + } } }; + expect(await materializeGit(context)).toMatchObject({ build: "deferred" }); + expect(await readFile(path.join(context.destination, "package/secret.txt"), "utf8")).toBe("omit\n"); + expect(await readFile(path.join(context.destination, "readme.txt"), "utf8")).toBe("past\n"); + }); + + it("refuses generated inputs spanning repositories instead of substituting the wrong tree", async () => { + const { root, context } = await fixture(), other = path.join(root, "other"); + await initTestGit(other); await commitTestGit(other, { "other.txt": "other repository\n" }, EARLY); + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { generated: { + command: ["never-run"], inputs: [{ root: "." }, { root: other }] + } } }; + await expect(materializeGit(context)).rejects.toThrow("multiple git repositories"); + }); + + it("recreates bundle symlinks and applies declared exclusions", async () => { + const { repository, context } = await fixture(); + await symlink("package.json", path.join(repository, "package/link")); + const ref = await commitTestGit(repository, {}, "2026-10-09T08:10:00Z"); + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { files: { root: "package", ref } } }; + expect(await materializeGit(context)).toMatchObject({ fidelity: "approximate" }); + expect(await readlink(path.join(context.destination, "link"))).toBe("package.json"); + expect(await readFile(path.join(context.destination, "link"), "utf8")).toBe("{}\n"); + await rm(context.destination, { recursive: true, force: true }); + context.input.declaration.build!.files!.exclude = ["link"]; + expect(await materializeGit(context)).toMatchObject({ fidelity: "approximate" }); + expect(await readdir(context.destination)).not.toContain("link"); + }); + + it("extracts a validated local prebuilt override", async () => { + const { root, context } = await fixture(), tar = path.join(root, "override.tar"); + const writer = await BundleTarWriter.create(tar); + await writer.begin("override.txt", 6, 0o644); await writer.data(Buffer.from("fresh\n")); await writer.end(); await writer.finish(); + context.input.overridden = true; + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", source: tar }; + expect(await materializeGit(context)).toMatchObject({ fidelity: "overridden", source: {} }); + expect(await readFile(path.join(context.destination, "override.txt"), "utf8")).toBe("fresh\n"); + }); + + it("rejects textual history indirections before following them", async () => { + const { repository, context } = await fixture(); + await writeFile(path.join(repository, ".git/objects/info/alternates"), "/outside/object-store\n"); + await expect(materializeGit(context)).rejects.toThrow("alternates"); + await rm(path.join(repository, ".git/objects/info/alternates")); + await writeFile(path.join(repository, ".git/config"), "[include]\npath = /outside/config\n"); + await expect(materializeGit(context)).rejects.toThrow("configuration indirection"); + await writeFile(path.join(repository, ".git/commondir"), "/outside/common\n"); + await expect(materializeGit(context)).rejects.toThrow("commondir"); + }); + + it("selects date boundaries by instant, rejects unsafe local locations", () => { + const at = "2026-10-09T10:20:31+02:00"; + expect(selectCommitAt([{ commit: "equal", instant: AT }, { commit: "later", instant: "2026-10-09T08:20:31.001Z" }], at)?.commit).toBe("equal"); + expect(selectCommitAt([{ commit: "later", instant: LATE }], AT)).toBeNull(); + expect(() => localRepositoryPath("git@host:repo", "/tmp")).toThrow("network"); + expect(() => localRepositoryPath("file://remote/tmp/repo", "/tmp")).toThrow("remote file hosts"); + }); + + it("supports standalone bare repositories", async () => { + const { root, repository, past, context } = await fixture(); + const bare = path.join(root, "bare"); await mkdir(bare); + testGit(root, ["clone", "--quiet", "--bare", repository, bare]); + await rm(repository, { recursive: true }); + await mkdir(path.dirname(repository), { recursive: true }); + testGit(root, ["clone", "--quiet", "--bare", bare, repository]); + expect((await materializeGit(context)).source.commit).toBe(past); + }); +}); diff --git a/src/compiler/training/rewind/git.ts b/src/compiler/training/rewind/git.ts new file mode 100644 index 00000000..bd259417 --- /dev/null +++ b/src/compiler/training/rewind/git.ts @@ -0,0 +1,151 @@ +import { mkdir, rm } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import type { TeamWorkspaceResource } from "../../../manifest/index.js"; +import { deployedResource } from "./deployedResource.js"; +import { historyKey } from "./history.js"; +import { assertLocalGitRepository } from "./gitHistory.js"; +import { assertCommitNotFuture, commitAt, commitInstant, resolveRewindCommit, rewindGit } from "./gitProcess.js"; +import { exportGitTree, extractOverrideBundle } from "./gitTree.js"; +import { resolveRecordedCommit } from "./recordedCommit.js"; +import { materializeRecordedBundle } from "./recordedBundle.js"; +import { recordActivationErrors } from "./recordInput.js"; +import type { MaterializedInput } from "./contract.js"; +import type { InputContext, InputResult } from "./types.js"; + +export { commitAt, commitInstant, resolveRewindCommit, rewindGit, rewindGitEnvironment, rewindGitSync, selectCommitAt } from "./gitProcess.js"; +export { exportGitTree } from "./gitTree.js"; +export { assertLocalGitRepository } from "./gitHistory.js"; + +type GitResource = Extract; +type BundleResource = Extract; +const FULL_OBJECT = /^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/u; + +export const localRepositoryPath = (location: string, base: string): string => { + if (location.startsWith("file://")) { + const url = new URL(location); + if (url.hostname && url.hostname !== "localhost") throw new Error("Rewind only reads local repositories; remote file hosts are forbidden"); + return fileURLToPath(url); + } + if (/^[A-Za-z][A-Za-z0-9+.-]*:/u.test(location) || /^[^/]+@[^:]+:/u.test(location)) throw new Error("Rewind only reads local repositories; network access is forbidden"); + return path.resolve(base, location); +}; + +/** Import only the transitive object set of the selected commit; no source refs or config survive. */ +export const materializeCleanRepository = async (repository: string, commit: string, destination: string): Promise => { + const format = (await rewindGit(repository, ["rev-parse", "--show-object-format"])).toString().trim(); + const pack = await rewindGit(repository, ["pack-objects", "--revs", "--stdout"], Buffer.from(`${commit}\n`)); + await mkdir(destination, { recursive: true }); + await rewindGit(destination, ["init", "--quiet", "--template=", "--initial-branch=moment", `--object-format=${format}`]); + await rewindGit(destination, ["config", "core.logAllRefUpdates", "false"]); + await rewindGit(destination, ["index-pack", "--stdin"], pack); + await rewindGit(destination, ["update-ref", "refs/heads/moment", commit]); + await rewindGit(destination, ["read-tree", commit]); + await rewindGit(destination, ["checkout-index", "--all", "--force"]); + // read-tree may reuse cached stat data from an existing index; rebuild it from scratch. + await rm(path.join(destination, ".git/index")); + await rewindGit(destination, ["read-tree", commit]); + for (const name of ["FETCH_HEAD", "ORIG_HEAD", "logs", "packed-refs"]) await rm(path.join(destination, ".git", name), { recursive: true, force: true }); +}; + +const selectorOf = (resource: GitResource | BundleResource): string => resource.kind === "git" + ? resource.ref || (resource.branch ? `refs/heads/${resource.branch}` : resource.tag ? `refs/tags/${resource.tag}` : "HEAD") + : resource.build?.files?.ref ?? "HEAD"; + +const localRoot = (resource: GitResource | BundleResource): string | undefined => resource.kind === "git" ? resource.url + : resource.build?.files?.root ?? resource.build?.dependencies?.directory ?? resource.build?.generated?.cwd ?? resource.build?.generated?.inputs[0]?.root; + +/** Preserve the authored root's prefix when its checkout exists; otherwise use a plain manifest-relative root. */ +const bundlePrefix = async (root: string, base: string): Promise => { + const absolute = path.resolve(base, root); + try { return (await rewindGit(absolute, ["rev-parse", "--show-prefix"])).toString().trim().replace(/\/$/u, ""); } + catch { + if (path.isAbsolute(root) || root.split(/[\\/]/u).includes("..")) throw new Error(`Cannot determine historical repository prefix for bundle root ${root}; keep its local checkout or use a repository-relative root`); + return root.replace(/^\.\//u, "").replace(/\/$/u, "").replace(/^\.$/u, ""); + } +}; + +const materializeBundle = async (resource: BundleResource, context: InputContext, repository: string, commit: string): Promise<{ + trees: NonNullable; deferred: boolean +}> => { + const build = resource.build; + const trees: NonNullable = []; + if (!build) trees.push({ root: "" }); + else if (build.files) trees.push({ root: await bundlePrefix(build.files.root, context.input.declarationBase), exclude: build.files.exclude }); + else if (build.dependencies) trees.push({ root: await bundlePrefix(build.dependencies.directory, context.input.declarationBase) }); + else { + const origins = await Promise.all(build.generated!.inputs.map(async (input) => { + try { return (await rewindGit(path.resolve(context.input.declarationBase, input.root), ["rev-parse", "--show-toplevel"])).toString().trim(); } + catch { return null; } // Missing relative roots are mapped against the retained repository below. + })); + if (new Set(origins.filter((origin) => origin !== null)).size > 1) { + throw new Error("Generated bundle inputs span multiple git repositories; one recorded repository/commit cannot rewind them correctly"); + } + // Keep repository-relative paths: trial preparation can restore the declared cwd/input relationship. + for (const input of build.generated!.inputs) { + if (input.ref !== undefined) throw new Error("Generated bundle inputs cannot pin a ref: the deferred command reads the work tree"); + const root = await bundlePrefix(input.root, context.input.declarationBase); + trees.push({ root, prefix: root, exclude: input.exclude }); + } + } + for (const tree of trees) await exportGitTree(repository, commit, context.destination, { ...tree, mergeIdentical: true }); + return { trees, deferred: Boolean(build?.dependencies || build?.generated) }; +}; + +export const materializeGit = async (context: InputContext): Promise => { + const { input, history } = context; + if (input.section !== "resources" || input.declaration.kind === "volume") throw new Error("Git rewind requires a git or bundle resource"); + const resource = input.declaration, selector = selectorOf(resource), pinned = FULL_OBJECT.test(selector); + const matchingRecord = input.overridden ? undefined : context.record?.resources.find((entry) => entry.id === input.id); + const activationErrors = matchingRecord ? recordActivationErrors(context.record) : []; + const candidateCommits = input.overridden ? [] : context.candidateRecords?.flatMap((candidate) => + candidate.resources.flatMap((entry) => entry.id === input.id && entry.kind === "git" ? [entry.commit] : [])) ?? []; + if (matchingRecord && matchingRecord.kind !== resource.kind) throw new Error(`Moment resource ${input.id} kind disagrees with its declaration`); + const deployed = !input.overridden && !matchingRecord && !(resource.kind === "git" && pinned) + ? await deployedResource(context) : null; + if (deployed && deployed.pin.kind !== resource.kind) throw new Error(`Deployment resource ${input.id} kind disagrees with its declaration`); + const effectivePin = matchingRecord ?? deployed?.pin; + const bundlePin = effectivePin?.kind === "bundle" ? effectivePin : undefined; + if (bundlePin && await materializeRecordedBundle(history, bundlePin.sha256, context.destination)) { + return { fidelity: activationErrors.length || deployed && !deployed.exact ? "approximate" : "exact", basis: deployed ? "deploy-log" : "record", + source: { sha256: bundlePin.sha256 }, notes: [...(deployed?.notes ?? []), ...activationErrors], instant: context.at }; + } + if (deployed && bundlePin) throw new Error(`Deployment bundle ${bundlePin.sha256} could not be reproduced from retained archive bytes`); + const bundleNote = bundlePin ? `Recorded bundle ${bundlePin.sha256} could not be reproduced from retained archive bytes.` : undefined; + const gitPin = effectivePin?.kind === "git" ? effectivePin : undefined; + const record = pinned ? undefined : gitPin; + if (input.overridden && resource.kind === "bundle" && resource.source) { + await extractOverrideBundle(localRepositoryPath(resource.source, input.declarationBase), context.destination, resource.sha256); + return { fidelity: "overridden", basis: "override", source: {}, notes: [], instant: context.at }; + } + if (resource.kind === "bundle" && resource.source) throw new Error(`${bundleNote ?? "Prebuilt bundle has no matching recorded archive"} and cannot be rewound; provide an override`); + let repository = input.overridden ? null : await history?.repository(`git/${historyKey(input)}`); + const fromHistory = Boolean(repository); + if ((gitPin || candidateCommits.length) && !repository) throw new Error(`Missing git history for recorded resource ${input.id}`); + if (!repository && (pinned || input.overridden)) { + const declared = localRoot(resource); + if (declared) repository = localRepositoryPath(declared, input.declarationBase); + } + if (!repository) throw new Error(`Missing git history for ${input.id}; unpinned resources require --history`); + await assertLocalGitRepository(repository, !input.overridden && history && repository.startsWith(`${history.root}${path.sep}`) ? history : undefined); + for (const commit of candidateCommits) await resolveRecordedCommit(repository, commit); + const recordedCommit = gitPin ? await resolveRecordedCommit(repository, gitPin.commit) : null; + const selected = pinned || record || input.overridden + ? await (record ? Promise.resolve(recordedCommit!) : resolveRewindCommit(repository, selector)) + .then(async (commit) => ({ commit, instant: await commitInstant(repository!, commit) })) + : await commitAt(repository, selector, context.at); + if (!selected) throw new Error(`No first-parent commit for ${selector} exists at or before ${context.at}`); + await assertCommitNotFuture(repository, selected.commit, context.at); + const bundle = resource.kind === "bundle" ? await materializeBundle(resource, context, repository, selected.commit) : undefined; + if (resource.kind === "git") await materializeCleanRepository(repository, selected.commit, context.destination); + return { + fidelity: input.overridden ? "overridden" : resource.kind === "bundle" ? "approximate" + : record && activationErrors.length ? "approximate" : deployed ? deployed.exact ? "exact" : "approximate" : pinned || record ? "exact" : "approximate", + basis: input.overridden ? "override" : pinned ? "pinned" : deployed ? "deploy-log" : record ? "record" : "selector-at", + source: { commit: selected.commit, ...(!fromHistory ? { repository } : {}), ...(bundle ? { trees: bundle.trees } : {}) }, instant: selected.instant, + notes: [...(deployed?.notes ?? []), ...(record || bundle ? activationErrors : []), ...(pinned || record || input.overridden ? [] : ["Commit dates do not establish when a commit reached the selected ref."]), + ...(!input.overridden && resource.kind === "bundle" ? [bundleNote ?? "Source reconstruction has no matching recorded bundle digest."] : [])], + ...(bundle?.deferred ? { build: "deferred" } : {}) + }; +}; diff --git a/src/compiler/training/rewind/gitArchive.test.ts b/src/compiler/training/rewind/gitArchive.test.ts new file mode 100644 index 00000000..1ecb91b3 --- /dev/null +++ b/src/compiler/training/rewind/gitArchive.test.ts @@ -0,0 +1,70 @@ +import { lstat, mkdir, readFile, readlink, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { resolveFeedSource } from "../../../volume/index.js"; +import { materializeFeed } from "./feed.js"; +import { materializeGit } from "./git.js"; +import { exportFeedGitTree } from "./gitArchive.js"; +import { commitTestGit, gitTestRoot, initTestGit, testGit, testMomentRecord } from "./git.test-helper.js"; +import { History } from "./history.js"; +import type { InputContext } from "./types.js"; + +const BEFORE = "2026-10-09T07:00:00Z", AT = "2026-10-09T08:20:31Z", AFTER = "2026-10-10T00:00:00Z"; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); + +describe("raw historical blob extraction", () => { + it.each(["feed", "bundle"] as const)("preserves literal export-subst and export-ignore bytes in a %s after a new tag", async (kind) => { + const root = await gitTestRoot(); roots.push(root); + const relative = kind === "feed" ? "feeds/agent%3Awriter/input/repo" : "git/agent%3Awriter/input"; + const repository = path.join(root, "history", relative); + await initTestGit(repository); + const commit = await commitTestGit(repository, { + ".gitattributes": "refs.txt export-subst\nhidden.txt export-ignore\n", + "refs.txt": "$Format:%D$\n", "hidden.txt": "raw retained bytes\n" + }, BEFORE); + // This ref did not exist at the historical moment; archive expansion would leak it. + testGit(repository, ["tag", "-a", "created-after-the-wake", "-m", "later tag", commit], AFTER); + await mkdir(path.join(repository, ".git/info"), { recursive: true }); + await writeFile(path.join(repository, ".git/info/attributes"), "hidden.txt export-ignore\n"); + const context: InputContext = { at: AT, destination: path.join(root, "out"), history: new History(path.join(root, "history")), + record: { ...testMomentRecord(), resources: [] }, + input: { section: "resources", id: "input", declaredBy: "agent:writer", overridden: false, declarationBase: repository, + declaration: kind === "feed" + ? { id: "input", kind: "volume", mode: "mutable", mount: "./input", name: "input", feed: { git: { repo: repository } } } + : { id: "input", kind: "bundle", mode: "readonly", mount: "./input", build: { files: { root: "." } } } } }; + let output = context.destination; + if (kind === "feed") { + const revision = resolveFeedSource({ kind: "git", repo: repository, ref: commit, fetch: false }).revision; + await writeFile(path.join(repository, "../landings.jsonl"), `${JSON.stringify({ version: "spawnfile.volume-feed-landing.v1", resource: "input", volume: "input", source: { kind: "git", repo: repository }, revision, tree: `trees/${revision}`, commit, ref: "main", switching_at: BEFORE, landed_at: BEFORE })}\n`); + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", source: { revision, commit } }); + output = path.join(output, "current"); + } else expect(await materializeGit(context)).toMatchObject({ fidelity: "approximate", source: { commit } }); + expect(await readFile(path.join(output, "refs.txt"), "utf8")).toBe("$Format:%D$\n"); + expect(await readFile(path.join(output, "hidden.txt"), "utf8")).toBe("raw retained bytes\n"); + }); + + it("recreates symlinks and preserves binary blob bytes without conversion", async () => { + const root = await gitTestRoot(); roots.push(root); + const repository = path.join(root, "repo"); await initTestGit(repository); + const bytes = Buffer.from([0, 255, 10, 128]); + await writeFile(path.join(repository, "binary"), bytes); + await symlink("binary", path.join(repository, "link")); + const commit = await commitTestGit(repository, { ".gitattributes": "binary text eol=crlf\n" }, BEFORE); + const destination = path.join(root, "out"); + await exportFeedGitTree(repository, commit, destination); + expect(await readFile(path.join(destination, "binary"))).toEqual(bytes); + expect((await lstat(path.join(destination, "link"))).isSymbolicLink()).toBe(true); + expect(await readlink(path.join(destination, "link"))).toBe("binary"); + }); + + it("refuses gitlinks instead of silently dropping submodule contents", async () => { + const root = await gitTestRoot(); roots.push(root); + const repository = path.join(root, "repo"); await initTestGit(repository); + const parent = await commitTestGit(repository, { "file": "before\n" }, BEFORE); + testGit(repository, ["update-index", "--add", "--cacheinfo", `160000,${parent},nested`]); + testGit(repository, ["commit", "--quiet", "-m", "gitlink"], BEFORE); + await expect(exportFeedGitTree(repository, testGit(repository, ["rev-parse", "HEAD"]), path.join(root, "out"))) + .rejects.toThrow(/nested.*submodules need their own history/u); + }); +}); diff --git a/src/compiler/training/rewind/gitArchive.ts b/src/compiler/training/rewind/gitArchive.ts new file mode 100644 index 00000000..0ae520d7 --- /dev/null +++ b/src/compiler/training/rewind/gitArchive.ts @@ -0,0 +1,7 @@ +import { exportGitTree, safeCommittedPath } from "./gitTree.js"; + +/** Retain raw blob bytes: archive attributes may disclose refs added after the moment. */ +export const exportFeedGitTree = async (repository: string, commit: string, destination: string, paths?: string[]): Promise => { + for (const selected of paths ?? []) safeCommittedPath(selected); + await exportGitTree(repository, commit, destination, { paths }); +}; diff --git a/src/compiler/training/rewind/gitHistory.ts b/src/compiler/training/rewind/gitHistory.ts new file mode 100644 index 00000000..dd0fe497 --- /dev/null +++ b/src/compiler/training/rewind/gitHistory.ts @@ -0,0 +1,49 @@ +import { lstat, readFile, readdir } from "node:fs/promises"; +import path from "node:path"; +import type { History } from "./history.js"; +import { rewindGit } from "./gitProcess.js"; + +/** Git's textual indirections are not covered by ordinary realpath containment. */ +export const assertLocalGitRepository = async (repository: string, history?: History): Promise => { + const exists = (file: string) => history ? history.path(path.relative(history.root, file)) : lstat(file).then(() => file, (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + const text = (file: string) => history ? history.text(path.relative(history.root, file)) : readFile(file, "utf8").catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + const dotGit = path.join(repository, ".git"); + if (history && await exists(dotGit) && !(await lstat(dotGit)).isDirectory()) throw new Error("History repository .git indirection files are not supported; retain a standalone repository"); + // `git -C` would discover an ancestor repository if this history directory is not itself one. + if (history && !await exists(dotGit) && (!await exists(path.join(repository, "HEAD")) || !await exists(path.join(repository, "objects")))) { + throw new Error("History entry is not a standalone git repository"); + } + // Check config includes before asking git anything: even rev-parse reads included config files. + if (history) { + const directory = await exists(dotGit) ? dotGit : repository; + // `commondir` is another textual redirect git follows during discovery, before any command runs. + for (const name of ["commondir", "objects/info/alternates", "objects/info/http-alternates", "shallow", "info/grafts"]) { + if (await text(path.join(directory, name))) throw new Error(`Rewind requires complete standalone git history; ${name} is unsupported`); + } + const configFile = path.join(directory, "config"); + const configText = await text(configFile); + if (configText && /^\s*\[\s*include(?:if)?\b/imu.test(configText)) throw new Error("Rewind refuses git configuration indirection include"); + } + // Resolve only after refusing textual worktree links that could leave the history root. + const gitDirectory = (await rewindGit(repository, ["rev-parse", "--absolute-git-dir"])).toString().trim(); + if (history) await history.path(path.relative(history.root, gitDirectory)); + for (const name of ["commondir", "objects/info/alternates", "objects/info/http-alternates", "shallow", "info/grafts"]) { + if (await text(path.join(gitDirectory, name))) throw new Error(`Rewind requires complete standalone git history; ${name} is unsupported`); + } + const config = (await rewindGit(repository, ["config", "--local", "--no-includes", "--list", "--null"])) + .toString().split("\0").filter(Boolean); + for (const entry of config) { + const key = entry.split("\n")[0]!.toLowerCase(); + if (key === "include.path" || /^includeif\..*\.path$/u.test(key) || key === "extensions.partialclone" || /^remote\..*\.promisor$/u.test(key)) { + throw new Error(`Rewind refuses git configuration indirection ${key}`); + } + } + const packs = path.join(gitDirectory, "objects/pack"); + if (await exists(packs) && (await readdir(packs)).some((name) => name.endsWith(".promisor"))) throw new Error("Rewind refuses incomplete promisor git history"); +}; diff --git a/src/compiler/training/rewind/gitProcess.test.ts b/src/compiler/training/rewind/gitProcess.test.ts new file mode 100644 index 00000000..8d4d14f6 --- /dev/null +++ b/src/compiler/training/rewind/gitProcess.test.ts @@ -0,0 +1,37 @@ +import { rm } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { commitTestGit, gitTestRoot, initTestGit, testGit } from "./git.test-helper.js"; +import { commitAt, selectCommitAt } from "./gitProcess.js"; + +const EARLY = "2026-10-09T08:00:00Z", AT = "2026-10-09T08:20:31Z", LATE = "2026-10-09T09:00:00Z"; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const repository = async (): Promise => { + const root = await gitTestRoot(); roots.push(root); + const repo = path.join(root, "repo"); await initTestGit(repo); return repo; +}; + +describe("historical commit selection", () => { + it.each(["author", "committer"] as const)("steps back past a commit with a future %s date", async (role) => { + const repo = await repository(), past = await commitTestGit(repo, { "readme.txt": "past\n" }, EARLY); + await commitTestGit(repo, { "readme.txt": "future\n" }, AT); + testGit(repo, ["commit", "--amend", "--no-edit", `--date=${role === "author" ? LATE : AT}`], role === "committer" ? LATE : AT); + expect(await commitAt(repo, "main", AT)).toMatchObject({ commit: past, instant: "2026-10-09T08:00:00.000Z" }); + }); + + it("has no eligible state when every ancestor is future-authored", async () => { + const repo = await repository(); await commitTestGit(repo, { "readme.txt": "future\n" }, EARLY); + testGit(repo, ["commit", "--amend", "--no-edit", `--date=${LATE}`], EARLY); + expect(await commitAt(repo, "main", AT)).toBeNull(); + }); + + it("filters author and committer dates before choosing the newest eligible candidate", () => { + const eligible = { commit: "eligible", instant: EARLY, authorInstant: AT }; + expect(selectCommitAt([ + { commit: "future-author", instant: AT, authorInstant: LATE }, + { commit: "future-committer", instant: LATE, authorInstant: EARLY }, eligible + ], AT)).toEqual(eligible); + expect(selectCommitAt([{ commit: "equal", instant: AT, authorInstant: "2026-10-09T10:20:31+02:00" }], AT)?.commit).toBe("equal"); + }); +}); diff --git a/src/compiler/training/rewind/gitProcess.ts b/src/compiler/training/rewind/gitProcess.ts new file mode 100644 index 00000000..bdf3b4f1 --- /dev/null +++ b/src/compiler/training/rewind/gitProcess.ts @@ -0,0 +1,88 @@ +import { execFile, execFileSync } from "node:child_process"; +import { compareInstants } from "./instant.js"; + +const LIMIT = 268_435_456; + +/** Do not inherit GIT_DIR, injected -c options, credentials or user/system configuration. */ +export const rewindGitEnvironment = (): NodeJS.ProcessEnv => ({ + PATH: process.env.PATH, + ...(process.env.SystemRoot ? { SystemRoot: process.env.SystemRoot } : {}), + LC_ALL: "C", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_SYSTEM: "/dev/null", GIT_CONFIG_GLOBAL: "/dev/null", + GIT_NO_REPLACE_OBJECTS: "1", GIT_OPTIONAL_LOCKS: "0", GIT_TERMINAL_PROMPT: "0", GIT_NO_LAZY_FETCH: "1", + GIT_ATTR_NOSYSTEM: "1", GIT_CONFIG_COUNT: "0" +}); + +const argumentsFor = (repo: string, args: readonly string[]): string[] => [ + "-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", "-c", "core.autocrlf=false", + "-c", "core.logAllRefUpdates=false", "-c", "core.attributesFile=/dev/null", "-c", "protocol.allow=never", "-c", "protocol.file.allow=always", + "-c", "maintenance.auto=false", "-c", "gc.auto=0", "-c", "safe.directory=*", "-C", repo, ...args +]; + +/** Bounded argv-only git invocation. Local transport is the only enabled protocol. */ +export const rewindGit = (repo: string, args: readonly string[], input?: Buffer): Promise => + new Promise((resolve, reject) => { + const child = execFile("git", argumentsFor(repo, args), { + encoding: "buffer", env: rewindGitEnvironment(), maxBuffer: LIMIT, timeout: 60_000 + }, (error, stdout, stderr) => { + if (error) reject(new Error(`Rewind git ${args[0]} failed: ${stderr.toString("utf8").trim() || error.message}`)); + else resolve(stdout); + }); + child.stdin?.on("error", () => undefined); + child.stdin?.end(input); + }); + +/** Adapter for the existing synchronous feed revision function; it never fetches. */ +export const rewindGitSync = (repo: string, args: readonly string[]): Buffer => + execFileSync("git", argumentsFor(repo, args), { + env: rewindGitEnvironment(), maxBuffer: LIMIT, stdio: ["ignore", "pipe", "pipe"], timeout: 60_000 + }); + +export interface SelectedCommit { commit: string; instant: string; authorInstant?: string } + +/** Git's pretty formatter drops negative epochs. Read the signed raw dates instead. */ +const commitDates = async (repo: string, commit: string): Promise<{ author: string; committer: string }> => { + const headers = (await rewindGit(repo, ["cat-file", "commit", commit])).toString().split("\n\n", 1)[0]!; + const date = (role: string): string => { + const match = new RegExp(`^${role} .* (-?\\d+) [+-]\\d{4}$`, "mu").exec(headers); + const seconds = match ? Number(match[1]) : NaN; + if (!Number.isSafeInteger(seconds) || !Number.isFinite(new Date(seconds * 1000).getTime())) { + throw new Error(`Commit ${commit} has an invalid ${role} date`); + } + return new Date(seconds * 1000).toISOString(); + }; + return { author: date("author"), committer: date("committer") }; +}; +export const commitInstant = async (repo: string, commit: string): Promise => (await commitDates(repo, commit)).committer; + +/** Validate the selected source before any copy/export, regardless of the materialized layout. */ +export const assertCommitNotFuture = async (repo: string, commit: string, at: string): Promise => { + const dates = await commitDates(repo, commit); + if (Object.values(dates).some((instant) => compareInstants(instant, at) > 0)) { + throw new Error(`Selected commit ${commit} has an author or committer date after ${at}`); + } +}; + +export const resolveRewindCommit = async (repo: string, selector: string): Promise => { + if (!selector || selector.startsWith("-") || /[\0\r\n]/u.test(selector)) throw new Error("Invalid git selector"); + const ref = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/iu.test(selector) ? selector.toLowerCase() : selector; + return (await rewindGit(repo, ["rev-parse", "--verify", "--end-of-options", `${ref}^{commit}`])).toString().trim(); +}; + +/** Dates approximate ref history; they cannot establish when a commit reached a moving ref. */ +export const commitAt = async (repo: string, selector: string, at: string): Promise => { + let head: string; + try { head = await resolveRewindCommit(repo, selector); } catch { return null; } + const commits = (await rewindGit(repo, ["log", "--first-parent", "--format=%H", head, "--"])) + .toString().trim().split("\n").filter(Boolean); + const dated: SelectedCommit[] = []; + for (const commit of commits) { + const dates = await commitDates(repo, commit); + dated.push({ commit, instant: dates.committer, authorInstant: dates.author }); + } + return selectCommitAt(dated, at); +}; + +/** Stable ties preserve the first-parent order returned by git. */ +export const selectCommitAt = (commits: readonly SelectedCommit[], at: string): SelectedCommit | null => + commits.filter((entry) => compareInstants(entry.instant, at) <= 0 && compareInstants(entry.authorInstant ?? entry.instant, at) <= 0) + .reduce((latest, entry) => latest === null || compareInstants(entry.instant, latest.instant) > 0 ? entry : latest, null); diff --git a/src/compiler/training/rewind/gitTree.ts b/src/compiler/training/rewind/gitTree.ts new file mode 100644 index 00000000..e05d48b7 --- /dev/null +++ b/src/compiler/training/rewind/gitTree.ts @@ -0,0 +1,85 @@ +import { lstat, mkdir, readFile, readlink, stat, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { createHash } from "node:crypto"; + +import { compileExcludePatterns } from "../../workspaceBundleFiles.js"; +import { validateWorkspaceBundleTar } from "../../workspaceBundleArtifacts.js"; +import { rewindGit } from "./gitProcess.js"; + +export const safeCommittedPath = (value: string): string => { + if (!value || value.startsWith("/") || value.includes("\\") || value.includes("\0") || value.split("/").some((part) => !part || part === "." || part === ".." || part.toLowerCase() === ".git")) { + throw new Error(`Unsafe committed path: ${value}`); + } + return value; +}; + +/** Never traverse a symlink left by an earlier generated input while merging trees. */ +const parentDirectory = async (root: string, relative: string): Promise => { + let directory = root; + for (const segment of relative.split("/").slice(0, -1)) { + directory = path.join(directory, segment); + const existing = await lstat(directory).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); + if (existing && !existing.isDirectory()) throw new Error(`Committed entry traverses a non-directory: ${relative}`); + if (!existing) await mkdir(directory); + } +}; + +export interface ExportTreeOptions { root?: string; exclude?: readonly string[]; paths?: readonly string[]; prefix?: string; mergeIdentical?: boolean } + +/** Materialize blobs by object id, without invoking archive attributes, filters or extraction tools. */ +export const exportGitTree = async (repo: string, commit: string, destination: string, options: ExportTreeOptions = {}): Promise => { + const root = options.root && options.root !== "." ? safeCommittedPath(options.root.replace(/\/$/u, "")) : ""; + const tree = root ? `${commit}:${root}` : `${commit}^{tree}`; + if ((await rewindGit(repo, ["cat-file", "-t", tree])).toString().trim() !== "tree") throw new Error(`Bundle root ${root || "."} is not a tree at ${commit}`); + const entries = (await rewindGit(repo, ["ls-tree", "-rz", tree])).toString("utf8").split("\0").filter(Boolean); + const excluded = compileExcludePatterns(options.exclude); + const prefix = options.prefix ? safeCommittedPath(options.prefix) : ""; + await mkdir(destination, { recursive: true }); + for (const entry of entries) { + const tab = entry.indexOf("\t"), name = safeCommittedPath(entry.slice(tab + 1)); + const [mode, kind, oid] = entry.slice(0, tab).split(" "); + if (excluded(name) || (options.paths && !options.paths.some((selected) => name === selected || name.startsWith(`${selected}/`)))) continue; + if (kind !== "blob" || !["100644", "100755", "120000"].includes(mode!)) throw new Error(`Committed path ${name} is not a supported file or symlink (submodules need their own history)`); + const target = path.join(destination, prefix, name), bytes = await rewindGit(repo, ["cat-file", "blob", oid!]); + await parentDirectory(destination, [prefix, name].filter(Boolean).join("/")); + if (options.mergeIdentical) { + const present = await lstat(target).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); + if (present) { + const identical = mode === "120000" ? present.isSymbolicLink() && await readlink(target) === bytes.toString("utf8") + : present.isFile() && (present.mode & 0o111 ? "100755" : "100644") === mode && (await readFile(target)).equals(bytes); + if (!identical) throw new Error(`Conflicting generated bundle inputs at ${name}`); + continue; + } + } + if (mode === "120000") await symlink(bytes.toString("utf8"), target); + else await writeFile(target, bytes, { flag: "wx", mode: mode === "100755" ? 0o755 : 0o644 }); + } +}; + +/** Overrides use validated local ustar bytes, never tar commands or the network. */ +export const extractOverrideBundle = async (file: string, destination: string, expected?: string): Promise => { + const info = await stat(file); + if (!info.isFile() || info.size > 536_870_912) throw new Error("Override bundle must be a bounded local tar file"); + const bytes = await readFile(file); + if (expected && `sha256:${createHash("sha256").update(bytes).digest("hex")}` !== expected) throw new Error("Override bundle checksum mismatch"); + await extractBundleBytes(bytes, destination); +}; + +/** Validated archive bytes are extracted only after digest validation by the caller. */ +export const extractBundleBytes = async (bytes: Buffer, destination: string): Promise => { + validateWorkspaceBundleTar(bytes); + await mkdir(destination, { recursive: true }); + const text = (buffer: Buffer) => buffer.toString("utf8").replace(/\0.*$/su, ""); + for (let offset = 0; offset < bytes.length && !bytes.subarray(offset, offset + 512).every((byte) => byte === 0);) { + const header = bytes.subarray(offset, offset + 512), prefix = text(header.subarray(345, 500)), name = text(header.subarray(0, 100)); + const target = path.join(destination, safeCommittedPath(`${prefix ? `${prefix}/` : ""}${name}`.replace(/\/$/u, ""))); + const size = Number.parseInt(text(header.subarray(124, 136)).trim(), 8); + if (header[156] === 53) await mkdir(target, { recursive: true }); + else { + await mkdir(path.dirname(target), { recursive: true }); + const executable = Number.parseInt(text(header.subarray(100, 108)).trim(), 8) & 0o111; + await writeFile(target, bytes.subarray(offset + 512, offset + 512 + size), { flag: "wx", mode: executable ? 0o755 : 0o644 }); + } + offset += 512 + Math.ceil(size / 512) * 512; + } +}; diff --git a/src/compiler/training/rewind/history.test.ts b/src/compiler/training/rewind/history.test.ts new file mode 100644 index 00000000..144a0e88 --- /dev/null +++ b/src/compiler/training/rewind/history.test.ts @@ -0,0 +1,116 @@ +import { chmod, mkdir, mkdtemp, readFile, readlink, realpath, rm, stat, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { z } from "zod"; +import { emptyRecord } from "./fixtures.test-helper.js"; +import { momentRecordSchema } from "../../../recording/index.js"; +import { History, historyKey, openHistory, safeSegment } from "./history.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const fixture = async () => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "spawnfile-rewind-history-"))); + roots.push(root); + const directory = path.join(root, "history"), outside = path.join(root, "outside"); + await mkdir(directory); await mkdir(outside); + await writeFile(path.join(outside, "secret"), "outside bytes"); + return { root, directory, outside, history: (await openHistory(directory))! }; +}; +const record = emptyRecord("agent:writer"); + +describe("history containment and strict reads", () => { + it("allows missing history and entries but rejects non-directory roots and non-file reads", async () => { + const { directory, history } = await fixture(); + expect(await openHistory()).toBeUndefined(); + expect(await history.path("missing/file")).toBeNull(); + expect(await history.bytes("missing")).toBeNull(); + expect(await history.jsonl("missing", z.object({}).strict())).toBeNull(); + await writeFile(path.join(directory, "file"), "bytes"); + await expect(openHistory(path.join(directory, "file"))).rejects.toThrow(/must be a directory/u); + await expect(history.text(".")).rejects.toThrow(/regular file/u); + }); + + it.each(["../outside/secret", "child/../../outside/secret", "/etc/passwd", "bad\\path", "bad\0path"])( + "refuses lexical escape or invalid history path %s", async (name) => { + const { history } = await fixture(); + await expect(history.text(name)).rejects.toThrow(/escapes root|Unsafe history path/u); + } + ); + + it("rejects outside symlinks, including missing descendants and outside links back inside", async () => { + const { root, directory, outside, history } = await fixture(); + await writeFile(path.join(directory, "inside"), "retained"); + await symlink(path.join(outside, "secret"), path.join(directory, "escape")); + await symlink(outside, path.join(directory, "external-directory")); + await symlink(path.join(directory, "inside"), path.join(root, "back-inside")); + for (const name of ["escape", "external-directory/secret", "external-directory/missing"]) { + await expect(history.bytes(name)).rejects.toThrow(/History symlink escape/u); + } + await expect(history.bytes("../back-inside")).rejects.toThrow(/History path escapes root/u); + }); + + it("distinguishes dangling links from missing data and permits contained links", async () => { + const { directory, history } = await fixture(); + await writeFile(path.join(directory, "real"), "retained"); + await symlink("real", path.join(directory, "alias")); + await symlink("missing", path.join(directory, "dangling")); + expect(await history.text("alias")).toBe("retained"); + await expect(history.text("dangling")).rejects.toThrow(/Dangling history symlink/u); + }); + + it.each(["{broken", '{"id":"second","unknown":true}', ""])( + "names the exact malformed JSONL line for %j", async (second) => { + const { directory, history } = await fixture(); + await writeFile(path.join(directory, "records.jsonl"), `{"id":"first"}\n${second}\n`); + await expect(history.jsonl("records.jsonl", z.object({ id: z.string() }).strict())) + .rejects.toThrow(/Invalid history records.jsonl:2:/u); + } + ); + + it("loads every moment JSONL filename and ignores other files", async () => { + const { directory, history } = await fixture(); + await mkdir(path.join(directory, "moments")); + await writeFile(path.join(directory, "moments/z-any-name.jsonl"), `${JSON.stringify(record)}\n`); + await writeFile(path.join(directory, "moments/a-capture.jsonl"), `${JSON.stringify({ ...record, agent: "agent:other" })}\n`); + await writeFile(path.join(directory, "moments/ignore.txt"), "not JSON"); + expect((await history.records()).map((entry) => entry.agent)).toEqual(["agent:other", "agent:writer"]); + }); + + it.each([ + { ...record, unexpected: true }, + { ...record, wake: { ...record.wake, text: "not in the record format" } }, + { ...record, resources: [{ id: "same", kind: "git", commit: "1".repeat(40) }, { id: "same", kind: "git", commit: "2".repeat(40) }] } + ])("strictly validates record fields and identities", async (invalid) => { + const { directory, history } = await fixture(); + await writeFile(path.join(directory, "invalid.jsonl"), `${JSON.stringify(invalid)}\n`); + await expect(history.jsonl("invalid.jsonl", momentRecordSchema)).rejects.toThrow(/invalid.jsonl:1:/u); + }); + + it("checks nested repository entries before granting a repository path", async () => { + const { directory, outside, history } = await fixture(); + await mkdir(path.join(directory, "repo", "objects"), { recursive: true }); + await symlink(path.join(outside, "secret"), path.join(directory, "repo", "objects", "escape")); + await expect(history.repository("repo")).rejects.toThrow(/History symlink escape/u); + }); + + it("copies retained bytes, executable mode, and internal relative links through the boundary", async () => { + const { root, directory, history } = await fixture(); + await mkdir(path.join(directory, "tree")); + await writeFile(path.join(directory, "tree", "script"), "retained bytes\n"); + await chmod(path.join(directory, "tree", "script"), 0o755); + await symlink("script", path.join(directory, "tree", "alias")); + const destination = path.join(root, "copy"); + await history.copyTree("tree", destination); + expect(await readFile(path.join(destination, "script"), "utf8")).toBe("retained bytes\n"); + expect((await stat(path.join(destination, "script"))).mode & 0o777).toBe(0o755); + expect(await readlink(path.join(destination, "alias"))).toBe("script"); + }); + + it("encodes declaring node IDs and rejects unsafe input segments", () => { + expect(historyKey({ declaredBy: "agent:writer", id: "archive" })).toBe("agent%3Awriter/archive"); + for (const value of ["", ".", "..", "a/b", "a\\b", "bad\0id"]) expect(() => safeSegment(value)).toThrow(/Unsafe history identity/u); + expect(safeSegment("archive-1")).toBe("archive-1"); + expect(new History("/fixture").root).toBe("/fixture"); + }); +}); diff --git a/src/compiler/training/rewind/history.ts b/src/compiler/training/rewind/history.ts new file mode 100644 index 00000000..e87e68a3 --- /dev/null +++ b/src/compiler/training/rewind/history.ts @@ -0,0 +1,181 @@ +import { chmod, lstat, mkdir, readFile, readdir, readlink, realpath, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { z } from "zod"; +import { momentRecordSchema, parseMomentRecord, parseWakeMoments, type MomentRecord, type WakeMomentRow } from "../../../recording/index.js"; +import { deployResourceLogPath, deployResourcesAt, deployResourcesSchema, type DeployResources } from "../../../deployment/index.js"; +import { readFeedLandings, type FeedLanding } from "../../../volume/index.js"; + +export const inside = (root: string, target: string): boolean => { + const relative = path.relative(root, target); + return relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative); +}; +export const safeSegment = (value: string): string => { + if (!value || value === "." || value === ".." || /[/\\\0]/u.test(value)) throw new Error(`Unsafe history identity ${JSON.stringify(value)}`); + return value; +}; +export const historyKey = (identity: { declaredBy: string; id: string }): string => + `${safeSegment(encodeURIComponent(identity.declaredBy))}/${safeSegment(identity.id)}`; +const reason = (error: unknown): string => error instanceof Error ? error.message : String(error); + +/** All reads of the consumer history layout pass through this containment boundary. */ +export class History { + readonly root: string; + constructor(root: string) { this.root = root; } + async path(relative: string): Promise { + if (path.isAbsolute(relative) || relative.includes("\\") || relative.includes("\0")) throw new Error(`Unsafe history path ${relative}`); + const target = path.resolve(this.root, relative); + if (!inside(this.root, target)) throw new Error(`History path escapes root: ${relative}`); + let ancestor = target; + while (true) { + try { + const physical = await realpath(ancestor); + if (!inside(this.root, physical)) throw new Error(`History symlink escape: ${relative}`); + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if (ancestor === this.root) throw new Error(`History root no longer exists: ${this.root}`); + ancestor = path.dirname(ancestor); + } + } + try { + await lstat(target); + // A dangling symlink is not a missing history entry. + const physical = await realpath(target); + if (!inside(this.root, physical)) throw new Error(`History symlink escape: ${relative}`); + return target; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + const entry = await lstat(target).catch(() => null); + if (entry?.isSymbolicLink()) throw new Error(`Dangling history symlink: ${relative}`); + return null; + } + } + async bytes(relative: string): Promise { + const file = await this.path(relative); + if (file === null) return null; + if (!(await lstat(await realpath(file))).isFile()) throw new Error(`History entry must be a regular file: ${relative}`); + return readFile(file); + } + async text(relative: string): Promise { return (await this.bytes(relative))?.toString("utf8") ?? null; } + async jsonl(relative: string, schema: z.ZodType): Promise { + const payload = await this.text(relative); + if (payload === null) return null; + const lines = payload.split("\n"); + if (lines.at(-1) === "") lines.pop(); + return lines.map((line, index) => { + try { return schema.parse(JSON.parse(line)); } + catch (error) { throw new Error(`Invalid history ${relative}:${index + 1}: ${reason(error)}`); } + }); + } + async feedLandings(key: string, resource: string): Promise { + const relative = `${key}/landings.jsonl`, file = await this.path(relative); + if (!file) return []; + if (!(await lstat(await realpath(file))).isFile()) throw new Error(`History entry must be a regular file: ${relative}`); + const result = readFeedLandings(path.dirname(file)); + if (result.errors.length) throw new Error(`Invalid history ${result.errors.map((error) => `${relative}:${error.replace(/^(?:wake moments )?line /u, "")}`).join("; ")}`); + result.entries.forEach((entry, index) => { + if (entry.resource !== resource) throw new Error(`Invalid history ${relative}:${index + 1}: Landing resource disagrees with the history input`); + }); + return result.entries; + } + async wakeMoments(relative: string): Promise { + const payload = await this.text(relative); + if (payload === null) return null; + const result = parseWakeMoments(payload); + if (result.errors.length) throw new Error(`Invalid history ${result.errors.map((error) => `${relative}:${error.replace(/^(?:wake moments )?line /u, "")}`).join("; ")}`); + return result.entries; + } + /** One imported deployment, preserving its original absolute identity without opening that path. */ + async deploymentResources(at: string, agentId?: string): Promise { + const directory = await this.path("deployments"); + if (!directory) return null; + const logs: string[] = []; + for (const name of (await readdir(directory)).sort()) { + const relative = `deployments/${name}`, entry = await this.path(relative); + if (!entry) throw new Error(`History entry vanished: ${relative}`); + if (name.endsWith(".resource-log.jsonl")) logs.push(relative); + else if ((await lstat(await realpath(entry))).isDirectory() && await this.path(`${relative}/resource-log.jsonl`)) logs.push(`${relative}/resource-log.jsonl`); + } + if (!logs.length) return null; + if (logs.length !== 1) throw new Error("Rewind history must contain one deployment resource log; multiple deployments are ambiguous"); + const relative = logs[0]!, entries = await this.jsonl(relative, deployResourcesSchema) ?? []; + if (!entries.length) return null; + const identities = new Set(entries.map((entry) => entry.deployment_id)); + const recordPath = entries[0]!.deployment_id; + if (identities.size !== 1 || !path.isAbsolute(recordPath) || path.resolve(recordPath) !== recordPath) { + throw new Error(`Invalid history ${relative}: expected one canonical deployment_id`); + } + const originalLog = deployResourceLogPath(recordPath); + const mirror = path.basename(recordPath) === "record.json" + ? `deployments/${path.basename(path.dirname(recordPath))}/${path.basename(originalLog)}` + : `deployments/${path.basename(originalLog)}`; + if (relative !== mirror) throw new Error(`Invalid history ${relative}: deployment_id requires ${mirror}`); + if (!agentId) throw new Error("Deployment resource lookup requires the compiled agent ID"); + return deployResourcesAt(entries, recordPath, at, agentId); + } + async records(): Promise { + const directory = await this.path("moments"); + if (!directory) return []; + const records: MomentRecord[] = []; + const seen = new Set(); + const walk = async (relative: string): Promise => { + const file = await this.path(relative); + if (!file) throw new Error(`History entry vanished: ${relative}`); + const physical = await realpath(file); + if ((await lstat(physical)).isDirectory()) { + if (seen.has(physical)) throw new Error(`Repeated moment history directory: ${relative}`); + seen.add(physical); + for (const name of (await readdir(file)).sort()) await walk(`${relative}/${name}`); + } else if (relative.endsWith(".jsonl")) records.push(...(await this.jsonl(relative, momentRecordSchema) ?? [])); + else if (relative.endsWith(".json")) { + try { records.push(parseMomentRecord((await this.text(relative))!)); } + catch (error) { throw new Error(`Invalid history ${relative}: ${reason(error)}`); } + } + }; + await walk("moments"); + return records; + } + /** Validate all tree entries before git or another local parser receives the directory. */ + async repository(relative: string): Promise { + const root = await this.path(relative); + if (!root) return null; + const seen = new Set(); + const walk = async (entry: string): Promise => { + const file = await this.path(entry); + if (!file) throw new Error(`History entry vanished: ${entry}`); + const physical = await realpath(file), stats = await lstat(physical); + if (seen.has(physical)) return; + seen.add(physical); + if (stats.isDirectory()) for (const name of await readdir(file)) await walk(`${entry}/${name}`); + else if (!stats.isFile()) throw new Error(`Special history file: ${entry}`); + }; + await walk(relative); + return root; + } + async copyTree(relative: string, destination: string): Promise { + const source = await this.path(relative); + if (!source) throw new Error(`Missing retained tree: ${relative}`); + const stats = await lstat(source); + if (!stats.isDirectory()) throw new Error(`Retained tree must be a real directory: ${relative}`); + const copy = async (entry: string, target: string): Promise => { + const file = await this.path(entry); + if (!file) throw new Error(`History entry vanished: ${entry}`); + const stat = await lstat(file); + if (stat.isSymbolicLink()) { await symlink(await readlink(file), target); return; } + if (stat.isDirectory()) { + await mkdir(target, { recursive: true }); + for (const name of (await readdir(file)).sort()) await copy(`${entry}/${name}`, path.join(target, name)); + } else if (stat.isFile()) { + await writeFile(target, (await this.bytes(entry))!); + await chmod(target, stat.mode & 0o777); + } else throw new Error(`Special history file: ${entry}`); + }; + await copy(relative, destination); + } +} +export const openHistory = async (directory?: string): Promise => { + if (directory === undefined) return undefined; + const root = await realpath(path.resolve(directory)); + if (!(await lstat(root)).isDirectory()) throw new Error(`History must be a directory: ${directory}`); + return new History(root); +}; diff --git a/src/compiler/training/rewind/historyLandings.test.ts b/src/compiler/training/rewind/historyLandings.test.ts new file mode 100644 index 00000000..f7719e4e --- /dev/null +++ b/src/compiler/training/rewind/historyLandings.test.ts @@ -0,0 +1,72 @@ +import { mkdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import * as volume from "../../../volume/index.js"; +import { materializeFeed } from "./feed.js"; +import { gitTestRoot } from "./git.test-helper.js"; +import { History } from "./history.js"; + +const roots: string[] = []; +afterEach(async () => { vi.restoreAllMocks(); await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const AT = "2026-10-09T08:20:31Z", BEFORE = "2026-10-09T08:00:00Z"; +const fixture = async () => { + const root = await gitTestRoot(); roots.push(root); + const key = "feeds/agent%3Awriter/news", directory = path.join(root, key), retained = path.join(directory, "retained"); + await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "news.txt"), "past\n"); + const revision = volume.digestDirectory(retained), tree = path.join(directory, "trees", revision); + await mkdir(tree, { recursive: true }); await writeFile(path.join(tree, "news.txt"), "past\n"); + const row: volume.FeedLanding = { version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "news", + revision, tree: `trees/${revision}`, source: { kind: "directory", directory: "/never-read" }, switching_at: BEFORE, landed_at: BEFORE }; + const history = new History(root); + const log = (rows: unknown[]) => writeFile(path.join(directory, "landings.jsonl"), rows.map((value) => JSON.stringify(value)).join("\n") + "\n"); + const materialize = (at = AT) => materializeFeed({ at, history, record: null, destination: path.join(root, "out"), input: { + section: "resources", id: "news", declaredBy: "agent:writer", declarationBase: root, overridden: false, + declaration: { id: "news", kind: "volume", mode: "mutable", mount: "./news", name: "news", feed: { directory: "/never-read" } } + } }); + return { history, key, row, log, materialize }; +}; + +describe("merged feed landing readers in rewind", () => { + it.each(["2026-10-09T08:20:30Z", AT, "2026-10-09T08:20:32Z"])("uses the merged inclusive ambiguity window at %s", async (at) => { + const f = await fixture(), read = vi.spyOn(volume, "readFeedLandings"), lookup = vi.spyOn(volume, "feedLandingAt"); + const incoming = { ...f.row, switching_at: "2026-10-09T08:20:30Z", landed_at: "2026-10-09T08:20:32Z" }; + await f.log([f.row, incoming]); + expect(await f.materialize(at)).toMatchObject({ fidelity: "approximate", basis: "landing-log", + source: { switchingAt: incoming.switching_at, landedAt: incoming.landed_at }, notes: [expect.stringContaining("ambiguous")] }); + expect(read).toHaveBeenCalledOnce(); expect(lookup).toHaveBeenCalledWith([f.row, incoming], at); + }); + + it("selects the newest transition even when an older overlapping window was appended last", async () => { + const f = await fixture(); + const newer = { ...f.row, switching_at: "2026-10-09T08:20:30Z", landed_at: "2026-10-09T08:20:32Z" }; + const older = { ...f.row, switching_at: "2026-10-09T08:20:29Z", landed_at: "2026-10-09T08:20:33Z" }; + await f.log([newer, older]); + expect(await f.materialize()).toMatchObject({ fidelity: "approximate", source: { landedAt: newer.landed_at } }); + }); + + it.each([false, true])("uses the last appended landing outside windows (reverse=%s)", async (reverse) => { + const f = await fixture(), second = { ...f.row, volume: "second", landed_at: "2026-10-09T10:00:00+02:00" }; + const rows = reverse ? [second, f.row] : [f.row, second]; await f.log(rows); + const result = await f.materialize(); + expect(result).toMatchObject({ fidelity: "exact", basis: "landing-log", source: { landedAt: rows[1]!.landed_at } }); + }); + + it("keeps a zero-duration window ambiguous at its exact instant", async () => { + const f = await fixture(); await f.log([f.row]); + expect(await f.materialize(BEFORE)).toMatchObject({ fidelity: "approximate", basis: "landing-log" }); + }); + + it.each(["missing boundary", "reversed boundary", "wrong tree", "wrong resource", "invalid line"])("refuses %s with its line number", async (change) => { + const f = await fixture(); + const bad = { ...f.row, ...(change === "missing boundary" ? { switching_at: undefined } : {}), + ...(change === "reversed boundary" ? { switching_at: AT } : {}), + ...(change === "wrong tree" ? { tree: f.row.revision } : {}), ...(change === "wrong resource" ? { resource: "other" } : {}) }; + await f.log([f.row, change === "invalid line" ? {} : bad]); + await expect(f.materialize()).rejects.toThrow(/landings.jsonl:2:/u); + }); + + it("keeps history containment before invoking the host reader", async () => { + const f = await fixture(); + await expect(f.history.feedLandings("../escape", "news")).rejects.toThrow("escapes root"); + }); +}); diff --git a/src/compiler/training/rewind/index.ts b/src/compiler/training/rewind/index.ts new file mode 100644 index 00000000..bee15df8 --- /dev/null +++ b/src/compiler/training/rewind/index.ts @@ -0,0 +1,15 @@ +export * from "./contract.js"; +export * from "./history.js"; +export * from "./inputs.js"; +export * from "./instant.js"; +export * from "./leakCheck.js"; +export * from "./materialize.js"; +export * from "./select.js"; +export * from "./taskClock.js"; +export { feedLandingAt } from "../../../volume/index.js"; +export { selectFeedCommit } from "./feed.js"; +export { selectCommitAt } from "./gitProcess.js"; +export { selectMemoryPrefix, parseMemoryLedger } from "./memory.js"; +export { rewindMoltnetJson } from "./moltnetJson.js"; +export { selectVolumeSnapshot } from "./volumes.js"; +export type * from "./types.js"; diff --git a/src/compiler/training/rewind/inputs.test.ts b/src/compiler/training/rewind/inputs.test.ts new file mode 100644 index 00000000..20585d4d --- /dev/null +++ b/src/compiler/training/rewind/inputs.test.ts @@ -0,0 +1,122 @@ +import { rm } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { loadTrainingProject, resolveTrainingPlan } from "../plan/index.js"; +import { + agentManifest, cleanTrainingProjects, dataFiles, example, memory, network, surfaces, target, + teamFiles, teamManifest, training, trainingProject, volume, writeTrainingFile +} from "../plan/fixtures.test-helper.js"; +import { resolveRewindInputs } from "./inputs.js"; + +afterEach(cleanTrainingProjects); +afterEach(() => vi.restoreAllMocks()); +const resolve = async (root: string, agent = "agent:a") => { + const project = await loadTrainingProject(root); + return { project, plan: resolveTrainingPlan(project, agent, "test") }; +}; +const identity = ({ section, id, declaredBy }: { section: string; id: string; declaredBy: string }) => ({ section, id, declaredBy }); + +describe("effective rewind inputs", () => { + it("includes every resource kind, accessible bank and attached network with P2 declaring identities", async () => { + const root = await trainingProject(teamFiles()); + await writeTrainingFile(root, "a/Spawnfile", agentManifest({ name: "a", training: undefined, surfaces, memory: [memory], + workspace: { docs: { system: "AGENTS.md" }, resources: [ + { id: "source", kind: "git", url: "./local-repo", mount: "./source", mode: "readonly" }, + { id: "bundle", kind: "bundle", source: "./local.tar", mount: "./bundle", mode: "readonly" } + ] } + })); + const { plan, project } = await resolve(root), before = structuredClone(project); + // All source files are gone: this is a pure projection of already loaded facts. + await rm(root, { recursive: true, force: true }); + const clock = vi.spyOn(Date, "now").mockImplementation(() => { throw Error("clock read"); }); + const first = resolveRewindInputs(plan, project, example.id); + expect(first.at).toBe("2026-10-09T08:20:31Z"); + expect(first.inputs.map(identity)).toEqual([ + { section: "memory", id: "self", declaredBy: "agent:a" }, + { section: "memory", id: "team-memory", declaredBy: "team:publication" }, + { section: "networks", id: "news", declaredBy: "team:publication" }, + { section: "resources", id: "bundle", declaredBy: "agent:a" }, + { section: "resources", id: "source", declaredBy: "agent:a" }, + { section: "resources", id: "edition-state", declaredBy: "team:publication" } + ]); + expect(first.inputs.every((input) => !input.overridden)).toBe(true); + expect(resolveRewindInputs(plan, project, `train:${example.id}`)).toEqual(first); + expect(project).toEqual(before); expect(clock).not.toHaveBeenCalled(); + }); + + it("replaces whole declarations from the loaded dataset and retains inherited identity", async () => { + const root = await trainingProject(teamFiles()); + const resourceOverride = { ...volume, mount: "./fresh" }; + const memoryOverride = { ...memory, id: "team-memory" }; + const networkOverride = { id: network.id, provider: "moltnet", rooms: [{ id: "assignment", members: ["a", "b"] }] }; + await writeTrainingFile(root, "training/train.yaml", { examples: [{ ...example, + resources: [resourceOverride], memory: [memoryOverride], networks: [networkOverride] + }] }); + const { plan, project } = await resolve(root), before = structuredClone(project); + const result = resolveRewindInputs(plan, project, example.id); + const replacements = result.inputs.filter((input) => input.overridden); + expect(replacements).toHaveLength(3); + for (const [section, declaration] of [["resources", resourceOverride], ["memory", memoryOverride], ["networks", networkOverride]] as const) { + expect(replacements.find((input) => input.section === section)).toEqual({ section, id: declaration.id, + declaredBy: "team:publication", declaration, overridden: true, declarationBase: path.join(root, "training") }); + } + expect(result.inputs.find((input) => input.id === "self")).toMatchObject({ overridden: false, declaredBy: "agent:a" }); + expect(project).toEqual(before); + }); + + it("deduplicates multiple access slots by bank source, without collapsing distinct same-id banks", async () => { + const root = await trainingProject(teamFiles()); + await writeTrainingFile(root, "Spawnfile", teamManifest({ networks: undefined, memory: [memory], + shared: { training: { ...training, targets: [] } }, members: [{ id: "a", ref: "a" }, { id: "alias", ref: "a" }] })); + await writeTrainingFile(root, "a/Spawnfile", agentManifest({ name: "a", memory: [memory], training: { + targets: [{ ...target, datasets: { train: "../training/train.yaml", test: "../training/test.yaml" } }] + } })); + const { plan, project } = await resolve(root); + const agent = project.compilePlan.nodes.find((node) => node.id === "agent:a")!; + expect(agent.value.kind).toBe("agent"); + if (agent.value.kind !== "agent") throw Error("expected agent"); + expect(agent.value.memoryAccess!.filter((entry) => entry.declaringKind === "team")).toHaveLength(2); + expect(resolveRewindInputs(plan, project, example.id).inputs.map(identity)).toEqual([ + { section: "memory", id: "self", declaredBy: "agent:a" }, + { section: "memory", id: "self", declaredBy: "team:publication" } + ]); + }); + + it("includes representative attachments synthesized from a parent team's network", async () => { + const root = await trainingProject({ ...dataFiles, + Spawnfile: teamManifest({ shared: undefined, memory: undefined, + networks: [{ ...network, rooms: [{ id: "assignment", members: ["child"] }] }], members: [{ id: "child", ref: "child" }] }), + "child/Spawnfile": teamManifest({ name: "child", shared: undefined, networks: undefined, memory: undefined, + members: [{ id: "writer", ref: "writer" }] }), + ...Object.fromEntries(Object.entries(dataFiles).map(([name, value]) => [`child/writer/${name}`, value])), + "child/writer/Spawnfile": agentManifest() + }); + const { plan, project } = await resolve(root, "agent:writer"); + expect(resolveRewindInputs(plan, project, example.id).inputs.map(identity)).toEqual([ + { section: "networks", id: "news", declaredBy: "team:publication" } + ]); + }); + + it("excludes inaccessible banks and unattached networks", async () => { + const root = await trainingProject(teamFiles()); + await writeTrainingFile(root, "Spawnfile", teamManifest({ memory: [{ ...memory, id: "private", access: { members: ["b"] } }] })); + await writeTrainingFile(root, "a/Spawnfile", agentManifest({ name: "a", training: undefined, memory: [memory] })); + const { plan, project } = await resolve(root); + expect(resolveRewindInputs(plan, project, example.id).inputs.map(identity)).toEqual([ + { section: "memory", id: "self", declaredBy: "agent:a" }, + { section: "resources", id: "edition-state", declaredBy: "team:publication" } + ]); + }); + + it("accepts equivalent loaded instants and refuses stale or missing example data", async () => { + const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest() }); + const { plan, project } = await resolve(root, "agent:writer"); + const raw = project.datasets[path.join(root, "training/train.yaml")]!.value.examples[0]!; + raw.at = "2026-10-09T08:20:31Z"; + expect(resolveRewindInputs(plan, project, example.id).at).toBe(raw.at); + raw.at = "2026-10-09T08:20:31.001Z"; + expect(() => resolveRewindInputs(plan, project, example.id)).toThrow(/Loaded example disagrees with plan/u); + delete project.datasets[path.join(root, "training/train.yaml")]; + expect(() => resolveRewindInputs(plan, project, example.id)).toThrow(/Loaded example disagrees with plan/u); + }); +}); diff --git a/src/compiler/training/rewind/inputs.ts b/src/compiler/training/rewind/inputs.ts new file mode 100644 index 00000000..e9d48835 --- /dev/null +++ b/src/compiler/training/rewind/inputs.ts @@ -0,0 +1,64 @@ +import path from "node:path"; +import type { LoadedTrainingProject, TrainingPlan, TrainingAgentNode } from "../plan/index.js"; +import { resolveTrainingOverride, resolveTrainingResource } from "../plan/references.js"; +import { selectTrainingExample } from "./select.js"; +import { normalizeInstant } from "./instant.js"; +import type { RewindInput } from "./types.js"; + +/** The same compiled resources, accessible bank sources and surface attachments P2 resolves. */ +export const resolveRewindInputs = ( + plan: TrainingPlan, project: LoadedTrainingProject, selector: string +) => { + const selected = selectTrainingExample(plan, selector); + const agent = project.compilePlan.nodes.find((node): node is TrainingAgentNode => + node.value.kind === "agent" && node.id === plan.agent.id); + if (!agent) throw new Error(`No compiled training agent ${plan.agent.id}`); + const raw = project.datasets[selected.sourcePath]?.value.examples.find((example) => example.id === selected.id); + if (!raw || normalizeInstant(raw.at) !== normalizeInstant(selected.example.at)) throw new Error(`Loaded example disagrees with plan: ${selector}`); + const inputs: RewindInput[] = []; + const declarationBase = (id: string): string => { + const node = project.compilePlan.nodes.find((candidate) => candidate.id === id); + if (!node) throw new Error(`Missing declaring node ${id}`); + return path.dirname(node.value.kind === "agent" ? node.value.sourcePath ?? node.value.source : node.value.source); + }; + for (const resource of agent.value.workspaceResources ?? []) { + const { declaredBy } = resolveTrainingResource(project.compilePlan, agent, resource.id, "Rewind resource"); + inputs.push({ section: "resources", id: resource.id, declaredBy, declarationBase: declarationBase(declaredBy), + declaration: resource, overridden: false }); + } + const banks = new Set(); + for (const entry of agent.value.memoryAccess ?? []) { + const key = JSON.stringify([entry.source, entry.bank.id]); + if (banks.has(key)) continue; + banks.add(key); + const node = project.compilePlan.nodes.find((candidate) => candidate.value.source === entry.source); + if (!node) throw new Error(`Memory bank ${entry.bank.id} has no declaring node`); + inputs.push({ section: "memory", id: entry.bank.id, declaredBy: node.id, + declarationBase: declarationBase(node.id), declaration: entry.bank, overridden: false }); + } + const networks = new Set(); + for (const attachment of agent.value.surfaces?.moltnet ?? []) { + const sources = new Set([attachment.teamSource, ...Object.keys(attachment.contextRooms ?? {})]); + for (const node of project.compilePlan.nodes) { + if (node.value.kind !== "team" || !sources.has(node.value.source)) continue; + for (const network of node.value.networks ?? []) { + if (network.id !== attachment.network) continue; + const key = JSON.stringify([node.id, network.id]); + if (networks.has(key)) continue; + networks.add(key); + inputs.push({ section: "networks", id: network.id, declaredBy: node.id, + declarationBase: declarationBase(node.id), declaration: network, overridden: false }); + } + } + } + for (const section of ["resources", "memory", "networks"] as const) { + for (const declaration of raw[section] ?? []) { + const identity = resolveTrainingOverride(project.compilePlan, agent, section, declaration.id, `Rewind override ${selector}`); + const index = inputs.findIndex((input) => input.section === section && input.id === identity.id && input.declaredBy === identity.declaredBy); + if (index === -1) throw new Error(`Missing effective override ${section}:${identity.id}`); + inputs[index] = { ...inputs[index], declaration, declarationBase: path.dirname(selected.sourcePath), overridden: true } as RewindInput; + } + } + inputs.sort((a, b) => a.section.localeCompare(b.section) || a.declaredBy.localeCompare(b.declaredBy) || a.id.localeCompare(b.id)); + return { inputs, selected, at: normalizeInstant(raw.at), agentSlug: agent.slug }; +}; diff --git a/src/compiler/training/rewind/instant.test.ts b/src/compiler/training/rewind/instant.test.ts new file mode 100644 index 00000000..1efc70e1 --- /dev/null +++ b/src/compiler/training/rewind/instant.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import { compareInstants, instantNanos, normalizeInstant } from "./instant.js"; + +describe("rewind instant comparisons", () => { + it("compares RFC3339Nano fractions numerically when string order is reversed", () => { + const earlier = "2026-10-09T08:20:31.1Z", later = "2026-10-09T08:20:31.11Z"; + expect(earlier > later).toBe(true); + expect(compareInstants(earlier, later)).toBe(-1); + expect(compareInstants(later, earlier)).toBe(1); + expect(compareInstants("2026-10-09T08:20:31.100000000Z", earlier)).toBe(0); + }); + + it("preserves nanosecond boundaries and offset-equivalent instants", () => { + const before = "2026-10-09T08:20:31.123456789Z", after = "2026-10-09T08:20:31.123456790Z"; + expect(instantNanos(after) - instantNanos(before)).toBe(1n); + expect(compareInstants(before, "2026-10-09T10:20:31.123456789+02:00")).toBe(0); + expect(compareInstants(after, before)).toBe(1); + }); + + it.each([ + ["2026-10-09T10:20:31.120000000+02:00", "2026-10-09T08:20:31.12Z"], + ["2026-10-09T08:20Z", "2026-10-09T08:20:00Z"], + ["2026-10-09T00:01:00+14:00", "2026-10-08T10:01:00Z"], + ["1969-12-31T23:59:59.999999999Z", "1969-12-31T23:59:59.999999999Z"] + ])("normalizes %s to UTC without losing precision", (input, expected) => { + expect(normalizeInstant(input)).toBe(expected); + expect(compareInstants(input, expected)).toBe(0); + }); + + it.each(["invalid", "2026-02-30T00:00:00Z", "2026-10-09T08:20:31", "2026-10-09T08:20:31.1234567891Z"])( + "refuses unsupported or invalid instant %s", (value) => { + expect(() => instantNanos(value)).toThrow(); + } + ); +}); diff --git a/src/compiler/training/rewind/instant.ts b/src/compiler/training/rewind/instant.ts new file mode 100644 index 00000000..44b77e06 --- /dev/null +++ b/src/compiler/training/rewind/instant.ts @@ -0,0 +1,23 @@ +import { trainingInstantSchema } from "../../../manifest/index.js"; + +/** RFC3339Nano ordering, including offsets and fractions Date.parse would truncate. */ +export const instantNanos = (value: string): bigint => { + trainingInstantSchema.parse(value); + const match = /^(.*T\d{2}:\d{2})(?::(\d{2})(?:\.(\d{1,9}))?)?(Z|[+-]\d{2}:\d{2})$/u.exec(value); + if (!match) throw new Error(`Unsupported instant ${value}: at most nanosecond precision is required`); + const seconds = Date.parse(`${match[1]}:${match[2] ?? "00"}${match[4]}`); + if (!Number.isFinite(seconds)) throw new Error(`Invalid instant ${value}`); + return BigInt(seconds) * 1_000_000n + BigInt((match[3] ?? "").padEnd(9, "0")); +}; +export const compareInstants = (left: string, right: string): number => { + const a = instantNanos(left), b = instantNanos(right); + return a < b ? -1 : a > b ? 1 : 0; +}; +export const normalizeInstant = (value: string): string => { + const ns = instantNanos(value), seconds = ns / 1_000_000_000n; + const fraction = (ns % 1_000_000_000n + 1_000_000_000n) % 1_000_000_000n; + const floorSeconds = ns < 0n && fraction !== 0n ? seconds - 1n : seconds; + const base = new Date(Number(floorSeconds * 1000n)).toISOString().replace(/\.000Z$/u, ""); + const tail = fraction.toString().padStart(9, "0").replace(/0+$/u, ""); + return `${base}${tail ? `.${tail}` : ""}Z`; +}; diff --git a/src/compiler/training/rewind/leakCheck.test.ts b/src/compiler/training/rewind/leakCheck.test.ts new file mode 100644 index 00000000..87c179ae --- /dev/null +++ b/src/compiler/training/rewind/leakCheck.test.ts @@ -0,0 +1,139 @@ +import { lstat, lutimes, mkdir, readFile, readdir, rm, symlink, utimes, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { cleanTrainingProjects } from "../plan/fixtures.test-helper.js"; +import type { MaterializedMoment, LeakFinding } from "./contract.js"; +import { AT, FUTURE, memoryEvent, rewindFixture, testGit } from "./fixtures.test-helper.js"; +import { leakCheckTrainingMoment } from "./leakCheck.js"; +import { finalizeTrainingMoment, materializeTrainingMoment } from "./materialize.js"; +import { rewindGit } from "./gitProcess.js"; +import { stampTree } from "./timestamps.js"; + +afterEach(cleanTrainingProjects); +type Fixture = Awaited>; +type Mutation = (fixture: Fixture, moment: MaterializedMoment) => Promise; +const inputPath = (fixture: Fixture, moment: MaterializedMoment, kind: string) => + path.join(fixture.options.out, moment.inputs.find((input) => input.kind === kind)!.path!); +const patchNetwork = async (fixture: Fixture, moment: MaterializedMoment, patch: (state: Record) => void) => { + const file = path.join(inputPath(fixture, moment, "moltnet"), "state.json"); + const state = JSON.parse(await readFile(file, "utf8")); patch(state); await writeFile(file, JSON.stringify(state)); +}; +const cases: Array<[string, LeakFinding["class"], Mutation]> = [ + ["future file mtime", "file-time", async (fixture) => { await writeFile(path.join(fixture.options.out, "future"), "future"); }], + ["future directory mtime", "file-time", async (fixture) => { await mkdir(path.join(fixture.options.out, "new-directory")); }], + ["future symlink mtime", "file-time", async (fixture) => { await symlink("moment.json", path.join(fixture.options.out, "new-link")); }], + ["future report atime", "file-time", async () => {}], + ["future report mtime", "file-time", async () => {}], + ["future file atime", "file-time", async (fixture) => { await writeFile(path.join(fixture.options.out, "future"), "future"); }], + ["future directory atime", "file-time", async (fixture) => { await mkdir(path.join(fixture.options.out, "new-directory")); }], + ["future symlink atime", "file-time", async (fixture) => { await symlink("moment.json", path.join(fixture.options.out, "new-link")); }], + ["extra unreachable future commit", "git-object", async (fixture, moment) => { + const code = inputPath(fixture, moment, "git"); + const pack = await rewindGit(fixture.gitRepo, ["pack-objects", "--revs", "--stdout"], Buffer.from(`${fixture.futureCommit}\n`)); + await rewindGit(code, ["index-pack", "--stdin"], pack); + }], + ["extra tag", "git-object", async (fixture, moment) => { testGit(inputPath(fixture, moment, "git"), ["tag", "leaked", fixture.commit]); }], + ["extra remote", "git-object", async (fixture, moment) => { testGit(inputPath(fixture, moment, "git"), ["remote", "add", "leaked", fixture.gitRepo]); }], + ["reachable commit future author only", "git-object", async (fixture, moment) => { + const code = inputPath(fixture, moment, "git"); + testGit(code, ["commit", "--quiet", "--allow-empty", "--date=2026-10-10T00:00:00Z", "-m", "future author"]); + moment.inputs.find((input) => input.kind === "git")!.source.commit = testGit(code, ["rev-parse", "HEAD"]); + await writeFile(path.join(fixture.options.out, "moment.json"), JSON.stringify(moment)); + }], + ["reachable commit future committer only", "git-object", async (fixture, moment) => { + const code = inputPath(fixture, moment, "git"); + testGit(code, ["commit", "--quiet", "--allow-empty", "--date=2026-10-09T07:00:00Z", "-m", "future committer"], "2026-10-10T00:00:00Z"); + moment.inputs.find((input) => input.kind === "git")!.source.commit = testGit(code, ["rev-parse", "HEAD"]); + await writeFile(path.join(fixture.options.out, "moment.json"), JSON.stringify(moment)); + }], + ["future memory event", "memory-event", async (fixture, moment) => { + const file = path.join(inputPath(fixture, moment, "memory"), "memory/events.jsonl"); + const event = JSON.parse((await readFile(file, "utf8")).trim()); + await writeFile(file, `${JSON.stringify({ ...event, createdAt: FUTURE })}\n`); + }], + ["memory seq gap", "memory-event", async (fixture, moment) => { + await writeFile(path.join(inputPath(fixture, moment, "memory"), "memory/events.jsonl"), `${JSON.stringify(memoryEvent(3))}\n`); + }], + ["memory audit state", "memory-event", async (fixture, moment) => { + await writeFile(path.join(inputPath(fixture, moment, "memory"), "memory/index.sqlite"), "future derived state"); + }], + ["future Moltnet message", "moltnet-message", async (fixture, moment) => patchNetwork(fixture, moment, (state) => { + state.room_messages.assignment = [{ id: "future", network_id: "news", target: { kind: "room", room_id: "assignment" }, + from: { type: "agent", id: "writer" }, parts: [{ kind: "text", text: "future message" }], created_at: FUTURE }]; + })], + ["future Moltnet event", "moltnet-event", async (fixture, moment) => patchNetwork(fixture, moment, (state) => { + state.attachment_delivery_events = [{ id: "future", type: "test", network_id: "news", created_at: FUTURE }]; + })], + ["future Moltnet room", "moltnet-room", async (fixture, moment) => patchNetwork(fixture, moment, (state) => { + state.rooms.future = { id: "future", name: "Future", created_at: FUTURE }; + })], + ["escaping symlink", "symlink-escape", async (fixture) => { await symlink("../outside", path.join(fixture.options.out, "escape")); }], + ["absolute symlink", "symlink-escape", async (fixture) => { await symlink(fixture.root, path.join(fixture.options.out, "escape")); }], + ["future dated path", "dated-path", async (fixture) => { await mkdir(path.join(fixture.options.out, "edition-2026-10-11")); }], + ["tampered snapshot", "snapshot", async (fixture, moment) => { await writeFile(path.join(inputPath(fixture, moment, "volume"), "saved.txt"), "tampered"); }], + ["future feed landing", "feed-identity", async (fixture, moment) => { + const file = path.join(inputPath(fixture, moment, "feed"), ".spawnfile-feed.json"); + const identity = JSON.parse(await readFile(file, "utf8")); identity.landed_at = FUTURE; await writeFile(file, JSON.stringify(identity)); + }], + ["feed current outside its volume", "feed-identity", async (fixture, moment) => { + const link = path.join(inputPath(fixture, moment, "feed"), "current"); + await rm(link); await symlink("../code", link); + }] +]; + +const intendedFinding = (name: string, fixture: Fixture, moment: MaterializedMoment): { path: string; detail: string | RegExp } => { + const relative = (kind: string) => moment.inputs.find((input) => input.kind === kind)!.path!; + if (name.startsWith("future ") && /[am]time$/u.test(name)) return { + path: name.includes("report") ? "moment.json" : name.includes("directory") ? "new-directory" : name.includes("symlink") ? "new-link" : "future", + detail: `${name.endsWith("atime") ? "Access" : "Modification"} time is later than the moment` + }; + const findings: Record = { + "extra unreachable future commit": { path: relative("git"), detail: `Object ${fixture.futureCommit} is not reachable from ${fixture.commit}` }, + "extra tag": { path: relative("git"), detail: "Exactly one branch and HEAD must name the materialized commit" }, + "extra remote": { path: relative("git"), detail: "Forbidden git configuration remote.leaked.url" }, + "reachable commit future author only": { path: relative("git"), detail: /^Commit [a-f0-9]+ has a future author date$/u }, + "reachable commit future committer only": { path: relative("git"), detail: /^Commit [a-f0-9]+ has a future committer date$/u }, + "future memory event": { path: relative("memory"), detail: "Future memory event at line 1" }, + "memory seq gap": { path: relative("memory"), detail: "Noncontiguous memory sequence at line 1" }, + "memory audit state": { path: relative("memory"), detail: "Unexpected payload entry index.sqlite" }, + "future Moltnet message": { path: `${relative("moltnet")}/state.json`, detail: "Message future is later than the moment" }, + "future Moltnet event": { path: `${relative("moltnet")}/state.json`, detail: "Attachment event future is later than the moment" }, + "future Moltnet room": { path: `${relative("moltnet")}/state.json`, detail: "Room future is later than the moment" }, + "escaping symlink": { path: "escape", detail: 'Unsafe symlink target "../outside"' }, + "absolute symlink": { path: "escape", detail: `Unsafe symlink target ${JSON.stringify(fixture.root)}` }, + "future dated path": { path: "edition-2026-10-11", detail: "Calendar date 2026-10-11 is later than the latest possible local date 2026-10-09" }, + "tampered snapshot": { path: relative("volume"), detail: "Snapshot digest mismatch for saved.txt" }, + "future feed landing": { path: relative("feed"), detail: "Feed landed_at is later than the moment" }, + "feed current outside its volume": { path: relative("feed"), detail: "Feed current does not point to its tree inside the volume" } + }; + return findings[name]!; +}; + +describe("independent disk leak check mutation guarantees", () => { + it.each(cases)("refuses %s with class %s", async (name, expectedClass, mutate) => { + const fixture = await rewindFixture(); + const moment = await materializeTrainingMoment(fixture.options); + expect(moment.status).toBe("materialized"); + await mutate(fixture, moment); + const expected = intendedFinding(name, fixture, moment); + // Restore every parent and unrelated entry; time mutations change only the intended field on the target. + await stampTree(fixture.options.out, AT, AT); + if (expectedClass === "file-time") { + const target = path.join(fixture.options.out, expected.path); + const setTimes = (await lstat(target)).isSymbolicLink() ? lutimes : utimes; + await setTimes(target, new Date(name.endsWith("atime") ? FUTURE : AT), new Date(name.endsWith("mtime") ? FUTURE : AT)); + } + const check = await leakCheckTrainingMoment(fixture.options.out, { historyDir: fixture.historyDir }); + expect(check.ok).toBe(false); + expect(check.findings).toContainEqual(expect.objectContaining({ class: expectedClass, path: expected.path, + detail: typeof expected.detail === "string" ? expected.detail : expect.stringMatching(expected.detail) })); + if (expectedClass === "file-time") expect(check.findings.filter((finding) => finding.class === "file-time")) + .toEqual([expect.objectContaining({ class: expectedClass, ...expected })]); + const out = path.join(fixture.root, "rejected"); + const refused = await finalizeTrainingMoment(fixture.options.out, out, { historyDir: fixture.historyDir }); + expect(refused.status).toBe("refused"); + expect(refused.leakCheck.ok).toBe(false); + expect(refused.inputs.every((input) => input.path === null)).toBe(true); + expect(await readdir(out)).toEqual(["leak-check.json", "moment.json"]); + }); +}); diff --git a/src/compiler/training/rewind/leakCheck.ts b/src/compiler/training/rewind/leakCheck.ts new file mode 100644 index 00000000..0eb64e0e --- /dev/null +++ b/src/compiler/training/rewind/leakCheck.ts @@ -0,0 +1,50 @@ +import { readFile, realpath } from "node:fs/promises"; +import path from "node:path"; +import { materializedMomentSchema, leakCheckSchema, type LeakCheck, type LeakFinding, type MaterializedMoment } from "./contract.js"; +import { openHistory } from "./history.js"; +import { diskPath, inspectBundleFiles, restoreInspectedAccessTimes, snapshotBundleFiles } from "./leakFiles.js"; +import { inspectGitBundle } from "./leakGit.js"; +import { inspectInputProvenance } from "./leakProvenance.js"; +import { addInputFinding, inspectFeedBundle, inspectMemoryBundle, inspectMoltnetBundle, inspectVolumeBundle } from "./leakInputs.js"; + +export interface LeakCheckOptions { historyDir?: string } + +/** Independent acceptance boundary: only disk reports, disk payloads and disk history are trusted. */ +export const inspectTrainingMoment = async ( + bundle: string, options: LeakCheckOptions = {}, skipRefused = false +): Promise<{ moment: MaterializedMoment; check: LeakCheck | null }> => { + const root = await realpath(path.resolve(bundle)); + const files = await snapshotBundleFiles(root); + try { + const moment = materializedMomentSchema.parse(JSON.parse(await readFile(await diskPath(root, "moment.json"), "utf8"))); + if (skipRefused && moment.status === "refused") return { moment, check: null }; + const report: LeakCheck = { version: "spawnfile.leak-check.v1", at: moment.effectiveAt, ok: false, + checked: { files: 0, gitObjects: 0, memoryEvents: 0, moltnetMessages: 0, moltnetEvents: 0, snapshots: 0 }, findings: [] }; + await inspectBundleFiles(root, moment.inputs, report, files); + for (const input of moment.inputs) { + const kind: LeakFinding["class"] = input.kind === "memory" ? "memory-event" + : input.kind === "moltnet" ? "moltnet-message" : input.kind === "feed" ? "feed-identity" + : input.kind === "volume" ? "snapshot" : "git-object"; + try { + if (!input.path || input.refusal || !input.fidelity) throw new Error("Input is not materialized"); + const directory = await diskPath(root, input.path); + // Do not hand git/SQLite a payload with escaping paths, even for a failed check. + if (report.findings.some((finding) => finding.class === "symlink-escape" + && (finding.path === input.path || finding.path.startsWith(`${input.path}/`)))) continue; + if (["git", "bundle", "feed"].includes(input.kind)) await inspectInputProvenance(root, input, report, options.historyDir); + if (input.kind === "git") await inspectGitBundle(directory, input, report); + if (input.kind === "memory") await inspectMemoryBundle(root, input, report); + if (input.kind === "feed") await inspectFeedBundle(root, input, report, moment, await openHistory(options.historyDir)); + if (input.kind === "moltnet") await inspectMoltnetBundle(root, input, report); + if (input.kind === "volume") await inspectVolumeBundle(root, input, report, await openHistory(options.historyDir)); + } catch (error) { + addInputFinding(report, input, kind, error instanceof Error ? error.message : String(error), input.path ?? "."); + } + } + report.ok = report.findings.length === 0; + return { moment, check: leakCheckSchema.parse(report) }; + } finally { await restoreInspectedAccessTimes(files); } +}; + +export const leakCheckTrainingMoment = async (bundle: string, options: LeakCheckOptions = {}): Promise => + (await inspectTrainingMoment(bundle, options)).check!; diff --git a/src/compiler/training/rewind/leakFeed.ts b/src/compiler/training/rewind/leakFeed.ts new file mode 100644 index 00000000..ec8b37cc --- /dev/null +++ b/src/compiler/training/rewind/leakFeed.ts @@ -0,0 +1,27 @@ +import { feedLandingAt, type FeedIdentity } from "../../../volume/index.js"; +import type { MaterializedInput, MaterializedMoment } from "./contract.js"; +import { historyKey, type History } from "./history.js"; +import { compareInstants } from "./instant.js"; +import { recordInputErrors } from "./recordInput.js"; +import { selectMomentRecord } from "./select.js"; + +/** Publication metadata can follow the bound only when independent history proves the incoming pin/window. */ +export const permitsFeedPublicationTime = async ( + identity: FeedIdentity, input: MaterializedInput, moment: MaterializedMoment, history?: History +): Promise => { + if (!history || identity.landed_at !== input.source.landedAt) return false; + if (input.basis === "record" && input.fidelity === "exact" && moment.record.matched) { + const record = selectMomentRecord(await history.records(), moment.agent, moment.at, [input]); + const pin = record?.feeds.find((entry) => entry.id === input.id); + return Boolean(record && pin && compareInstants(record.wake.startedAt, moment.effectiveAt) === 0 + && !recordInputErrors(record, input).length && pin.revision === identity.revision + && pin.tree.replace(/^trees\//u, "") === identity.tree.replace(/^trees\//u, "") + && pin.commit === input.source.commit && compareInstants(pin.landedAt, identity.landed_at) === 0); + } + if (input.basis !== "landing-log" || input.fidelity !== "approximate") return false; + const landing = feedLandingAt(await history.feedLandings(`feeds/${historyKey(input)}`, input.id), moment.effectiveAt); + return Boolean(landing?.ambiguous && landing.revision === identity.revision + && landing.tree.replace(/^trees\//u, "") === identity.tree.replace(/^trees\//u, "") + && landing.commit === input.source.commit && landing.switching_at === input.source.switchingAt + && compareInstants(landing.landed_at, identity.landed_at) === 0); +}; diff --git a/src/compiler/training/rewind/leakFiles.ts b/src/compiler/training/rewind/leakFiles.ts new file mode 100644 index 00000000..454de73e --- /dev/null +++ b/src/compiler/training/rewind/leakFiles.ts @@ -0,0 +1,100 @@ +import type { BigIntStats } from "node:fs"; +import { lstat, lutimes, readdir, readlink, realpath, utimes } from "node:fs/promises"; +import path from "node:path"; +import { inside } from "./history.js"; +import { instantNanos } from "./instant.js"; +import type { LeakCheck, MaterializedInput } from "./contract.js"; + +export const inputAtPath = (inputs: MaterializedInput[], relative: string): MaterializedInput | undefined => + inputs.find((input) => input.path !== null && (relative === input.path || relative.startsWith(`${input.path}/`))); + +export const diskPath = async (root: string, relative: string): Promise => { + const candidate = path.resolve(root, relative); + if (!inside(root, candidate) || path.isAbsolute(relative)) throw new Error(`Bundle path escapes root: ${relative}`); + if (!inside(root, await realpath(candidate))) throw new Error(`Bundle symlink escapes root: ${relative}`); + return candidate; +}; + +/** Capture metadata before any inspection read can advance an entry's access time. */ +export const snapshotBundleFiles = async (root: string): Promise> => { + const files = new Map(); + const visit = async (file: string): Promise => { + const stats = await lstat(file, { bigint: true }); + files.set(file, stats); + if (stats.isDirectory()) for (const name of (await readdir(file)).sort()) await visit(path.join(file, name)); + }; + await visit(root); + return files; +}; + +/** Preserve the inspected access times, including future ones; never hide an existing violation. */ +export const restoreInspectedAccessTimes = async (files: Map): Promise => { + for (const [file, stats] of [...files].reverse()) { + // Numeric strings preserve sub-millisecond precision; negative numeric arguments mean "now" in Node. + await (stats.isSymbolicLink() ? lutimes : utimes)(file, + String(Number(stats.atimeNs / 1_000n) / 1_000_000), String(Number(stats.mtimeNs / 1_000n) / 1_000_000)); + } +}; + +/** No symlinks are traversed. Inspection never reads a target outside the bundle. */ +export const inspectBundleFiles = async (root: string, inputs: MaterializedInput[], report: LeakCheck, files: Map): Promise => { + const at = instantNanos(report.at); + const latestDate = new Date(Math.min(253_402_300_799_999, Date.parse(report.at) + 14 * 3600_000)).toISOString().slice(0, 10); + const visit = async (file: string): Promise => { + const relative = path.relative(root, file).split(path.sep).join("/") || "."; + const owner = inputAtPath(inputs, relative); + const input = owner ? { section: owner.section, id: owner.id, declaredBy: owner.declaredBy } : null; + const add = (kind: "bundle-entry" | "file-time" | "symlink-escape" | "dated-path" | "git-object", detail: string) => { + report.findings.push({ input, class: kind, path: relative, detail }); + }; + const stats = files.get(file)!; + if (!owner && relative !== ".") { + const reportFile = ["moment.json", "leak-check.json", "task-clock.json"].includes(relative) && stats.isFile(); + const wrapper = stats.isDirectory() && inputs.some((entry) => entry.path?.startsWith(`${relative}/`)); + if (!reportFile && !wrapper) add("bundle-entry", "Unexpected entry outside the materialized inputs and reports"); + } + report.checked.files++; + if (stats.mtimeNs > at) add("file-time", "Modification time is later than the moment"); + if (stats.atimeNs > at) add("file-time", "Access time is later than the moment"); + const allowedGitDirectory = owner?.kind === "git" && relative === `${owner.path}/.git` && stats.isDirectory(); + if (path.basename(file).toLowerCase() === ".git" && !allowedGitDirectory) { + add("git-object", "Unexpected .git repository metadata outside the materialized git resource repository"); + } + const head = files.get(path.join(file, "HEAD")); + const objects = files.get(path.join(file, "objects")); + const refs = files.get(path.join(file, "refs")); + if (stats.isDirectory() && head && (head.isFile() || head.isSymbolicLink()) + && objects && (objects.isDirectory() || objects.isSymbolicLink()) + && refs && (refs.isDirectory() || refs.isSymbolicLink()) && !allowedGitDirectory) { + add("git-object", "Unexpected bare repository (HEAD, objects/, refs/) outside the materialized git resource repository"); + } + for (const match of path.basename(file).matchAll(/\d{4}-\d{2}-\d{2}/gu)) { + const date = match[0]; + const parsed = Date.parse(`${date}T00:00:00Z`); + if (Number.isFinite(parsed) && new Date(parsed).toISOString().startsWith(date) && date > latestDate) { + add("dated-path", `Calendar date ${date} is later than the latest possible local date ${latestDate}`); + } + } + if (stats.isSymbolicLink()) { + const link = await readlink(file); + if (path.isAbsolute(link) || !inside(root, path.resolve(path.dirname(file), link))) { + add("symlink-escape", `Unsafe symlink target ${JSON.stringify(link)}`); return; + } + // Even a dangling leaf can cross an existing symlink in a parent component. + let target = path.resolve(path.dirname(file), link); + for (;;) { + try { + if (!inside(root, await realpath(target))) add("symlink-escape", "Symlink resolves outside the bundle"); + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + add("symlink-escape", `Cannot resolve symlink: ${String(error)}`); break; + } + if (target === root) break; + target = path.dirname(target); + } + } + } else if (!stats.isDirectory() && !stats.isFile()) add("symlink-escape", "Special files are not allowed in a moment bundle"); + }; + for (const file of files.keys()) await visit(file); +}; diff --git a/src/compiler/training/rewind/leakGit.ts b/src/compiler/training/rewind/leakGit.ts new file mode 100644 index 00000000..7e08a3d3 --- /dev/null +++ b/src/compiler/training/rewind/leakGit.ts @@ -0,0 +1,61 @@ +import { lstat, readFile, readdir } from "node:fs/promises"; +import path from "node:path"; +import { rewindGit } from "./gitProcess.js"; +import { instantNanos } from "./instant.js"; +import type { LeakCheck, MaterializedInput } from "./contract.js"; +import { inspectGitIndexTimes, inspectGitWorktree } from "./leakGitPayload.js"; + +/** Enumerate every stored object, independently of the materializer's chosen-object list. */ +export const inspectGitBundle = async (repository: string, input: MaterializedInput, report: LeakCheck): Promise => { + const add = (detail: string) => report.findings.push({ + input: { section: input.section, id: input.id, declaredBy: input.declaredBy }, + class: "git-object", path: input.path!, detail + }); + const gitDir = path.join(repository, ".git"); + if (!(await lstat(gitDir)).isDirectory()) throw new Error("The materialized repository must have its own .git directory"); + const names = new Set(await readdir(gitDir)); + for (const name of ["FETCH_HEAD", "ORIG_HEAD", "logs", "packed-refs", "commondir", "shallow"]) { + if (names.has(name)) add(`Forbidden git metadata ${name}`); + } + const config = await readFile(path.join(gitDir, "config"), "utf8"); + if (/^\s*\[\s*(?:include|includeif)\b/imu.test(config)) throw new Error("Git configuration includes are forbidden"); + for (const name of ["objects/info/alternates", "objects/info/http-alternates", "info/grafts"]) { + const exists = await lstat(path.join(gitDir, name)).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (exists) throw new Error(`Git object indirection ${name} is forbidden`); + } + const localConfig = (await rewindGit(repository, ["config", "--local", "--no-includes", "--list", "--null"])) + .toString().split("\0").filter(Boolean); + for (const entry of localConfig) { + const key = entry.split("\n")[0]!.toLowerCase(); + if (key.startsWith("remote.") || key === "core.worktree" || key === "extensions.partialclone") add(`Forbidden git configuration ${key}`); + } + const refs = (await rewindGit(repository, ["for-each-ref", "--format=%(refname) %(objectname)"])).toString().trim().split("\n").filter(Boolean); + const head = (await readFile(path.join(gitDir, "HEAD"), "utf8")).trim(); + const commit = input.source.commit; + if (!commit || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/u.test(commit)) throw new Error("Missing materialized commit identity"); + if (refs.length !== 1 || !refs[0]!.startsWith("refs/heads/") || refs[0]!.split(" ")[1] !== commit + || head !== `ref: ${refs[0]?.split(" ")[0]}`) add("Exactly one branch and HEAD must name the materialized commit"); + const reachable = new Set((await rewindGit(repository, ["rev-list", "--objects", commit, "--"])) + .toString().split("\n").filter(Boolean).map((line) => line.split(" ")[0])); + const objects = (await rewindGit(repository, ["cat-file", "--batch-all-objects", "--batch-check=%(objectname) %(objecttype)"])) + .toString().trim().split("\n").filter(Boolean); + report.checked.gitObjects += objects.length; + for (const row of objects) { + const [oid, kind] = row.split(" "); + if (!reachable.has(oid)) add(`Object ${oid} is not reachable from ${commit}`); + if (kind !== "commit") continue; + const object = (await rewindGit(repository, ["cat-file", "commit", oid!])).toString(); + const headers = object.split("\n\n", 1)[0]!; + for (const role of ["author", "committer"]) { + const match = new RegExp(`^${role} .* (-?\\d+) [+-]\\d{4}$`, "mu").exec(headers); + if (!match) { add(`Commit ${oid} has no valid ${role} date`); continue; } + if (BigInt(match[1]!) * 1_000_000_000n > instantNanos(report.at)) add(`Commit ${oid} has a future ${role} date`); + } + } + for (const inspect of [() => inspectGitWorktree(repository, commit), () => inspectGitIndexTimes(repository, report.at)]) { + try { await inspect(); } catch (error) { add(error instanceof Error ? error.message : String(error)); } + } +}; diff --git a/src/compiler/training/rewind/leakGitPayload.test.ts b/src/compiler/training/rewind/leakGitPayload.test.ts new file mode 100644 index 00000000..63163877 --- /dev/null +++ b/src/compiler/training/rewind/leakGitPayload.test.ts @@ -0,0 +1,94 @@ +import { createHash } from "node:crypto"; +import { chmod, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { LeakCheck, MaterializedInput } from "./contract.js"; +import { materializeCleanRepository } from "./git.js"; +import { commitTestGit, gitTestRoot, initTestGit, testGit } from "./git.test-helper.js"; +import { inspectGitBundle } from "./leakGit.js"; +import { stampTree } from "./timestamps.js"; +import * as gitProcess from "./gitProcess.js"; + +const AT = "2026-10-09T08:20:31Z", BEFORE = "2026-10-09T08:00:00Z"; +const roots: string[] = []; +afterEach(async () => { vi.restoreAllMocks(); await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const fixture = async () => { + const root = await gitTestRoot(); roots.push(root); + const source = path.join(root, "source"), repository = path.join(root, "output"); + await initTestGit(source); + await symlink("data.txt", path.join(source, "link")); + const commit = await commitTestGit(source, { "data.txt": "past bytes", ".gitignore": "ignored.txt\n", "folder/child": "child" }, BEFORE); + await materializeCleanRepository(source, commit, repository); + const input: MaterializedInput = { section: "resources", declaredBy: "agent:writer", id: "code", kind: "git", + fidelity: "exact", basis: "record", source: { commit }, path: "resources/code", notes: [], refusal: null }; + const inspect = async () => { + const report: LeakCheck = { version: "spawnfile.leak-check.v1", at: AT, ok: false, + checked: { files: 0, gitObjects: 0, memoryEvents: 0, moltnetMessages: 0, moltnetEvents: 0, snapshots: 0 }, findings: [] }; + await inspectGitBundle(repository, input, report); + return report.findings; + }; + return { source, repository, inspect }; +}; + +describe("independent git payload inspection", () => { + it("rebuilds the checked-out index without materialization-time stat fields", async () => { + const f = await fixture(); + const debug = testGit(f.repository, ["ls-files", "--debug"]); + const times = [...debug.matchAll(/(?:ctime|mtime): (\d+):(\d+)/gu)]; + expect(times.length).toBeGreaterThan(0); + expect(times.every((match) => match[1] === "0" && match[2] === "0")).toBe(true); + expect(await f.inspect()).toEqual([]); + }); + + it("clears refreshed checkout stat fields before publishing", async () => { + const run = gitProcess.rewindGit; + // Some git versions leave checkout-index stat fields zero already. Populate real + // checkout stats at this boundary so deleting the rebuild cannot silently pass. + const checkout = vi.spyOn(gitProcess, "rewindGit").mockImplementation(async (repo, args, input) => { + const result = await run(repo, args, input); + if (args[0] === "checkout-index") { + await run(repo, ["update-index", "--refresh"]); + const populated = (await run(repo, ["ls-files", "--debug"])).toString(); + expect([...populated.matchAll(/ctime: (\d+):/gu)].some((match) => Number(match[1]) > Date.parse(AT) / 1000)).toBe(true); + } + return result; + }); + const f = await fixture(); checkout.mockRestore(); + expect(await f.inspect()).toEqual([]); + const times = [...testGit(f.repository, ["ls-files", "--debug"]).matchAll(/(?:ctime|mtime): (\d+):(\d+)/gu)]; + expect(times.length).toBeGreaterThan(0); + expect(times.every((match) => match[1] === "0" && match[2] === "0")).toBe(true); + }); + + it.each(["changed", "missing", "untracked", "ignored", "extra-directory", "mode", "symlink", "index-disguised"]) + ("rejects %s worktree content against local ls-tree blobs", async (mutation) => { + const f = await fixture(); + expect(await f.inspect()).toEqual([]); + // Source history is unavailable: this comparison must use the bundle repository itself. + await rm(f.source, { recursive: true }); + if (["changed", "index-disguised"].includes(mutation)) await writeFile(path.join(f.repository, "data.txt"), "later data"); + if (mutation === "index-disguised") { + testGit(f.repository, ["add", "data.txt"]); + testGit(f.repository, ["update-index", "--assume-unchanged", "data.txt"]); + } + if (mutation === "missing") await rm(path.join(f.repository, "data.txt")); + if (mutation === "untracked" || mutation === "ignored") await writeFile(path.join(f.repository, `${mutation}.txt`), "later data"); + if (mutation === "extra-directory") await mkdir(path.join(f.repository, "extra")); + if (mutation === "mode") await chmod(path.join(f.repository, "data.txt"), 0o755); + if (mutation === "symlink") { await rm(path.join(f.repository, "link")); await symlink("folder/child", path.join(f.repository, "link")); } + await stampTree(f.repository, BEFORE, AT); + expect(await f.inspect()).toContainEqual(expect.objectContaining({ class: "git-object", detail: expect.stringMatching(/git worktree/iu) })); + }); + + it.each(["ctime", "mtime"])("rejects future index %s after filesystem timestamps are stamped", async (field) => { + const f = await fixture(), file = path.join(f.repository, ".git/index"); + expect(await f.inspect()).toEqual([]); + const bytes = await readFile(file); + expect(bytes.subarray(0, 4).toString()).toBe("DIRC"); + expect(bytes.readUInt32BE(8)).toBeGreaterThan(0); + bytes.writeUInt32BE(Math.floor(Date.parse(AT) / 1000) + 1, 12 + (field === "ctime" ? 0 : 8)); + createHash("sha1").update(bytes.subarray(0, -20)).digest().copy(bytes, bytes.length - 20); + await writeFile(file, bytes); await stampTree(f.repository, BEFORE, AT); + expect(await f.inspect()).toContainEqual(expect.objectContaining({ class: "git-object", detail: expect.stringContaining(`Git index ${field} is later`) })); + }); +}); diff --git a/src/compiler/training/rewind/leakGitPayload.ts b/src/compiler/training/rewind/leakGitPayload.ts new file mode 100644 index 00000000..4da53c09 --- /dev/null +++ b/src/compiler/training/rewind/leakGitPayload.ts @@ -0,0 +1,50 @@ +import { lstat, readFile, readdir, readlink } from "node:fs/promises"; +import path from "node:path"; +import { rewindGit } from "./gitProcess.js"; +import { safeCommittedPath } from "./gitTree.js"; +import { instantNanos } from "./instant.js"; + +/** Compare against objects in the output repository, never the index or status' cached stat data. */ +export const inspectGitWorktree = async (repository: string, commit: string): Promise => { + const entries = new Map(); + for (const row of (await rewindGit(repository, ["ls-tree", "-rtz", `${commit}^{tree}`])).toString().split("\0").filter(Boolean)) { + const tab = row.indexOf("\t"), name = safeCommittedPath(row.slice(tab + 1)); + const [mode, kind, oid] = row.slice(0, tab).split(" "); + entries.set(name, { mode: mode!, kind: kind!, oid: oid! }); + } + const visit = async (directory: string, prefix = ""): Promise => { + for (const name of await readdir(directory)) { + if (!prefix && name === ".git") continue; + const relative = prefix ? `${prefix}/${name}` : name, file = path.join(directory, name); + const expected = entries.get(relative), stats = await lstat(file); + if (!expected) throw new Error(`Unexpected git worktree entry ${relative}`); + entries.delete(relative); + if (expected.kind === "tree" && stats.isDirectory()) { await visit(file, relative); continue; } + if (expected.kind !== "blob" || !["100644", "100755", "120000"].includes(expected.mode)) { + throw new Error(`Unsupported git worktree entry ${relative}`); + } + const bytes = await rewindGit(repository, ["cat-file", "blob", expected.oid]); + const matches = expected.mode === "120000" + ? stats.isSymbolicLink() && (await readlink(file, { encoding: "buffer" })).equals(bytes) + : stats.isFile() && Boolean(stats.mode & 0o111) === (expected.mode === "100755") && (await readFile(file)).equals(bytes); + if (!matches) throw new Error(`Git worktree entry mismatch: ${relative}`); + } + }; + await visit(repository); + if (entries.size) throw new Error(`Missing git worktree entries: ${[...entries.keys()].join(", ")}`); +}; + +/** NUL terminates each pathname, so path text cannot masquerade as --debug stat metadata. */ +export const inspectGitIndexTimes = async (repository: string, at: string): Promise => { + const debug = (await rewindGit(repository, ["ls-files", "--debug", "-z"])).toString(); + for (let cursor = 0; cursor < debug.length;) { + const end = debug.indexOf("\0", cursor); + const metadata = end < 0 ? null : /^ ctime: (\d+):(\d+)\n mtime: (\d+):(\d+)\n dev: [^\n]+\n uid: [^\n]+\n size: [^\n]+\n/u.exec(debug.slice(end + 1)); + if (!metadata) throw new Error("Invalid git index stat metadata"); + for (const [label, offset] of [["ctime", 1], ["mtime", 3]] as const) { + const nanos = BigInt(metadata[offset]!) * 1_000_000_000n + BigInt(metadata[offset + 1]!); + if (nanos > instantNanos(at)) throw new Error(`Git index ${label} is later than the moment: ${debug.slice(cursor, end)}`); + } + cursor = end + 1 + metadata[0].length; + } +}; diff --git a/src/compiler/training/rewind/leakInputs.ts b/src/compiler/training/rewind/leakInputs.ts new file mode 100644 index 00000000..f17efbd5 --- /dev/null +++ b/src/compiler/training/rewind/leakInputs.ts @@ -0,0 +1,121 @@ +import { lstat, readFile, readdir, readlink, realpath } from "node:fs/promises"; +import path from "node:path"; +import { feedIdentityFindings, type FeedIdentity } from "../../../volume/feedRecord.js"; +import type { LeakCheck, LeakFinding, MaterializedInput, MaterializedMoment } from "./contract.js"; +import { digestDirectory } from "../../../volume/index.js"; +import { assertPreparedFeedRevision, feedGitRevision, verifyPlainFeedTree } from "./feedContent.js"; +import { assertLocalGitRepository } from "./gitHistory.js"; +import { permitsFeedPublicationTime } from "./leakFeed.js"; +import { historyKey, inside, safeSegment, type History } from "./history.js"; +import { compareInstants } from "./instant.js"; +import { diskPath } from "./leakFiles.js"; +import { assertDirectoryEntries } from "./leakLayout.js"; +import { parseMemoryLedger } from "./memory.js"; +import { inspectMoltnetJson } from "./moltnetJson.js"; +import { inspectMoltnetSqlite } from "./moltnetSqlite.js"; +import { readSnapshotManifest, snapshotBytes, snapshotHistoryPath, snapshotTreeFindings } from "./snapshotManifest.js"; + +export const addInputFinding = (report: LeakCheck, input: MaterializedInput, kind: LeakFinding["class"], detail: string, file = input.path!): void => { + report.findings.push({ input: { section: input.section, id: input.id, declaredBy: input.declaredBy }, class: kind, path: file, detail }); +}; + +export const inspectMemoryBundle = async (root: string, input: MaterializedInput, report: LeakCheck): Promise => { + const home = await diskPath(root, input.path!); + await assertDirectoryEntries(home, { memory: "directory" }); + await assertDirectoryEntries(path.join(home, "memory"), { "events.jsonl": "file" }); + const ledger = parseMemoryLedger(await readFile(await diskPath(root, `${input.path}/memory/events.jsonl`), "utf8")); + report.checked.memoryEvents += ledger.events.length; + let native = false; + for (const [index, event] of ledger.events.entries()) { + if (event.seq === undefined ? native : event.seq !== index + 1) { + addInputFinding(report, input, "memory-event", `Noncontiguous memory sequence at line ${index + 1}`); + } + native ||= event.seq !== undefined; + if (compareInstants(event.createdAt, report.at) > 0) addInputFinding(report, input, "memory-event", `Future memory event at line ${index + 1}`); + if (index > 0 && compareInstants(event.createdAt, ledger.events[index - 1]!.createdAt) < 0) { + addInputFinding(report, input, "memory-event", `Memory clock regression at line ${index + 1}`); + } + } + if (input.source.seq !== ledger.events.length) addInputFinding(report, input, "memory-event", "Ledger length disagrees with the materialized sequence"); +}; + +export const inspectFeedBundle = async (root: string, input: MaterializedInput, report: LeakCheck, moment?: MaterializedMoment, history?: History): Promise => { + const directory = await diskPath(root, input.path!); + if (input.fidelity === "overridden") { + if ((await readdir(directory)).length) addInputFinding(report, input, "feed-identity", "Overridden feed volume must be empty"); + return; + } + const identity: FeedIdentity = JSON.parse(await readFile(await diskPath(root, `${input.path}/.spawnfile-feed.json`), "utf8")); + await assertDirectoryEntries(directory, { current: "symlink", ".spawnfile-feed.json": "file", trees: "directory" }); + for (const issue of feedIdentityFindings(identity)) addInputFinding(report, input, "feed-identity", issue); + if (compareInstants(identity.landed_at, report.at) > 0 && !(moment && await permitsFeedPublicationTime(identity, input, moment, history))) { + addInputFinding(report, input, "feed-identity", "Feed landed_at is later than the moment"); + } + if (identity.resource !== input.id || identity.revision !== input.source.revision) { + addInputFinding(report, input, "feed-identity", "Feed identity disagrees with the moment input"); + } + const current = path.join(directory, "current"); + if (!(await lstat(current)).isSymbolicLink()) throw new Error("Feed current must be a relative symlink"); + const link = await readlink(current); + if (link !== identity.tree || !/^trees\/[a-f0-9]{64}(?:\.[1-9][0-9]*)?$/u.test(link) + || !inside(path.join(directory, "trees"), await realpath(current))) { + addInputFinding(report, input, "feed-identity", "Feed current does not point to its tree inside the volume"); + } else if (!(await lstat(await realpath(current))).isDirectory()) addInputFinding(report, input, "feed-identity", "Feed tree must be a directory"); + else { + await assertDirectoryEntries(path.join(directory, "trees"), { [path.basename(link)]: "directory" }); + const prepared = identity.source.prepared !== undefined || input.source.prepared === true; + if (prepared && (input.fidelity !== "approximate" || !input.notes.includes("prepared output not content-verifiable"))) { + addInputFinding(report, input, "feed-identity", "Prepared feed must be approximate: prepared output not content-verifiable"); + } + if (identity.source.prepared) assertPreparedFeedRevision(identity.revision, identity.source.prepared); + if (identity.source.kind === "directory" && !prepared && digestDirectory(await realpath(current)) !== identity.revision) { + addInputFinding(report, input, "feed-identity", "Directory feed revision disagrees with retained bytes"); + } + if (identity.source.kind === "git") { + const repository = await history?.repository(`feeds/${historyKey(input)}/repo`); + if (!repository) throw new Error("Missing feed source history repository for content verification"); + await assertLocalGitRepository(repository, history); + if (!prepared) await verifyPlainFeedTree(repository, identity, await realpath(current)); + else if (identity.source.prepared && feedGitRevision(repository, identity.source.commit, identity.source.ref, identity.source.paths) + !== identity.source.prepared.source_revision) throw new Error("Prepared feed source revision disagrees with selected commit and paths"); + } + } +}; + +export const inspectMoltnetBundle = async (root: string, input: MaterializedInput, report: LeakCheck): Promise => { + const directory = await diskPath(root, input.path!); + const stores = (await readdir(directory)).filter((name) => name === "state.json" || name === "moltnet.sqlite"); + if (stores.length !== 1) throw new Error("Expected exactly one Moltnet store"); + await assertDirectoryEntries(directory, { [stores[0]!]: "file" }); + const store = await diskPath(root, `${input.path}/${stores[0]}`); + const result = stores[0] === "state.json" ? await inspectMoltnetJson(store, report.at) : await inspectMoltnetSqlite(store, report.at); + report.checked.moltnetMessages += result.messages; + report.checked.moltnetEvents += result.events; + for (const finding of result.findings) addInputFinding(report, input, finding.class, finding.detail, path.relative(root, finding.path)); +}; + +export const inspectVolumeBundle = async (root: string, input: MaterializedInput, report: LeakCheck, history?: History): Promise => { + const directory = await diskPath(root, input.path!); + if (input.fidelity === "overridden") { + for (const name of await readdir(directory)) addInputFinding(report, input, "snapshot", `Unexpected snapshot entry ${name}`); + return; + } + if (!history || !input.source.snapshot || !input.source.agentSlug) throw new Error("Snapshot verification requires its history, agent slug and selected snapshot"); + if (input.source.snapshot.endsWith(".partial")) throw new Error("Selected snapshot is partial"); + const relative = snapshotHistoryPath(input.source.agentSlug, input.id, input.source.snapshot); + const manifest = await readSnapshotManifest(history, relative, input.id, input.source.snapshot); + const retained = await history.path(relative); + if (!manifest || !retained) throw new Error("Snapshot tree or manifest is missing"); + if (compareInstants(manifest.started_at, report.at) > 0) throw new Error("Selected snapshot is later than the moment"); + const rows = await history.wakeMoments(`recording/${safeSegment(input.source.agentSlug)}/wake-moments.jsonl`); + if (!rows?.some((row) => compareInstants(row.started_at, manifest.started_at) === 0 + && row.snapshots.some((entry) => entry.id === input.id && entry.snapshot === input.source.snapshot))) { + throw new Error("Selected snapshot has no matching wake row"); + } + report.checked.snapshots++; + for (const detail of await snapshotTreeFindings(retained, manifest, { retained: true })) addInputFinding(report, input, "snapshot", `Retained ${detail}`); + for (const detail of await snapshotTreeFindings(directory, manifest, { + readBytes: async (name) => readFile(await diskPath(root, `${input.path}/${name}`)), + expectedBytes: (name) => snapshotBytes(history, relative, name) + })) addInputFinding(report, input, "snapshot", detail); +}; diff --git a/src/compiler/training/rewind/leakLayout.test.ts b/src/compiler/training/rewind/leakLayout.test.ts new file mode 100644 index 00000000..1ce84c66 --- /dev/null +++ b/src/compiler/training/rewind/leakLayout.test.ts @@ -0,0 +1,35 @@ +import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { cleanTrainingProjects } from "../plan/fixtures.test-helper.js"; +import { AT, rewindFixture } from "./fixtures.test-helper.js"; +import { leakCheckTrainingMoment } from "./leakCheck.js"; +import { finalizeTrainingMoment, materializeTrainingMoment } from "./materialize.js"; +import { stampTree } from "./timestamps.js"; + +afterEach(cleanTrainingProjects); +describe("closed rewind payload layouts", () => { + it.each(["root", "wrapper", "memory-root", "memory-ledger", "moltnet", "feed-root", "feed-tree"]) + ("rejects a stamped extra entry at %s", async (location) => { + const f = await rewindFixture(), moment = await materializeTrainingMoment(f.options); + expect(moment.status, JSON.stringify(moment.inputs)).toBe("materialized"); + const input = moment.inputs.find((entry) => entry.kind === (location.startsWith("memory") ? "memory" + : location.startsWith("feed") ? "feed" : "moltnet"))!; + let directory = location === "root" ? f.options.out : location === "wrapper" ? path.join(f.options.out, "resources") + : path.join(f.options.out, input.path!, location === "memory-ledger" ? "memory" : location === "feed-tree" ? "trees" : ""); + if (location === "feed-tree") { + directory = path.join(directory, "a".repeat(64)); await mkdir(directory); + } + await writeFile(path.join(directory, "unrelated.txt"), "post-instant bytes"); + await stampTree(f.options.out, AT, AT); + const report = await leakCheckTrainingMoment(f.options.out, { historyDir: f.historyDir }); + const expectedClass = location === "root" || location === "wrapper" ? "bundle-entry" + : location.startsWith("memory") ? "memory-event" : location.startsWith("feed") ? "feed-identity" : "moltnet-message"; + expect(report.findings).toContainEqual(expect.objectContaining({ class: expectedClass, detail: expect.stringMatching(/Unexpected/u) })); + expect(report.findings.some((finding) => finding.class === "file-time")).toBe(false); + const out = path.join(f.root, "refused"); + expect((await finalizeTrainingMoment(f.options.out, out, { historyDir: f.historyDir })).status).toBe("refused"); + expect(await readdir(out)).toEqual(["leak-check.json", "moment.json"]); + expect(JSON.parse(await readFile(path.join(out, "leak-check.json"), "utf8")).ok).toBe(false); + }); +}); diff --git a/src/compiler/training/rewind/leakLayout.ts b/src/compiler/training/rewind/leakLayout.ts new file mode 100644 index 00000000..ccaba7e5 --- /dev/null +++ b/src/compiler/training/rewind/leakLayout.ts @@ -0,0 +1,17 @@ +import { lstat, readdir } from "node:fs/promises"; +import path from "node:path"; + +/** Closed native layouts cannot carry unrelated retained bytes, even when their dates are stamped. */ +export const assertDirectoryEntries = async ( + directory: string, expected: Record +): Promise => { + const names = await readdir(directory); + for (const name of names) { + if (!Object.hasOwn(expected, name)) throw new Error(`Unexpected payload entry ${name}`); + const stat = await lstat(path.join(directory, name)), kind = expected[name]; + if (!(kind === "file" ? stat.isFile() : kind === "directory" ? stat.isDirectory() : stat.isSymbolicLink())) { + throw new Error(`Payload entry ${name} must be a ${kind}`); + } + } + for (const name of Object.keys(expected)) if (!names.includes(name)) throw new Error(`Missing payload entry ${name}`); +}; diff --git a/src/compiler/training/rewind/leakProvenance.test.ts b/src/compiler/training/rewind/leakProvenance.test.ts new file mode 100644 index 00000000..ef9b1bc3 --- /dev/null +++ b/src/compiler/training/rewind/leakProvenance.test.ts @@ -0,0 +1,141 @@ +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { resolveFeedSource } from "../../../volume/index.js"; +import type { MaterializedMoment } from "./contract.js"; +import { materializeFeed } from "./feed.js"; +import { materializeGit } from "./git.js"; +import * as gitProcess from "./gitProcess.js"; +import { commitTestGit, gitTestRoot, initTestGit, testGit } from "./git.test-helper.js"; +import { History } from "./history.js"; +import { leakCheckTrainingMoment } from "./leakCheck.js"; +import { finalizeTrainingMoment } from "./materialize.js"; +import { stampTree } from "./timestamps.js"; +import type { InputContext, InputResult } from "./types.js"; + +const AT = "2026-10-09T08:20:31Z", BEFORE = "2026-10-09T08:00:00Z", AFTER = "2026-10-11T00:00:00Z"; +const roots: string[] = []; +afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); +type Kind = "git" | "bundle" | "feed"; +const fixture = async (kind: Kind) => { + const root = await gitTestRoot(); roots.push(root); + const historyDir = path.join(root, "history"), key = "agent%3Awriter/source"; + const repository = path.join(historyDir, kind === "feed" ? `feeds/${key}/repo` : `git/${key}`); + await initTestGit(repository); + const commit = await commitTestGit(repository, { "data.txt": "past bytes\n" }, BEFORE); + const bundle = path.join(root, "stage"), relative = `resources/${key}`, destination = path.join(bundle, relative); + const declaration = kind === "feed" + ? { id: "source", kind: "volume" as const, mode: "mutable" as const, mount: "./data", name: "source", feed: { git: { repo: repository } } } + : kind === "git" ? { id: "source", kind, mode: "readonly" as const, mount: "./data", url: repository } + : { id: "source", kind, mode: "readonly" as const, mount: "./data", build: { files: { root: repository } } }; + const context: InputContext = { at: AT, destination, history: new History(historyDir), record: null, + input: { section: "resources", id: "source", declaredBy: "agent:writer", declarationBase: root, declaration, overridden: false } }; + const landing = async (selected: string) => { + const revision = resolveFeedSource({ kind: "git", fetch: false, repo: repository, ref: selected }).revision; + const directory = path.dirname(repository), retained = path.join(directory, "trees", revision); + await mkdir(retained, { recursive: true }); + await writeFile(path.join(retained, "data.txt"), "past bytes\n"); + await writeFile(path.join(directory, "landings.jsonl"), JSON.stringify({ version: "spawnfile.volume-feed-landing.v1", resource: "source", volume: "source", source: { kind: "git", repo: repository }, revision, tree: `trees/${revision}`, commit: selected, + ref: "main", switching_at: BEFORE, landed_at: BEFORE }) + "\n"); + await stampTree(retained, BEFORE, AT); + }; + const stage = async () => { + const result: InputResult = await (kind === "feed" ? materializeFeed(context) : materializeGit(context)); + const moment: MaterializedMoment = { version: "spawnfile.materialized-moment.v1", status: "materialized", agent: "agent:writer", + example: { id: "example", role: "test", dataset: "dataset" }, at: AT, effectiveAt: AT, record: { matched: false }, leakCheck: { ok: false, findings: 0 }, + inputs: [{ section: "resources", id: "source", declaredBy: "agent:writer", kind, fidelity: result.fidelity, + basis: result.basis, source: result.source, path: relative, notes: [], refusal: null }] }; + await writeFile(path.join(bundle, "moment.json"), JSON.stringify(moment)); + await stampTree(bundle, BEFORE, AT); + return moment; + }; + const future = (role: string) => { + testGit(repository, ["commit", "--allow-empty", "-m", "future", `--date=${role === "author" ? AFTER : BEFORE}`], role === "committer" ? AFTER : BEFORE); + return testGit(repository, ["rev-parse", "HEAD"]); + }; + if (kind === "feed") await landing(commit); + return { root, historyDir, repository, bundle, relative, destination, context, commit, landing, stage, future }; +}; + +describe("independent source commit provenance", () => { + it.each(["author", "committer"])("refuses retained future feed bytes when the producer's %s guard is deleted", async (role) => { + const f = await fixture("feed"), commit = f.future(role); + await f.landing(commit); // Eligible landing, future source, retained bytes with normalized timestamps. + const guard = vi.spyOn(gitProcess, "assertCommitNotFuture").mockResolvedValue(undefined); + const moment = await f.stage(); guard.mockRestore(); + expect(moment.inputs[0]).toMatchObject({ fidelity: "exact", source: { commit } }); + expect(await readFile(path.join(f.destination, "current/data.txt"), "utf8")).toBe("past bytes\n"); + await stampTree(f.bundle, BEFORE, AT); + const out = path.join(f.root, "refused"); + expect((await finalizeTrainingMoment(f.bundle, out, { historyDir: f.historyDir })).status).toBe("refused"); + const check = JSON.parse(await readFile(path.join(out, "leak-check.json"), "utf8")); + expect(check.findings).toEqual([{ input: { section: "resources", id: "source", declaredBy: "agent:writer" }, + class: "git-object", path: f.relative, detail: `Provenance commit ${commit} has a future ${role} date` }]); + expect(await readdir(out)).toEqual(["leak-check.json", "moment.json"]); + }); + + it.each((["git", "bundle", "feed"] as const).flatMap((kind) => ["author", "committer", "blob"].map((role) => ({ kind, role })))) + ("re-reads $kind source provenance and refuses $role on disk", async ({ kind, role }) => { + const f = await fixture(kind), moment = await f.stage(); + expect((await leakCheckTrainingMoment(f.bundle, { historyDir: f.historyDir })).findings).toEqual([]); + const commit = role === "blob" ? testGit(f.repository, ["rev-parse", `${f.commit}:data.txt`]) : f.future(role); + moment.inputs[0]!.source.commit = commit; + await writeFile(path.join(f.bundle, "moment.json"), JSON.stringify(moment)); + if (kind === "feed") { + const file = path.join(f.destination, ".spawnfile-feed.json"), identity = JSON.parse(await readFile(file, "utf8")); + identity.source.commit = commit; await writeFile(file, JSON.stringify(identity)); + } + await stampTree(f.bundle, BEFORE, AT); + const check = await leakCheckTrainingMoment(f.bundle, { historyDir: f.historyDir }); + expect(check.findings).toContainEqual(expect.objectContaining({ class: "git-object", path: f.relative, + detail: role === "blob" ? `Provenance object ${commit} must be a commit object; found blob` + : `Provenance commit ${commit} has a future ${role} date` })); + expect(check.findings.some((finding) => finding.class === "file-time")).toBe(false); + }); + + it.each(["git", "bundle"] as const)("keeps local pinned %s provenance verifiable without a history directory", async (kind) => { + const f = await fixture(kind); + if (f.context.input.section !== "resources") throw new Error("Expected resource"); + const resource = f.context.input.declaration; + if (resource.kind === "git") resource.ref = f.commit; + if (resource.kind === "bundle") resource.build!.files!.ref = f.commit; + f.context.history = undefined; + const moment = await f.stage(); + expect(moment.inputs[0]!.source).toMatchObject({ commit: f.commit, repository: f.repository }); + expect((await leakCheckTrainingMoment(f.bundle)).findings).toEqual([]); + await rm(f.repository, { recursive: true }); + expect((await leakCheckTrainingMoment(f.bundle)).findings).toContainEqual(expect.objectContaining({ class: "git-object" })); + }); + + it("refuses inconsistent feed commit identities even when both commits are historical", async () => { + const f = await fixture("feed"); await f.stage(); + const other = await commitTestGit(f.repository, {}, BEFORE); + const file = path.join(f.destination, ".spawnfile-feed.json"), identity = JSON.parse(await readFile(file, "utf8")); + identity.source.commit = other; await writeFile(file, JSON.stringify(identity)); + await stampTree(f.bundle, BEFORE, AT); + expect((await leakCheckTrainingMoment(f.bundle, { historyDir: f.historyDir })).findings) + .toContainEqual(expect.objectContaining({ class: "git-object", detail: "Feed source commit disagrees with moment provenance" })); + }); + + it.each(["selector-at", "pinned", "dependencies", "generated"])("compares %s bundle payload with historical roots and exclusions", async (selection) => { + const f = await fixture("bundle"); + const commit = await commitTestGit(f.repository, { "package/keep.txt": "kept", "package/omit.txt": "excluded", "other.txt": "other" }, BEFORE); + if (f.context.input.section !== "resources" || f.context.input.declaration.kind !== "bundle") throw new Error("Expected bundle"); + f.context.input.declarationBase = f.repository; + f.context.input.declaration.build = selection === "dependencies" ? { dependencies: { directory: "package", image: "node:22" } } + : selection === "generated" ? { generated: { command: ["never-run"], inputs: [{ root: "package", exclude: ["omit.txt"] }, { root: ".", exclude: ["package/**"] }] } } + : { files: { root: "package", exclude: ["omit.txt"], ...(selection === "pinned" ? { ref: commit } : {}) } }; + const moment = await f.stage(); + expect(moment.inputs[0]!.source.trees).toBeDefined(); + expect((await leakCheckTrainingMoment(f.bundle, { historyDir: f.historyDir })).findings).toEqual([]); + // The gate must inspect bytes despite a valid receipt and producer-stamped times. + await writeFile(path.join(f.destination, selection === "generated" ? "package/keep.txt" : "keep.txt"), "future payload"); + await stampTree(f.bundle, BEFORE, AT); + const check = await leakCheckTrainingMoment(f.bundle, { historyDir: f.historyDir }); + expect(check.findings).toContainEqual(expect.objectContaining({ class: "git-object", detail: "Bundle bytes disagree with selected commit, roots and exclusions" })); + expect(check.findings.some((finding) => finding.class === "file-time")).toBe(false); + }); +}); diff --git a/src/compiler/training/rewind/leakProvenance.ts b/src/compiler/training/rewind/leakProvenance.ts new file mode 100644 index 00000000..0c40bc1a --- /dev/null +++ b/src/compiler/training/rewind/leakProvenance.ts @@ -0,0 +1,72 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { FeedIdentity } from "../../../volume/feedRecord.js"; +import type { LeakCheck, MaterializedInput } from "./contract.js"; +import { assertLocalGitRepository } from "./gitHistory.js"; +import { rewindGit } from "./gitProcess.js"; +import { historyKey, openHistory } from "./history.js"; +import { instantNanos } from "./instant.js"; +import { diskPath } from "./leakFiles.js"; +import { addInputFinding } from "./leakInputs.js"; +import { verifyRecordedBundle } from "./recordedBundle.js"; +import { digestDirectory } from "../../../volume/index.js"; +import { exportGitTree } from "./gitTree.js"; + +const inspectBundleTree = async (repository: string, commit: string, directory: string, input: MaterializedInput): Promise => { + if (!input.source.trees?.length) throw new Error("Missing bundle source tree selection"); + const scratch = await mkdtemp(path.join(os.tmpdir(), "spawnfile-bundle-check-")); + try { + for (const tree of input.source.trees) await exportGitTree(repository, commit, scratch, { ...tree, mergeIdentical: true }); + if (digestDirectory(directory) !== digestDirectory(scratch)) throw new Error("Bundle bytes disagree with selected commit, roots and exclusions"); + } finally { await rm(scratch, { recursive: true, force: true }); } +}; + +/** Read receipts and source objects afresh; never trust the producer's date guard or stamped files. */ +export const inspectInputProvenance = async ( + root: string, input: MaterializedInput, report: LeakCheck, historyDir?: string +): Promise => { + const add = (detail: string) => addInputFinding(report, input, "git-object", detail); + try { + if (input.kind === "bundle" && (input.basis === "record" || input.basis === "deploy-log") && input.source.sha256) { + await verifyRecordedBundle(await openHistory(historyDir), input.source.sha256, await diskPath(root, input.path!)); + return; + } + const commit = input.source.commit; + if (input.kind === "feed") { + if (input.fidelity === "overridden") return; + const identity: FeedIdentity = JSON.parse(await readFile(await diskPath(root, `${input.path}/.spawnfile-feed.json`), "utf8")); + if (identity.source?.kind === "git") { + if (!commit || identity.source.commit !== commit) add("Feed source commit disagrees with moment provenance"); + } else if (!commit) return; // Directory feeds have no git provenance. + } else if (input.kind === "bundle" && input.fidelity === "overridden" && !commit && !input.source.repository) { + return; // Explicit prebuilt tar overrides have no source commit. + } + if (!commit || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/u.test(commit)) throw new Error("Missing or invalid source commit provenance"); + let repository: string; + if (input.source.repository) { + // Only locally resolved pins/overrides may lack an entry in the history layout. + if (input.kind === "feed" || input.basis !== "pinned" && input.fidelity !== "overridden" + || !path.isAbsolute(input.source.repository)) throw new Error("Invalid local provenance repository"); + repository = input.source.repository; + await assertLocalGitRepository(repository); + } else { + const history = await openHistory(historyDir); + const key = input.kind === "feed" ? `feeds/${historyKey(input)}/repo` : `git/${historyKey(input)}`; + const retained = await history?.repository(key); + if (!retained) throw new Error(`Missing source history repository for provenance: ${key}`); + repository = retained; + await assertLocalGitRepository(repository, history); + } + const type = (await rewindGit(repository, ["cat-file", "-t", commit])).toString().trim(); + if (type !== "commit") throw new Error(`Provenance object ${commit} must be a commit object; found ${type}`); + const headers = (await rewindGit(repository, ["cat-file", "commit", commit])).toString().split("\n\n", 1)[0]!; + report.checked.gitObjects++; + for (const role of ["author", "committer"]) { + const match = new RegExp(`^${role} .* (-?\\d+) [+-]\\d{4}$`, "mu").exec(headers); + if (!match) add(`Provenance commit ${commit} has no valid ${role} date`); + else if (BigInt(match[1]!) * 1_000_000_000n > instantNanos(report.at)) add(`Provenance commit ${commit} has a future ${role} date`); + } + if (input.kind === "bundle") await inspectBundleTree(repository, commit, await diskPath(root, input.path!), input); + } catch (error) { add(error instanceof Error ? error.message : String(error)); } +}; diff --git a/src/compiler/training/rewind/leakRepositories.test.ts b/src/compiler/training/rewind/leakRepositories.test.ts new file mode 100644 index 00000000..456ac8e7 --- /dev/null +++ b/src/compiler/training/rewind/leakRepositories.test.ts @@ -0,0 +1,65 @@ +import { cp, mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { cleanTrainingProjects } from "../plan/fixtures.test-helper.js"; +import { AT, rewindFixture, testGit } from "./fixtures.test-helper.js"; +import { leakCheckTrainingMoment } from "./leakCheck.js"; +import { finalizeTrainingMoment, materializeTrainingMoment } from "./materialize.js"; +import { stampTree } from "./timestamps.js"; +import { digestDirectory } from "../../../volume/index.js"; + +afterEach(cleanTrainingProjects); +const unexpected = "Unexpected .git repository metadata outside the materialized git resource repository"; + +describe("repository discovery across every bundle input", () => { + it("refuses a future repository inside retained feed bytes after materializer timestamp stamping", async () => { + const fixture = await rewindFixture(); + const feedDirectory = path.dirname(fixture.feedRepo); + const landing = JSON.parse((await readFile(path.join(feedDirectory, "landings.jsonl"), "utf8")).trim()); + const retained = path.join(feedDirectory, "trees", landing.revision); + await mkdir(retained, { recursive: true }); + await cp(fixture.gitRepo, path.join(retained, "nested"), { recursive: true }); + await stampTree(retained, AT, AT); + // A plain directory revision can match malicious repository bytes; repository discovery must still reject them. + landing.revision = digestDirectory(retained); landing.tree = `trees/${landing.revision}`; + landing.source = { kind: "directory", directory: "never-read" }; delete landing.commit; delete landing.ref; + await rename(retained, path.join(feedDirectory, landing.tree)); + await writeFile(path.join(feedDirectory, "landings.jsonl"), JSON.stringify(landing) + "\n"); + const result = await materializeTrainingMoment(fixture.options); + expect(result.status).toBe("refused"); + const check = JSON.parse(await readFile(path.join(fixture.options.out, "leak-check.json"), "utf8")); + expect(check.findings).toContainEqual(expect.objectContaining({ class: "git-object", + path: `resources/team%3Apublication/news/trees/${landing.revision}/nested/.git`, detail: unexpected })); + expect(check.findings.filter((finding: { class: string }) => finding.class !== "git-object")).toEqual([]); + expect(await readdir(fixture.options.out)).toEqual(["leak-check.json", "moment.json"]); + }); + + it.each(["repository", "gitfile", "bare"] as const)("refuses an unexpected %s inside a bundle with all timestamps backdated", async (kind) => { + const fixture = await rewindFixture(), moment = await materializeTrainingMoment(fixture.options); + expect(moment.status).toBe("materialized"); + const input = moment.inputs.find((entry) => entry.kind === "git")!; + input.kind = "bundle"; + const directory = path.join(fixture.options.out, input.path!); + await rm(path.join(directory, ".git"), { recursive: true }); + const nested = path.join(directory, "nested"); + let relative = `${input.path}/nested/.git`, detail = unexpected; + if (kind === "repository") await cp(fixture.gitRepo, nested, { recursive: true }); + else if (kind === "gitfile") { + await mkdir(nested); + // Detection must refuse this metadata without following the textual pointer. + await writeFile(path.join(nested, ".git"), `gitdir: ${fixture.gitRepo}/.git\n`); + } else { + testGit(fixture.root, ["clone", "--quiet", "--bare", "--no-local", fixture.gitRepo, nested]); + relative = `${input.path}/nested`; + detail = "Unexpected bare repository (HEAD, objects/, refs/) outside the materialized git resource repository"; + } + await writeFile(path.join(fixture.options.out, "moment.json"), JSON.stringify(moment)); + await stampTree(fixture.options.out, AT, AT); + const check = await leakCheckTrainingMoment(fixture.options.out, { historyDir: fixture.historyDir }); + expect(check.findings).toContainEqual(expect.objectContaining({ class: "git-object", path: relative, detail })); + expect(check.findings.filter((finding) => finding.class !== "git-object")).toEqual([]); + const out = path.join(fixture.root, "rejected"); + expect((await finalizeTrainingMoment(fixture.options.out, out, { historyDir: fixture.historyDir })).status).toBe("refused"); + expect(await readdir(out)).toEqual(["leak-check.json", "moment.json"]); + }); +}); diff --git a/src/compiler/training/rewind/materialize.test.ts b/src/compiler/training/rewind/materialize.test.ts new file mode 100644 index 00000000..8f533d34 --- /dev/null +++ b/src/compiler/training/rewind/materialize.test.ts @@ -0,0 +1,215 @@ +import { cp, lstat, mkdir, readFile, readdir, rename, symlink, utimes, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { loadTrainingProject, resolveTrainingPlan } from "../plan/index.js"; +import { agentManifest, cleanTrainingProjects, dataFiles, example, trainingProject } from "../plan/fixtures.test-helper.js"; +import { materializeTrainingMoment, resolveTrainingMoment } from "./materialize.js"; +import { leakCheckTrainingMoment } from "./leakCheck.js"; +import * as gitMaterializer from "./git.js"; +import * as volumeMaterializer from "./volumes.js"; +import { rewindGit } from "./gitProcess.js"; +import { selectTrainingExample } from "./select.js"; +import { AT, BEFORE, FUTURE, emptyRecord, jsonl, rewindFixture, testGit } from "./fixtures.test-helper.js"; + +afterEach(cleanTrainingProjects); +afterEach(() => vi.restoreAllMocks()); +vi.mock("node:fs/promises", async () => { + const actual = await vi.importActual("node:fs/promises"); + return { ...actual, rename: vi.fn(actual.rename) }; +}); + +describe("materializeTrainingMoment", () => { + it.each(["at", "effectiveAt"])("refuses out-of-range %s before creating output or dispatching inputs", async (field) => { + const f = await rewindFixture(), selected = selectTrainingExample(f.plan, f.options.example), original = selected.example; + const dispatch = vi.spyOn(gitMaterializer, "materializeGit"); + for (const instant of ["1969-12-31T23:59:59.999999999Z", "9999-12-31T23:59:59.999000001Z"]) { + const record = emptyRecord(); + if (field === "at") { + original.at = instant; + f.project.datasets[selected.sourcePath]!.value.examples.find((entry) => entry.id === selected.id)!.at = instant; + } + else { record.wake.startedAt = instant; await jsonl(f.historyDir, "moments/range.jsonl", [record]); } + const out = path.join(f.root, "not-created", "out"); + expect(() => resolveTrainingMoment(f.plan, f.project, f.options.example, field === "effectiveAt" ? [record] : [])) + .toThrow(/must be within 1970.*9999/u); + await expect(materializeTrainingMoment({ ...f.options, out })).rejects.toThrow(/must be within 1970.*9999/u); + expect(dispatch).not.toHaveBeenCalled(); + expect(await readdir(f.root)).not.toContain("not-created"); + } + }); + + it("preserves a pre-1970 source commit on the final published bundle", async () => { + const f = await rewindFixture(), old = "1960-01-01T00:00:00Z"; + testGit(f.gitRepo, ["read-tree", f.commit]); await symlink("news.txt", path.join(f.gitRepo, "old-link")); + testGit(f.gitRepo, ["add", "old-link"]); + const tree = testGit(f.gitRepo, ["write-tree"]); + const raw = `tree ${tree}\nauthor Rewind -315619200 +0000\ncommitter Rewind -315619200 +0000\n\nOld source\n`; + // Git's authoring CLI refuses negative dates; retained raw historical objects can still contain them. + const commit = (await rewindGit(f.gitRepo, ["hash-object", "--literally", "-t", "commit", "-w", "--stdin"], Buffer.from(raw))).toString().trim(); + const input = resolveTrainingMoment(f.plan, f.project, f.options.example).inputs.find((entry) => entry.id === "code")!; + if (input.section !== "resources" || input.declaration.kind !== "git") throw new Error("Expected git"); + input.declaration.ref = commit; delete input.declaration.branch; + const result = await materializeTrainingMoment(f.options); + expect(result, JSON.stringify(result.inputs)).toMatchObject({ status: "materialized", leakCheck: { ok: true } }); + const directory = path.join(f.options.out, result.inputs.find((entry) => entry.id === "code")!.path!); + for (const name of ["", "news.txt", "old-link"]) { + const stats = await lstat(path.join(directory, name)); + expect(stats.mtimeMs, name).toBe(Date.parse(old)); expect(stats.atimeMs, name).toBe(Date.parse(old)); + } + expect((await leakCheckTrainingMoment(f.options.out, { historyDir: f.historyDir })).findings).toEqual([]); + expect((await lstat(path.join(directory, "news.txt"))).mtimeMs).toBe(Date.parse(old)); + }); + + it("rejects timestamps changed during rename on the published bundle", async () => { + const f = await rewindFixture(), actual = await vi.importActual("node:fs/promises"); + vi.mocked(rename).mockImplementationOnce(async (from, to) => { + await actual.rename(from, to); + await utimes(path.join(String(to), "resources/team%3Apublication/code/news.txt"), new Date(FUTURE), new Date(BEFORE)); + }); + const result = await materializeTrainingMoment(f.options); + expect(result).toMatchObject({ status: "refused", leakCheck: { ok: false } }); + const check = JSON.parse(await readFile(path.join(f.options.out, "leak-check.json"), "utf8")); + expect(check.findings).toContainEqual(expect.objectContaining({ class: "file-time", path: "resources/team%3Apublication/code/news.txt", + detail: "Access time is later than the moment" })); + expect(await readdir(f.options.out)).toEqual(["leak-check.json", "moment.json"]); + }); + it("stages every effective input, verifies disk, and preserves source times", async () => { + const fixture = await rewindFixture(); + const resolved = resolveTrainingMoment(fixture.plan, fixture.project, example.id); + expect(resolved.inputs.map((input) => `${input.section}:${input.id}`)).toEqual([ + "memory:self", "networks:news", "resources:code", "resources:edition-state", "resources:news" + ]); + const result = await materializeTrainingMoment(fixture.options); + expect(result.status).toBe("materialized"); + expect(result.leakCheck).toEqual({ ok: true, findings: 0 }); + expect(result.inputs.map((input) => input.fidelity)).toEqual(["approximate", "overridden", "approximate", "approximate", "exact"]); + const inspectTimes = async (file: string): Promise => { + const stats = await lstat(file); + expect(stats.mtimeMs, file).toBeLessThanOrEqual(Date.parse(AT)); + expect(stats.atimeMs, file).toBeLessThanOrEqual(Date.parse(AT)); + if (stats.isDirectory()) for (const name of await readdir(file)) await inspectTimes(path.join(file, name)); + }; + expect((await leakCheckTrainingMoment(fixture.options.out, { historyDir: fixture.historyDir })).ok).toBe(true); + await inspectTimes(fixture.options.out); + for (const input of result.inputs.filter((item) => item.kind !== "moltnet")) { + expect((await lstat(path.join(fixture.options.out, input.path!))).mtime.toISOString()).toBe(new Date(BEFORE).toISOString()); + } + expect((await readdir(fixture.root)).filter((name) => name.includes(".rewind-"))).toEqual([]); + expect(await readFile(path.join(fixture.gitRepo, "future.txt"), "utf8")).toContain("after wake"); + }); + it("uses equal-instant records and accepts an existing empty output directory", async () => { + const fixture = await rewindFixture(); + await mkdir(fixture.options.out); + const record = { ...emptyRecord("agent:a", "2026-10-09T10:20:31+02:00"), + resources: [{ id: "code", kind: "git", commit: fixture.commit }], + memory: [{ id: "self", declaredBy: "agent:a", seq: 1 }], + volumes: [{ id: "edition-state", snapshot: "20261009T070000000Z-execution-17" }] }; + await jsonl(fixture.historyDir, "moments/anything.jsonl", [record]); + const result = await materializeTrainingMoment(fixture.options); + expect(result.status).toBe("materialized"); + expect(result.record.matched).toBe(true); + expect(result.inputs.filter((input) => input.fidelity === "approximate")).toEqual([]); + }); + it("refuses a suppressed candidate's noncommit only for its pinned input", async () => { + const fixture = await rewindFixture(); + const code = resolveTrainingMoment(fixture.plan, fixture.project, example.id).inputs.find((input) => input.id === "code")!; + if (code.section !== "resources" || code.declaration.kind !== "git") throw new Error("Expected git resource"); + code.declaration.ref = fixture.commit; delete code.declaration.branch; + const blob = testGit(fixture.gitRepo, ["rev-parse", `${fixture.commit}:news.txt`]); + const first = { ...emptyRecord(), resources: [{ id: "code", kind: "git", commit: fixture.commit }] }; + await jsonl(fixture.historyDir, "moments/candidates.jsonl", [first, + { ...first, resources: [{ id: "code", kind: "git", commit: blob }] }]); + let snapshotBeforeDiscard = ""; + const correct = volumeMaterializer.materializeVolume; + vi.spyOn(volumeMaterializer, "materializeVolume").mockImplementation(async (context) => { + const result = await correct(context); + snapshotBeforeDiscard = await readFile(path.join(context.destination, "saved.txt"), "utf8"); + return result; + }); + const result = await materializeTrainingMoment(fixture.options); + expect(result.status).toBe("refused"); + expect(result.inputs.find((input) => input.id === "code")!.refusal) + .toBe(`Recorded object ${blob} must be a commit object; found blob`); + expect(result.inputs.filter((input) => input.refusal !== null).map((input) => input.id)).toEqual(["code"]); + expect(result.inputs.find((input) => input.id === "edition-state")) + .toMatchObject({ fidelity: "approximate", source: { snapshot: "20261009T070000000Z-execution-17" }, refusal: null }); + expect(snapshotBeforeDiscard).toBe("snapshot bytes\n"); + expect(await readdir(fixture.options.out)).toEqual(["moment.json"]); + }); + it("excludes other agents, other wakes, and unrelated inputs from candidate commit validation", async () => { + const fixture = await rewindFixture(); + const code = resolveTrainingMoment(fixture.plan, fixture.project, example.id).inputs.find((input) => input.id === "code")!; + if (code.section !== "resources" || code.declaration.kind !== "git") throw new Error("Expected git resource"); + code.declaration.ref = fixture.commit; delete code.declaration.branch; + const blob = testGit(fixture.gitRepo, ["rev-parse", `${fixture.commit}:news.txt`]); + const first = { ...emptyRecord(), resources: [{ id: "code", kind: "git", commit: fixture.commit }] }; + await jsonl(fixture.historyDir, "moments/candidates.jsonl", [first, + { ...emptyRecord("agent:other"), resources: [{ id: "code", kind: "git", commit: blob }] }, + { ...emptyRecord("agent:a", FUTURE), resources: [{ id: "code", kind: "git", commit: blob }] }, + { ...first, resources: [...first.resources, { id: "unrelated", kind: "git", commit: blob }] }]); + expect((await materializeTrainingMoment(fixture.options)).status).toBe("materialized"); + }); + it("collects all missing-history refusals and leaves only moment.json", async () => { + const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest({ + memory: [{ id: "self", store: { kind: "memory" } }], + workspace: { docs: { system: "AGENTS.md" }, resources: [ + { id: "code", kind: "git", mode: "readonly", mount: "./code", url: "https://invalid.example/code.git" }, + { id: "prebuilt", kind: "bundle", mode: "readonly", mount: "./prebuilt", source: "archive.tar" }, + { id: "news", kind: "volume", name: "news", mode: "mutable", mount: "./news", feed: { directory: "." } } + ] } + }) }); + const project = await loadTrainingProject(root), plan = resolveTrainingPlan(project, "agent:writer", "test"); + const out = path.join(root, "out"), result = await materializeTrainingMoment({ project, plan, example: example.id, out }); + expect(result.status).toBe("refused"); + expect(result.inputs).toHaveLength(4); + expect(result.inputs.every((input) => input.refusal && input.path === null)).toBe(true); + expect(result.inputs.map((input) => input.refusal).join(" ")).toMatch(/Prebuilt bundle/); + expect(await readdir(out)).toEqual(["moment.json"]); + expect((await readdir(root)).filter((name) => name.includes(".rewind-"))).toEqual([]); + const missing = await materializeTrainingMoment({ project, plan, example: example.id, + out: path.join(root, "missing-history-out"), historyDir: path.join(root, "does-not-exist") }); + expect(missing.status).toBe("refused"); + expect(missing.inputs.every((input) => input.refusal?.startsWith("Cannot open history:"))).toBe(true); + }); + it("refuses contradictory matching records and malformed history lines without a usable payload", async () => { + const fixture = await rewindFixture(); + await jsonl(fixture.historyDir, "moments/conflict.jsonl", [emptyRecord(), { + ...emptyRecord(), memory: [{ id: "self", declaredBy: "agent:a", seq: 1 }] + }]); + const refused = await materializeTrainingMoment(fixture.options); + expect(refused.inputs.filter((input) => input.fidelity !== "overridden") + .every((input) => input.refusal?.includes("Conflicting moment records"))).toBe(true); + expect(refused.inputs.find((input) => input.section === "networks")!.fidelity).toBe("overridden"); + expect(await readdir(fixture.options.out)).toEqual(["moment.json"]); + await jsonl(fixture.historyDir, "moments/conflict.jsonl", [{ invalid: true }]); + const malformed = await materializeTrainingMoment({ ...fixture.options, out: path.join(fixture.root, "out-2") }); + expect(malformed.inputs[0]!.refusal).toContain("moments/conflict.jsonl:1"); + }); + it("does not overwrite nonempty output or output symlinks", async () => { + const fixture = await rewindFixture(); + await mkdir(fixture.options.out); await writeFile(path.join(fixture.options.out, "keep"), "keep"); + await expect(materializeTrainingMoment(fixture.options)).rejects.toThrow(/empty directory/); + expect(await readFile(path.join(fixture.options.out, "keep"), "utf8")).toBe("keep"); + const link = path.join(fixture.root, "out-link"); await symlink(fixture.options.out, link); + await expect(materializeTrainingMoment({ ...fixture.options, out: link })).rejects.toThrow(/empty directory/); + const alias = path.join(fixture.root, "history-alias"); await symlink(fixture.historyDir, alias); + await expect(materializeTrainingMoment({ ...fixture.options, out: path.join(alias, "output") })).rejects.toThrow(/must not overlap/); + expect(await readdir(fixture.historyDir)).not.toContain("output"); + }); + it("catches a deliberately broken git producer that copies full history", async () => { + const fixture = await rewindFixture(), correct = gitMaterializer.materializeGit; + vi.spyOn(gitMaterializer, "materializeGit").mockImplementation(async (context) => { + const result = await correct(context); + await cp(path.join(fixture.gitRepo, ".git"), path.join(context.destination, ".git"), { recursive: true }); + return result; + }); + const result = await materializeTrainingMoment(fixture.options); + expect(result.status).toBe("refused"); + expect(result.inputs.find((input) => input.id === "code")!.refusal).toContain("git-object"); + expect(await readdir(fixture.options.out)).toEqual(["leak-check.json", "moment.json"]); + expect(JSON.parse(await readFile(path.join(fixture.options.out, "leak-check.json"), "utf8")).findings) + .toEqual(expect.arrayContaining([expect.objectContaining({ class: "git-object", path: "resources/team%3Apublication/code", + detail: `Object ${fixture.futureCommit} is not reachable from ${fixture.commit}` })])); + expect(result.at).toBe(AT); + }); +}); diff --git a/src/compiler/training/rewind/materialize.ts b/src/compiler/training/rewind/materialize.ts new file mode 100644 index 00000000..8e2eaf0a --- /dev/null +++ b/src/compiler/training/rewind/materialize.ts @@ -0,0 +1,198 @@ +import { lstat, mkdir, mkdtemp, readdir, realpath, rename, rm, utimes, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { LoadedTrainingProject, TrainingPlan } from "../plan/index.js"; +import { materializedMomentSchema, type MaterializedInput, type MaterializedMoment } from "./contract.js"; +import { type MomentRecord } from "../../../recording/index.js"; +import { materializeFeed } from "./feed.js"; +import { materializeGit } from "./git.js"; +import { historyKey, inside, openHistory, type History } from "./history.js"; +import { resolveRewindInputs } from "./inputs.js"; +import { inspectTrainingMoment, type LeakCheckOptions } from "./leakCheck.js"; +import { inspectBundleFiles, restoreInspectedAccessTimes, snapshotBundleFiles } from "./leakFiles.js"; +import { materializeMemory } from "./memory.js"; +import { materializeMoltnet } from "./moltnet.js"; +import { matchingMomentRecords, originalMomentRecords, selectMomentRecord } from "./select.js"; +import { hasRecordPin, recordForInput } from "./recordInput.js"; +import { assertTaskClockInstant, createTaskClock } from "./taskClock.js"; +import { restoreAccessTimes, stampTree } from "./timestamps.js"; +import type { InputContext, InputResult, RewindInput } from "./types.js"; +import { materializeVolume } from "./volumes.js"; + +export interface MaterializeTrainingMomentOptions { + plan: TrainingPlan; + project: LoadedTrainingProject; + example: string; + historyDir?: string; + out: string; +} + +/** Pure core for a per-trial caller: effective declarations, example identity and matching record. */ +export const resolveTrainingMoment = ( + plan: TrainingPlan, project: LoadedTrainingProject, example: string, records: MomentRecord[] = [] +) => { + const resolved = resolveRewindInputs(plan, project, example); + assertTaskClockInstant(resolved.at); + const candidateRecords = originalMomentRecords(matchingMomentRecords(records, plan.agent.id, resolved.at)); + const record = selectMomentRecord(candidateRecords, plan.agent.id, resolved.at, resolved.inputs.filter((input) => !input.overridden)); + const effectiveAt = record?.wake.startedAt ?? resolved.at; + assertTaskClockInstant(effectiveAt); + return { ...resolved, candidateRecords, record, effectiveAt }; +}; + +const reason = (error: unknown): string => error instanceof Error ? error.message : String(error); +const serialize = (value: unknown): string => `${JSON.stringify(value, null, 2)}\n`; +const kindOf = (input: RewindInput): MaterializedInput["kind"] => input.section === "memory" ? "memory" + : input.section === "networks" ? "moltnet" : input.declaration.kind === "volume" && input.declaration.feed ? "feed" : input.declaration.kind; +const reportInput = (input: RewindInput): MaterializedInput => ({ + section: input.section, id: input.id, declaredBy: input.declaredBy, kind: kindOf(input), fidelity: null, + basis: "refused", source: {}, path: null, notes: [], refusal: null +}); +const dispatch = (context: InputContext): Promise => { + const { input } = context; + if (input.section === "memory") return materializeMemory(context); + if (input.section === "networks") return materializeMoltnet(context); + if (input.declaration.kind !== "volume") return materializeGit(context); + return input.declaration.feed ? materializeFeed(context) : materializeVolume(context); +}; + +const assertEmptyOutput = async (out: string): Promise => { + const existing = await lstat(out).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (existing && (!existing.isDirectory() || (await readdir(out)).length)) throw new Error("Rewind --out must not exist or must be an empty directory"); +}; + +const discardPayload = async (staging: string, moment: MaterializedMoment): Promise => { + moment.status = "refused"; + for (const input of moment.inputs) input.path = null; + for (const name of await readdir(staging)) if (name !== "moment.json" && name !== "leak-check.json") { + await rm(path.join(staging, name), { recursive: true, force: true }); + } +}; + +/** A separately callable disk acceptance gate, also used to test deliberately broken producers. */ +export const finalizeTrainingMoment = async (staging: string, out: string, options: LeakCheckOptions = {}): Promise => { + // Capture disk metadata before reading even moment.json; inspection preserves access times. + const { moment, check } = await inspectTrainingMoment(staging, options, true); + if (check) { + moment.leakCheck = { ok: check.ok, findings: check.findings.length }; + if (!check.ok) { + for (const finding of check.findings) { + const affected = finding.input ? moment.inputs.filter((input) => input.section === finding.input!.section + && input.id === finding.input!.id && input.declaredBy === finding.input!.declaredBy) : moment.inputs; + for (const input of affected) input.refusal = [input.refusal, `${finding.class}: ${finding.path}: ${finding.detail}`].filter(Boolean).join("; "); + } + await discardPayload(staging, moment); + } + await writeFile(path.join(staging, "leak-check.json"), serialize(check)); + await stampTree(path.join(staging, "leak-check.json"), moment.effectiveAt, moment.effectiveAt); + } else await discardPayload(staging, moment); + await writeFile(path.join(staging, "moment.json"), serialize(materializedMomentSchema.parse(moment))); + await stampTree(path.join(staging, "moment.json"), moment.effectiveAt, moment.effectiveAt); + await utimes(staging, new Date(moment.effectiveAt), new Date(moment.effectiveAt)); + await restoreAccessTimes(staging, moment.effectiveAt); + await assertEmptyOutput(out); + await rename(staging, out); + if (check?.ok) { + // The post-restoration, post-rename payload is the acceptance target. + try { + const files = await snapshotBundleFiles(out); + const published = { ...check, checked: { ...check.checked, files: 0 }, findings: [] as typeof check.findings }; + try { await inspectBundleFiles(out, moment.inputs, published, files); } + finally { await restoreInspectedAccessTimes(files); } + if (published.findings.length) { + check.findings.push(...published.findings); check.ok = false; + moment.leakCheck = { ok: false, findings: check.findings.length }; + for (const finding of published.findings) for (const input of moment.inputs.filter((entry) => !finding.input + || entry.section === finding.input.section && entry.id === finding.input.id && entry.declaredBy === finding.input.declaredBy)) { + input.refusal = `${finding.class}: ${finding.path}: ${finding.detail}`; + } + await discardPayload(out, moment); + await writeFile(path.join(out, "moment.json"), serialize(materializedMomentSchema.parse(moment))); + await stampTree(path.join(out, "moment.json"), moment.effectiveAt, moment.effectiveAt); + } + check.checked.files = published.checked.files; + await writeFile(path.join(out, "leak-check.json"), serialize(check)); + await stampTree(path.join(out, "leak-check.json"), moment.effectiveAt, moment.effectiveAt); + await utimes(out, new Date(moment.effectiveAt), new Date(moment.effectiveAt)); + } catch (error) { + await rm(out, { recursive: true, force: true }); + throw error; + } + } + return moment; +}; + +/** Local-only rewind. No compilation artifacts, model invocation, auth, shell or network access. */ +export const materializeTrainingMoment = async (options: MaterializeTrainingMomentOptions): Promise => { + const { plan, project, example } = options; + const resolved = resolveRewindInputs(plan, project, example); + assertTaskClockInstant(resolved.at); + let out = path.resolve(options.out); + await assertEmptyOutput(out); + let history: History | undefined, historyError: string | undefined; + try { history = await openHistory(options.historyDir); } + catch (error) { historyError = `Cannot open history: ${reason(error)}`; } + let record: MomentRecord | null = null, recordError: string | undefined; + let candidateRecords: MomentRecord[] = []; + try { + candidateRecords = matchingMomentRecords(await history?.records() ?? [], plan.agent.id, resolved.at); + record = selectMomentRecord(candidateRecords, plan.agent.id, resolved.at, + resolved.inputs.filter((input) => !input.overridden)); + } catch (error) { recordError = reason(error); } + const attempts = candidateRecords.length; + candidateRecords = originalMomentRecords(candidateRecords); + const effectiveAt = record?.wake.startedAt ?? resolved.at; + assertTaskClockInstant(effectiveAt); + await mkdir(path.dirname(out), { recursive: true }); + out = path.join(await realpath(path.dirname(out)), path.basename(out)); + if (history && (inside(history.root, out) || inside(out, history.root))) throw new Error("Rewind output and history must not overlap"); + const staging = await mkdtemp(path.join(path.dirname(out), `.${path.basename(out)}.rewind-`)); + try { + const moment: MaterializedMoment = { + version: "spawnfile.materialized-moment.v1", status: "materialized", agent: plan.agent.id, + example: { id: resolved.selected.id, role: resolved.selected.role, dataset: resolved.selected.dataset }, + at: resolved.at, effectiveAt, inputs: [], record: { matched: record !== null, ...(record ? { + attempts, selectedStartedAt: record.wake.startedAt, executionId: record.wake.executionId, selection: "earliest-startedAt" as const + } : {}) }, leakCheck: { ok: false, findings: 0 } + }; + for (const input of resolved.inputs) { + const entry = reportInput(input); + moment.inputs.push(entry); + try { + if (!input.overridden && (historyError || recordError)) throw new Error(historyError ?? recordError); + const relative = `${input.section === "networks" ? "moltnet" : input.section}/${historyKey(input)}`; + const destination = path.join(staging, relative); + const usableRecord = recordForInput(record, input); + const result = await dispatch({ input, at: effectiveAt, agentSlug: resolved.agentSlug, agentId: plan.agent.id, + record: usableRecord, candidateRecords: candidateRecords.filter((candidate) => recordForInput(candidate, input)), history, destination }); + if (record && !record.exact && !input.overridden && !hasRecordPin(usableRecord, input)) { + result.fidelity = "approximate"; + result.notes.push(...(record.errors ?? [])); + } + await stampTree(destination, result.instant, effectiveAt); + Object.assign(entry, { fidelity: result.fidelity, basis: result.basis, source: result.source, + notes: result.notes, path: relative, ...(result.build ? { build: result.build } : {}) }); + } catch (error) { + entry.refusal = reason(error); + moment.status = "refused"; + } + } + if (recordError && resolved.inputs.length === 0) throw new Error(recordError); + await writeFile(path.join(staging, "task-clock.json"), serialize(createTaskClock(effectiveAt))); + await writeFile(path.join(staging, "moment.json"), serialize(moment)); + // Set only unstamped wrapper directories; per-input source times remain intact. + const wrappers = async (directory: string): Promise => { + const relative = path.relative(staging, directory).split(path.sep).join("/"); + if (moment.inputs.some((input) => input.path === relative)) return; + for (const entry of await readdir(directory, { withFileTypes: true })) { + if (entry.isDirectory()) await wrappers(path.join(directory, entry.name)); + else if (directory === staging) await stampTree(path.join(directory, entry.name), effectiveAt, effectiveAt); + } + await utimes(directory, new Date(effectiveAt), new Date(effectiveAt)); + }; + await wrappers(staging); + return await finalizeTrainingMoment(staging, out, { historyDir: options.historyDir }); + } finally { await rm(staging, { recursive: true, force: true }); } +}; diff --git a/src/compiler/training/rewind/memory.test.ts b/src/compiler/training/rewind/memory.test.ts new file mode 100644 index 00000000..dffcf125 --- /dev/null +++ b/src/compiler/training/rewind/memory.test.ts @@ -0,0 +1,92 @@ +import { mkdtemp, readFile, readdir, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { openHistory } from "./history.js"; +import { materializeMemory, parseMemoryLedger, selectMemoryPrefix } from "./memory.js"; +import { AT, FUTURE, emptyRecord, jsonl, memoryEvent } from "./fixtures.test-helper.js"; +import type { InputContext, RewindInput } from "./types.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const setup = async (): Promise => { + const root = await mkdtemp(path.join(os.tmpdir(), "rewind-memory-")); roots.push(root); + const input: RewindInput = { section: "memory", id: "self", declaredBy: "agent:a", declarationBase: root, + declaration: { id: "self", store: { kind: "memory" } }, overridden: false }; + return { input, at: AT, record: null, history: await openHistory(root), destination: path.join(root, "out") }; +}; +const base = "memory/agent%3Aa/self"; +const ledger = (...events: unknown[]) => parseMemoryLedger(events.map((e) => JSON.stringify(e)).join("\n") + "\n"); + +describe("memory rewind", () => { + it("cuts at the first future event and never filters a clock regression back into the bank", () => { + const source = ledger(memoryEvent(1), memoryEvent(2, FUTURE), memoryEvent(3)); + expect(selectMemoryPrefix(source, AT).events.map((event) => event.seq)).toEqual([1]); + expect(() => selectMemoryPrefix(source, AT, 3)).toThrow(/later than/); + }); + it("refuses seq gaps, prefix time regressions, invalid framing, strict unknown fields and resumed legacy runs", () => { + expect(() => selectMemoryPrefix(ledger(memoryEvent(2)), AT)).toThrow(/contiguous/); + expect(() => selectMemoryPrefix(ledger(memoryEvent(1, AT), memoryEvent(2)), AT)).toThrow(/regresses/); + expect(() => parseMemoryLedger(JSON.stringify(memoryEvent(1)))).toThrow(/LF/); + expect(() => ledger({ ...memoryEvent(1), extra: true })).toThrow(/invalid Mneme event/); + expect(() => selectMemoryPrefix(ledger(memoryEvent(1), { ...memoryEvent(2), seq: undefined }), AT)).toThrow(/leading run/); + expect(selectMemoryPrefix(ledger({ ...memoryEvent(1), seq: undefined }, memoryEvent(2)), AT).events).toHaveLength(2); + }); + it("rejects malformed JSON, invalid createdAt instants and duplicate JSON keys", () => { + expect(() => parseMemoryLedger("{broken}\n")).toThrow(/memory\/events.jsonl:1: invalid Mneme event/); + expect(() => ledger({ ...memoryEvent(1), createdAt: "not an instant" })).toThrow(/createdAt/); + expect(() => parseMemoryLedger(`${JSON.stringify(memoryEvent(1))}\n\n`)).toThrow(/memory\/events.jsonl:2/); + const payload = JSON.stringify(memoryEvent(1)); + for (const duplicate of [payload.replace('"seq":1', '"seq":2,"s\\u0065q":1'), + payload.replace('"agentId":"a"', '"agentId":"b","agentId":"a"')]) { + expect(() => parseMemoryLedger(`${duplicate}\n`)).toThrow(/duplicate JSON key/); + } + }); + it("materializes an exact record prefix without an index or a Mneme runtime dependency", async () => { + const context = await setup(); + await jsonl(context.history!.root, `${base}/events.jsonl`, [memoryEvent(1), memoryEvent(2, AT)]); + context.record = { ...emptyRecord(), memory: [{ id: "self", declaredBy: "agent:a", seq: 1 }] }; + const result = await materializeMemory(context); + expect(result).toMatchObject({ fidelity: "exact", basis: "record", source: { seq: 1 } }); + expect(result.source).not.toHaveProperty("generation"); + expect(await readdir(path.join(context.destination, "memory"))).toEqual(["events.jsonl"]); + expect(await readFile(path.join(context.destination, "memory/events.jsonl"), "utf8")).toBe(`${JSON.stringify(memoryEvent(1))}\n`); + expect(result.notes.join(" ")).toContain("without a Mneme runtime dependency"); + expect(result.notes.join(" ")).toContain("Checksum verification is skipped"); + }); + it("includes the boundary and labels the timestamp prefix approximate", async () => { + const context = await setup(); + await jsonl(context.history!.root, `${base}/events.jsonl`, [memoryEvent(1, AT), memoryEvent(2, FUTURE)]); + expect(await materializeMemory(context)).toMatchObject({ fidelity: "approximate", basis: "created-at-prefix", source: { seq: 1 } }); + }); + it.each([{ generation: 0, seq: 0 }, { generation: 0, seq: 1 }, { generation: 1, seq: 0 }, { generation: 1, seq: 1 }])( + "refuses unsupported recorded memory generation $generation at seq $seq", async ({ generation, seq }) => { + const context = await setup(); + await jsonl(context.history!.root, `${base}/events.jsonl`, [memoryEvent(1)]); + context.record = { ...emptyRecord(), memory: [{ id: "self", declaredBy: "agent:a", seq, generation }] }; + await expect(materializeMemory(context)).rejects.toThrow(/Mneme generations are unsupported/); + }); + it.each(["missing", "malformed", "no history"])("uses seq zero as an exact empty bank with %s ledger", async (state) => { + const context = await setup(); + if (state === "malformed") await jsonl(context.history!.root, `${base}/events.jsonl`, [{ bad: true }]); + if (state === "no history") context.history = undefined; + context.record = { ...emptyRecord(), memory: [{ id: "self", declaredBy: "agent:a", seq: 0 }] }; + expect(await materializeMemory(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { seq: 0 } }); + expect(await readdir(path.join(context.destination, "memory"))).toEqual(["events.jsonl"]); + expect(await readFile(path.join(context.destination, "memory/events.jsonl"), "utf8")).toBe(""); + }); + it("creates an empty override without history and refuses a missing non-overridden bank", async () => { + const context = await setup(); + await expect(materializeMemory(context)).rejects.toThrow(/Missing memory ledger/); + context.history = undefined; + await expect(materializeMemory(context)).rejects.toThrow(/requires --history/); + context.input.overridden = true; + expect(await materializeMemory(context)).toMatchObject({ fidelity: "overridden", source: { seq: 0 } }); + expect(await readFile(path.join(context.destination, "memory/events.jsonl"), "utf8")).toBe(""); + }); + it("validates malformed suffix history even when the selected prefix would be empty", async () => { + const context = await setup(); + await jsonl(context.history!.root, `${base}/events.jsonl`, [memoryEvent(1, FUTURE), { bad: true }]); + await expect(materializeMemory(context)).rejects.toThrow(/events.jsonl:2/); + }); +}); diff --git a/src/compiler/training/rewind/memory.ts b/src/compiler/training/rewind/memory.ts new file mode 100644 index 00000000..72e33ee0 --- /dev/null +++ b/src/compiler/training/rewind/memory.ts @@ -0,0 +1,76 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { historyKey } from "./history.js"; +import { compareInstants } from "./instant.js"; +import { parseMemoryLedgerEvent, type MemoryLedgerEvent } from "./memorySchemas.js"; +import type { InputContext, InputResult } from "./types.js"; + +export interface MemoryLedger { lines: string[]; events: MemoryLedgerEvent[] } +const reason = (error: unknown): string => error instanceof Error ? error.message : String(error); + +export const parseMemoryLedger = (payload: string, label = "memory/events.jsonl"): MemoryLedger => { + if (payload === "") return { lines: [], events: [] }; + if (!payload.endsWith("\n")) throw new Error(`${label}: ledger must end with LF`); + const lines = payload.slice(0, -1).split("\n"); + const events = lines.map((line, index) => { + try { return parseMemoryLedgerEvent(line); } + catch (error) { throw new Error(`${label}:${index + 1}: invalid Mneme event: ${reason(error)}`); } + }); + return { lines, events }; +}; + +/** The first future event is a boundary; a clock regression after it cannot reopen the cut. */ +export const selectMemoryPrefix = (ledger: MemoryLedger, at: string, seq?: number): MemoryLedger => { + if (seq !== undefined && (!Number.isSafeInteger(seq) || seq < 0 || seq > ledger.events.length)) { + throw new Error(`Memory record seq ${seq} exceeds or does not identify a ledger prefix`); + } + const firstFuture = ledger.events.findIndex((event) => compareInstants(event.createdAt, at) > 0); + const length = seq ?? (firstFuture < 0 ? ledger.events.length : firstFuture); + const events = ledger.events.slice(0, length); + let sawNative = false; + events.forEach((event, index) => { + if (event.seq !== undefined) { + if (event.seq !== index + 1) throw new Error(`Memory ledger line ${index + 1}: seq ${event.seq} is not contiguous`); + sawNative = true; + } else if (sawNative) throw new Error(`Memory ledger line ${index + 1}: seq-less legacy events must form a leading run`); + if (compareInstants(event.createdAt, at) > 0) throw new Error(`Memory ledger line ${index + 1}: createdAt is later than the moment`); + if (index > 0 && compareInstants(event.createdAt, events[index - 1]!.createdAt) < 0) { + throw new Error(`Memory ledger line ${index + 1}: createdAt regresses within the selected prefix`); + } + }); + return { events, lines: ledger.lines.slice(0, length) }; +}; + +export const materializeMemory = async (context: InputContext): Promise => { + const { input, at, record, history, destination } = context; + if (input.section !== "memory") throw new Error("Memory materialization requires a memory input"); + const directory = path.join(destination, "memory"); + if (input.overridden) { + await mkdir(directory, { recursive: true }); + await writeFile(path.join(directory, "events.jsonl"), ""); + return { fidelity: "overridden", basis: "override", source: { seq: 0 }, notes: [], instant: at }; + } + const pin = record?.memory.find((entry) => entry.id === input.id && entry.declaredBy === input.declaredBy); + if (pin?.generation !== undefined) throw new Error(`Memory bank ${input.id}: Mneme generations are unsupported`); + if (pin?.seq === 0) { + await mkdir(directory, { recursive: true }); + await writeFile(path.join(directory, "events.jsonl"), ""); + return { fidelity: "exact", basis: "record", source: { seq: 0 }, + notes: ["The record pins an empty bank (seq: 0); no history ledger is required."], instant: at }; + } + if (!history) throw new Error(`Memory bank ${input.id} requires --history or an override`); + const ledgerPath = `memory/${historyKey(input)}/events.jsonl`; + const payload = await history.text(ledgerPath); + if (payload === null) throw new Error(`Missing memory ledger ${ledgerPath}`); + const prefix = selectMemoryPrefix(parseMemoryLedger(payload, ledgerPath), at, pin?.seq); + await mkdir(directory, { recursive: true }); + await writeFile(path.join(directory, "events.jsonl"), prefix.lines.length ? `${prefix.lines.join("\n")}\n` : ""); + return { + fidelity: pin ? "exact" : "approximate", basis: pin ? "record" : "created-at-prefix", + source: { seq: prefix.events.length }, + notes: ["Spawnfile structurally validated the selected ledger without a Mneme runtime dependency; no derived or authority audit state is included.", + "Mneme 0.1.1 rebuilds its index from the ledger before every index read.", + "Checksum verification is skipped: Mneme 0.1.1 hashes the original input before normalization, which cannot be reconstructed exactly from stored events."], + instant: prefix.events.at(-1)?.createdAt ?? at + }; +}; diff --git a/src/compiler/training/rewind/memoryPackage.test.ts b/src/compiler/training/rewind/memoryPackage.test.ts new file mode 100644 index 00000000..6e142d72 --- /dev/null +++ b/src/compiler/training/rewind/memoryPackage.test.ts @@ -0,0 +1,84 @@ +import { execFile } from "node:child_process"; +import { cp, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { ModuleKind, ScriptTarget, preProcessFile, transpileModule } from "typescript"; +import { afterEach, expect, it } from "vitest"; +import { AT, jsonl, memoryEvent } from "./fixtures.test-helper.js"; + +const root = fileURLToPath(new URL("../../../../", import.meta.url)); +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); }); +const run = promisify(execFile); + +/** Emit only this runtime dependency closure; the child receives JS and production packages. */ +const emitMemoryPackage = async (directory: string): Promise => { + const seen = new Set(), sourceRoot = path.join(root, "src"); + const emit = async (source: string): Promise => { + if (seen.has(source)) return; + seen.add(source); + const relative = path.relative(sourceRoot, source); + if (relative.startsWith("..") || path.isAbsolute(relative)) throw new Error(`Unexpected package source ${source}`); + const output = path.join(directory, "dist", relative.replace(/\.ts$/u, ".js")); + const { outputText } = transpileModule(await readFile(source, "utf8"), { + fileName: source, compilerOptions: { module: ModuleKind.ESNext, target: ScriptTarget.ES2023 } + }); + await mkdir(path.dirname(output), { recursive: true }); + await writeFile(output, outputText); + for (const { fileName } of preProcessFile(outputText, true, true).importedFiles) { + if (fileName.startsWith(".")) await emit(path.resolve(path.dirname(source), fileName.replace(/\.js$/u, ".ts"))); + } + }; + await emit(path.join(sourceRoot, "compiler/training/rewind/memory.ts")); + const manifest = JSON.parse(await readFile(path.join(root, "package.json"), "utf8")) as { + dependencies: Record; + }; + await writeFile(path.join(directory, "package.json"), JSON.stringify(manifest)); + const copied = new Set(); + const copyDependency = async (name: string): Promise => { + if (copied.has(name)) return; + copied.add(name); + const source = await realpath(path.join(root, "node_modules", name)); + await cp(source, path.join(directory, "node_modules", name), { + recursive: true, dereference: true, + filter: (entry) => !path.relative(source, entry).split(path.sep).includes("node_modules") + }); + const dependency = JSON.parse(await readFile(path.join(source, "package.json"), "utf8")) as { + dependencies?: Record; + }; + for (const child of Object.keys(dependency.dependencies ?? {})) await copyDependency(child); + }; + for (const name of Object.keys(manifest.dependencies)) await copyDependency(name); +}; + +it("materializes nonempty historical memory in an isolated production package without development dependencies", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "rewind-memory-package-")); roots.push(directory); + await emitMemoryPackage(directory); + const history = path.join(directory, "history"), event = memoryEvent(1); + await jsonl(history, "memory/agent%3Aa/self/events.jsonl", [event]); + await writeFile(path.join(directory, "check.mjs"), ` +import assert from "node:assert/strict"; +import { readFile, readdir } from "node:fs/promises"; +import { materializeMemory } from "spawnfile/dist/compiler/training/rewind/memory.js"; +import { openHistory } from "spawnfile/dist/compiler/training/rewind/history.js"; +const manifest = JSON.parse(await readFile("package.json", "utf8")); +for (const name of Object.keys(manifest.devDependencies)) { + assert.throws(() => import.meta.resolve(name), { code: "ERR_MODULE_NOT_FOUND" }); +} +const result = await materializeMemory({ + input: { section: "memory", id: "self", declaredBy: "agent:a", declarationBase: process.cwd(), + declaration: { id: "self", store: { kind: "memory" } }, overridden: false }, + at: ${JSON.stringify(AT)}, record: null, history: await openHistory("history"), destination: "out" +}); +assert.equal(result.source.seq, 1); +assert.equal(await readFile("out/memory/events.jsonl", "utf8"), ${JSON.stringify(`${JSON.stringify(event)}\n`)}); +assert.deepEqual(await readdir("out/memory"), ["events.jsonl"]); +console.log("PRODUCTION_MEMORY_REWIND_OK"); +`); + const result = await run(process.execPath, ["check.mjs"], { + cwd: directory, env: { ...process.env, NODE_ENV: "production", NODE_PATH: "", NODE_OPTIONS: "" }, timeout: 15_000 + }); + expect(result.stdout.trim()).toBe("PRODUCTION_MEMORY_REWIND_OK"); +}); diff --git a/src/compiler/training/rewind/memorySchemas.ts b/src/compiler/training/rewind/memorySchemas.ts new file mode 100644 index 00000000..9e437bcc --- /dev/null +++ b/src/compiler/training/rewind/memorySchemas.ts @@ -0,0 +1,47 @@ +import { z } from "zod"; +import { trainingInstantSchema } from "../../../manifest/index.js"; + +const safeSequence = z.number().int().nonnegative().refine(Number.isSafeInteger); +/** Mirrors the public MemoryEvent contract and Mneme 0.1.1's strict ledger reader. */ +export const memoryEventSchema = z.object({ + id: z.string().min(1), + type: z.enum(["memory.observed", "memory.claimed", "memory.registered", "memory.summarized", + "memory.recalled", "memory.located", "memory.denied", "memory.forgotten", "memory.promoted", "memory.consolidated"]), + createdAt: trainingInstantSchema, + principal: z.object({ agentId: z.string().min(1), + scope: z.enum(["global", "team", "room", "pair", "task", "role", "artifact"]), qualifier: z.string().optional() + }).strict(), + scope: z.string(), visibility: z.enum(["private", "pair", "team", "room", "global", "public", "sealed"]), + source: z.string(), + content: z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("text"), text: z.string() }).strict(), + z.object({ kind: z.literal("claim"), subject: z.string(), predicate: z.string(), object: z.string() }).strict(), + z.object({ kind: z.literal("decision"), decision: z.string(), rationale: z.string().optional() }).strict(), + z.object({ kind: z.literal("artifact"), description: z.string(), path: z.string().optional(), uri: z.string().optional() }).strict(), + z.object({ kind: z.literal("relationship"), from: z.string(), relation: z.string(), to: z.string() }).strict() + ]), + tags: z.array(z.string()), entities: z.array(z.string()), + sensitivity: z.enum(["normal", "sensitive", "secret"]), confidence: z.number().finite().optional(), + ttl: z.string().optional(), parentEventIds: z.array(z.string()), checksum: z.string().min(1), + seq: safeSequence.refine((value) => value > 0).optional(), memoryId: z.string().optional(), + origin: z.enum(["awake", "dream", "system"]).optional(), highWaterSeq: safeSequence.optional() +}).strict(); +export type MemoryLedgerEvent = z.infer; + +/** Mneme rejects duplicate JSON keys, including escaped and nested duplicates. */ +export const parseMemoryLedgerEvent = (line: string): MemoryLedgerEvent => { + const value: unknown = JSON.parse(line); + const objects: (Set | null)[] = []; + let string = ""; + for (const [token] of line.matchAll(/"(?:\\.|[^"\\])*"|[{}\[\]:]/gu)) { + if (token === "{") objects.push(new Set()); + else if (token === "[") objects.push(null); + else if (token === "}" || token === "]") objects.pop(); + else if (token === ":") { + const key = JSON.parse(string) as string, keys = objects.at(-1)!; + if (keys.has(key)) throw new Error(`duplicate JSON key ${JSON.stringify(key)}`); + keys.add(key); + } else string = token; + } + return memoryEventSchema.parse(value); +}; diff --git a/src/compiler/training/rewind/moltnet.ts b/src/compiler/training/rewind/moltnet.ts new file mode 100644 index 00000000..01c6af61 --- /dev/null +++ b/src/compiler/training/rewind/moltnet.ts @@ -0,0 +1,49 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { historyKey } from "./history.js"; +import { compareInstants } from "./instant.js"; +import { emptyMoltnetSnapshot, moltnetMessages, parseMoltnetSnapshot, rewindMoltnetJson } from "./moltnetJson.js"; +import { rewindMoltnetSqlite } from "./moltnetSqlite.js"; +import type { InputContext, InputResult } from "./types.js"; + +export const materializeMoltnet = async (context: InputContext): Promise => { + const { input, at, record, history, destination } = context; + if (input.section !== "networks") throw new Error("Moltnet materialization requires a network input"); + if (input.overridden) { + await mkdir(destination, { recursive: true }); + await writeFile(path.join(destination, "state.json"), `${JSON.stringify(emptyMoltnetSnapshot(input.declaration, at), null, 2)}\n`); + return { fidelity: "overridden", basis: "override", source: {}, notes: ["Fresh network state is stored as a JSON snapshot with the declared rooms."], instant: at }; + } + const pin = record?.moltnet.find((entry) => entry.network === input.id); + if (pin && compareInstants(pin.cursor, at) !== 0) { + throw new Error(`Moltnet network ${input.id}: recorded cursor must equal wake.startedAt (${at})`); + } + if (!history) throw new Error(`Moltnet network ${input.id} requires --history or an override`); + const base = `moltnet/${historyKey(input)}`; + const json = await history.bytes(`${base}/state.json`), sqlite = await history.bytes(`${base}/moltnet.sqlite`); + if (!json && !sqlite) throw new Error(`Missing Moltnet store ${base}/state.json or moltnet.sqlite`); + if (json && sqlite) throw new Error(`Ambiguous Moltnet history: both state.json and moltnet.sqlite exist for ${input.id}`); + await mkdir(destination, { recursive: true }); + let messages: Array<{ id: string; created_at: string }>; + if (json) { + const cut = rewindMoltnetJson(parseMoltnetSnapshot(JSON.parse(json.toString("utf8"))), at); + messages = moltnetMessages(cut); + await writeFile(path.join(destination, "state.json"), `${JSON.stringify(cut, null, 2)}\n`); + } else { + const file = path.join(destination, "moltnet.sqlite"); + await writeFile(file, sqlite!); + // History readers never open the source database: SQLite recovery is confined to these copies. + for (const suffix of ["-wal", "-journal"]) { + const bytes = await history.bytes(`${base}/moltnet.sqlite${suffix}`); + if (bytes) await writeFile(`${file}${suffix}`, bytes); + } + messages = await rewindMoltnetSqlite(file, at); + } + const retainedIds = new Set(messages.map((message) => message.id)); + const missing = pin?.messageIds?.filter((id) => !retainedIds.has(id)) ?? []; + if (missing.length) throw new Error(`Moltnet network ${input.id}: recorded wake message IDs ${missing.join(", ")} are not retained after the cursor cut`); + return { fidelity: pin ? "exact" : "approximate", basis: pin ? "record-cursor" : "created-at-cut", + source: pin ? { cursor: pin.cursor } : {}, instant: at, + notes: ["Future agent registrations and their memberships are removed; retained registration updates are clamped. Other membership history is unavailable (best effort).", + "Room removals without a recorded removal instant retain the current removal state; current JSON Moltnet uses a zero-time marker."] }; +}; diff --git a/src/compiler/training/rewind/moltnetJson.test.ts b/src/compiler/training/rewind/moltnetJson.test.ts new file mode 100644 index 00000000..dd2fe760 --- /dev/null +++ b/src/compiler/training/rewind/moltnetJson.test.ts @@ -0,0 +1,154 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { afterEach, describe, expect, it } from "vitest"; +import { materializeMoltnet } from "./moltnet.js"; +import { momentRecordSchema } from "../../../recording/index.js"; +import { openHistory } from "./history.js"; +import { emptyMoltnetSnapshot, inspectMoltnetJson, moltnetMessages, parseMoltnetSnapshot, rewindMoltnetJson } from "./moltnetJson.js"; +import type { MoltnetMessage } from "./moltnetSchemas.js"; +import type { InputContext } from "./types.js"; + +const at = "2026-10-09T10:00:00.1Z", before = "2026-10-09T10:00:00Z", future = "2026-10-09T10:00:00.11Z"; +const network = { id: "news", provider: "moltnet" as const, rooms: [{ id: "assignment", members: ["writer"] }] }; +const message = (id: string, created_at: string, target: MoltnetMessage["target"] = { kind: "room", room_id: "assignment" }): MoltnetMessage => + ({ id, network_id: "news", from: { type: "agent", id: "writer" }, target, parts: [{ kind: "text", text: id }], created_at }); +const temps: string[] = []; +afterEach(async () => { await Promise.all(temps.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); }); + +describe("Moltnet JSON rewind", () => { + it("cuts RFC3339Nano times chronologically, removes future topology, recomputes counters and remaps delivery cursors", () => { + const source = emptyMoltnetSnapshot(network, before); + source.room_messages.assignment = [message("old", before), message("boundary", at), message("future", future)]; + source.rooms.later = { id: "later", name: "later", created_at: future }; + source.room_messages.later = [message("regressed-room", before, { kind: "room", room_id: "later" })]; + for (const [id, parent] of [["kept", "old"], ["dropped", "future"]]) { + source.threads[id] = { id, room_id: "assignment", parent_message_id: parent, message_count: 99, last_message_at: future }; + source.thread_messages[id] = [message(`${id}-old`, before, { kind: "thread", room_id: "assignment", thread_id: id }), + message(`${id}-future`, future, { kind: "thread", room_id: "assignment", thread_id: id })]; + } + source.direct_messages.dm = [message("dm-root-future", future, { kind: "dm", dm_id: "dm" }), + message("dm-regression", before, { kind: "dm", dm_id: "dm" })]; + source.direct_members.dm = { writer: {}, brass: {} }; + source.attachment_delivery_events = [ + { id: "e1", type: "message.created", network_id: "news", created_at: before, message: message("old", before) }, + { id: "e2", type: "message.created", network_id: "news", created_at: future, message: message("future", future) }, + { id: "e3", type: "message.created", network_id: "news", created_at: at, message: message("boundary", at) } + ]; + source.attachment_delivery_cursors = { writer: 3, brass: 2 }; + const cut = rewindMoltnetJson(source, at); + expect(Object.keys(cut.rooms)).toEqual(["assignment"]); + expect(cut.threads.kept).toMatchObject({ message_count: 1, last_message_at: before }); + expect(cut.threads.dropped).toBeUndefined(); expect(cut.direct_messages).toEqual({}); expect(cut.direct_members).toEqual({}); + expect(cut.attachment_delivery_events.map((event) => event.id)).toEqual(["e1", "e3"]); + expect(cut.attachment_delivery_cursors).toEqual({ writer: 2, brass: 1 }); + expect(moltnetMessages(cut).map((entry) => entry.id)).toEqual(["old", "boundary", "kept-old"]); + expect(source.threads.kept?.message_count).toBe(99); + }); + + it("restores known later removals, keeps zero-time unknown removals and refuses missing restoration metadata", () => { + const source = emptyMoltnetSnapshot(network, before); + source.removed_rooms.assignment = future; + const room = source.rooms.assignment!; delete source.rooms.assignment; + expect(() => rewindMoltnetJson(source, at)).toThrow("room metadata was not retained"); + source.attachment_delivery_events = [{ id: "room-created", type: "room.created", network_id: "news", created_at: before, room }]; + expect(rewindMoltnetJson(source, at).rooms.assignment).toEqual(room); + source.removed_rooms.assignment = "0001-01-01T00:00:00Z"; + expect(rewindMoltnetJson(source, at).removed_rooms.assignment).toBe("0001-01-01T00:00:00Z"); + }); + + it("refuses restoration from future room events even when the embedded creation date is old", () => { + const source = emptyMoltnetSnapshot(network, before), room = source.rooms.assignment!; + delete source.rooms.assignment; source.removed_rooms.assignment = future; + source.attachment_delivery_events = [{ id: "future-room", type: "room.updated", network_id: "news", created_at: future, + room: { ...room, name: "FUTURE ROOM NAME", members: ["future-agent"] } }]; + expect(() => rewindMoltnetJson(source, at)).toThrow("room metadata was not retained"); + source.attachment_delivery_events.push({ id: "old-room", type: "room.created", network_id: "news", created_at: before, room }); + expect(rewindMoltnetJson(source, at).rooms.assignment).toEqual(room); + }); + + it("drops future registrations with their memberships and cursors while clamping retained updates", async () => { + const source = emptyMoltnetSnapshot(network, before); + for (const [id, created_at, updated_at] of [["writer", before, future], ["boundary", at, at], ["later", future, future]]) { + source.agents[id] = { agent_id: id, network_id: "news", actor_uid: id, actor_uri: id, created_at, updated_at }; + } + source.agents.undated = { agent_id: "undated", network_id: "news", actor_uid: "undated", actor_uri: "undated" }; + source.rooms.assignment!.members = ["writer", "boundary", "later", "unregistered"]; + source.direct_messages.dm = [message("dm-old", before, { kind: "dm", dm_id: "dm" })]; + source.direct_members.dm = { writer: {}, boundary: {}, later: {}, unregistered: {} }; + source.attachment_delivery_cursors = { writer: 0, later: 0 }; + const cut = rewindMoltnetJson(source, at); + expect(Object.keys(cut.agents)).toEqual(["writer", "boundary", "undated"]); + expect(cut.agents.writer).toMatchObject({ created_at: before, updated_at: at }); + expect(cut.agents.boundary).toMatchObject({ created_at: at, updated_at: at }); + expect(cut.agents.undated).not.toHaveProperty("updated_at"); + expect(cut.rooms.assignment!.members).toEqual(["writer", "boundary", "unregistered"]); + expect(cut.direct_members.dm).toEqual({ writer: {}, boundary: {}, unregistered: {} }); + expect(cut.attachment_delivery_cursors).toEqual({ writer: 0 }); + expect(source.agents.writer!.updated_at).toBe(future); expect(source.agents.later).toBeDefined(); + expect(source.rooms.assignment!.members).toContain("later"); expect(source.direct_members.dm).toHaveProperty("later"); + const root = await mkdtemp(path.join(os.tmpdir(), "rewind-moltnet-agents-")); temps.push(root); + const file = path.join(root, "state.json"); await writeFile(file, JSON.stringify(cut)); + expect((await inspectMoltnetJson(file, at)).findings).toEqual([]); + }); + + it.each(["created_at", "updated_at"] as const)("independently flags an agent's future %s", async (field) => { + const root = await mkdtemp(path.join(os.tmpdir(), "rewind-moltnet-agent-leak-")); temps.push(root); + const file = path.join(root, "state.json"), source = emptyMoltnetSnapshot(network, before); + source.agents.writer = { agent_id: "writer", network_id: "news", actor_uid: "uid", actor_uri: "uri", + created_at: before, updated_at: before, [field]: future }; + const bytes = JSON.stringify(source); await writeFile(file, bytes); + expect((await inspectMoltnetJson(file, at)).findings).toEqual([ + { class: "moltnet-event", path: file, detail: `Agent writer has a future ${field}` } + ]); + expect(await readFile(file, "utf8")).toBe(bytes); + }); + + it("materializes exact cursor and approximate cuts, keeps history unchanged and makes fresh overridden rooms", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "rewind-moltnet-")); temps.push(root); + const bank = path.join(root, "moltnet/team%3Aorg/news"); await mkdir(bank, { recursive: true }); + const source = emptyMoltnetSnapshot(network, before); + source.room_messages.assignment = [message("wake", before), message("boundary", at), message("future", future)]; + const bytes = JSON.stringify(source); await writeFile(path.join(bank, "state.json"), bytes); + const context: InputContext = { at, record: null, history: await openHistory(root), destination: path.join(root, "out"), + input: { section: "networks", id: "news", declaredBy: "team:org", overridden: false, declarationBase: root, declaration: network } }; + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "approximate", basis: "created-at-cut" }); + expect(await readFile(path.join(bank, "state.json"), "utf8")).toBe(bytes); + expect(await inspectMoltnetJson(path.join(context.destination, "state.json"), at)).toMatchObject({ messages: 2, findings: [] }); + context.record = momentRecordSchema.parse({ version: "spawnfile.moment-record.v1", agent: "agent:writer", + wake: { at, startedAt: at, executionId: "turn-1", deliveries: [{ deliveryId: "moltnet:wake", kind: "message", occurredAt: at }], + surface: "moltnet", room: "assignment", from: "brass" }, recordedAt: at, exact: true, + resources: [], feeds: [], memory: [], moltnet: [{ network: "news", cursor: at, messageIds: ["wake", "boundary"] }], volumes: [] }); + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "exact", basis: "record-cursor" }); + expect(moltnetMessages(parseMoltnetSnapshot(JSON.parse(await readFile(path.join(context.destination, "state.json"), "utf8")))) + .map((entry) => entry.id)).toEqual(["wake", "boundary"]); + context.record.moltnet[0]!.cursor = "2026-10-09T12:00:00.100000000+02:00"; + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "exact", basis: "record-cursor" }); + for (const cursor of [before, future]) { + context.record.moltnet[0]!.cursor = cursor; + await expect(materializeMoltnet(context)).rejects.toThrow(/cursor must equal.*startedAt/); + } + context.record.moltnet[0]!.cursor = at; + for (const messageIds of [["missing"], ["wake", "future"]]) { + context.record.moltnet[0]!.messageIds = messageIds; + await expect(materializeMoltnet(context)).rejects.toThrow(/wake message.*not retained/); + } + delete context.record.moltnet[0]!.messageIds; + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "exact", basis: "record-cursor" }); + context.input.overridden = true; context.history = undefined; + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "overridden" }); + const output = parseMoltnetSnapshot(JSON.parse(await readFile(path.join(context.destination, "state.json"), "utf8"))); + expect(output.rooms.assignment?.members).toEqual(["writer"]); expect(moltnetMessages(output)).toEqual([]); + }); + + it("independently detects mutated messages, events and rooms on disk", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "rewind-moltnet-leak-")); temps.push(root); + const file = path.join(root, "state.json"), snapshot = emptyMoltnetSnapshot(network, before); + await writeFile(file, JSON.stringify(snapshot)); expect((await inspectMoltnetJson(file, at)).findings).toEqual([]); + snapshot.room_messages.assignment = [message("future", future)]; + snapshot.rooms.future = { id: "future", name: "future", created_at: future }; + snapshot.attachment_delivery_events = [{ id: "future", type: "message.created", network_id: "news", created_at: future }]; + await writeFile(file, JSON.stringify(snapshot)); + expect((await inspectMoltnetJson(file, at)).findings.map((finding) => finding.class)).toEqual(["moltnet-message", "moltnet-room", "moltnet-event"]); + }); +}); diff --git a/src/compiler/training/rewind/moltnetJson.ts b/src/compiler/training/rewind/moltnetJson.ts new file mode 100644 index 00000000..8a4ed0ba --- /dev/null +++ b/src/compiler/training/rewind/moltnetJson.ts @@ -0,0 +1,170 @@ +import { readFile } from "node:fs/promises"; +import type { TeamNetwork } from "../../../manifest/index.js"; +import { compareInstants } from "./instant.js"; +import { moltnetSnapshotSchema, type MoltnetSnapshot, type MoltnetEvent, + type MoltnetMessage, type MoltnetInspection, type MoltnetFinding } from "./moltnetSchemas.js"; + +export const moltnetMessages = (snapshot: MoltnetSnapshot): MoltnetMessage[] => + [snapshot.room_messages, snapshot.thread_messages, snapshot.direct_messages] + .flatMap((group) => Object.values(group).flat()); + +export const parseMoltnetSnapshot = (value: unknown): MoltnetSnapshot => { + const snapshot = moltnetSnapshotSchema.parse(value); + const ids = new Set(); + for (const message of moltnetMessages(snapshot)) { + if (ids.has(message.id)) throw new Error(`Moltnet snapshot has duplicate message ${message.id}`); + ids.add(message.id); + } + for (const group of [snapshot.rooms, snapshot.threads]) { + for (const [id, value] of Object.entries(group)) { + if (id !== value.id) throw new Error(`Moltnet snapshot key ${id} disagrees with id ${value.id}`); + } + } + return snapshot; +}; + +export const emptyMoltnetSnapshot = (network: TeamNetwork, at: string): MoltnetSnapshot => + parseMoltnetSnapshot({ + rooms: Object.fromEntries(network.rooms.map((room) => [room.id, { + ...room, name: room.name ?? room.id, network_id: network.id, + fqid: `molt://${network.id}/rooms/${room.id}`, created_at: at + }])), removed_rooms: {}, room_messages: {}, threads: {}, thread_messages: {}, + direct_messages: {}, direct_members: {}, agents: {}, attachment_delivery_events: [], + attachment_delivery_cursors: {} + }); + +export const latestMoltnetMessage = (messages: T[]): T | undefined => + messages.reduce((latest, item) => !latest + || compareInstants(item.created_at, latest.created_at) > 0 ? item : latest, undefined); + +const eventFuture = (event: MoltnetEvent, at: string): boolean => [event.created_at, + event.message?.created_at, event.room?.created_at, event.thread?.last_message_at, + event.dm?.last_message_at, event.pairing?.diagnostics?.checked_at] + .some((instant) => instant !== undefined && compareInstants(instant, at) > 0); + +/** Pure cut. Delivery offsets must be remapped after removing interior future events. */ +export const rewindMoltnetJson = (source: MoltnetSnapshot, at: string): MoltnetSnapshot => { + const snapshot = structuredClone(source); + const futureAgents = new Set(Object.values(source.agents) + .filter((agent) => agent.created_at && compareInstants(agent.created_at, at) > 0).map((agent) => agent.agent_id)); + for (const [key, agent] of Object.entries(snapshot.agents)) { + if (futureAgents.has(agent.agent_id)) { + delete snapshot.agents[key]; delete snapshot.attachment_delivery_cursors[agent.agent_id]; + } else if (agent.updated_at && compareInstants(agent.updated_at, at) > 0) agent.updated_at = at; + } + const futureRooms = new Set(Object.values(source.rooms) + .filter((room) => compareInstants(room.created_at, at) > 0).map((room) => room.id)); + for (const id of futureRooms) { + delete snapshot.rooms[id]; delete snapshot.room_messages[id]; delete snapshot.removed_rooms[id]; + } + for (const [id, removedAt] of Object.entries(snapshot.removed_rooms)) { + if (compareInstants(removedAt, at) <= 0) continue; + if (!snapshot.rooms[id]) { + const candidates = source.attachment_delivery_events.filter((event) => event.room?.id === id + && compareInstants(event.room.created_at, at) <= 0) + .sort((a, b) => compareInstants(b.created_at, a.created_at)); + const room = candidates.find((event) => compareInstants(event.created_at, at) <= 0)?.room; + if (!room) throw new Error(`Cannot restore Moltnet room ${id}: removal is after the moment but room metadata was not retained`); + snapshot.rooms[id] = structuredClone(room); + } + delete snapshot.removed_rooms[id]; + } + for (const room of Object.values(snapshot.rooms)) { + if (room.members) room.members = room.members.filter((id) => !futureAgents.has(id)); + } + for (const members of Object.values(snapshot.direct_members)) { + for (const id of futureAgents) delete members[id]; + } + for (const group of [snapshot.room_messages, snapshot.thread_messages, snapshot.direct_messages]) { + for (const [id, messages] of Object.entries(group)) { + group[id] = messages.filter((message) => compareInstants(message.created_at, at) <= 0 + && !futureRooms.has(message.target.room_id ?? "")); + } + } + // A removed root can itself belong to a removed thread; resolve until no dangling root remains. + let changed = true; + while (changed) { + changed = false; + const retainedIds = new Set(moltnetMessages(snapshot).map((message) => message.id)); + for (const [id, thread] of Object.entries(snapshot.threads)) { + const messages = snapshot.thread_messages[id] ?? []; + const root = source.thread_messages[id]?.[0]; + if (!messages.length || futureRooms.has(thread.room_id) + || (thread.parent_message_id && !retainedIds.has(thread.parent_message_id)) + || (!thread.parent_message_id && root && compareInstants(root.created_at, at) > 0)) { + delete snapshot.threads[id]; delete snapshot.thread_messages[id]; changed = true; + } else { + thread.message_count = messages.length; + thread.last_message_at = latestMoltnetMessage(messages)!.created_at; + } + } + } + for (const id of Object.keys(snapshot.thread_messages)) { + if (!snapshot.threads[id]) delete snapshot.thread_messages[id]; + } + for (const [id, messages] of Object.entries(snapshot.direct_messages)) { + const root = source.direct_messages[id]?.[0]; + if (!messages.length || (root && compareInstants(root.created_at, at) > 0)) { + delete snapshot.direct_messages[id]; delete snapshot.direct_members[id]; + } + } + for (const id of Object.keys(snapshot.direct_members)) { + if (!snapshot.direct_messages[id]) delete snapshot.direct_members[id]; + } + const finalIds = new Set(moltnetMessages(snapshot).map((message) => message.id)); + const retainedEvents = source.attachment_delivery_events.map((event, index) => ({ event, index })) + .filter(({ event }) => !eventFuture(event, at) + && (!event.message || finalIds.has(event.message.id)) + && (!event.room || !futureRooms.has(event.room.id)) + && (!event.thread || Boolean(snapshot.threads[event.thread.id])) + && (!event.dm || Boolean(snapshot.direct_messages[event.dm.id]))); + snapshot.attachment_delivery_events = retainedEvents.map(({ event }) => event); + for (const [agent, cursor] of Object.entries(snapshot.attachment_delivery_cursors)) { + snapshot.attachment_delivery_cursors[agent] = retainedEvents.filter(({ index }) => index < cursor).length; + } + return snapshot; +}; + +/** Independent timestamp inspection of event payloads, including embedded messages and rooms. */ +export const inspectMoltnetEvent = (event: MoltnetEvent, at: string, path: string): MoltnetFinding[] => { + const findings: MoltnetFinding[] = []; + const check = (instant: string | undefined, kind: MoltnetFinding["class"], detail: string) => { + if (instant !== undefined && compareInstants(instant, at) > 0) findings.push({ class: kind, path, detail }); + }; + check(event.created_at, "moltnet-event", `Attachment event ${event.id} is later than the moment`); + check(event.message?.created_at, "moltnet-message", `Event ${event.id} contains a future message`); + check(event.room?.created_at, "moltnet-room", `Event ${event.id} contains a future room`); + check(event.thread?.last_message_at, "moltnet-event", `Event ${event.id} contains a future thread counter`); + check(event.dm?.last_message_at, "moltnet-event", `Event ${event.id} contains a future DM counter`); + check(event.pairing?.diagnostics?.checked_at, "moltnet-event", `Event ${event.id} contains future pairing diagnostics`); + return findings; +}; + +export const inspectMoltnetJson = async (path: string, at: string): Promise => { + const snapshot = parseMoltnetSnapshot(JSON.parse(await readFile(path, "utf8"))); + const findings: MoltnetFinding[] = []; + for (const agent of Object.values(snapshot.agents)) { + for (const field of ["created_at", "updated_at"] as const) { + if (agent[field] && compareInstants(agent[field], at) > 0) findings.push({ class: "moltnet-event", path, + detail: `Agent ${agent.agent_id} has a future ${field}` }); + } + } + const messages = moltnetMessages(snapshot); + for (const message of messages) if (compareInstants(message.created_at, at) > 0) { + findings.push({ class: "moltnet-message", path, detail: `Message ${message.id} is later than the moment` }); + } + for (const room of Object.values(snapshot.rooms)) if (compareInstants(room.created_at, at) > 0) { + findings.push({ class: "moltnet-room", path, detail: `Room ${room.id} is later than the moment` }); + } + for (const thread of Object.values(snapshot.threads)) { + if (thread.last_message_at && compareInstants(thread.last_message_at, at) > 0) { + findings.push({ class: "moltnet-message", path, detail: `Thread ${thread.id} has a future counter` }); + } + } + for (const [id, instant] of Object.entries(snapshot.removed_rooms)) { + if (compareInstants(instant, at) > 0) findings.push({ class: "moltnet-room", path, + detail: `Room ${id} has a future removal marker` }); + } + for (const event of snapshot.attachment_delivery_events) findings.push(...inspectMoltnetEvent(event, at, path)); + return { messages: messages.length, events: snapshot.attachment_delivery_events.length, findings }; +}; diff --git a/src/compiler/training/rewind/moltnetSchemas.ts b/src/compiler/training/rewind/moltnetSchemas.ts new file mode 100644 index 00000000..c83b74df --- /dev/null +++ b/src/compiler/training/rewind/moltnetSchemas.ts @@ -0,0 +1,78 @@ +import { z } from "zod"; +import { instantNanos } from "./instant.js"; + +const text = z.string(); +const id = text.min(1); +export const moltnetInstantSchema = text.refine((value) => { + try { instantNanos(value); return true; } catch { return false; } +}, "Expected an RFC3339 instant with at most nanosecond precision"); +const instant = moltnetInstantSchema; +const strings = z.array(text); +const actor = z.object({ type: id, id, name: text.optional(), network_id: text.optional(), + fqid: text.optional(), credential_bound: z.boolean().optional() }).strict(); +const target = z.object({ kind: z.enum(["room", "thread", "dm"]), room_id: text.optional(), + thread_id: text.optional(), parent_message_id: text.optional(), dm_id: text.optional(), + participant_ids: strings.optional() }).strict(); +const part = z.object({ kind: id, text: text.optional(), media_type: text.optional(), + url: text.optional(), filename: text.optional(), data: z.record(text, z.unknown()).optional() }).strict(); + +export const moltnetMessageSchema = z.object({ + id, network_id: text, origin: z.object({ network_id: text, message_id: text, + received_via: text.optional() }).strict().optional(), target, from: actor, + parts: z.array(part).nullable(), mentions: strings.nullable().optional(), created_at: instant +}).strict(); +export const moltnetRoomSchema = z.object({ + id, network_id: text.optional(), fqid: text.optional(), name: text, + members: strings.nullable().optional(), visibility: text.optional(), write_policy: text.optional(), + federation: z.union([z.enum(["all", "none"]), strings]).nullable().optional(), + access: z.object({ can_read: z.boolean(), can_write: z.boolean(), reason: text.optional() }).strict().optional(), + created_at: instant +}).strict(); +export const moltnetThreadSchema = z.object({ + id, network_id: text.optional(), fqid: text.optional(), room_id: id, + parent_message_id: text.optional(), message_count: z.number().int().nonnegative(), + last_message_at: instant.optional() +}).strict(); +const direct = z.object({ id, network_id: text.optional(), fqid: text.optional(), + participant_ids: strings.optional(), message_count: z.number().int().nonnegative(), + last_message_at: instant.optional() }).strict(); +const agentEvent = z.object({ agent_id: id, network_id: text.optional(), fqid: text.optional(), + name: text.optional(), message_id: text.optional(), reason: text.optional(), target: target.optional(), + error: text.optional(), attempts: z.number().int().optional(), classification: text.optional() }).strict(); +const pairing = z.object({ id, remote_network_id: text, remote_network_name: text.optional(), + remote_base_url: text.optional(), status: text.optional(), token: text.optional(), + relay: z.object({ url: text, room: text.optional(), token: text.optional() }).strict().optional(), + diagnostics: z.object({ checked_at: instant.optional(), remote_version: text.optional(), + remote_network_id: text.optional(), remote_protocols: z.object({ + http: strings.optional(), attach: strings.optional(), pair: strings.optional() + }).strict().optional(), reason: text.optional(), message: text.optional() }).strict().optional() +}).strict(); +export const moltnetEventSchema = z.object({ id, type: id, network_id: text, + agent: agentEvent.optional(), message: moltnetMessageSchema.optional(), room: moltnetRoomSchema.optional(), + thread: moltnetThreadSchema.optional(), dm: direct.optional(), pairing: pairing.optional(), + replay_gap: z.object({ requested_event_id: text.optional(), oldest_event_id: text.optional(), + newest_event_id: text.optional() }).strict().optional(), created_at: instant +}).strict(); +const registration = z.object({ network_id: text, agent_id: id, actor_uid: text, actor_uri: text, + display_name: text.optional(), credential_key: text.optional(), created_at: instant.optional(), + updated_at: instant.optional() }).strict(); +const map = (value: T) => z.record(text, value).nullable().transform((v) => v ?? {}); +const messages = map(z.array(moltnetMessageSchema).nullable().transform((v) => v ?? [])); +export const moltnetSnapshotSchema = z.object({ + rooms: map(moltnetRoomSchema), removed_rooms: map(instant).optional().default({}), + room_messages: messages, threads: map(moltnetThreadSchema), thread_messages: messages, + direct_messages: messages, direct_members: map(map(z.object({}).strict())), + agents: map(registration).optional().default({}), + attachment_delivery_events: z.array(moltnetEventSchema).nullable().optional().transform((v) => v ?? []), + attachment_delivery_cursors: map(z.number().int().nonnegative()).optional().default({}) +}).strict(); + +export type MoltnetSnapshot = z.infer; +export type MoltnetMessage = z.infer; +export type MoltnetEvent = z.infer; +export interface MoltnetFinding { + class: "moltnet-message" | "moltnet-event" | "moltnet-room"; + path: string; + detail: string; +} +export interface MoltnetInspection { messages: number; events: number; findings: MoltnetFinding[] } diff --git a/src/compiler/training/rewind/moltnetSqlite.test.ts b/src/compiler/training/rewind/moltnetSqlite.test.ts new file mode 100644 index 00000000..d322bd4a --- /dev/null +++ b/src/compiler/training/rewind/moltnetSqlite.test.ts @@ -0,0 +1,209 @@ +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { DatabaseSync } from "node:sqlite"; +import path from "node:path"; +import os from "node:os"; +import { afterEach, describe, expect, it } from "vitest"; +import { materializeMoltnet } from "./moltnet.js"; +import { momentRecordSchema } from "../../../recording/index.js"; +import { openHistory } from "./history.js"; +import { inspectMoltnetSqlite } from "./moltnetSqlite.js"; +import type { InputContext } from "./types.js"; + +const at = "2026-10-09T10:00:00.1Z", before = "2026-10-09T10:00:00Z", future = "2026-10-09T10:00:00.11Z"; +const temps: string[] = []; +afterEach(async () => { await Promise.all(temps.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); }); + +const schema = ` +CREATE TABLE rooms (id TEXT PRIMARY KEY, network_id TEXT, fqid TEXT, name TEXT, created_at TEXT, deleted_at TEXT, + visibility TEXT DEFAULT 'private', write_policy TEXT DEFAULT 'members', federation TEXT DEFAULT '"none"'); +CREATE TABLE room_members (room_id TEXT, member_id TEXT); +CREATE TABLE threads (id TEXT PRIMARY KEY, network_id TEXT, fqid TEXT, room_id TEXT, parent_message_id TEXT, + message_count INTEGER, last_message_at TEXT); +CREATE TABLE dm_conversations (dm_id TEXT PRIMARY KEY, network_id TEXT, fqid TEXT, message_count INTEGER, last_message_at TEXT); +CREATE TABLE dm_participants (dm_id TEXT, participant_id TEXT); +CREATE TABLE messages (id TEXT PRIMARY KEY, network_id TEXT, target_kind TEXT, room_id TEXT, thread_id TEXT, + parent_message_id TEXT, dm_id TEXT, target_json TEXT, from_json TEXT, parts_json TEXT, mentions_json TEXT, + created_at TEXT, origin_json TEXT DEFAULT '{}'); +CREATE TABLE artifacts (id TEXT PRIMARY KEY, message_id TEXT, created_at TEXT); +CREATE TABLE agents (agent_id TEXT PRIMARY KEY, network_id TEXT, actor_uid TEXT, actor_uri TEXT, + display_name TEXT, credential_key TEXT, created_at TEXT, updated_at TEXT); +CREATE TABLE attachment_delivery_events (seq INTEGER PRIMARY KEY AUTOINCREMENT, event_id TEXT UNIQUE, event_json TEXT, created_at TEXT); +CREATE TABLE attachment_delivery_cursors (agent_id TEXT PRIMARY KEY, event_seq INTEGER, updated_at TEXT);`; +const room = (database: DatabaseSync, id: string, time = before, deleted: string | null = null) => { + database.prepare("INSERT INTO rooms (id, network_id, fqid, name, created_at, deleted_at) VALUES (?, 'news', ?, ?, ?, ?)") + .run(id, `molt://news/rooms/${id}`, id, time, deleted); + database.prepare("INSERT INTO room_members VALUES (?, 'writer')").run(id); +}; +const message = (database: DatabaseSync, id: string, time: string, target: Record = { kind: "room", room_id: "assignment" }) => { + database.prepare(`INSERT INTO messages (id, network_id, target_kind, room_id, thread_id, parent_message_id, dm_id, + target_json, from_json, parts_json, mentions_json, created_at) VALUES (?, 'news', ?, ?, ?, ?, ?, ?, ?, ?, '[]', ?)`) + .run(id, target.kind, target.room_id ?? null, target.thread_id ?? null, target.parent_message_id ?? null, + target.dm_id ?? null, JSON.stringify(target), JSON.stringify({ type: "agent", id: "writer" }), + JSON.stringify([{ kind: "text", text: id.startsWith("future") ? "SECRET_FUTURE_PAYLOAD" : id }]), time); +}; +const thread = (database: DatabaseSync, id: string, root: string) => database.prepare("INSERT INTO threads VALUES (?, 'news', ?, 'assignment', ?, 99, ?)") + .run(id, `molt://news/threads/${id}`, root, future); +const dm = (database: DatabaseSync, id: string) => { + database.prepare("INSERT INTO dm_conversations VALUES (?, 'news', ?, 99, ?)").run(id, `molt://news/dms/${id}`, future); + database.prepare("INSERT INTO dm_participants VALUES (?, 'writer')").run(id); +}; +const event = (database: DatabaseSync, id: string, time: string) => database.prepare("INSERT INTO attachment_delivery_events (event_id, event_json, created_at) VALUES (?, ?, ?)") + .run(id, JSON.stringify({ id, type: "agent.connected", network_id: "news", created_at: time }), time); +const fixture = async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "rewind-moltnet-sqlite-")); temps.push(root); + const bank = path.join(root, "moltnet/team%3Aorg/news"); await mkdir(bank, { recursive: true }); + const file = path.join(bank, "moltnet.sqlite"), database = new DatabaseSync(file); database.exec(schema); + const context: InputContext = { at, record: null, history: await openHistory(root), destination: path.join(root, "out"), + input: { section: "networks", id: "news", declaredBy: "team:org", overridden: false, declarationBase: root, + declaration: { id: "news", provider: "moltnet", rooms: [{ id: "assignment", members: ["writer"] }] } } }; + return { root, bank, file, database, context }; +}; + +describe("Moltnet SQLite rewind", () => { + it("cuts a copy, preserves nanos ordering, prunes topology/artifacts, restores rooms, and erases deleted payload bytes", async () => { + const { file, database, context } = await fixture(); + room(database, "assignment"); room(database, "future-room", future); room(database, "restored", before, future); room(database, "removed", before, before); + message(database, "old", before); message(database, "boundary", at); message(database, "future-root", future); + message(database, "regressed-room", before, { kind: "room", room_id: "future-room" }); + thread(database, "kept", "old"); thread(database, "dropped", "future-root"); + message(database, "thread-old", before, { kind: "thread", room_id: "assignment", thread_id: "kept" }); + message(database, "future-thread", future, { kind: "thread", room_id: "assignment", thread_id: "kept" }); + message(database, "thread-regressed", before, { kind: "thread", room_id: "assignment", thread_id: "dropped" }); + dm(database, "dm-kept"); dm(database, "dm-dropped"); + message(database, "dm-old", before, { kind: "dm", dm_id: "dm-kept" }); + message(database, "future-dm", future, { kind: "dm", dm_id: "dm-kept" }); + message(database, "future-dm-root", future, { kind: "dm", dm_id: "dm-dropped" }); + message(database, "dm-regressed", before, { kind: "dm", dm_id: "dm-dropped" }); + database.prepare("INSERT INTO artifacts VALUES (?, ?, ?)").run("old-artifact", "old", before); + database.prepare("INSERT INTO artifacts VALUES (?, ?, ?)").run("future-artifact", "future-root", future); + for (const agent of ["writer", "brass"]) database.prepare("INSERT INTO agents VALUES (?, 'news', 'uid', 'uri', NULL, 'key', ?, ?)").run(agent, before, future); + event(database, "event-old", before); event(database, "event-future", future); event(database, "event-boundary", at); + database.prepare("INSERT INTO attachment_delivery_cursors VALUES (?, ?, ?)").run("writer", 2, future); + database.prepare("INSERT INTO attachment_delivery_cursors VALUES (?, ?, ?)").run("brass", 3, future); + database.close(); + const sourceBytes = await readFile(file); + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "approximate", basis: "created-at-cut" }); + expect(await readFile(file)).toEqual(sourceBytes); + const output = path.join(context.destination, "moltnet.sqlite"), result = new DatabaseSync(output, { readOnly: true }); + try { + expect(result.prepare("SELECT id FROM messages ORDER BY id").all().map((row) => row.id)).toEqual(["boundary", "dm-old", "old", "thread-old"]); + expect(result.prepare("SELECT id, message_count, last_message_at FROM threads").all()).toEqual([{ id: "kept", message_count: 1, last_message_at: before }]); + expect(result.prepare("SELECT dm_id, message_count, last_message_at FROM dm_conversations").all()).toEqual([{ dm_id: "dm-kept", message_count: 1, last_message_at: before }]); + expect(result.prepare("SELECT id, deleted_at FROM rooms WHERE id IN ('restored', 'removed') ORDER BY id").all()) + .toEqual([{ id: "removed", deleted_at: before }, { id: "restored", deleted_at: null }]); + expect(result.prepare("SELECT id FROM rooms WHERE id = 'future-room'").get()).toBeUndefined(); + expect(result.prepare("SELECT * FROM artifacts").all()).toEqual([{ id: "old-artifact", message_id: "old", created_at: before }]); + expect(result.prepare("SELECT agent_id, event_seq, updated_at FROM attachment_delivery_cursors ORDER BY agent_id").all()) + .toEqual([{ agent_id: "brass", event_seq: 3, updated_at: at }, { agent_id: "writer", event_seq: 1, updated_at: at }]); + expect(result.prepare("SELECT count(*) AS total FROM agents").get()?.total).toBe(2); + } finally { result.close(); } + expect((await readFile(output)).includes(Buffer.from("SECRET_FUTURE_PAYLOAD"))).toBe(false); + expect(await readdir(context.destination)).toEqual(["moltnet.sqlite"]); + expect(await inspectMoltnetSqlite(output, at)).toMatchObject({ messages: 4, events: 2, findings: [] }); + context.record = momentRecordSchema.parse({ version: "spawnfile.moment-record.v1", agent: "agent:writer", + wake: { at, startedAt: at, executionId: "turn-1", deliveries: [{ deliveryId: "moltnet:old", kind: "message", occurredAt: at }], + surface: "moltnet", room: "assignment", from: "brass" }, recordedAt: at, exact: true, + resources: [], feeds: [], memory: [], moltnet: [{ network: "news", cursor: at, messageIds: ["old", "boundary"] }], volumes: [] }); + context.destination += "-exact"; + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "exact", basis: "record-cursor" }); + context.record.moltnet[0]!.messageIds = ["old", "future-root"]; + await expect(materializeMoltnet(context)).rejects.toThrow(/wake message.*future-root.*not retained/); + context.record.moltnet[0]!.messageIds = ["missing"]; + await expect(materializeMoltnet(context)).rejects.toThrow(/wake message.*missing.*not retained/); + delete context.record.moltnet[0]!.messageIds; + context.record.moltnet[0]!.cursor = "2026-10-09T12:00:00.100000000+02:00"; + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "exact", basis: "record-cursor" }); + for (const cursor of [before, future]) { + context.record.moltnet[0]!.cursor = cursor; + await expect(materializeMoltnet(context)).rejects.toThrow(/cursor must equal.*startedAt/); + } + }); + + it("drops future registrations and memberships, clamps retained updates and migration times, and preserves history bytes", async () => { + const { database, context, file } = await fixture(); + room(database, "assignment"); dm(database, "dm-kept"); + message(database, "dm-old", before, { kind: "dm", dm_id: "dm-kept" }); + for (const [id, time] of [["writer", before], ["boundary", at], ["later", future]]) { + database.prepare("INSERT INTO agents VALUES (?, 'news', ?, ?, NULL, 'key', ?, ?)").run(id, id, id, time, future); + } + for (const id of ["boundary", "later", "unregistered"]) { + database.prepare("INSERT INTO room_members VALUES ('assignment', ?)").run(id); + database.prepare("INSERT INTO dm_participants VALUES ('dm-kept', ?)").run(id); + } + database.prepare("INSERT INTO attachment_delivery_cursors VALUES ('later', 0, ?)").run(future); + database.exec("CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, name TEXT, applied_at TEXT)"); + for (const [version, time] of [[1, before], [2, at], [3, future]] as const) { + database.prepare("INSERT INTO schema_migrations VALUES (?, 'migration', ?)").run(version, time); + } + database.close(); const sourceBytes = await readFile(file); + await materializeMoltnet(context); + const output = path.join(context.destination, "moltnet.sqlite"), result = new DatabaseSync(output, { readOnly: true }); + try { + expect(result.prepare("SELECT agent_id, created_at, updated_at FROM agents ORDER BY agent_id").all()).toEqual([ + { agent_id: "boundary", created_at: at, updated_at: at }, { agent_id: "writer", created_at: before, updated_at: at } + ]); + expect(result.prepare("SELECT member_id FROM room_members ORDER BY member_id").all()) + .toEqual(["boundary", "unregistered", "writer"].map((member_id) => ({ member_id }))); + expect(result.prepare("SELECT participant_id FROM dm_participants ORDER BY participant_id").all()) + .toEqual(["boundary", "unregistered", "writer"].map((participant_id) => ({ participant_id }))); + expect(result.prepare("SELECT * FROM attachment_delivery_cursors").all()).toEqual([]); + expect(result.prepare("SELECT version, applied_at FROM schema_migrations ORDER BY version").all()).toEqual([ + { version: 1, applied_at: before }, { version: 2, applied_at: at }, { version: 3, applied_at: at } + ]); + } finally { result.close(); } + expect(await readFile(file)).toEqual(sourceBytes); + expect((await inspectMoltnetSqlite(output, at)).findings).toEqual([]); + }); + + it.each(["created_at", "updated_at"] as const)("independently flags a SQLite agent's future %s", async (field) => { + const { database, file } = await fixture(); + database.prepare("INSERT INTO agents VALUES ('writer', 'news', 'uid', 'uri', NULL, 'key', ?, ?)") + .run(field === "created_at" ? future : before, field === "updated_at" ? future : before); + database.close(); const bytes = await readFile(file); + expect((await inspectMoltnetSqlite(file, at)).findings).toEqual([ + { class: "moltnet-event", path: file, detail: `Agent writer has a future ${field}` } + ]); + expect(await readFile(file)).toEqual(bytes); + }); + + it("independently flags future schema migration times", async () => { + const { database, file } = await fixture(); + database.exec("CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, name TEXT, applied_at TEXT)"); + database.prepare("INSERT INTO schema_migrations VALUES (1, 'migration', ?)").run(future); + database.close(); const bytes = await readFile(file); + expect((await inspectMoltnetSqlite(file, at)).findings).toEqual([ + { class: "moltnet-event", path: file, detail: "Schema migration 1 has a future applied_at" } + ]); + expect(await readFile(file)).toEqual(bytes); + }); + + it("copies WAL state and independently detects message, room and attachment-event mutations without changing the file", async () => { + const { database, context } = await fixture(); + database.exec("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0"); + room(database, "assignment"); message(database, "cursor", at); + try { expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "approximate" }); } + finally { database.close(); } + const output = path.join(context.destination, "moltnet.sqlite"); + expect(await inspectMoltnetSqlite(output, at)).toMatchObject({ messages: 1, findings: [] }); + const mutate = new DatabaseSync(output); + room(mutate, "future", future); message(mutate, "future-message", future); event(mutate, "future-event", future); mutate.close(); + const bytes = await readFile(output), findings = (await inspectMoltnetSqlite(output, at)).findings; + expect(new Set(findings.map((finding) => finding.class))).toEqual(new Set(["moltnet-message", "moltnet-room", "moltnet-event"])); + expect(await readFile(output)).toEqual(bytes); expect(await readdir(context.destination)).toEqual(["moltnet.sqlite"]); + await writeFile(`${output}-wal`, "uninspected bytes"); + expect((await inspectMoltnetSqlite(output, at)).findings).toMatchObject([{ class: "moltnet-event", detail: expect.stringContaining("sidecar") }]); + expect(await readFile(`${output}-wal`, "utf8")).toBe("uninspected bytes"); + }); + + it("refuses unknown persistent state rather than silently retaining it", async () => { + const { database, context } = await fixture(); + database.exec("CREATE TABLE unknown_future_state (payload TEXT)"); database.close(); + await expect(materializeMoltnet(context)).rejects.toThrow("Unsupported Moltnet SQLite table unknown_future_state"); + }); + + it("refuses unrecognized table columns rather than silently retaining added state", async () => { + const { database, context } = await fixture(); + database.exec("ALTER TABLE messages ADD COLUMN later_edits TEXT"); database.close(); + await expect(materializeMoltnet(context)).rejects.toThrow("Unsupported Moltnet SQLite column messages.later_edits"); + }); +}); diff --git a/src/compiler/training/rewind/moltnetSqlite.ts b/src/compiler/training/rewind/moltnetSqlite.ts new file mode 100644 index 00000000..2411a943 --- /dev/null +++ b/src/compiler/training/rewind/moltnetSqlite.ts @@ -0,0 +1,139 @@ +import { lstat, rm } from "node:fs/promises"; +import type { DatabaseSync } from "node:sqlite"; +import { compareInstants } from "./instant.js"; +import { inspectMoltnetEvent, latestMoltnetMessage, moltnetMessages, rewindMoltnetJson } from "./moltnetJson.js"; +import { readMoltnetSqlite, readMoltnetSqliteMigrations, sqliteInteger, sqliteText, sqliteJson } from "./moltnetSqliteData.js"; +import { moltnetEventSchema, type MoltnetFinding, type MoltnetInspection, type MoltnetMessage } from "./moltnetSchemas.js"; + +const openSqlite = async (file: string, readOnly = false): Promise => { + let sqlite: typeof import("node:sqlite"); + try { sqlite = await import("node:sqlite"); } + catch { throw new Error("Moltnet SQLite rewind requires node:sqlite (supported Node >=22.19); this Node runtime does not expose it"); } + return new sqlite.DatabaseSync(file, { readOnly, enableForeignKeyConstraints: false, allowExtension: false }); +}; + +/** Mutates only a caller-created COPY. secure_delete and VACUUM remove future content from free pages. */ +export const rewindMoltnetSqlite = async (file: string, at: string): Promise => { + const database = await openSqlite(file); + try { + const source = readMoltnetSqlite(database), cut = rewindMoltnetJson(source, at); + const messages = moltnetMessages(cut), keepMessages = new Set(messages.map((message) => message.id)); + const eventRows = database.prepare("SELECT * FROM attachment_delivery_events ORDER BY seq").all(); + const keepEvents = new Set(cut.attachment_delivery_events.map((event) => event.id)); + // The database column and the embedded event each carry an instant; both must pass the cut. + const remainingEvents = eventRows.filter((row) => keepEvents.has(sqliteText(row, "event_id")) + && compareInstants(sqliteText(row, "created_at"), at) <= 0); + const remainingSeq = new Set(remainingEvents.map((row) => sqliteInteger(row, "seq"))); + database.exec("PRAGMA secure_delete=ON; PRAGMA foreign_keys=OFF; BEGIN IMMEDIATE"); + try { + for (const [id, agent] of Object.entries(source.agents)) { + if (!cut.agents[id]) { + database.prepare("DELETE FROM agents WHERE agent_id = ?").run(id); + database.prepare("DELETE FROM room_members WHERE member_id = ?").run(id); + database.prepare("DELETE FROM dm_participants WHERE participant_id = ?").run(id); + database.prepare("DELETE FROM attachment_delivery_cursors WHERE agent_id = ?").run(id); + } else if (agent.updated_at !== cut.agents[id].updated_at) { + database.prepare("UPDATE agents SET updated_at = ? WHERE agent_id = ?").run(cut.agents[id].updated_at!, id); + } + } + for (const row of readMoltnetSqliteMigrations(database)) { + if (compareInstants(sqliteText(row, "applied_at"), at) > 0) { + database.prepare("UPDATE schema_migrations SET applied_at = ? WHERE version = ?").run(at, row.version!); + } + } + const dropMessage = database.prepare("DELETE FROM messages WHERE id = ?"); + for (const message of moltnetMessages(source)) if (!keepMessages.has(message.id)) dropMessage.run(message.id); + const dropRoom = database.prepare("DELETE FROM rooms WHERE id = ?"), restoreRoom = database.prepare("UPDATE rooms SET deleted_at = NULL WHERE id = ?"); + for (const id of Object.keys(source.rooms)) { + if (!cut.rooms[id]) dropRoom.run(id); + else if (source.removed_rooms[id] && !cut.removed_rooms[id]) restoreRoom.run(id); + } + database.exec("DELETE FROM room_members WHERE room_id NOT IN (SELECT id FROM rooms)"); + const dropThread = database.prepare("DELETE FROM threads WHERE id = ?"); + const updateThread = database.prepare("UPDATE threads SET message_count = ?, last_message_at = ? WHERE id = ?"); + for (const id of Object.keys(source.threads)) { + const thread = cut.threads[id]; + if (!thread) dropThread.run(id); else updateThread.run(thread.message_count, thread.last_message_at!, id); + } + const dropDm = database.prepare("DELETE FROM dm_conversations WHERE dm_id = ?"); + const updateDm = database.prepare("UPDATE dm_conversations SET message_count = ?, last_message_at = ? WHERE dm_id = ?"); + for (const row of database.prepare("SELECT dm_id FROM dm_conversations").all()) { + const id = sqliteText(row, "dm_id"), retained = cut.direct_messages[id] ?? []; + if (!retained.length) dropDm.run(id); else updateDm.run(retained.length, latestMoltnetMessage(retained)!.created_at, id); + } + database.exec("DELETE FROM dm_participants WHERE dm_id NOT IN (SELECT dm_id FROM dm_conversations)"); + database.exec("DELETE FROM artifacts WHERE message_id NOT IN (SELECT id FROM messages)"); + const dropArtifact = database.prepare("DELETE FROM artifacts WHERE id = ?"); + for (const row of database.prepare("SELECT id, created_at FROM artifacts").all()) { + if (compareInstants(sqliteText(row, "created_at"), at) > 0) dropArtifact.run(sqliteText(row, "id")); + } + const dropEvent = database.prepare("DELETE FROM attachment_delivery_events WHERE seq = ?"); + for (const row of eventRows) if (!remainingSeq.has(sqliteInteger(row, "seq"))) dropEvent.run(sqliteInteger(row, "seq")); + const updateCursor = database.prepare("UPDATE attachment_delivery_cursors SET event_seq = ?, updated_at = ? WHERE agent_id = ?"); + for (const row of database.prepare("SELECT * FROM attachment_delivery_cursors").all()) { + const cursor = sqliteInteger(row, "event_seq"); + const clamped = remainingEvents.reduce((value, event) => { + const seq = sqliteInteger(event, "seq"); return seq <= cursor ? Math.max(value, seq) : value; + }, 0); + const updated = sqliteText(row, "updated_at"); + updateCursor.run(clamped, compareInstants(updated, at) > 0 ? at : updated, sqliteText(row, "agent_id")); + } + const hasSequence = database.prepare("SELECT name FROM sqlite_schema WHERE name = 'sqlite_sequence'").get(); + if (hasSequence) database.prepare("UPDATE sqlite_sequence SET seq = ? WHERE name = 'attachment_delivery_events'") + .run(remainingEvents.reduce((value, row) => Math.max(value, sqliteInteger(row, "seq")), 0)); + database.exec("COMMIT"); + } catch (error) { database.exec("ROLLBACK"); throw error; } + database.exec("PRAGMA wal_checkpoint(TRUNCATE); PRAGMA journal_mode=DELETE; VACUUM"); + if (database.prepare("PRAGMA integrity_check").get()?.integrity_check !== "ok") throw new Error("Rewound Moltnet SQLite integrity check failed"); + return messages; + } finally { + database.close(); + for (const suffix of ["-wal", "-shm", "-journal"]) await rm(`${file}${suffix}`, { force: true }); + } +}; + +/** Reopens on-disk SQLite read-only; no materializer state or cut helper supplies the verdict. */ +export const inspectMoltnetSqlite = async (file: string, at: string): Promise => { + for (const suffix of ["-wal", "-shm", "-journal"]) { + const entry = await lstat(`${file}${suffix}`).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + // The materialized database is checkpointed. Refuse sidecars before SQLite can recover or touch them. + if (entry) return { messages: 0, events: 0, findings: [{ class: "moltnet-event", path: `${file}${suffix}`, + detail: "Materialized Moltnet SQLite has an uninspected journal/WAL sidecar" }] }; + } + const database = await openSqlite(file, true); + try { + const snapshot = readMoltnetSqlite(database), messages = moltnetMessages(snapshot); + const findings: MoltnetFinding[] = []; + const check = (instant: string, kind: MoltnetFinding["class"], detail: string) => { + if (compareInstants(instant, at) > 0) findings.push({ class: kind, path: file, detail }); + }; + for (const agent of Object.values(snapshot.agents)) { + for (const field of ["created_at", "updated_at"] as const) { + if (agent[field]) check(agent[field], "moltnet-event", `Agent ${agent.agent_id} has a future ${field}`); + } + } + for (const row of readMoltnetSqliteMigrations(database)) { + check(sqliteText(row, "applied_at"), "moltnet-event", `Schema migration ${row.version} has a future applied_at`); + } + for (const message of messages) check(message.created_at, "moltnet-message", `Message ${message.id} is later than the moment`); + for (const room of Object.values(snapshot.rooms)) check(room.created_at, "moltnet-room", `Room ${room.id} is later than the moment`); + for (const [id, instant] of Object.entries(snapshot.removed_rooms)) check(instant, "moltnet-room", `Room ${id} has a future removal marker`); + for (const row of database.prepare("SELECT id, created_at FROM artifacts").all()) { + check(sqliteText(row, "created_at"), "moltnet-message", `Artifact ${row.id} is later than the moment`); + } + for (const [table, id] of [["threads", "id"], ["dm_conversations", "dm_id"]]) { + for (const row of database.prepare(`SELECT ${id}, last_message_at FROM ${table}`).all()) { + check(sqliteText(row, "last_message_at"), "moltnet-message", `${table} ${row[id]} has a future counter`); + } + } + const events = database.prepare("SELECT * FROM attachment_delivery_events").all(); + for (const row of events) { + check(sqliteText(row, "created_at"), "moltnet-event", `Attachment event ${row.event_id} is later than the moment`); + findings.push(...inspectMoltnetEvent(moltnetEventSchema.parse(sqliteJson(row, "event_json")), at, file)); + } + return { messages: messages.length, events: events.length, findings }; + } finally { database.close(); } +}; diff --git a/src/compiler/training/rewind/moltnetSqliteData.ts b/src/compiler/training/rewind/moltnetSqliteData.ts new file mode 100644 index 00000000..ff505121 --- /dev/null +++ b/src/compiler/training/rewind/moltnetSqliteData.ts @@ -0,0 +1,108 @@ +import type { DatabaseSync, SQLOutputValue } from "node:sqlite"; +import { parseMoltnetSnapshot } from "./moltnetJson.js"; +import { moltnetEventSchema, type MoltnetSnapshot } from "./moltnetSchemas.js"; + +export type SqliteRow = Record; +export const sqliteText = (row: SqliteRow, key: string): string => { + if (typeof row[key] !== "string") throw new Error(`Invalid Moltnet SQLite ${key}: expected text`); + return row[key]; +}; +export const sqliteInteger = (row: SqliteRow, key: string): number => { + const value = row[key]; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`Invalid Moltnet SQLite ${key}: expected nonnegative safe integer`); + } + return value; +}; +export const sqliteJson = (row: SqliteRow, key: string): unknown => { + try { return JSON.parse(sqliteText(row, key)); } + catch (error) { throw new Error(`Invalid Moltnet SQLite ${key}: ${(error as Error).message}`); } +}; +export const readMoltnetSqliteMigrations = (database: DatabaseSync): SqliteRow[] => + database.prepare("SELECT name FROM sqlite_schema WHERE name = 'schema_migrations'").get() + ? database.prepare("SELECT version, applied_at FROM schema_migrations").all() : []; +const allowedColumns: Record = { + rooms: ["id", "network_id", "fqid", "name", "created_at", "deleted_at", "visibility", "write_policy", "federation"], + room_members: ["room_id", "member_id"], + threads: ["id", "network_id", "fqid", "room_id", "parent_message_id", "message_count", "last_message_at"], + messages: ["id", "network_id", "target_kind", "room_id", "thread_id", "parent_message_id", "dm_id", "target_json", + "from_json", "parts_json", "mentions_json", "created_at", "origin_json"], + dm_conversations: ["dm_id", "network_id", "fqid", "message_count", "last_message_at"], + dm_participants: ["dm_id", "participant_id"], + artifacts: ["id", "network_id", "fqid", "message_id", "target_kind", "room_id", "thread_id", "dm_id", "target_json", + "part_index", "kind", "media_type", "filename", "url", "created_at"], + agents: ["agent_id", "network_id", "actor_uid", "actor_uri", "display_name", "credential_key", "created_at", "updated_at"], + attachment_delivery_events: ["seq", "event_id", "event_json", "created_at"], + attachment_delivery_cursors: ["agent_id", "event_seq", "updated_at"], + schema_migrations: ["version", "name", "applied_at"], sqlite_sequence: ["name", "seq"] +}; + +/** A snapshot is an offline store: unknown tables, views or triggers cannot be safely rewound. */ +export const validateSqliteLayout = (database: DatabaseSync): void => { + for (const row of database.prepare("SELECT type, name FROM sqlite_schema WHERE type IN ('table', 'view', 'trigger')").all()) { + const table = sqliteText(row, "name"); + if (row.type !== "table" || !Object.hasOwn(allowedColumns, table)) { + throw new Error(`Unsupported Moltnet SQLite ${row.type} ${row.name}; cannot establish its historical contents`); + } + for (const column of database.prepare(`PRAGMA table_info(${table})`).all()) { + if (!allowedColumns[table]!.includes(sqliteText(column, "name"))) { + throw new Error(`Unsupported Moltnet SQLite column ${table}.${column.name}; cannot establish its historical contents`); + } + } + } +}; + +export const readMoltnetSqlite = (database: DatabaseSync): MoltnetSnapshot => { + validateSqliteLayout(database); + const rooms: Record = {}, removed: Record = {}, threads: Record = {}; + const roomMessages: Record = {}, threadMessages: Record = {}, directMessages: Record = {}; + const directMembers: Record> = {}, agents: Record = {}; + const roomMembers = database.prepare("SELECT room_id, member_id FROM room_members").all(); + for (const row of database.prepare("SELECT * FROM rooms").all()) { + const id = sqliteText(row, "id"); + rooms[id] = { id, network_id: row.network_id, fqid: row.fqid, name: row.name, created_at: row.created_at, + members: roomMembers.filter((member) => member.room_id === id).map((member) => member.member_id), + ...(row.visibility !== undefined ? { visibility: row.visibility } : {}), + ...(row.write_policy !== undefined ? { write_policy: row.write_policy } : {}), + ...(row.federation !== undefined ? { federation: sqliteJson(row, "federation") } : {}) }; + if (row.deleted_at !== null && row.deleted_at !== undefined && row.deleted_at !== "") removed[id] = row.deleted_at; + } + for (const row of database.prepare("SELECT * FROM threads").all()) threads[sqliteText(row, "id")] = { + id: row.id, network_id: row.network_id, fqid: row.fqid, room_id: row.room_id, + parent_message_id: row.parent_message_id, message_count: row.message_count, last_message_at: row.last_message_at + }; + // rowid preserves append order for a DM's first/root message; random message IDs do not. + for (const row of database.prepare("SELECT * FROM messages ORDER BY rowid").all()) { + const target = sqliteJson(row, "target_json") as Record; + for (const [column, key] of [["target_kind", "kind"], ["room_id", "room_id"], ["thread_id", "thread_id"], + ["parent_message_id", "parent_message_id"], ["dm_id", "dm_id"]]) { + if ((row[column] ?? "") !== (target?.[key] ?? "")) throw new Error(`Moltnet SQLite message ${row.id}: ${column} disagrees with target_json`); + } + const message = { id: row.id, network_id: row.network_id, target, + from: sqliteJson(row, "from_json"), parts: sqliteJson(row, "parts_json"), mentions: sqliteJson(row, "mentions_json"), + created_at: row.created_at, ...(row.origin_json !== undefined && row.origin_json !== "{}" + ? { origin: sqliteJson(row, "origin_json") } : {}) }; + const group = row.target_kind === "room" ? roomMessages : row.target_kind === "thread" ? threadMessages + : row.target_kind === "dm" ? directMessages : undefined; + if (!group) throw new Error(`Unknown Moltnet SQLite message target kind ${row.target_kind}`); + const key = sqliteText(row, row.target_kind === "room" ? "room_id" : row.target_kind === "thread" ? "thread_id" : "dm_id"); + (group[key] ??= []).push(message); + } + for (const row of database.prepare("SELECT dm_id, participant_id FROM dm_participants").all()) { + (directMembers[sqliteText(row, "dm_id")] ??= {})[sqliteText(row, "participant_id")] = {}; + } + for (const row of database.prepare("SELECT * FROM agents").all()) agents[sqliteText(row, "agent_id")] = { + ...row, ...(row.display_name === null ? { display_name: "" } : {}) + }; + const eventRows = database.prepare("SELECT * FROM attachment_delivery_events ORDER BY seq").all(); + const events = eventRows.map((row) => { + const event = moltnetEventSchema.parse(sqliteJson(row, "event_json")); + if (event.id !== row.event_id) throw new Error(`Moltnet SQLite event ${row.event_id} disagrees with event_json`); + return event; + }); + const cursors = Object.fromEntries(database.prepare("SELECT * FROM attachment_delivery_cursors").all() + .map((row) => [sqliteText(row, "agent_id"), eventRows.filter((event) => sqliteInteger(event, "seq") <= sqliteInteger(row, "event_seq")).length])); + return parseMoltnetSnapshot({ rooms, removed_rooms: removed, threads, room_messages: roomMessages, + thread_messages: threadMessages, direct_messages: directMessages, direct_members: directMembers, + agents, attachment_delivery_events: events, attachment_delivery_cursors: cursors }); +}; diff --git a/src/compiler/training/rewind/recordInput.test.ts b/src/compiler/training/rewind/recordInput.test.ts new file mode 100644 index 00000000..b58fbfb9 --- /dev/null +++ b/src/compiler/training/rewind/recordInput.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { emptyRecord } from "./fixtures.test-helper.js"; +import { recordInputErrors } from "./recordInput.js"; +import type { InputIdentity } from "./types.js"; + +const memory: InputIdentity = { section: "memory", id: "self", declaredBy: "agent:a" }; +const otherBank: InputIdentity = { ...memory, declaredBy: "team:publication" }; +const feed: InputIdentity = { section: "resources", id: "news", declaredBy: "team:publication" }; +const network: InputIdentity = { ...feed, section: "networks" }; + +describe("record input error attribution", () => { + it.each([ + ["memory agent:a/self: capture failed", memory, otherBank], + ["memory self (agent:a): capture failed", memory, otherBank], + ["memory self declaredBy=agent:a: capture failed", memory, otherBank], + ["memory/agent%3Aa/self: capture failed", memory, otherBank], + ["feed news: capture failed", feed, network], + ["moltnet news: capture failed", network, feed] + ] as const)("honors qualified input identity in %s", (error, affected, healthy) => { + const record = emptyRecord(); record.exact = false; record.errors = [error]; + expect(recordInputErrors(record, affected)).toEqual([error]); + expect(recordInputErrors(record, healthy)).toEqual([]); + }); + + it("keeps unqualified errors conservative while respecting complete ID tokens", () => { + const record = emptyRecord(); record.errors = ["self: capture failed", "other-self: failed", "myself: failed"]; + expect(recordInputErrors(record, memory)).toEqual([record.errors[0]]); + expect(recordInputErrors(record, otherBank)).toEqual([record.errors[0]]); + }); +}); diff --git a/src/compiler/training/rewind/recordInput.ts b/src/compiler/training/rewind/recordInput.ts new file mode 100644 index 00000000..9942df6e --- /dev/null +++ b/src/compiler/training/rewind/recordInput.ts @@ -0,0 +1,50 @@ +import type { MomentRecord } from "../../../recording/index.js"; +import type { InputIdentity, RewindInput } from "./types.js"; + +const sections: Record = { + resource: "resources", resources: "resources", git: "resources", bundle: "resources", bundles: "resources", + feed: "resources", feeds: "resources", volume: "resources", volumes: "resources", + memory: "memory", bank: "memory", banks: "memory", moltnet: "networks", network: "networks", networks: "networks" +}; +const ownerPattern = "(?:agent|team|org|organization):[A-Za-z0-9_.:/-]+?"; +const qualifiedPrefix = new RegExp(`(?:^|[^A-Za-z0-9_.-])(${Object.keys(sections).join("|")})` + + `(?:[\\s/]+(${ownerPattern}))?[\\s/:]+$`, "iu"); +const ownerPrefix = new RegExp(`(?:^|[^A-Za-z0-9_.-])(${ownerPattern})[\\s/]+$`, "u"); +const ownerSuffix = new RegExp(`^\\s*(?:\\(\\s*|declaredBy\\s*[=:]\\s*|@\\s*)(${ownerPattern})(?=[\\s):,;]|$)`, "u"); + +/** Merged recording keeps deployment uncertainty at record level, rather than naming each pin. */ +export const recordActivationErrors = (record: MomentRecord | null): string[] => + (record?.errors ?? []).filter((error) => error.includes("Deployment activation is non-exact")); + +/** Errors are free text in P3: honor explicit qualifications; otherwise conservatively match the whole ID. */ +export const recordInputErrors = (record: MomentRecord, input: InputIdentity): string[] => { + const escaped = input.id.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const namesInput = new RegExp(`(? { + if (recordActivationErrors(record).includes(error)) return false; + let decoded = error; + try { decoded = decodeURIComponent(error); } catch { /* A literal percent is valid free text. */ } + return [...decoded.matchAll(namesInput)].some((match) => { + const prefix = decoded.slice(0, match.index); + const qualification = qualifiedPrefix.exec(prefix); + if (qualification && sections[qualification[1]!.toLowerCase()] !== input.section) return false; + const owner = qualification?.[2] ?? ownerPrefix.exec(prefix)?.[1] + ?? ownerSuffix.exec(decoded.slice(match.index + match[0].length))?.[1]; + return !owner || owner === input.declaredBy; + }); + }); +}; + +/** Do not discard healthy pins just because another input made the whole record inexact. */ +export const recordForInput = (record: MomentRecord | null, input: InputIdentity): MomentRecord | null => + record && !recordInputErrors(record, input).length ? record : null; + +export const recordPins = (record: MomentRecord | null, input: InputIdentity | RewindInput) => { + if (!record) return []; + if (input.section === "memory") return record.memory.filter((entry) => entry.id === input.id && entry.declaredBy === input.declaredBy); + if (input.section === "networks") return record.moltnet.filter((entry) => entry.network === input.id); + const pins = !("declaration" in input) ? [...record.resources, ...record.feeds, ...record.volumes] + : input.declaration.kind !== "volume" ? record.resources : input.declaration.feed ? record.feeds : record.volumes; + return pins.filter((entry) => entry.id === input.id); +}; +export const hasRecordPin = (record: MomentRecord | null, input: InputIdentity | RewindInput): boolean => recordPins(record, input).length > 0; diff --git a/src/compiler/training/rewind/recordedBundle.test.ts b/src/compiler/training/rewind/recordedBundle.test.ts new file mode 100644 index 00000000..059fc4f3 --- /dev/null +++ b/src/compiler/training/rewind/recordedBundle.test.ts @@ -0,0 +1,69 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { BundleTarWriter } from "../../workspaceBundleTar.js"; +import { materializeGit } from "./git.js"; +import { commitTestGit, gitTestRoot, initTestGit, testMomentRecord } from "./git.test-helper.js"; +import { History } from "./history.js"; +import { verifyRecordedBundle } from "./recordedBundle.js"; +import type { InputContext } from "./types.js"; + +const AT = "2026-10-09T08:20:31Z", EARLY = "2026-10-09T07:00:00Z"; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const fixture = async () => { + const root = await gitTestRoot(); roots.push(root); + const historyRoot = path.join(root, "history"), archive = path.join(root, "bundle.tar"); + await mkdir(path.join(historyRoot, "bundles"), { recursive: true }); + const writer = await BundleTarWriter.create(archive); + await writer.begin("tools/run", 6, 0o755); await writer.data(Buffer.from("saved\n")); await writer.end(); await writer.finish(); + const bytes = await readFile(archive), sha256 = `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + const context: InputContext = { at: AT, destination: path.join(root, "out"), history: new History(historyRoot), + record: { ...testMomentRecord(), resources: [{ id: "tools", kind: "bundle", sha256 }] }, + input: { section: "resources", id: "tools", declaredBy: "agent:writer", declarationBase: root, overridden: false, + declaration: { id: "tools", kind: "bundle", mode: "readonly", mount: "./tools", source: "never-read-current.tar" } } }; + return { root, historyRoot, context, bytes, sha256 }; +}; + +describe("recorded bundle archive", () => { + it("is exact only from the matching retained tar without a repository", async () => { + const f = await fixture(); + await writeFile(path.join(f.historyRoot, `bundles/${f.sha256.slice(7)}.tar`), f.bytes); + expect(await materializeGit(f.context)).toMatchObject({ fidelity: "exact", basis: "record", source: { sha256: f.sha256 } }); + expect(await readFile(path.join(f.context.destination, "tools/run"), "utf8")).toBe("saved\n"); + await verifyRecordedBundle(f.context.history, f.sha256, f.context.destination); + await writeFile(path.join(f.context.destination, "tools/run"), "other\n"); + await expect(verifyRecordedBundle(f.context.history, f.sha256, f.context.destination)).rejects.toThrow("Recorded bundle entry mismatch"); + }); + + it.each(["missing", "mismatch"])("falls back approximately when a recorded digest is %s even with a pinned source commit", async (kind) => { + const f = await fixture(), repository = path.join(f.historyRoot, "git/agent%3Awriter/tools"); + await initTestGit(repository); + const commit = await commitTestGit(repository, { "source.txt": "historical source\n" }, EARLY); + f.context.input.declaration = { id: "tools", kind: "bundle", mode: "readonly", mount: "./tools", build: { files: { root: repository, ref: commit } } }; + if (kind === "mismatch") await writeFile(path.join(f.historyRoot, `bundles/${f.sha256.slice(7)}.tar`), Buffer.concat([f.bytes, Buffer.alloc(512)])); + const result = await materializeGit(f.context); + expect(result).toMatchObject({ fidelity: "approximate", source: { commit }, notes: expect.arrayContaining([expect.stringContaining("could not be reproduced")]) }); + expect(await readFile(path.join(f.context.destination, "source.txt"), "utf8")).toBe("historical source\n"); + }); + + it("refuses a prebuilt resource when its recorded archive is unavailable", async () => { + const f = await fixture(); + await expect(materializeGit(f.context)).rejects.toThrow("could not be reproduced"); + }); + + it.each(["escape", "symlink", "hardlink", "device"])("refuses a digest-matching tar with a %s entry before writing output", async (kind) => { + const f = await fixture(), bytes = Buffer.from(f.bytes); + if (kind === "escape") { bytes.fill(0, 0, 100); bytes.write("../../escape", 0); } + else bytes[156] = { symlink: 50, hardlink: 49, device: 51 }[kind]!; + bytes.fill(32, 148, 156); + const sum = bytes.subarray(0, 512).reduce((total, byte) => total + byte, 0); + bytes.write(`${sum.toString(8).padStart(6, "0")}\0 `, 148, "ascii"); + const sha256 = `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + f.context.record!.resources = [{ id: "tools", kind: "bundle", sha256 }]; + await writeFile(path.join(f.historyRoot, `bundles/${sha256.slice(7)}.tar`), bytes); + await expect(materializeGit(f.context)).rejects.toThrow(/unsafe tar entry/); + await expect(readdir(f.context.destination)).rejects.toThrow(); + }); +}); diff --git a/src/compiler/training/rewind/recordedBundle.ts b/src/compiler/training/rewind/recordedBundle.ts new file mode 100644 index 00000000..0b608565 --- /dev/null +++ b/src/compiler/training/rewind/recordedBundle.ts @@ -0,0 +1,54 @@ +import { createHash } from "node:crypto"; +import { lstat, readFile, readdir } from "node:fs/promises"; +import path from "node:path"; +import { validateWorkspaceBundleTar } from "../../workspaceBundleArtifacts.js"; +import { recordedSha256Schema } from "../../../recording/index.js"; +import { extractBundleBytes } from "./gitTree.js"; +import type { History } from "./history.js"; + +const recordedBytes = async (history: History | undefined, sha256: string): Promise => { + recordedSha256Schema.parse(sha256); + const bytes = await history?.bytes(`bundles/${sha256.slice(7)}.tar`); + if (!bytes || bytes.length > 536_870_912 || `sha256:${createHash("sha256").update(bytes).digest("hex")}` !== sha256) return null; + return bytes; +}; + +export const materializeRecordedBundle = async (history: History | undefined, sha256: string, destination: string): Promise => { + const bytes = await recordedBytes(history, sha256); + if (!bytes) return false; + await extractBundleBytes(bytes, destination); + return true; +}; + +/** Acceptance reopens the digest-pinned archive and compares disk bytes, without trusting producer metadata. */ +export const verifyRecordedBundle = async (history: History | undefined, sha256: string, directory: string): Promise => { + const bytes = await recordedBytes(history, sha256); + if (!bytes) throw new Error(`Recorded bundle ${sha256} could not be reproduced`); + validateWorkspaceBundleTar(bytes); + const text = (field: Buffer) => field.toString("utf8").replace(/\0.*$/su, ""); + const files = new Map(), directories = new Set(); + for (let offset = 0; offset < bytes.length && !bytes.subarray(offset, offset + 512).every((byte) => byte === 0);) { + const header = bytes.subarray(offset, offset + 512), prefix = text(header.subarray(345, 500)); + const name = `${prefix ? `${prefix}/` : ""}${text(header.subarray(0, 100))}`.replace(/\/$/u, ""); + const size = Number.parseInt(text(header.subarray(124, 136)).trim(), 8); + for (let parent = path.posix.dirname(name); parent !== "."; parent = path.posix.dirname(parent)) directories.add(parent); + if (header[156] === 53) directories.add(name); + else files.set(name, { bytes: bytes.subarray(offset + 512, offset + 512 + size), executable: Boolean(Number.parseInt(text(header.subarray(100, 108)).trim(), 8) & 0o111) }); + offset += 512 + Math.ceil(size / 512) * 512; + } + const walk = async (folder: string, prefix = ""): Promise => { + for (const name of await readdir(folder)) { + const relative = prefix ? `${prefix}/${name}` : name, full = path.join(folder, name), stats = await lstat(full); + if (stats.isDirectory() && directories.delete(relative)) await walk(full, relative); + else { + const expected = files.get(relative); + if (!stats.isFile() || !expected || Boolean(stats.mode & 0o111) !== expected.executable || !(await readFile(full)).equals(expected.bytes)) { + throw new Error(`Recorded bundle entry mismatch: ${relative}`); + } + files.delete(relative); + } + } + }; + await walk(directory); + if (files.size || directories.size) throw new Error(`Missing recorded bundle entries: ${[...files.keys(), ...directories].join(", ")}`); +}; diff --git a/src/compiler/training/rewind/recordedCommit.test.ts b/src/compiler/training/rewind/recordedCommit.test.ts new file mode 100644 index 00000000..b946b0e2 --- /dev/null +++ b/src/compiler/training/rewind/recordedCommit.test.ts @@ -0,0 +1,237 @@ +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { momentRecordSchema } from "../../../recording/index.js"; +import { materializeFeed } from "./feed.js"; +import { materializeGit } from "./git.js"; +import { commitTestGit, gitTestRoot, initTestGit, testGit, testMomentRecord } from "./git.test-helper.js"; +import { History } from "./history.js"; +import { resolveRecordedCommit } from "./recordedCommit.js"; +import type { InputContext } from "./types.js"; +import { resolveFeedSource, isFeedLanding } from "../../../volume/index.js"; + +const AT = "2026-10-09T08:20:31Z", EARLY = "2026-10-09T08:00:00Z", REVISION = "e".repeat(64); +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const landing = (commit: string) => ({ version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "org-news", source: { kind: "git", repo: "recorded-repo" }, revision: REVISION, tree: `trees/${REVISION}`, commit, ref: "main", paths: [], switching_at: EARLY, landed_at: EARLY }); +const feedRecord = (commit: string, revision = REVISION) => ({ id: "news", revision, tree: `trees/${revision}`, commit, paths: [], landedAt: EARLY }); +const recordedValue = (field: "resources" | "feeds" | "landing", commit: string): string | undefined => { + if (field === "landing") { + const row = landing(commit); + if (!isFeedLanding(row)) throw new Error("Expected a full 40- or 64-hex commit object id in a valid landing"); + return row.commit; + } + const value = momentRecordSchema.parse({ ...testMomentRecord(), [field]: field === "resources" + ? [{ id: "archive", kind: "git", commit }] + : [{ id: "news", revision: REVISION, tree: `trees/${REVISION}`, commit, paths: [], landedAt: EARLY }] }); + const entry = value[field][0]!; + return "commit" in entry ? entry.commit : undefined; +}; +const fixture = async () => { + const root = await gitTestRoot(); roots.push(root); + const repository = path.join(root, "history/git/agent%3Awriter/archive"); + await initTestGit(repository); + const past = await commitTestGit(repository, { "readme.txt": "recorded bytes\n" }, EARLY); + testGit(repository, ["tag", "--annotate", "record-tag", "--message", "not a commit object", past]); + const objects = { blob: testGit(repository, ["rev-parse", `${past}:readme.txt`]), + tree: testGit(repository, ["rev-parse", `${past}^{tree}`]), tag: testGit(repository, ["rev-parse", "refs/tags/record-tag"]) }; + const context: InputContext = { at: AT, destination: path.join(root, "output"), history: new History(path.join(root, "history")), + record: { ...testMomentRecord(), resources: [{ id: "archive", kind: "git", commit: past }] }, + input: { section: "resources", declaredBy: "agent:writer", id: "archive", declarationBase: root, overridden: false, + declaration: { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: repository } } }; + return { root, repository, past, objects, context }; +}; +const retainedFeed = async () => { + const source = await fixture(); + const directory = path.join(source.root, "history/feeds/agent%3Awriter/news"), repository = path.join(directory, "repo"); + await mkdir(directory, { recursive: true }); + testGit(source.root, ["clone", "--quiet", "--bare", source.repository, repository]); + const revision = resolveFeedSource({ kind: "git", repo: repository, ref: source.past, fetch: false }).revision; + const retained = path.join(directory, "trees", revision); + await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "readme.txt"), "recorded bytes\n"); + const context: InputContext = { at: AT, destination: path.join(source.root, "feed-output"), history: source.context.history, record: null, + input: { section: "resources", declaredBy: "agent:writer", id: "news", declarationBase: source.root, overridden: false, + declaration: { id: "news", kind: "volume", mode: "mutable", mount: "./news", name: "org-news", + feed: { git: { repo: "never-fetch" }, prepare: { host: true, command: ["never-run"] } } } } }; + const log = async (commit: string) => writeFile(path.join(directory, "landings.jsonl"), `${JSON.stringify({ ...landing(commit), revision, tree: `trees/${revision}` })}\n`); + return { ...source, repository, directory, context, log, revision }; +}; + +describe("recorded commit contracts", () => { + it("requires the merged landing contract's lowercase full commit IDs", () => { + for (const length of [40, 64]) { + expect(recordedValue("landing", "a".repeat(length))).toBe("a".repeat(length)); + expect(() => recordedValue("landing", "A".repeat(length))).toThrow(); + } + }); + it.each(["resources", "feeds"] as const)("normalizes full SHA-1 and SHA-256 %s commits to lowercase", (field) => { + for (const length of [40, 64]) expect(recordedValue(field, "AbCDef12".repeat(length / 8))).toBe("abcdef12".repeat(length / 8)); + }); + + it.each(["resources", "feeds", "landing"] as const)("rejects moving selectors, abbreviated and malformed %s commit IDs", (field) => { + for (const value of ["HEAD", "main", "refs/tags/old", "abcdef1", "a".repeat(39), "a".repeat(41), "a".repeat(63), "a".repeat(65), "g".repeat(40)]) { + expect(() => recordedValue(field, value), `${field}: ${value}`).toThrow("Expected a full 40- or 64-hex commit object id"); + } + }); + + it("directory landing logs omit git commit fields", () => { + const { commit: _commit, ref: _ref, paths: _paths, ...entry } = landing("a".repeat(40)); + expect(isFeedLanding({ ...entry, source: { kind: "directory", directory: "/source" } })).toBe(true); + }); +}); + +describe("recorded git commit objects", () => { + const kind = "git" as const; + it("materializes the full recorded ID case-insensitively", async () => { + const { past, context } = await fixture(); + context.record!.resources[0] = { id: "archive", kind: "git", commit: past.toUpperCase() }; + expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: past } }); + expect(await readFile(path.join(context.destination, "readme.txt"), "utf8")).toBe("recorded bytes\n"); + }); + + it.each(["blob", "tree", "tag"] as const)("refuses a recorded %s object without peeling it", async (type) => { + const { objects, context } = await fixture(); + context.record!.resources[0] = { id: "archive", kind: "git", commit: objects[type] }; + await expect(materializeGit(context)).rejects.toThrow(`Recorded object ${objects[type]} must be a commit object; found ${type}`); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it("refuses moving and abbreviated records even when their selectors resolve", async () => { + const { past, context } = await fixture(); + for (const commit of ["HEAD", past.slice(0, 12)]) { + context.record!.resources[0] = { id: "archive", kind: "git", commit }; + await expect(materializeGit(context)).rejects.toThrow("Expected a full 40- or 64-hex commit object id"); + } + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + const pin = (context: InputContext, commit: string) => { + if (context.input.section !== "resources" || context.input.declaration.kind === "volume") throw new Error("Expected git-backed resource"); + context.input.declaration = kind === "git" ? { ...context.input.declaration, kind, url: "never-read", ref: commit } + : { id: "archive", kind, mode: "readonly", mount: "./archive", build: { files: { root: ".", ref: commit } } }; + }; + + it.each(["blob", "tree", "tag"] as const)("refuses a matching %s record despite a valid declared pin", async (type) => { + const { context, past, objects } = await fixture(); + pin(context, past); context.record!.resources[0] = { id: "archive", kind: "git", commit: objects[type] }; + await expect(materializeGit(context)).rejects.toThrow(`Recorded object ${objects[type]} must be a commit object; found ${type}`); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it("keeps pin precedence after verifying a different valid recorded commit", async () => { + const { context, repository, past } = await fixture(); + const next = await commitTestGit(repository, { "readme.txt": "other recorded commit\n" }, "2026-10-09T08:10:00Z"); + pin(context, past); context.record!.resources[0] = { id: "archive", kind: "git", commit: next.toUpperCase() }; + expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "pinned", source: { commit: past } }); + expect(await readFile(path.join(context.destination, "readme.txt"), "utf8")).toBe("recorded bytes\n"); + }); + + it("does not verify unrelated recorded inputs", async () => { + const { context, past, objects } = await fixture(); + pin(context, past); context.record!.resources[0] = { id: "other", kind, commit: objects.blob }; + expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "pinned", source: { commit: past } }); + }); +}); + +describe("recorded commit validation authority", () => { + it("requires history for a matching pin record even if the local declared repository has the commit", async () => { + const { context, repository, past } = await fixture(); + context.history = undefined; + context.input.declaration = { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: repository, ref: past }; + await expect(materializeGit(context)).rejects.toThrow("Missing git history for recorded resource archive"); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it("does not verify the replaced input's recorded commit for an override", async () => { + const { context, repository, past, objects } = await fixture(); + context.input.overridden = true; context.history = undefined; context.record!.resources[0] = { id: "archive", kind: "git", commit: objects.blob }; + context.candidateRecords = [context.record!]; + context.input.declaration = { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: repository, ref: past }; + expect(await materializeGit(context)).toMatchObject({ fidelity: "overridden", source: { commit: past } }); + }); +}); + +describe("retained feed recorded commit objects", () => { + it("normalizes moment IDs before comparing with lowercase landings and reports the commit object", async () => { + const { context, past, log, revision } = await retainedFeed(); + await log(past); + context.record = { ...testMomentRecord(), feeds: [feedRecord(past.toUpperCase(), revision)] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", source: { commit: past } }); + expect(await readFile(path.join(context.destination, "current/readme.txt"), "utf8")).toBe("recorded bytes\n"); + }); + + it.each(["blob", "tree", "tag"] as const)("refuses a retained tree whose recorded commit is a %s", async (type) => { + const { context, objects, log } = await retainedFeed(); + await log(objects[type]); + await expect(materializeFeed(context)).rejects.toThrow(`Recorded object ${objects[type]} must be a commit object; found ${type}`); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it("refuses retained bytes when a recorded commit is absent from its history repository", async () => { + const { context, log } = await retainedFeed(); + await log("0".repeat(40)); + await expect(materializeFeed(context)).rejects.toThrow("Rewind git cat-file failed"); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it("requires the history repository even when retained tree bytes exist", async () => { + const { context, past, repository, log } = await retainedFeed(); + await log(past); await rm(repository, { recursive: true }); + await expect(materializeFeed(context)).rejects.toThrow("Missing feed source repository for news"); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it.each(["blob", "tree", "tag"] as const)("validates an unselected landing's %s commit object", async (type) => { + const { context, past, objects, directory } = await retainedFeed(); + const entries = [landing(past), { ...landing(objects[type]), landed_at: "2026-10-09T09:00:00Z" }]; + await writeFile(path.join(directory, "landings.jsonl"), `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`); + await expect(materializeFeed(context)).rejects.toThrow(`Recorded object ${objects[type]} must be a commit object; found ${type}`); + await expect(readdir(context.destination)).rejects.toThrow(); + }); +}); + +describe("feed moment records without landing logs", () => { + const fallbackFeed = async () => { + const value = await retainedFeed(), input = value.context.input; + if (input.section !== "resources" || input.declaration.kind !== "volume") throw new Error("Expected volume feed"); + input.declaration.feed = { git: { repo: "never-fetch", ref: "main" } }; + return value; + }; + + it.each(["blob", "tree", "tag"] as const)("refuses matching %s moment commits before feed-rule fallback", async (type) => { + const { context, objects } = await fallbackFeed(); + context.record = { ...testMomentRecord(), feeds: [feedRecord(objects[type])] }; + await expect(materializeFeed(context)).rejects.toThrow(`Recorded object ${objects[type]} must be a commit object; found ${type}`); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it("uses a verified moment commit without a landing log", async () => { + const { context, past, revision } = await fallbackFeed(); + context.record = { ...testMomentRecord(), feeds: [feedRecord(past.toUpperCase(), revision)] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: past } }); + }); + + it("does not verify unrelated feed moment records", async () => { + const { context, objects, past } = await fallbackFeed(); + context.record = { ...testMomentRecord(), feeds: [{ ...feedRecord(objects.blob), id: "other" }] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", source: { commit: past } }); + }); + + it("does not verify the replaced feed's moment record for an override", async () => { + const { context, objects } = await fallbackFeed(); + context.input.overridden = true; context.history = undefined; + context.record = { ...testMomentRecord(), feeds: [feedRecord(objects.blob)] }; + context.candidateRecords = [context.record]; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "overridden", basis: "override", source: {} }); + }); +}); + +describe("recorded commit object format", () => { + it("verifies full SHA-256 commit IDs without changing object format", async () => { + const root = await gitTestRoot(); roots.push(root); + testGit(root, ["init", "--quiet", "--initial-branch=main", "--template=", "--object-format=sha256"]); + const commit = await commitTestGit(root, { "readme.txt": "SHA-256\n" }, EARLY); + expect(commit).toHaveLength(64); + expect(await resolveRecordedCommit(root, commit.toUpperCase())).toBe(commit); + }); +}); diff --git a/src/compiler/training/rewind/recordedCommit.ts b/src/compiler/training/rewind/recordedCommit.ts new file mode 100644 index 00000000..466fc957 --- /dev/null +++ b/src/compiler/training/rewind/recordedCommit.ts @@ -0,0 +1,10 @@ +import { recordedCommitSchema } from "../../../recording/index.js"; +import { rewindGit } from "./gitProcess.js"; + +/** Recorded identity names the commit object itself, never a selector or a peeled tag. */ +export const resolveRecordedCommit = async (repository: string, value: string): Promise => { + const commit = recordedCommitSchema.parse(value); + const type = (await rewindGit(repository, ["cat-file", "-t", commit])).toString().trim(); + if (type !== "commit") throw new Error(`Recorded object ${commit} must be a commit object; found ${type}`); + return commit; +}; diff --git a/src/compiler/training/rewind/recordingFormat.test.ts b/src/compiler/training/rewind/recordingFormat.test.ts new file mode 100644 index 00000000..dc4fa8be --- /dev/null +++ b/src/compiler/training/rewind/recordingFormat.test.ts @@ -0,0 +1,159 @@ +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { momentRecordSchema, momentFileName, withMomentStore, writeMomentRecord, type WakeMomentRow } from "../../../recording/index.js"; +import * as rewindContract from "./contract.js"; +import { AT, BEFORE, emptyRecord, jsonl, memoryEvent, rewindFixture } from "./fixtures.test-helper.js"; +import { gitTestRoot } from "./git.test-helper.js"; +import { History } from "./history.js"; +import { materializeTrainingMoment } from "./materialize.js"; +import { selectMomentRecord } from "./select.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const recorded = () => ({ + version: "spawnfile.moment-record.v1" as const, agent: "agent:a", + wake: { at: AT, startedAt: AT, executionId: "execution-17", + deliveries: [{ deliveryId: "delivery-17", kind: "schedule", occurredAt: AT }] }, + recordedAt: AT, exact: true, resources: [], feeds: [], memory: [], moltnet: [], volumes: [] +}); + +describe("P3 recording format", () => { + it("leaves the moment record schema owned only by the recording barrel", () => { + expect(rewindContract).not.toHaveProperty("momentRecordSchema"); + }); + it("accepts the recording shape and rejects the superseded sketch", () => { + const value = { ...recorded(), exact: false, errors: ["volume state: capture failed"], + resources: [{ id: "git", kind: "git", commit: "a".repeat(40) }, { id: "bundle", kind: "bundle", sha256: `sha256:${"b".repeat(64)}` }], + feeds: [{ id: "directory", revision: "c".repeat(64), tree: `trees/${"c".repeat(64)}.1`, landedAt: BEFORE }], + memory: [{ id: "self", declaredBy: "agent:a", seq: 0 }], + moltnet: [{ network: "news", cursor: AT, messageIds: ["wake-message"] }] }; + expect(momentRecordSchema.parse(value)).toEqual(value); + for (const invalid of [{ ...value, exact: undefined }, { ...value, wake: { at: AT } }, + { ...value, resources: [{ id: "bundle", kind: "bundle", commit: "a".repeat(40) }] }, + { ...value, moltnet: [{ network: "news", cursor: "message-id" }] }]) { + expect(momentRecordSchema.safeParse(invalid).success).toBe(false); + } + }); + + it("selects the original attempt by earliest startedAt, not capture or append order", () => { + const first = { ...recorded(), memory: [{ id: "self", declaredBy: "agent:a", seq: 1 }] }; + const replay = { ...first, wake: { ...first.wake, at: "2026-10-09T10:20:31+02:00", startedAt: "2026-10-09T09:00:00Z" }, + memory: [{ id: "self", declaredBy: "agent:a", seq: 9 }] }; + expect(selectMomentRecord([replay, first], "agent:a", AT)).toBe(first); + }); + + it("reads nested per-file JSON moments as well as JSONL", async () => { + const root = await gitTestRoot(); roots.push(root); + await mkdir(path.join(root, "moments/a/day"), { recursive: true }); + await writeFile(path.join(root, "moments/a/day/turn.json"), JSON.stringify(recorded())); + await jsonl(root, "moments/other/turns.jsonl", [{ ...recorded(), agent: "agent:other" }]); + expect((await new History(root).records()).map((entry) => entry.agent)).toEqual(["agent:a", "agent:other"]); + }); +}); + +describe("recording pins in a materialized trial", () => { + it.each([false, true])("consumes equal executionId+startedAt replay files and refuses differing decisions=%s", async (conflicting) => { + const f = await rewindFixture(); roots.push(f.root); + const first = emptyRecord(); first.memory = [{ id: "self", declaredBy: "agent:a", seq: 0 }]; + const second = structuredClone(first); + second.wake.deliveries[0]!.deliveryId = "replay-delivery"; + if (conflicting) second.memory[0]!.seq = 1; + const row = (record: typeof first): WakeMomentRow => ({ version: "noopolis.daimon.wake-moment.v1", + agent_id: record.agent, execution_id: record.wake.executionId, started_at: record.wake.startedAt, + deliveries: record.wake.deliveries.map((delivery) => ({ acceptance_id: delivery.deliveryId, + delivery_id: delivery.deliveryId, kind: delivery.kind, occurred_at: delivery.occurredAt })), snapshots: [] }); + const slug = f.project.compilePlan.nodes.find((node) => node.id === f.plan.agent.id)!.slug; + await withMomentStore(path.join(f.historyDir, "record.json"), slug, async (directory) => { + for (const record of [second, first]) expect(await writeMomentRecord(directory, record, row(record))).toBe(true); + expect(await readdir(directory)).toEqual(expect.arrayContaining([momentFileName(row(first)), momentFileName(row(second))])); + }); + expect(momentFileName(row(first))).not.toBe(momentFileName(row(second))); + const result = await materializeTrainingMoment(f.options); + if (conflicting) { + expect(result.status).toBe("refused"); + expect(result.inputs.find((input) => input.kind === "memory")!.refusal).toMatch(/Conflicting moment records.*memory agent:a\/self/u); + expect(await readdir(f.options.out)).toEqual(["moment.json"]); + } else { + expect(result.status, JSON.stringify(result.inputs)).toBe("materialized"); + expect(result.record).toMatchObject({ matched: true, attempts: 2, executionId: first.wake.executionId, selectedStartedAt: AT }); + expect(result.inputs.find((input) => input.kind === "memory")).toMatchObject({ fidelity: "exact", source: { seq: 0 } }); + } + }); + + it.each(["json", "jsonl"])("applies shared recording validation to .%s discovery", async (suffix) => { + const root = await gitTestRoot(); roots.push(root); + const record = { ...recorded(), memory: [{ id: "self", declaredBy: "agent:a", seq: Number.MAX_SAFE_INTEGER + 1 }] }; + await mkdir(path.join(root, "moments/a"), { recursive: true }); + await writeFile(path.join(root, `moments/a/invalid.${suffix}`), `${JSON.stringify(record)}\n`); + await expect(new History(root).records()).rejects.toThrow(/Invalid history moments\/a\/invalid/u); + }); + + it.each([false, true])("refuses conflicting copies instead of choosing the empty pin or nonempty fallback (reverse=%s)", async (reverse) => { + const f = await rewindFixture(); roots.push(f.root); + const healthy = emptyRecord(); healthy.memory = [{ id: "self", declaredBy: "agent:a", seq: 0 }]; + const errored = structuredClone(healthy); errored.exact = false; errored.errors = ["memory self: sequence capture failed"]; + await jsonl(f.historyDir, "memory/agent%3Aa/self/events.jsonl", [memoryEvent(1)]); + await jsonl(f.historyDir, "moments/copies.jsonl", reverse ? [errored, healthy] : [healthy, errored]); + const result = await materializeTrainingMoment(f.options); + expect(result.status).toBe("refused"); + expect(result.inputs.find((input) => input.id === "self")!.refusal).toMatch(/Conflicting moment records.*memory agent:a\/self/u); + expect(await readdir(f.options.out)).toEqual(["moment.json"]); + }); + it.each([false, true])("accepts incoming feed publication metadata after the bound with record=%s", async (withRecord) => { + const f = await rewindFixture(); roots.push(f.root); + const log = "feeds/team%3Apublication/news/landings.jsonl"; + const landing = JSON.parse((await readFile(path.join(f.historyDir, log), "utf8")).trim()); + landing.switching_at = AT; landing.landed_at = "2026-10-09T08:20:32Z"; + if (withRecord) { + const record = emptyRecord(); record.exact = false; record.errors = ["volume edition-state: snapshot failed"]; + record.feeds = [{ id: "news", revision: landing.revision, tree: landing.tree, commit: f.commit, ref: "main", landedAt: landing.landed_at }]; + await jsonl(f.historyDir, "moments/incoming.jsonl", [record]); + await rm(path.join(f.historyDir, log)); + } else await jsonl(f.historyDir, log, [landing]); + const result = await materializeTrainingMoment(f.options); + expect(result, JSON.stringify(result.inputs)).toMatchObject({ status: "materialized", leakCheck: { ok: true } }); + expect(result.inputs.find((input) => input.kind === "feed")).toMatchObject({ fidelity: withRecord ? "exact" : "approximate", + basis: withRecord ? "record" : "landing-log", source: { landedAt: landing.landed_at } }); + }); + + it("uses startedAt for task time and leak bounds, reports original selection and keeps wake at", async () => { + const f = await rewindFixture(); roots.push(f.root); + const first = emptyRecord(); first.wake.startedAt = "2026-10-09T08:21:00Z"; + first.memory = [{ id: "self", declaredBy: "agent:a", seq: 2 }]; + const replay = structuredClone(first); replay.wake.startedAt = "2026-10-09T09:00:00Z"; replay.memory[0]!.seq = 3; + await jsonl(f.historyDir, "moments/attempts.jsonl", [replay, first]); + const result = await materializeTrainingMoment(f.options); + expect(result, JSON.stringify(result.inputs)).toMatchObject({ status: "materialized", at: AT, effectiveAt: first.wake.startedAt, + record: { matched: true, attempts: 2, selectedStartedAt: first.wake.startedAt, selection: "earliest-startedAt" } }); + expect(result.inputs.find((input) => input.kind === "memory")).toMatchObject({ fidelity: "exact", source: { seq: 2 } }); + expect(JSON.parse(await readFile(path.join(f.options.out, "task-clock.json"), "utf8")).origin).toBe(first.wake.startedAt); + expect(JSON.parse(await readFile(path.join(f.options.out, "leak-check.json"), "utf8"))).toMatchObject({ at: first.wake.startedAt, ok: true }); + }); + + it("keeps healthy pins exact in an inexact record and annotates absent pins", async () => { + const f = await rewindFixture(); roots.push(f.root); + const record = emptyRecord(); record.exact = false; record.errors = ["volume edition-state: snapshot failed"]; + record.resources = [{ id: "code", kind: "git", commit: f.commit }]; + record.memory = [{ id: "self", declaredBy: "agent:a", seq: 0 }]; + await jsonl(f.historyDir, "moments/inexact.jsonl", [record]); + const result = await materializeTrainingMoment(f.options); + expect(result.status).toBe("materialized"); + expect(result.inputs.find((input) => input.id === "code")).toMatchObject({ fidelity: "exact", basis: "record" }); + expect(result.inputs.find((input) => input.id === "self")).toMatchObject({ fidelity: "exact", basis: "record", source: { seq: 0 } }); + expect(result.inputs.find((input) => input.id === "edition-state")).toMatchObject({ fidelity: "approximate", + notes: expect.arrayContaining(record.errors) }); + }); + + it("does not trust a present pin named by the record errors", async () => { + const f = await rewindFixture(); roots.push(f.root); + const record = emptyRecord(); record.exact = false; record.errors = ["memory self: sequence capture failed"]; + record.memory = [{ id: "self", declaredBy: "agent:a", seq: 0 }]; + await jsonl(f.historyDir, "moments/inexact.jsonl", [record]); + await jsonl(f.historyDir, "memory/agent%3Aa/self/events.jsonl", [memoryEvent(1)]); + const result = await materializeTrainingMoment(f.options); + expect(result.status).toBe("materialized"); + expect(result.inputs.find((input) => input.id === "self")).toMatchObject({ fidelity: "approximate", basis: "created-at-prefix", + source: { seq: 1 }, notes: expect.arrayContaining(record.errors) }); + }); +}); diff --git a/src/compiler/training/rewind/select.test.ts b/src/compiler/training/rewind/select.test.ts new file mode 100644 index 00000000..9143738c --- /dev/null +++ b/src/compiler/training/rewind/select.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, it } from "vitest"; +import type { TrainingPlan } from "../plan/index.js"; +import type { MomentRecord } from "../../../recording/index.js"; +import { selectMomentRecord, selectTrainingExample } from "./select.js"; + +const at = "2026-10-09T08:20:31Z"; +const record = (): MomentRecord => ({ + version: "spawnfile.moment-record.v1", agent: "agent:writer", + wake: { at, startedAt: at, executionId: "execution-17", deliveries: [{ deliveryId: "delivery-17", kind: "message", occurredAt: at }], surface: "moltnet", room: "assignment", from: "brass" }, recordedAt: at, exact: true, + resources: [{ id: "repo", kind: "git", commit: "a".repeat(40) }], + feeds: [{ id: "news", revision: "rev", tree: "tree", commit: "b".repeat(40), paths: ["published"], landedAt: at }], + memory: [{ id: "self", declaredBy: "agent:writer", seq: 2, generation: 0 }], + moltnet: [{ network: "network", cursor: at }], volumes: [{ id: "state", snapshot: "snapshot" }] +}); +const dataset = (name: string, id: string) => ({ sourcePath: `/project/${name}.yaml`, path: `${name}.yaml`, + examples: [{ id, at, wake: { surface: "moltnet" as const, room: "assignment", from: "brass", text: "write" }, overrides: [] }] }); +const plan = (): TrainingPlan => ({ + version: "spawnfile.training-plan.v1", producer: { package: "spawnfile", version: "test" }, + project: { root: "/project", manifest: "/project/Spawnfile" }, agent: { id: "agent:writer", name: "writer" }, + record: null, optimizer: null, graders: [], targets: [{ declaredBy: "agent:writer", grader: "writing", + files: [{ sourcePath: "/project/AGENTS.md", path: "AGENTS.md", consumers: ["agent:writer"] }], + datasets: { train: dataset("train", "example"), validation: null, test: dataset("test", "held-out") } + }] +}); + +describe("pure rewind record selection", () => { + it("matches equal instants across offsets and ignores capture time and unused wake payload", () => { + const first = record(), second = structuredClone(first); + second.wake = { ...second.wake, at: "2026-10-09T10:20:31+02:00", room: "other", messageId: "later-capture" }; + second.recordedAt = "2026-10-11T00:00:00Z"; + second.feeds[0]!.landedAt = "2026-10-09T10:20:31.000000000+02:00"; + expect(selectMomentRecord([first, second], "agent:writer", "2026-10-09T03:20:31-05:00")).toBe(first); + }); + + it("requires exact agent and wake instant, including sub-millisecond differences", () => { + expect(selectMomentRecord([record()], "writer", at)).toBeNull(); + expect(selectMomentRecord([record()], "agent:writer", "2026-10-09T08:20:31.000000001Z")).toBeNull(); + expect(selectMomentRecord([], "agent:writer", at)).toBeNull(); + }); + + it.each([false, true])("refuses effective pin/fallback disagreements in either order (reverse=%s)", (reverse) => { + for (const section of ["memory", "resources", "networks"] as const) { + const id = section === "memory" ? "self" : section === "resources" ? "news" : "network"; + const first = record(), second = record(); + second.exact = false; second.errors = [`${section} ${id}: capture failed`]; + const copies = reverse ? [second, first] : [first, second]; + expect(() => selectMomentRecord(copies, "agent:writer", at, [{ section, id, declaredBy: "agent:writer" }])) + .toThrow(new RegExp(`Conflicting moment records.*${section} agent:writer/${id}.*effective pin/fallback`, "u")); + } + }); + + it.each([false, true])("compares fallback fidelity when exact differs (reverse=%s)", (reverse) => { + const first = record(), second = record(); second.exact = false; + const inputs = [{ section: "resources" as const, id: "absent-feed", declaredBy: "agent:writer" }]; + expect(() => selectMomentRecord(reverse ? [second, first] : [first, second], "agent:writer", at, inputs)) + .toThrow(/absent-feed.*effective pin\/fallback/u); + }); + + it("ignores unrelated errors and exact flags for healthy pins, and compares usable rather than errored pins", () => { + const first = record(), second = record(); + second.exact = false; second.errors = ["volume unrelated: failed", "memory self (team:other): failed"]; + const inputs = [{ section: "memory" as const, id: "self", declaredBy: "agent:writer" }]; + expect(selectMomentRecord([first, second], "agent:writer", at, inputs)).toBe(first); + first.exact = false; first.errors = ["memory self: failed"]; + second.errors = ["memory self: another failure"]; second.memory[0]!.seq = 99; + expect(selectMomentRecord([first, second], "agent:writer", at, inputs)).toBe(first); + }); + + const changes: Array<[string, (value: MomentRecord) => void]> = [ + ["resource commit", (value) => { value.resources[0] = { id: "repo", kind: "git", commit: "c".repeat(40) }; }], + ["feed revision", (value) => { value.feeds[0]!.revision = "different"; }], + ["feed paths", (value) => { value.feeds[0]!.paths = ["private"]; }], + ["memory sequence", (value) => { value.memory[0]!.seq = 3; }], + ["memory generation", (value) => { value.memory[0]!.generation = 1; }], + ["network cursor", (value) => { value.moltnet[0]!.cursor = "2026-10-10T00:00:00Z"; }], + ["volume snapshot", (value) => { value.volumes[0]!.snapshot = "different"; }] + ]; + it.each(changes)("refuses conflicting %s fields", (_name, change) => { + const first = record(), second = structuredClone(first); + change(second); + expect(() => selectMomentRecord([first, second], "agent:writer", at)).toThrow(/Conflicting moment records.*agent:writer/u); + }); + + it("compares only inputs being used and retains memory declaring-node identity", () => { + const first = record(), second = structuredClone(first); + second.resources[0] = { id: "repo", kind: "git", commit: "c".repeat(40) }; + second.memory.push({ id: "self", declaredBy: "team:other", generation: 9, seq: 88 }); + const selected = [{ section: "memory" as const, id: "self", declaredBy: "agent:writer" }]; + expect(selectMomentRecord([first, second], "agent:writer", at, selected)).toBe(first); + second.memory[0]!.seq = 3; + expect(() => selectMomentRecord([first, second], "agent:writer", at, selected)).toThrow(/Conflicting/u); + }); + + it("accepts reordered record entries without mutating its inputs", () => { + const first = record(); + first.resources.push({ id: "another", kind: "bundle", sha256: `sha256:${"d".repeat(64)}` }); + const second = structuredClone(first); + second.resources.reverse(); + const before = structuredClone([first, second]); + expect(selectMomentRecord([first, second], "agent:writer", at)).toBe(first); + expect([first, second]).toEqual(before); + }); + + it.each(["git", "bundle"] as const)("refuses tied %s pins with different activation fidelity", (kind) => { + const first = record(), second = structuredClone(first); + first.resources = second.resources = [kind === "git" ? { id: "repo", kind, commit: "a".repeat(40) } + : { id: "repo", kind, sha256: `sha256:${"a".repeat(64)}` }]; + second.exact = false; second.errors = ["Deployment activation is non-exact"]; + expect(() => selectMomentRecord([first, second], "agent:writer", at)).toThrow("different effective pin/fallback decisions"); + }); + + it("refuses conflicting bundle digests even when the declared source has a full commit pin", () => { + const first = record(), second = record(); + first.resources = [{ id: "tools", kind: "bundle", sha256: `sha256:${"a".repeat(64)}` }]; + second.resources = [{ id: "tools", kind: "bundle", sha256: `sha256:${"b".repeat(64)}` }]; + expect(() => selectMomentRecord([first, second], "agent:writer", at, [{ + section: "resources", id: "tools", declaredBy: "agent:writer", overridden: false, declarationBase: "/project", + declaration: { id: "tools", kind: "bundle", mode: "readonly", mount: "./tools", build: { files: { root: ".", ref: "c".repeat(40) } } } + }])).toThrow("Conflicting moment records"); + }); + + it("does not compare record fields superseded by a full commit pin or another resource kind", () => { + const first = record(), second = structuredClone(first); + second.resources[0] = { id: "repo", kind: "git", commit: "c".repeat(40) }; + second.feeds[0]!.revision = "other"; + expect(selectMomentRecord([first, second], "agent:writer", at, [{ + section: "resources", id: "repo", declaredBy: "agent:writer", overridden: false, declarationBase: "/project", + declaration: { id: "repo", kind: "git", mode: "readonly", mount: "./repo", url: "/repo", ref: "a".repeat(40) } + }, { + section: "resources", id: "news", declaredBy: "agent:writer", overridden: false, declarationBase: "/project", + declaration: { id: "news", kind: "volume", mode: "mutable", mount: "./news" } + }])).toBe(first); + }); +}); + +describe("pure rewind example selection", () => { + it.each(["example", "train:example"])("selects %s and deduplicates repeated target references", (selector) => { + const source = plan(); + source.targets.push(structuredClone(source.targets[0]!)); + const before = structuredClone(source); + expect(selectTrainingExample(source, selector)).toMatchObject({ id: "example", role: "train", dataset: "train.yaml" }); + expect(source).toEqual(before); + }); + + it("lists ambiguous roles and permits an exact role qualifier", () => { + const source = plan(); + source.targets[0]!.datasets.test = dataset("test", "example"); + expect(() => selectTrainingExample(source, "example")).toThrow( + "Ambiguous training example example; candidates: test:example (test.yaml), train:example (train.yaml)" + ); + expect(selectTrainingExample(source, "test:example").dataset).toBe("test.yaml"); + }); + + it("still refuses a qualified id shared by distinct datasets and lists both candidates", () => { + const source = plan(), other = structuredClone(source.targets[0]!); + other.datasets.train = dataset("another-train", "example"); + source.targets.push(other); + expect(() => selectTrainingExample(source, "train:example")).toThrow( + "candidates: train:example (another-train.yaml), train:example (train.yaml)" + ); + }); + + it.each(["missing", "validation:example", "train:", "other:example"])("refuses unknown selector %s", (selector) => { + expect(() => selectTrainingExample(plan(), selector)).toThrow(`No training example matches ${selector}`); + }); +}); diff --git a/src/compiler/training/rewind/select.ts b/src/compiler/training/rewind/select.ts new file mode 100644 index 00000000..599c5e20 --- /dev/null +++ b/src/compiler/training/rewind/select.ts @@ -0,0 +1,75 @@ +import type { TrainingPlan } from "../plan/index.js"; +import type { MomentRecord } from "../../../recording/index.js"; +import { compareInstants, normalizeInstant } from "./instant.js"; +import { hasRecordPin, recordActivationErrors, recordForInput, recordPins } from "./recordInput.js"; +import type { InputIdentity, RewindInput } from "./types.js"; + +export type DatasetRole = "train" | "validation" | "test"; +export interface SelectedExample { + id: string; role: DatasetRole; dataset: string; sourcePath: string; + example: TrainingPlan["targets"][number]["datasets"]["train"]["examples"][number]; +} +/** Shared targets referencing the same role/dataset/example are one trial. */ +export const selectTrainingExample = (plan: TrainingPlan, selector: string): SelectedExample => { + const match = /^(train|validation|test):(.*)$/u.exec(selector); + const role = match?.[1], id = match?.[2] ?? selector; + const candidates = new Map(); + for (const target of plan.targets) for (const candidateRole of ["train", "validation", "test"] as const) { + const dataset = target.datasets[candidateRole]; + if (!dataset || (role && candidateRole !== role)) continue; + for (const example of dataset.examples) { + if (example.id !== id) continue; + const candidate = { id, role: candidateRole, dataset: dataset.path, sourcePath: dataset.sourcePath, example }; + candidates.set(JSON.stringify([candidateRole, dataset.sourcePath, id]), candidate); + } + } + const found = [...candidates.values()]; + if (found.length !== 1) throw new Error(found.length === 0 + ? `No training example matches ${selector}` + : `Ambiguous training example ${selector}; candidates: ${found.map((item) => `${item.role}:${id} (${item.dataset})`).sort().join(", ")}`); + return found[0]!; +}; +const canonical = (value: unknown): string => JSON.stringify(value, (_key, item: unknown) => + item && typeof item === "object" && !Array.isArray(item) + ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a.localeCompare(b))) : item); + +export const matchingMomentRecords = (records: readonly MomentRecord[], agent: string, at: string): MomentRecord[] => + records.filter((record) => record.agent === agent && compareInstants(record.wake.at, at) === 0); + +/** Replays can share executionId AND startedAt. Keep every earliest tie for per-input agreement. */ +export const originalMomentRecords = (records: readonly MomentRecord[]): MomentRecord[] => { + const first = records.reduce((earliest, entry) => + !earliest || compareInstants(entry.wake.startedAt, earliest.wake.startedAt) < 0 ? entry : earliest, null); + return first ? records.filter((entry) => compareInstants(entry.wake.startedAt, first.wake.startedAt) === 0) : []; +}; + +/** Capture time and wake payload are not used to materialize data. Normalize actual used instants. */ +export const selectMomentRecord = ( + records: MomentRecord[], agent: string, at: string, inputs?: Array +): MomentRecord | null => { + const matches = originalMomentRecords(matchingMomentRecords(records, agent, at)); + if (!matches.length) return null; + const identities: Array = inputs ?? matches.flatMap((record): InputIdentity[] => [ + ...[...record.resources, ...record.feeds, ...record.volumes].map(({ id }) => ({ section: "resources" as const, id, declaredBy: agent })), + ...record.memory.map(({ id, declaredBy }) => ({ section: "memory" as const, id, declaredBy })), + ...record.moltnet.map(({ network }) => ({ section: "networks" as const, id: network, declaredBy: agent })) + ]); + for (const input of identities) { + if ("overridden" in input && input.overridden) continue; + const declaredPin = "declaration" in input && input.section === "resources" && input.declaration.kind === "git" + && /^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/u.test(input.declaration.ref ?? ""); + const decision = (record: MomentRecord): string => { + const usable = recordForInput(record, input); + const pins = recordPins(usable, input).map((pin) => "landedAt" in pin ? { ...pin, landedAt: normalizeInstant(pin.landedAt) } + : "cursor" in pin ? { ...pin, cursor: normalizeInstant(pin.cursor) } : pin); + return canonical({ pin: declaredPin ? "declared-pin" : pins.sort((a, b) => canonical(a).localeCompare(canonical(b))), + approximateActivation: !declaredPin && pins.some((pin) => "kind" in pin && ["git", "bundle"].includes(pin.kind)) + && recordActivationErrors(usable).length > 0, + approximateFallback: !record.exact && !hasRecordPin(usable, input) }); + }; + if (matches.some((record) => decision(record) !== decision(matches[0]!))) { + throw new Error(`Conflicting moment records for ${agent} at ${normalizeInstant(at)}: ${input.section} ${input.declaredBy}/${input.id} has different effective pin/fallback decisions`); + } + } + return matches[0]!; +}; diff --git a/src/compiler/training/rewind/snapshotFixtures.test-helper.ts b/src/compiler/training/rewind/snapshotFixtures.test-helper.ts new file mode 100644 index 00000000..9184df1b --- /dev/null +++ b/src/compiler/training/rewind/snapshotFixtures.test-helper.ts @@ -0,0 +1,52 @@ +import { chmod, lstat, mkdir, readdir, readlink, writeFile, appendFile, symlink } from "node:fs/promises"; +import path from "node:path"; + +interface SnapshotFixture { + agentSlug: string; agentId: string; rootId: string; snapshot: string; startedAt: string; + files: Record; executionId?: string; + directories?: Record; symlinks?: Record; +} + +/** Recorder-format data only: no runtime dependency on Daimon. */ +export const writeRecordedVolumeSnapshot = async (historyRoot: string, options: SnapshotFixture) => { + const recording = path.join(historyRoot, "recording", options.agentSlug); + const base = path.join(recording, "snapshots", options.rootId); + const directory = path.join(base, options.snapshot), manifestPath = `${directory}.manifest.json`; + const rowsPath = path.join(recording, "wake-moments.jsonl"); + await mkdir(directory, { recursive: true, mode: 0o700 }); + for (const [relative, mode] of Object.entries(options.directories ?? {})) { + await mkdir(path.join(directory, relative), { recursive: true }); await chmod(path.join(directory, relative), mode); + } + for (const [relative, text] of Object.entries(options.files)) { + await mkdir(path.dirname(path.join(directory, relative)), { recursive: true }); + await writeFile(path.join(directory, relative), text); + } + for (const [relative, target] of Object.entries(options.symlinks ?? {})) { + await mkdir(path.dirname(path.join(directory, relative)), { recursive: true }); + await symlink(target, path.join(directory, relative)); + } + const entries: Record = {}; + const visit = async (relative: string): Promise => { + for (const name of await readdir(path.join(directory, relative))) { + const rel = relative ? `${relative}/${name}` : name, file = path.join(directory, rel); + const stat = await lstat(file, { bigint: true }); + const type = stat.isDirectory() ? "directory" : stat.isSymbolicLink() ? "symlink" : "file"; + entries[rel] = { type, mode: Number(stat.mode), size: String(stat.size), source: { + dev: String(stat.dev), ino: String(stat.ino), size: String(stat.size), + mtimeNs: String(stat.mtimeNs), ctimeNs: String(stat.ctimeNs), mode: Number(stat.mode) + }, ...(type === "symlink" ? { target: await readlink(file) } : {}) }; + if (type === "directory") await visit(rel); + } + }; + await visit(""); + const manifest = { version: "noopolis.daimon.wake-moment.v1", id: options.rootId, snapshot: options.snapshot, + started_at: options.startedAt, sourcePath: `/deployment/${options.rootId}`, entries }; + await writeFile(manifestPath, `${JSON.stringify(manifest)}\n`); + await writeFile(`${directory}.complete`, ""); + await appendFile(rowsPath, `${JSON.stringify({ version: manifest.version, agent_id: options.agentId, + execution_id: options.executionId ?? "execution", started_at: options.startedAt, + deliveries: [{ acceptance_id: "acceptance", delivery_id: "delivery", kind: "manual", occurred_at: options.startedAt }], + snapshots: [{ id: options.rootId, snapshot: options.snapshot, files: Object.keys(options.files).length, + copied: Object.keys(options.files).length, linked: 0, skipped: 0 }] })}\n`); + return { directory, manifestPath, rowsPath, manifest }; +}; diff --git a/src/compiler/training/rewind/snapshotManifest.ts b/src/compiler/training/rewind/snapshotManifest.ts new file mode 100644 index 00000000..0a88a760 --- /dev/null +++ b/src/compiler/training/rewind/snapshotManifest.ts @@ -0,0 +1,120 @@ +import { createHash } from "node:crypto"; +import { chmod, lstat, mkdir, readFile, readdir, readlink, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { z } from "zod"; +import { trainingInstantSchema } from "../../../manifest/index.js"; +import { inside, safeSegment, type History } from "./history.js"; + +const unsigned = z.string().regex(/^(?:0|[1-9][0-9]*)$/u); +const nanos = z.string().regex(/^-?(?:0|[1-9][0-9]*)$/u); +const mode = z.number().int().min(0).max(0xffff); +const relativePath = z.string().min(1).refine((value) => !value.includes("\\") && !value.includes("\0") + && !path.posix.isAbsolute(value) && !value.split("/").some((part) => !part || part === "." || part === ".."), +"Snapshot paths must be normalized relative POSIX paths"); +const sourceIdentity = z.object({ dev: unsigned, ino: unsigned, size: unsigned, mtimeNs: nanos, ctimeNs: nanos, mode }).strict(); +const entrySchema = z.object({ type: z.enum(["file", "directory", "symlink"]), mode, size: unsigned, + source: sourceIdentity, target: z.string().refine((value) => !value.includes("\0")).optional() }).strict().superRefine((entry, ctx) => { + if ((entry.type === "symlink") !== (entry.target !== undefined)) ctx.addIssue({ code: "custom", message: "Only snapshot symlinks require a target" }); + if (entry.mode !== entry.source.mode || entry.size !== entry.source.size) ctx.addIssue({ code: "custom", message: "Snapshot source identity disagrees with entry metadata" }); + const typeMode = { file: 0o100000, directory: 0o040000, symlink: 0o120000 }[entry.type]; + if ((entry.mode & 0o170000) !== typeMode) ctx.addIssue({ code: "custom", message: "Snapshot mode disagrees with entry type" }); +}); + +/** Daimon's manifest records source identity, not the identity of retained/copied inodes. */ +export const snapshotManifestSchema = z.object({ version: z.literal("noopolis.daimon.wake-moment.v1"), id: z.string().min(1), + snapshot: z.string().min(1), started_at: trainingInstantSchema, + sourcePath: z.string().refine((value) => path.posix.isAbsolute(value) && !value.includes("\0"), "Expected an absolute snapshot sourcePath"), + entries: z.record(relativePath, entrySchema) }).strict().superRefine((manifest, ctx) => { + for (const name of Object.keys(manifest.entries)) { + for (let parent = path.posix.dirname(name); parent !== "."; parent = path.posix.dirname(parent)) { + if (manifest.entries[parent]?.type !== "directory") ctx.addIssue({ code: "custom", message: `Snapshot parent ${parent} must be a recorded directory` }); + } + } +}); +export type SnapshotManifest = z.infer; + +export const snapshotHistoryPath = (agentSlug: string, id: string, snapshot: string): string => + `recording/${safeSegment(agentSlug)}/snapshots/${safeSegment(id === "." || id === ".." ? id.replaceAll(".", "%2E") : id)}/${safeSegment(snapshot)}`; + +export const readSnapshotManifest = async (history: History, relative: string, id: string, snapshot: string): Promise => { + // Daimon publishes this marker only after the tree and manifest are durable. + if (await history.bytes(`${relative}.complete`) === null) return null; + const bytes = await history.text(`${relative}.manifest.json`); + if (bytes === null) return null; + let manifest: SnapshotManifest; + try { manifest = snapshotManifestSchema.parse(JSON.parse(bytes)); } + catch (error) { throw new Error(`Invalid snapshot manifest ${relative}.manifest.json: ${error instanceof Error ? error.message : String(error)}`); } + if (manifest.id !== id || manifest.snapshot !== snapshot) throw new Error(`Snapshot manifest identity mismatch at ${relative}.manifest.json`); + return manifest; +}; + +interface TreeInspection { + retained?: boolean; + readBytes?: (relative: string) => Promise; + expectedBytes?: (relative: string) => Promise; +} +/** No symlink traversal. A separate call inspects the copied tree, independently of the producer. */ +export const snapshotTreeFindings = async (root: string, manifest: SnapshotManifest, options: TreeInspection = {}): Promise => { + if (!(await lstat(root)).isDirectory()) throw new Error("Snapshot tree must be a real directory"); + const findings: string[] = [], found = new Set(); + const visit = async (relative: string): Promise => { + for (const leaf of (await readdir(path.join(root, relative))).sort()) { + const name = relative ? `${relative}/${leaf}` : leaf, file = path.join(root, name); + const stat = await lstat(file, { bigint: true }), expected = manifest.entries[name]; + found.add(name); + const type = stat.isFile() ? "file" : stat.isDirectory() ? "directory" : stat.isSymbolicLink() ? "symlink" : "special"; + if (!expected) findings.push(`Unexpected snapshot entry ${name}`); + else if (type !== expected.type) findings.push(`Snapshot type mismatch for ${name}`); + else { + // Recorder directories are deliberately 0700. Restore source modes while copying. + const wantedMode = options.retained && type === "file" ? expected.mode & 0o777 : expected.mode & 0o7777; + if (!(options.retained && type === "directory") && (Number(stat.mode) & 0o7777) !== wantedMode) findings.push(`Snapshot mode mismatch for ${name}`); + // Directory inode sizes describe source filesystem bookkeeping, not tree content. + if (type !== "directory" && stat.size.toString() !== expected.size) findings.push(`Snapshot size mismatch for ${name}`); + if (type === "symlink" && await readlink(file) !== expected.target) findings.push(`Snapshot symlink target mismatch for ${name}`); + if (type === "file" && options.expectedBytes) { + const actual = options.readBytes ? await options.readBytes(name) : await readFile(file); + const original = await options.expectedBytes(name); + const digest = (bytes: Buffer): string => createHash("sha256").update(bytes).digest("hex"); + if (digest(actual) !== digest(original)) findings.push(`Snapshot digest mismatch for ${name}`); + } + } + if (type === "directory") await visit(name); + } + }; + await visit(""); + for (const name of Object.keys(manifest.entries)) if (!found.has(name)) findings.push(`Missing snapshot entry ${name}`); + return findings; +}; + +export const snapshotBytes = async (history: History, relative: string, name: string): Promise => { + const bytes = await history.bytes(`${relative}/${name}`); + if (bytes === null) throw new Error(`Missing snapshot entry ${name}`); + return bytes; +}; + +/** Retained files may be hard-linked to each other; bundle files are always new copies. */ +export const copySnapshotTree = async (history: History, relative: string, destination: string, manifest: SnapshotManifest): Promise => { + const source = await history.path(relative); + if (!source) throw new Error(`Missing snapshot tree ${relative}`); + const findings = await snapshotTreeFindings(source, manifest, { retained: true }); + if (findings.length) throw new Error(findings.join("; ")); + await mkdir(destination, { recursive: true, mode: 0o700 }); + const entries = Object.entries(manifest.entries).sort(([a], [b]) => a.split("/").length - b.split("/").length || a.localeCompare(b)); + for (const [name, entry] of entries) { + const target = path.join(destination, name); + if (entry.type === "directory") await mkdir(target, { mode: 0o700 }); + else if (entry.type === "symlink") { + if (path.isAbsolute(entry.target!) || !inside(destination, path.resolve(path.dirname(target), entry.target!))) { + throw new Error(`Snapshot symlink escapes root: ${name}`); + } + await symlink(entry.target!, target); + } else { + await writeFile(target, await snapshotBytes(history, relative, name), { flag: "wx" }); + await chmod(target, entry.mode & 0o7777); + } + } + for (const [name, entry] of entries.reverse()) if (entry.type === "directory") await chmod(path.join(destination, name), entry.mode & 0o7777); + const copied = await snapshotTreeFindings(destination, manifest, { expectedBytes: (name) => snapshotBytes(history, relative, name) }); + if (copied.length) throw new Error(copied.join("; ")); +}; diff --git a/src/compiler/training/rewind/taskClock.test.ts b/src/compiler/training/rewind/taskClock.test.ts new file mode 100644 index 00000000..ba1f7926 --- /dev/null +++ b/src/compiler/training/rewind/taskClock.test.ts @@ -0,0 +1,80 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { clockAwareEnvironment, createTaskClock, harnessTaskClockEnvironment, processClockEnvironment } from "./taskClock.js"; + +afterEach(() => vi.restoreAllMocks()); +const helpers = [harnessTaskClockEnvironment, processClockEnvironment, clockAwareEnvironment]; +// Shared Spawnfile/Daimon contract vectors, also documented in specs/TRAINING.md. +const vectors: Array<[string, number, string]> = [ + ["2001-01-01T00:00:00.000Z", 1_821_692_800_250, "-843385600.250"], + ["2001-01-01T00:00:00.000Z", 1_821_692_860_250, "-843385660.250"], + ["2001-01-01T00:00:00.250Z", 978_307_200_000, "+0.250"], + ["2001-01-01T00:00:00.000Z", 978_307_200_001, "-0.001"], + ["2001-01-01T00:00:00.000Z", 978_307_200_000, "+0"], + ["2001-01-01T00:00:00.000Z", 978_307_199_000, "+1"] +]; + +describe("training task clock contract", () => { + it.each(["1969-12-31T23:59:59.999999999Z", "1970-01-01T00:00:00+00:01", "9999-12-31T23:59:59.999000001Z", "9999-12-31T23:59:59-00:01"]) + ("refuses the out-of-range trial instant %s in every helper", (origin) => { + expect(() => createTaskClock(origin)).toThrow(/must be within 1970.*9999/u); + for (const helper of helpers) expect(() => helper(origin, 0)).toThrow(/must be within 1970.*9999/u); + }); + + it.each(["1970-01-01T00:00:00Z", "1969-12-31T23:00:00-01:00", "9999-12-31T23:59:59.999Z"]) + ("accepts the inclusive trial boundary %s", (origin) => { + expect(() => createTaskClock(origin)).not.toThrow(); + for (const helper of helpers) expect(() => helper(origin, 0)).not.toThrow(); + }); + it("normalizes the origin and gives each process exactly one clock owner", () => { + const origin = "2026-10-09T10:20:31.120000000+02:00", anchor = 1_790_000_000_123; + expect(createTaskClock(origin)).toEqual({ version: "spawnfile.task-clock.v1", origin: "2026-10-09T08:20:31.12Z" }); + expect(harnessTaskClockEnvironment(origin, anchor)).toEqual({ + NOOPOLIS_TASK_CLOCK: '{"version":"noopolis.task-clock.v1","origin":"2026-10-09T08:20:31.12Z","anchorEpochMs":1790000000123}' + }); + expect(clockAwareEnvironment(origin, anchor)).toEqual({ + MNEME_CLOCK_ORIGIN: "2026-10-09T08:20:31.12Z", MNEME_CLOCK_ANCHOR_MS: "1790000000123" + }); + expect(Object.keys(processClockEnvironment(origin, anchor)).sort()).toEqual(["FAKETIME", "FAKETIME_DONT_FAKE_MONOTONIC"]); + for (const helper of helpers) { + const env = helper(origin, anchor); + expect(["NOOPOLIS_TASK_CLOCK", "MNEME_CLOCK_ORIGIN", "FAKETIME"].filter((key) => key in env)).toHaveLength(1); + } + }); + + it.each(vectors)("matches the shared offset for %s anchored at %s", (origin, anchor, expected) => { + const processEnv = processClockEnvironment(origin, anchor); + expect(processEnv).toEqual({ FAKETIME: expected, FAKETIME_DONT_FAKE_MONOTONIC: "1" }); + const contract = JSON.parse(harnessTaskClockEnvironment(origin, anchor).NOOPOLIS_TASK_CLOCK!); + const aware = clockAwareEnvironment(origin, anchor); + // Daimon's task-clock formula, including a later independently started process. + for (const delay of [0, 1, 60_123]) { + const realNow = anchor + delay; + const daimonNow = Date.parse(contract.origin) + (realNow - contract.anchorEpochMs); + expect(realNow + Number(processEnv.FAKETIME) * 1000).toBe(daimonNow); + expect(Date.parse(aware.MNEME_CLOCK_ORIGIN!) + realNow - Number(aware.MNEME_CLOCK_ANCHOR_MS)).toBe(daimonNow); + expect(daimonNow).toBe(Date.parse(origin) + delay); + } + }); + + it("changes the relative offset when the shared anchor changes", () => { + const origin = "2001-01-01T00:00:00.000Z"; + expect(processClockEnvironment(origin, 1_821_692_800_250).FAKETIME).toBe("-843385600.250"); + expect(processClockEnvironment(origin, 1_821_692_860_250).FAKETIME).toBe("-843385660.250"); + }); + + it("uses only its supplied anchor without wall-clock reads, sleeping, or process environment mutation", () => { + const environment = { ...process.env }; + const now = vi.spyOn(Date, "now").mockImplementation(() => { throw Error("read live clock"); }); + const timeout = vi.spyOn(globalThis, "setTimeout").mockImplementation(() => { throw Error("started timeout"); }); + const interval = vi.spyOn(globalThis, "setInterval").mockImplementation(() => { throw Error("started interval"); }); + for (const helper of helpers) { + expect(helper("2026-10-09T08:20:31Z", 0)).toEqual(helper("2026-10-09T10:20:31+02:00", 0)); + } + expect(now).not.toHaveBeenCalled(); expect(timeout).not.toHaveBeenCalled(); expect(interval).not.toHaveBeenCalled(); + expect(process.env).toEqual(environment); + }); + + it.each([NaN, Infinity, 1.5, Number.MAX_SAFE_INTEGER + 1])("refuses non-integer or unsafe anchor %s", (anchor) => { + for (const helper of helpers) expect(() => helper("2026-10-09T08:20:31Z", anchor)).toThrow(/anchorEpochMs must be a safe integer/u); + }); +}); diff --git a/src/compiler/training/rewind/taskClock.ts b/src/compiler/training/rewind/taskClock.ts new file mode 100644 index 00000000..b70af7fd --- /dev/null +++ b/src/compiler/training/rewind/taskClock.ts @@ -0,0 +1,43 @@ +import { instantNanos, normalizeInstant } from "./instant.js"; +import { taskClockSchema } from "./contract.js"; + +/** Shared ecosystem range applies to trial instants, not historical source timestamps. */ +export const assertTaskClockInstant = (instant: string): void => { + const nanos = instantNanos(instant); + if (nanos < 0n || nanos > 253_402_300_799_999_000_000n) { + throw new Error(`Trial instant ${instant} must be within 1970-01-01T00:00:00Z .. 9999-12-31T23:59:59.999Z`); + } +}; +const taskOrigin = (origin: string): string => { + assertTaskClockInstant(origin); + return normalizeInstant(origin); +}; +export const createTaskClock = (origin: string) => taskClockSchema.parse({ + version: "spawnfile.task-clock.v1", origin: taskOrigin(origin) +}); +const clockOrigin = (origin: string, anchorEpochMs: number): string => { + if (!Number.isSafeInteger(anchorEpochMs)) throw new Error("Task clock anchorEpochMs must be a safe integer"); + return taskOrigin(origin); +}; + +/** Top-level Daimon only: its clock stays real and it derives each child's clock environment. */ +export const harnessTaskClockEnvironment = (origin: string, anchorEpochMs: number): Record => ({ + NOOPOLIS_TASK_CLOCK: JSON.stringify({ version: "noopolis.task-clock.v1", origin: clockOrigin(origin, anchorEpochMs), anchorEpochMs }) +}); + +/** Child/tool wall clock. Trial preparation supplies libfaketime LD_PRELOAD and a dot-decimal locale. */ +export const processClockEnvironment = (origin: string, anchorEpochMs: number): Record => { + const normalized = clockOrigin(origin, anchorEpochMs); + const offsetMs = BigInt(Date.parse(normalized)) - BigInt(anchorEpochMs); + const magnitude = offsetMs < 0n ? -offsetMs : offsetMs; + const fraction = magnitude % 1000n; + return { + FAKETIME: `${offsetMs < 0n ? "-" : "+"}${magnitude / 1000n}${fraction ? `.${String(fraction).padStart(3, "0")}` : ""}`, + FAKETIME_DONT_FAKE_MONOTONIC: "1" + }; +}; + +/** Clock-aware child (e.g. Mneme) without libfaketime: apply the offset inside the process exactly once. */ +export const clockAwareEnvironment = (origin: string, anchorEpochMs: number): Record => ({ + MNEME_CLOCK_ORIGIN: clockOrigin(origin, anchorEpochMs), MNEME_CLOCK_ANCHOR_MS: String(anchorEpochMs) +}); diff --git a/src/compiler/training/rewind/timestamps.test.ts b/src/compiler/training/rewind/timestamps.test.ts new file mode 100644 index 00000000..5d493c56 --- /dev/null +++ b/src/compiler/training/rewind/timestamps.test.ts @@ -0,0 +1,24 @@ +import { lstat, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { gitTestRoot } from "./git.test-helper.js"; +import { restoreInspectedAccessTimes, snapshotBundleFiles } from "./leakFiles.js"; +import { restoreAccessTimes, stampTree } from "./timestamps.js"; + +const roots: string[] = [], old = "1960-01-01T00:00:00Z", at = "2026-10-09T08:20:31Z"; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); + +it.each(["inspection", "materializer"])("preserves negative file, link and directory epochs during %s restoration", async (helper) => { + const root = await gitTestRoot(); roots.push(root); + await writeFile(path.join(root, "data"), "past"); await symlink("data", path.join(root, "link")); + await stampTree(root, old, at); + const files = await snapshotBundleFiles(root); + await readFile(path.join(root, "data")); await readdir(root); + if (helper === "inspection") await restoreInspectedAccessTimes(files); + else await restoreAccessTimes(root, at); + for (const file of files.keys()) { + const stats = await lstat(file); + expect(stats.atimeMs, file).toBe(Date.parse(old)); + expect(stats.mtimeMs, file).toBe(Date.parse(old)); + } +}); diff --git a/src/compiler/training/rewind/timestamps.ts b/src/compiler/training/rewind/timestamps.ts new file mode 100644 index 00000000..dc68f2fd --- /dev/null +++ b/src/compiler/training/rewind/timestamps.ts @@ -0,0 +1,25 @@ +import { lstat, lutimes, readdir, utimes } from "node:fs/promises"; +import path from "node:path"; +import { compareInstants } from "./instant.js"; + +/** Round down to milliseconds so filesystem precision cannot put a stamp after the instant. */ +export const stampTree = async (root: string, instant: string, at: string): Promise => { + const stamp = new Date(compareInstants(instant, at) > 0 ? at : instant); + const visit = async (file: string): Promise => { + const stats = await lstat(file); + if (stats.isDirectory()) for (const name of await readdir(file)) await visit(path.join(file, name)); + await (stats.isSymbolicLink() ? lutimes : utimes)(file, stamp, stamp); + }; + await visit(root); +}; + +/** Disk verification reads may update atime. Restore access time without changing verified mtimes. */ +export const restoreAccessTimes = async (root: string, at: string): Promise => { + const visit = async (file: string): Promise => { + const stats = await lstat(file); + if (stats.isDirectory()) for (const name of await readdir(file)) await visit(path.join(file, name)); + await (stats.isSymbolicLink() ? lutimes : utimes)(file, + new Date(Math.min(stats.mtimeMs, Date.parse(at))), new Date(stats.mtimeMs)); + }; + await visit(root); +}; diff --git a/src/compiler/training/rewind/types.ts b/src/compiler/training/rewind/types.ts new file mode 100644 index 00000000..d3b9c991 --- /dev/null +++ b/src/compiler/training/rewind/types.ts @@ -0,0 +1,31 @@ +import type { MemoryBank, TeamNetwork, TeamWorkspaceResource } from "../../../manifest/index.js"; +import type { ResolvedMemoryBank } from "../../types.js"; +import type { MaterializedInput } from "./contract.js"; +import type { MomentRecord } from "../../../recording/index.js"; +import type { History } from "./history.js"; + +export interface InputIdentity { section: "resources" | "memory" | "networks"; id: string; declaredBy: string } +interface InputBase extends InputIdentity { overridden: boolean; declarationBase: string } +export type RewindInput = InputBase & ( + | { section: "resources"; declaration: TeamWorkspaceResource } + | { section: "memory"; declaration: MemoryBank | ResolvedMemoryBank } + | { section: "networks"; declaration: TeamNetwork } +); +export interface InputContext { + input: RewindInput; + at: string; + agentSlug?: string; + agentId?: string; + record: MomentRecord | null; + candidateRecords?: readonly MomentRecord[]; + history: History | undefined; + destination: string; +} +export interface InputResult { + fidelity: "exact" | "approximate" | "overridden"; + basis: string; + source: MaterializedInput["source"]; + notes: string[]; + instant: string; + build?: "deferred"; +} diff --git a/src/compiler/training/rewind/volumes.test.ts b/src/compiler/training/rewind/volumes.test.ts new file mode 100644 index 00000000..41343433 --- /dev/null +++ b/src/compiler/training/rewind/volumes.test.ts @@ -0,0 +1,160 @@ +import { chmod, lstat, mkdtemp, mkdir, readFile, readdir, readlink, realpath, rm, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { openHistory } from "./history.js"; +import { type LeakCheck, type MaterializedInput } from "./contract.js"; +import { momentRecordSchema } from "../../../recording/index.js"; +import { inspectVolumeBundle } from "./leakInputs.js"; +import { writeRecordedVolumeSnapshot } from "./snapshotFixtures.test-helper.js"; +import type { InputContext } from "./types.js"; +import { materializeVolume } from "./volumes.js"; + +const at = "2026-10-09T10:00:00Z", before = "2026-10-09T09:00:00Z", future = "2026-10-09T10:00:00.001Z"; +const temps: string[] = []; +afterEach(async () => { await Promise.all(temps.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); }); +const fixtures = async () => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "rewind-volumes-"))); temps.push(root); + const context: InputContext = { at, record: null, history: await openHistory(root), destination: path.join(root, "out"), + agentSlug: "writer-compiled-slug", agentId: "agent:writer", + input: { section: "resources", id: "state", declaredBy: "team:org", declarationBase: root, overridden: false, + declaration: { id: "state", kind: "volume", mount: "./state", mode: "mutable" } } }; + const snapshot = (name: string, startedAt: string, files: Record = {}) => writeRecordedVolumeSnapshot(root, + { agentSlug: context.agentSlug!, agentId: context.agentId!, rootId: "state", snapshot: name, startedAt, files }); + return { root, context, snapshot }; +}; +const record = (snapshot: string) => momentRecordSchema.parse({ version: "spawnfile.moment-record.v1", agent: "agent:writer", + wake: { at, startedAt: at, executionId: "execution", deliveries: [{ deliveryId: "wake", kind: "message", occurredAt: at }] }, + recordedAt: at, exact: true, resources: [], feeds: [], memory: [], moltnet: [], volumes: [{ id: "state", snapshot }] }); +const report = (): LeakCheck => ({ version: "spawnfile.leak-check.v1", at, ok: false, + checked: { files: 0, gitObjects: 0, memoryEvents: 0, moltnetMessages: 0, moltnetEvents: 0, snapshots: 0 }, findings: [] }); + +describe("unfed recorded volume trees", () => { + it("requires the merged completion marker in both selection and independent acceptance", async () => { + const { root, context, snapshot } = await fixtures(); + const saved = await snapshot("20261009T090000000Z-completed", before, { data: "bytes" }); + const result = await materializeVolume(context); + await rm(`${saved.directory}.complete`); + context.destination += "-incomplete"; + await expect(materializeVolume(context)).rejects.toThrow("No complete volume snapshot"); + const input: MaterializedInput = { section: "resources", id: "state", declaredBy: "team:org", kind: "volume", path: "out", + fidelity: result.fidelity, basis: result.basis, source: result.source, notes: [], refusal: null }; + await expect(inspectVolumeBundle(root, input, report(), context.history)).rejects.toThrow("Snapshot tree or manifest is missing"); + }); + + it("uses the merged row parser's nonempty delivery requirement", async () => { + const { context, snapshot } = await fixtures(); + const saved = await snapshot("20261009T090000000Z-deliveries", before, { data: "bytes" }); + const row = JSON.parse(await readFile(saved.rowsPath, "utf8")); row.deliveries = []; + await writeFile(saved.rowsPath, `${JSON.stringify(row)}\n`); + await expect(materializeVolume(context)).rejects.toThrow(/wake-moments.jsonl:1:/u); + }); + + it("selects by inclusive row started_at, copies a full tree, and uses the named record exactly", async () => { + const { context, snapshot } = await fixtures(); + await snapshot("20261009T080000000Z-first", before, { old: "old" }); + const newest = await snapshot("20261009T070000000Z-newest", "2026-10-09T12:00:00+02:00", { "nested/new": "new" }); + await snapshot("20261009T060000000Z-future", future, { later: "future" }); + expect(await materializeVolume(context)).toMatchObject({ fidelity: "approximate", basis: "snapshot-at", + source: { snapshot: "20261009T070000000Z-newest", agentSlug: "writer-compiled-slug" } }); + expect(await readdir(context.destination)).toEqual(["nested"]); + expect(await readFile(path.join(context.destination, "nested/new"), "utf8")).toBe("new"); + expect((await lstat(path.join(context.destination, "nested/new"))).ino) + .not.toBe((await lstat(path.join(newest.directory, "nested/new"))).ino); + context.destination += "-record"; context.record = record("20261009T080000000Z-first"); + expect(await materializeVolume(context)).toMatchObject({ fidelity: "exact", basis: "record" }); + expect(await readFile(path.join(context.destination, "old"), "utf8")).toBe("old"); + }); + + it("ignores incomplete and .partial trees and chooses the last row when instants tie", async () => { + const { context, snapshot } = await fixtures(); + await snapshot("20261009T090000000Z-first", before, { data: "first" }); + await snapshot("20261009T090000000Z-tie", before, { data: "last" }); + const incomplete = await snapshot("20261009T090000000Z-incomplete", at, { data: "unfinished" }); + await rm(incomplete.manifestPath); + await snapshot("20261009T090000000Z-incoming.partial", at, { data: "partial" }); + expect(await materializeVolume(context)).toMatchObject({ source: { snapshot: "20261009T090000000Z-tie" } }); + expect(await readFile(path.join(context.destination, "data"), "utf8")).toBe("last"); + context.record = record("20261009T090000000Z-incoming.partial"); context.destination += "-record"; + await expect(materializeVolume(context)).rejects.toThrow(/partial/i); + }); + + it("creates an empty overridden volume without history", async () => { + const { context } = await fixtures(); context.history = undefined; context.input.overridden = true; + expect(await materializeVolume(context)).toMatchObject({ fidelity: "overridden", basis: "override" }); + expect(await readdir(context.destination)).toEqual([]); + }); + + it("refuses missing history, missing rows, future-only snapshots, and absent named snapshots", async () => { + const { context, snapshot } = await fixtures(); + await expect(materializeVolume(context)).rejects.toThrow("Missing volume snapshot history"); + await snapshot("20261009T090000000Z-future", future); + await expect(materializeVolume(context)).rejects.toThrow("No complete volume snapshot"); + context.record = record("20261009T090000000Z-missing"); + await expect(materializeVolume(context)).rejects.toThrow("Recorded volume snapshot 20261009T090000000Z-missing"); + context.history = undefined; + await expect(materializeVolume(context)).rejects.toThrow("requires --history"); + }); + + it("refuses invalid rows with line numbers and mismatched manifest identity", async () => { + const { context, snapshot } = await fixtures(), saved = await snapshot("20261009T090000000Z-one", before, { data: "bytes" }); + await writeFile(saved.rowsPath, `${await readFile(saved.rowsPath, "utf8")}{}\n`); + await expect(materializeVolume(context)).rejects.toThrow("wake-moments.jsonl:2"); + await writeFile(saved.rowsPath, (await readFile(saved.rowsPath, "utf8")).split("\n")[0] + "\n"); + await writeFile(saved.manifestPath, JSON.stringify({ ...saved.manifest, id: "other" })); + await expect(materializeVolume(context)).rejects.toThrow("manifest identity"); + }); + + it.each(["extra", "missing", "type", "size", "mode", "unsafe"])("refuses manifest %s mismatches", async (mutation) => { + const { context, snapshot } = await fixtures(), saved = await snapshot("20261009T090000000Z-one", before, { data: "bytes" }); + if (mutation === "extra") await writeFile(path.join(saved.directory, "extra"), "extra"); + if (mutation === "missing") await rm(path.join(saved.directory, "data")); + if (mutation === "type") { await rm(path.join(saved.directory, "data")); await mkdir(path.join(saved.directory, "data")); } + if (mutation === "size") await writeFile(path.join(saved.directory, "data"), "shortened"); + if (mutation === "mode") await chmod(path.join(saved.directory, "data"), 0o600); + if (mutation === "unsafe") { + const manifest = JSON.parse(await readFile(saved.manifestPath, "utf8")); + manifest.entries["../escape"] = manifest.entries.data; await writeFile(saved.manifestPath, JSON.stringify(manifest)); + } + await expect(materializeVolume(context)).rejects.toThrow({ + extra: "Unexpected snapshot entry extra", missing: "Missing snapshot entry data", type: "Snapshot type mismatch for data", + size: "Snapshot size mismatch for data", mode: "Snapshot mode mismatch for data", unsafe: "Snapshot paths must be normalized" + }[mutation]!); + }); + + it("restores recorded directory modes without comparing nonportable directory sizes and verifies symlinks", async () => { + const { context, root } = await fixtures(); + const saved = await writeRecordedVolumeSnapshot(root, { agentSlug: context.agentSlug!, agentId: context.agentId!, rootId: "state", + snapshot: "20261009T090000000Z-tree", startedAt: before, files: { "nested/data": "bytes" }, directories: { nested: 0o750 }, symlinks: { link: "nested/data" } }); + const manifest = JSON.parse(await readFile(saved.manifestPath, "utf8")); + manifest.entries.nested.size = "999999"; manifest.entries.nested.source.size = "999999"; + await writeFile(saved.manifestPath, JSON.stringify(manifest)); await chmod(path.join(saved.directory, "nested"), 0o700); + await materializeVolume(context); + expect((await lstat(path.join(context.destination, "nested"))).mode & 0o777).toBe(0o750); + expect(await readlink(path.join(context.destination, "link"))).toBe("nested/data"); + manifest.entries.link.target = "other/data"; + await writeFile(saved.manifestPath, JSON.stringify(manifest)); context.destination += "-bad-link"; + await expect(materializeVolume(context)).rejects.toThrow(/symlink target/i); + }); + + it.each(["bytes", "mode", "extra", "missing", "target"])("independently rejects a bundle %s mutation", async (mutation) => { + const { root, context } = await fixtures(); + await writeRecordedVolumeSnapshot(root, { agentSlug: context.agentSlug!, agentId: context.agentId!, rootId: "state", + snapshot: "20261009T090000000Z-one", startedAt: before, files: { data: "bytes", other: "other" }, symlinks: { link: "data" } }); + const result = await materializeVolume(context); + if (mutation === "bytes") await writeFile(path.join(context.destination, "data"), "wrong"); + if (mutation === "mode") await chmod(path.join(context.destination, "data"), 0o600); + if (mutation === "extra") await writeFile(path.join(context.destination, "extra"), "extra"); + if (mutation === "missing") await rm(path.join(context.destination, "other")); + if (mutation === "target") { + await rm(path.join(context.destination, "link")); await symlink("other", path.join(context.destination, "link")); + } + const input: MaterializedInput = { section: "resources", id: "state", declaredBy: "team:org", kind: "volume", path: "out", + fidelity: result.fidelity, basis: result.basis, source: result.source, notes: [], refusal: null }; + const check = report(); await inspectVolumeBundle(root, input, check, context.history); + expect(check.findings).toContainEqual(expect.objectContaining({ class: "snapshot", path: "out", detail: expect.stringMatching({ + bytes: /Snapshot digest mismatch for data/, mode: /Snapshot mode mismatch for data/, extra: /Unexpected snapshot entry extra/, + missing: /Missing snapshot entry other/, target: /Snapshot symlink target mismatch for link/ + }[mutation]!) })); + }); +}); diff --git a/src/compiler/training/rewind/volumes.ts b/src/compiler/training/rewind/volumes.ts new file mode 100644 index 00000000..7eb0278e --- /dev/null +++ b/src/compiler/training/rewind/volumes.ts @@ -0,0 +1,55 @@ +import { lstat, mkdir } from "node:fs/promises"; +import { safeSegment, type History } from "./history.js"; +import { compareInstants } from "./instant.js"; +import { copySnapshotTree, readSnapshotManifest, snapshotHistoryPath, type SnapshotManifest } from "./snapshotManifest.js"; +import type { InputContext, InputResult } from "./types.js"; + +export interface SelectedVolumeSnapshot { snapshot: string; startedAt: string; manifest: SnapshotManifest; relative: string } + +/** A published row plus retained directory and manifest is complete; never infer time from its name. */ +export const selectVolumeSnapshot = async (history: History, agentSlug: string, agentId: string, id: string, + at: string, recordId?: string): Promise => { + const rowsPath = `recording/${safeSegment(agentSlug)}/wake-moments.jsonl`; + const rows = await history.wakeMoments(rowsPath); + if (!rows) throw new Error(`Missing volume snapshot history ${rowsPath}`); + const candidates = rows.flatMap((row, index) => row.agent_id !== agentId ? [] : row.snapshots.filter((entry) => entry.id === id) + .map((entry) => ({ row, entry, index }))) + .filter(({ row, entry }) => recordId === undefined ? compareInstants(row.started_at, at) <= 0 : entry.snapshot === recordId) + .sort((a, b) => compareInstants(b.row.started_at, a.row.started_at) || b.index - a.index); + for (const { row, entry } of candidates) { + if (entry.snapshot.endsWith(".partial")) { + if (recordId !== undefined) throw new Error(`Recorded volume snapshot ${recordId} is partial`); + continue; + } + if (compareInstants(row.started_at, at) > 0) throw new Error(`Recorded volume snapshot ${entry.snapshot} is later than the moment`); + const relative = snapshotHistoryPath(agentSlug, id, entry.snapshot), tree = await history.path(relative); + const manifest = await readSnapshotManifest(history, relative, id, entry.snapshot); + if (!tree || !manifest) { + if (recordId !== undefined) throw new Error(`Recorded volume snapshot ${recordId} is incomplete`); + continue; + } + if (!(await lstat(tree)).isDirectory()) throw new Error(`Snapshot tree must be a real directory: ${relative}`); + if (compareInstants(manifest.started_at, row.started_at) !== 0) throw new Error(`Snapshot manifest started_at disagrees with its wake row: ${relative}`); + return { snapshot: entry.snapshot, startedAt: row.started_at, manifest, relative }; + } + if (recordId !== undefined) throw new Error(`Recorded volume snapshot ${recordId} is absent from ${rowsPath}`); + throw new Error("No complete volume snapshot was taken at or before the moment"); +}; + +export const materializeVolume = async (context: InputContext): Promise => { + const { input, at, record, history, destination } = context; + if (input.section !== "resources" || input.declaration.kind !== "volume" || input.declaration.feed) { + throw new Error("Volume materialization requires an unfed volume input"); + } + if (input.overridden) { + await mkdir(destination, { recursive: true }); + return { fidelity: "overridden", basis: "override", source: {}, notes: [], instant: at }; + } + if (!history) throw new Error(`Volume ${input.id} requires --history or an override`); + if (!context.agentSlug || !context.agentId) throw new Error("Volume snapshots require the compiled agent slug and identity"); + const pin = record?.volumes.find((entry) => entry.id === input.id); + const selected = await selectVolumeSnapshot(history, context.agentSlug, context.agentId, input.id, at, pin?.snapshot); + await copySnapshotTree(history, selected.relative, destination, selected.manifest); + return { fidelity: pin ? "exact" : "approximate", basis: pin ? "record" : "snapshot-at", + source: { snapshot: selected.snapshot, agentSlug: context.agentSlug }, notes: [], instant: selected.startedAt }; +}; diff --git a/src/recording/AGENTS.md b/src/recording/AGENTS.md index 46916a45..810f075f 100644 --- a/src/recording/AGENTS.md +++ b/src/recording/AGENTS.md @@ -1,5 +1,6 @@ # Moment Recording +- `moment.ts` owns the shared strict moment-record schema, inferred type and parser; rewind consumes the public barrel. - `types.ts`, `parse.ts`, and `compose.ts` own the pure wake-row contract and pin composition. - `sync.ts` joins activation-scoped evidence; `read.ts` reads bounded byte chunks via the usage exec transport. - `history.ts` selects recording facts from the activation log; `deployment.ts` selects the live deployment and resolves feed histories. diff --git a/src/recording/index.ts b/src/recording/index.ts index bb65208c..3bd866df 100644 --- a/src/recording/index.ts +++ b/src/recording/index.ts @@ -1,4 +1,5 @@ export * from "./types.js"; +export * from "./moment.js"; export * from "./compose.js"; export * from "./parse.js"; export * from "./store.js"; diff --git a/src/recording/moment.test.ts b/src/recording/moment.test.ts new file mode 100644 index 00000000..6e249a02 --- /dev/null +++ b/src/recording/moment.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { composeMomentRecord, momentRecordSchema, parseMomentRecord, type ComposeMomentRecordInput, type MomentRecord } from "./index.js"; + +const at = "2026-10-09T08:20:31Z"; +const input = (): ComposeMomentRecordInput => ({ + row: { version: "noopolis.daimon.wake-moment.v1", agent_id: "agent:writer", execution_id: "execution", + started_at: at, deliveries: [{ acceptance_id: "accept", delivery_id: "wake", kind: "manual", occurred_at: at }], snapshots: [] }, + agentNodeId: "agent:writer", deploymentRecordPath: "/retained/deployments/desk.json", recordedAt: at, + deployResources: [], feedLandings: {}, memoryLedgers: {} +}); + +describe("shared moment record contract", () => { + it("round-trips the merged composer's output without a second consumer schema", () => { + const record: MomentRecord = composeMomentRecord(input()); + expect(record.exact).toBe(false); + expect(parseMomentRecord(JSON.stringify(record))).toEqual(record); + }); + + it("accepts only digest bundle pins and bounded memory cursors with optional generation", () => { + const record = { ...composeMomentRecord(input()), resources: [{ id: "tools", kind: "bundle", sha256: `sha256:${"a".repeat(64)}` }], + memory: [{ id: "self", declaredBy: "agent:writer", seq: 0, generation: 0 }] }; + expect(parseMomentRecord(JSON.stringify(record))).toEqual(record); + for (const invalid of [{ ...record, resources: [{ id: "tools", kind: "bundle", commit: "a".repeat(40) }] }, + { ...record, exact: undefined }, { ...record, memory: [{ ...record.memory[0], seq: Number.MAX_SAFE_INTEGER + 1 }] }, + { ...record, wake: { ...record.wake, deliveries: [] } }]) expect(momentRecordSchema.safeParse(invalid).success).toBe(false); + }); +}); diff --git a/src/recording/moment.ts b/src/recording/moment.ts new file mode 100644 index 00000000..2cc2d553 --- /dev/null +++ b/src/recording/moment.ts @@ -0,0 +1,40 @@ +import { z } from "zod"; +import { trainingInstantSchema } from "../manifest/index.js"; +import { MOMENT_RECORD_VERSION } from "./types.js"; + +const text = z.string().min(1); +const count = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER); +export const recordedSha256Schema = z.string().regex(/^sha256:[a-f0-9]{64}$/u); +export const recordedCommitSchema = z.string() + .regex(/^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/u, "Expected a full 40- or 64-hex commit object id") + .toLowerCase(); +const unique = (schema: T, key: (item: z.infer) => string) => + z.array(schema).superRefine((items, context) => { + const keys = new Set(); + items.forEach((item, index) => { + const id = key(item); + if (keys.has(id)) context.addIssue({ code: "custom", message: `Duplicate identity ${id}`, path: [index] }); + keys.add(id); + }); + }); +export const momentRecordSchema = z.object({ + version: z.literal(MOMENT_RECORD_VERSION), agent: text, + wake: z.object({ at: trainingInstantSchema, startedAt: trainingInstantSchema, executionId: text, + deliveries: z.array(z.object({ deliveryId: text, kind: text, occurredAt: trainingInstantSchema }).strict()).min(1), + surface: z.literal("moltnet").optional(), room: z.string().optional(), + from: z.string().optional(), messageId: text.optional() }).strict(), + recordedAt: trainingInstantSchema, exact: z.boolean(), errors: z.array(z.string()).optional(), + resources: unique(z.discriminatedUnion("kind", [ + z.object({ id: text, kind: z.literal("git"), commit: recordedCommitSchema, sha256: recordedSha256Schema.optional() }).strict(), + z.object({ id: text, kind: z.literal("bundle"), sha256: recordedSha256Schema }).strict() + ]), (r) => r.id), + feeds: unique(z.object({ id: text, revision: text, tree: text, commit: recordedCommitSchema.optional(), ref: text.optional(), + paths: z.array(text).optional(), landedAt: trainingInstantSchema, period: text.optional() }).strict(), (r) => r.id), + memory: unique(z.object({ id: text, declaredBy: text, seq: count, generation: count.optional() }).strict(), + (r) => JSON.stringify([r.declaredBy, r.id])), + moltnet: unique(z.object({ network: text, cursor: trainingInstantSchema, messageIds: z.array(text).optional() }).strict(), (r) => r.network), + volumes: unique(z.object({ id: text, snapshot: text }).strict(), (r) => r.id) +}).strict(); +export type MomentRecord = z.infer; + +export const parseMomentRecord = (payload: string): MomentRecord => momentRecordSchema.parse(JSON.parse(payload)); diff --git a/src/recording/types.ts b/src/recording/types.ts index 0fb84463..302322d8 100644 --- a/src/recording/types.ts +++ b/src/recording/types.ts @@ -29,25 +29,4 @@ export interface ComposeMomentRecordInput { recordedAt: string; } -export interface MomentRecord { - version: typeof MOMENT_RECORD_VERSION; - agent: string; - wake: { - at: string; - startedAt: string; - executionId: string; - deliveries: { deliveryId: string; kind: string; occurredAt: string }[]; - surface?: "moltnet"; - messageId?: string; - }; - recordedAt: string; - exact: boolean; - resources: ({ id: string; kind: "git"; commit: string; sha256?: string } - | { id: string; kind: "bundle"; sha256: string })[]; - feeds: { id: string; revision: string; tree: string; commit?: string; ref?: string; - paths?: string[]; landedAt: string; period?: string }[]; - memory: { id: string; declaredBy: string; seq: number }[]; - moltnet: { network: string; cursor: string; messageIds?: string[] }[]; - volumes: { id: string; snapshot: string }[]; - errors?: string[]; -} +export type { MomentRecord } from "./moment.js";