From aa8f49b41bdcc2dedfdf56a9d7886a45cbc21ea1 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sun, 11 Oct 2026 02:23:52 +0200 Subject: [PATCH 1/9] feat: materialize a training moment at its instant with a leak check --- src/compiler/training/AGENTS.md | 1 + src/compiler/training/index.ts | 1 + src/compiler/training/plan/load.test.ts | 13 +- src/compiler/training/rewind/AGENTS.md | 20 ++ src/compiler/training/rewind/CLAUDE.md | 1 + src/compiler/training/rewind/contract.ts | 63 ++++++ src/compiler/training/rewind/feed.test.ts | 157 +++++++++++++++ src/compiler/training/rewind/feed.ts | 129 ++++++++++++ .../training/rewind/fixtures.test-helper.ts | 68 +++++++ .../training/rewind/git.test-helper.ts | 30 +++ src/compiler/training/rewind/git.test.ts | 187 ++++++++++++++++++ src/compiler/training/rewind/git.ts | 120 +++++++++++ src/compiler/training/rewind/gitArchive.ts | 81 ++++++++ src/compiler/training/rewind/gitHistory.ts | 49 +++++ src/compiler/training/rewind/gitProcess.ts | 77 ++++++++ src/compiler/training/rewind/gitTree.ts | 69 +++++++ src/compiler/training/rewind/history.test.ts | 117 +++++++++++ src/compiler/training/rewind/history.ts | 120 +++++++++++ src/compiler/training/rewind/index.ts | 14 ++ src/compiler/training/rewind/inputs.test.ts | 122 ++++++++++++ src/compiler/training/rewind/inputs.ts | 64 ++++++ src/compiler/training/rewind/instant.test.ts | 35 ++++ src/compiler/training/rewind/instant.ts | 23 +++ .../training/rewind/leakCheck.test.ts | 96 +++++++++ src/compiler/training/rewind/leakCheck.ts | 39 ++++ src/compiler/training/rewind/leakFiles.ts | 62 ++++++ src/compiler/training/rewind/leakGit.ts | 57 ++++++ src/compiler/training/rewind/leakInputs.ts | 112 +++++++++++ .../training/rewind/materialize.test.ts | 107 ++++++++++ src/compiler/training/rewind/materialize.ts | 149 ++++++++++++++ src/compiler/training/rewind/memory.test.ts | 77 ++++++++ src/compiler/training/rewind/memory.ts | 122 ++++++++++++ src/compiler/training/rewind/memorySchemas.ts | 37 ++++ src/compiler/training/rewind/moltnet.ts | 44 +++++ .../training/rewind/moltnetJson.test.ts | 95 +++++++++ src/compiler/training/rewind/moltnetJson.ts | 158 +++++++++++++++ .../training/rewind/moltnetSchemas.ts | 78 ++++++++ .../training/rewind/moltnetSqlite.test.ts | 139 +++++++++++++ src/compiler/training/rewind/moltnetSqlite.ts | 116 +++++++++++ .../training/rewind/moltnetSqliteData.ts | 105 ++++++++++ src/compiler/training/rewind/select.test.ts | 120 +++++++++++ src/compiler/training/rewind/select.ts | 66 +++++++ .../training/rewind/taskClock.test.ts | 32 +++ src/compiler/training/rewind/taskClock.ts | 17 ++ src/compiler/training/rewind/timestamps.ts | 24 +++ src/compiler/training/rewind/types.ts | 27 +++ src/compiler/training/rewind/volumes.test.ts | 93 +++++++++ src/compiler/training/rewind/volumes.ts | 122 ++++++++++++ 48 files changed, 3652 insertions(+), 3 deletions(-) create mode 100644 src/compiler/training/rewind/AGENTS.md create mode 120000 src/compiler/training/rewind/CLAUDE.md create mode 100644 src/compiler/training/rewind/contract.ts create mode 100644 src/compiler/training/rewind/feed.test.ts create mode 100644 src/compiler/training/rewind/feed.ts create mode 100644 src/compiler/training/rewind/fixtures.test-helper.ts create mode 100644 src/compiler/training/rewind/git.test-helper.ts create mode 100644 src/compiler/training/rewind/git.test.ts create mode 100644 src/compiler/training/rewind/git.ts create mode 100644 src/compiler/training/rewind/gitArchive.ts create mode 100644 src/compiler/training/rewind/gitHistory.ts create mode 100644 src/compiler/training/rewind/gitProcess.ts create mode 100644 src/compiler/training/rewind/gitTree.ts create mode 100644 src/compiler/training/rewind/history.test.ts create mode 100644 src/compiler/training/rewind/history.ts create mode 100644 src/compiler/training/rewind/index.ts create mode 100644 src/compiler/training/rewind/inputs.test.ts create mode 100644 src/compiler/training/rewind/inputs.ts create mode 100644 src/compiler/training/rewind/instant.test.ts create mode 100644 src/compiler/training/rewind/instant.ts create mode 100644 src/compiler/training/rewind/leakCheck.test.ts create mode 100644 src/compiler/training/rewind/leakCheck.ts create mode 100644 src/compiler/training/rewind/leakFiles.ts create mode 100644 src/compiler/training/rewind/leakGit.ts create mode 100644 src/compiler/training/rewind/leakInputs.ts create mode 100644 src/compiler/training/rewind/materialize.test.ts create mode 100644 src/compiler/training/rewind/materialize.ts create mode 100644 src/compiler/training/rewind/memory.test.ts create mode 100644 src/compiler/training/rewind/memory.ts create mode 100644 src/compiler/training/rewind/memorySchemas.ts create mode 100644 src/compiler/training/rewind/moltnet.ts create mode 100644 src/compiler/training/rewind/moltnetJson.test.ts create mode 100644 src/compiler/training/rewind/moltnetJson.ts create mode 100644 src/compiler/training/rewind/moltnetSchemas.ts create mode 100644 src/compiler/training/rewind/moltnetSqlite.test.ts create mode 100644 src/compiler/training/rewind/moltnetSqlite.ts create mode 100644 src/compiler/training/rewind/moltnetSqliteData.ts create mode 100644 src/compiler/training/rewind/select.test.ts create mode 100644 src/compiler/training/rewind/select.ts create mode 100644 src/compiler/training/rewind/taskClock.test.ts create mode 100644 src/compiler/training/rewind/taskClock.ts create mode 100644 src/compiler/training/rewind/timestamps.ts create mode 100644 src/compiler/training/rewind/types.ts create mode 100644 src/compiler/training/rewind/volumes.test.ts create mode 100644 src/compiler/training/rewind/volumes.ts diff --git a/src/compiler/training/AGENTS.md b/src/compiler/training/AGENTS.md index d47925d4..6505ecc9 100644 --- a/src/compiler/training/AGENTS.md +++ b/src/compiler/training/AGENTS.md @@ -5,6 +5,7 @@ - `context.ts` resolves the full compiler graph and pins source files without compiling or launching it. - `resolution.ts` merges direct-member training metadata during traversal and preserves declaring sources. - `plan/` owns local authoring-file loading, pure plan resolution, and the public training-plan schema. +- `rewind/` materializes authored moments from contained local history, with explicit fidelity and an independent disk leak check; it never runs trials. - Preserve resolved inheritance and exact agent IDs. Never create a second agent declaration. - Source mappings are project-relative editable files, not runtime-native destinations or flattened prompts. - Do not serialize secret values, arbitrary environments or transport credentials. diff --git a/src/compiler/training/index.ts b/src/compiler/training/index.ts index 5bd21453..e5d9f970 100644 --- a/src/compiler/training/index.ts +++ b/src/compiler/training/index.ts @@ -1,6 +1,7 @@ export * from "./contract.js"; export * from "./context.js"; export * from "./plan/index.js"; +export * from "./rewind/index.js"; export type { ResolvedTraining, ResolvedTrainingGrader } from "./resolution.js"; export { parseTrainingMappedPreparation } from "./preparation/contract.js"; export type { TrainingMappedPreparation } from "./preparation/contract.js"; diff --git a/src/compiler/training/plan/load.test.ts b/src/compiler/training/plan/load.test.ts index 96300475..a72c4b6d 100644 --- a/src/compiler/training/plan/load.test.ts +++ b/src/compiler/training/plan/load.test.ts @@ -5,6 +5,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { buildCompilePlan } from "../../buildCompilePlan.js"; import { loadTrainingProject, createTrainingPlan } from "./load.js"; import { trainingPlanJsonSchema, trainingPlanSchema } from "./contract.js"; +import { materializedMomentJsonSchema, leakCheckJsonSchema, taskClockJsonSchema } from "../rewind/contract.js"; import { agentManifest, cleanTrainingProjects, dataFiles, example, target, teamManifest, training, trainingProject, writeTrainingFile @@ -76,10 +77,16 @@ describe("training authoring loader", () => { expect((await createTrainingPlan(root, options)).targets[0].files[0].path).toBe("C:notes.md"); }); - it("keeps the documented plan JSON Schema identical to the exported contract", async () => { + it.each([ + ["training-plan-schema", trainingPlanJsonSchema], + ["materialized-moment-schema", materializedMomentJsonSchema], + ["leak-check-schema", leakCheckJsonSchema], + ["task-clock-schema", taskClockJsonSchema] + ])("keeps the documented %s identical to the exported contract", async (marker, expected) => { const doc = await readFile(new URL("../../../../specs/TRAINING.md", import.meta.url), "utf8"); - const schema = doc.match(/\n```json\n([\s\S]*?)\n```/u)?.[1]; - expect(JSON.parse(schema!)).toEqual(trainingPlanJsonSchema); + const pattern = "\\n```json\\n([\\s\\S]*?)\\n```"; + const schema = doc.match(new RegExp(pattern, "u"))?.[1]; + expect(JSON.parse(schema!)).toEqual(expected); }); }); diff --git a/src/compiler/training/rewind/AGENTS.md b/src/compiler/training/rewind/AGENTS.md new file mode 100644 index 00000000..e9d1429b --- /dev/null +++ b/src/compiler/training/rewind/AGENTS.md @@ -0,0 +1,20 @@ +# Training Rewind + +- `contract.ts` owns strict moment, materialization, leak and task-clock documents and generated schemas. +- `history.ts` is the only boundary for reading history files, logs and retained bytes; every path must be lexically and physically contained. +- `select.ts` and `inputs.ts` resolve records, examples and effective inputs without I/O. +- `types.ts` defines input identities, declarations and per-kind contexts/results; `index.ts` exports the public materializer and pure selection/cut helpers. +- `git.ts` owns resources; `gitProcess.ts`, `gitHistory.ts`, `gitTree.ts`, and `gitArchive.ts` isolate argv execution, containment, blob trees, and feed archives. +- `feed.ts`, `memory.ts`, and `volumes.ts` own their cuts and history-log schemas; `memorySchemas.ts` validates native ledger events. +- `moltnet.ts` dispatches stores; `moltnetJson.ts`, `moltnetSqlite.ts`, `moltnetSqliteData.ts`, and `moltnetSchemas.ts` own snapshots, copied databases, SQL decoding, and strict native shapes. +- `materialize.ts` stages all inputs and publishes only after the independent disk reader in `leakCheck.ts` accepts them. +- `leakFiles.ts`, `leakGit.ts`, and `leakInputs.ts` inspect disk metadata, every git object, and native payloads independently of materialization decisions. +- `instant.ts` compares parsed nanosecond instants; `timestamps.ts` stamps metadata without following symlinks. +- `taskClock.ts` defines offset wall-clock environment only; timeouts remain real. libfaketime requires trial-owned LD_PRELOAD. +- Adjacent tests use local temporary repositories and fixtures. Never use Docker, production hosts or network fetches. +- `fixtures.test-helper.ts` and `git.test-helper.ts` build disposable real projects and repositories; package emission excludes them. +- Selector-at git history is approximate: committer dates cannot prove when a commit reached a ref. Feed-rule approximation cannot reconstruct freeze holds. +- Reuse P2 effective input identity resolution. No input may silently use present-day data. +- Mneme's root API validates the cut through a temporary rebuilt index, which is discarded: output contains neither derived indexes nor authority/audit state. Missing optional Mneme support refuses explicitly. +- Generated inputs spanning repositories refuse: this history contract carries one commit per resource. Never claim that one repository proves another repository's bytes. +- Keep modules under 400 lines, use named/barrel exports, and never import Mneme internals. diff --git a/src/compiler/training/rewind/CLAUDE.md b/src/compiler/training/rewind/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/src/compiler/training/rewind/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/src/compiler/training/rewind/contract.ts b/src/compiler/training/rewind/contract.ts new file mode 100644 index 00000000..6be30b3c --- /dev/null +++ b/src/compiler/training/rewind/contract.ts @@ -0,0 +1,63 @@ +import { z } from "zod"; +import { trainingInstantSchema } from "../../../manifest/index.js"; + +const text = z.string().min(1); +const count = z.number().int().nonnegative(); +export const inputIdentitySchema = z.object({ + section: z.enum(["resources", "memory", "networks"]), id: text, declaredBy: text +}).strict(); +const unique = (schema: T, key: (item: z.infer) => string) => + z.array(schema).superRefine((items, context) => { + const keys = new Set(); + items.forEach((item, index) => { + const id = key(item); + if (keys.has(id)) context.addIssue({ code: "custom", message: `Duplicate identity ${id}`, path: [index] }); + keys.add(id); + }); + }); +export const momentRecordSchema = z.object({ + version: z.literal("spawnfile.moment-record.v1"), agent: text, + wake: z.object({ at: trainingInstantSchema, surface: z.literal("moltnet"), room: z.string(), + from: z.string(), messageId: text.optional() }).strict(), + recordedAt: trainingInstantSchema, + resources: unique(z.object({ id: text, kind: z.enum(["git", "bundle"]), commit: text }).strict(), (r) => r.id), + feeds: unique(z.object({ id: text, revision: text, tree: text, commit: text, + paths: z.array(text), landedAt: trainingInstantSchema }).strict(), (r) => r.id), + memory: unique(z.object({ id: text, declaredBy: text, seq: count, generation: count }).strict(), + (r) => JSON.stringify([r.declaredBy, r.id])), + moltnet: unique(z.object({ network: text, cursor: text }).strict(), (r) => r.network), + volumes: unique(z.object({ id: text, snapshot: text }).strict(), (r) => r.id) +}).strict(); +export type MomentRecord = z.infer; + +export const materializedInputSchema = inputIdentitySchema.extend({ + kind: z.enum(["git", "bundle", "feed", "volume", "memory", "moltnet"]), + fidelity: z.enum(["exact", "approximate", "overridden"]).nullable(), basis: text, + source: z.object({ commit: text.optional(), revision: text.optional(), seq: count.optional(), + generation: count.optional(), snapshot: text.optional(), cursor: text.optional() }).strict(), + path: text.nullable(), notes: z.array(z.string()), refusal: text.nullable(), build: z.literal("deferred").optional() +}).strict(); +export type MaterializedInput = z.infer; +export const materializedMomentSchema = z.object({ + version: z.literal("spawnfile.materialized-moment.v1"), status: z.enum(["materialized", "refused"]), + agent: text, example: z.object({ id: text, role: z.enum(["train", "validation", "test"]), dataset: text }).strict(), + at: trainingInstantSchema, inputs: z.array(materializedInputSchema), record: z.object({ matched: z.boolean() }).strict(), + leakCheck: z.object({ ok: z.boolean(), findings: count }).strict() +}).strict(); +export type MaterializedMoment = z.infer; +export const leakFindingSchema = z.object({ + input: inputIdentitySchema.nullable(), + class: z.enum(["file-time", "symlink-escape", "git-object", "memory-event", "moltnet-message", + "moltnet-event", "moltnet-room", "feed-identity", "dated-path", "snapshot"]), path: text, detail: text +}).strict(); +export const leakCheckSchema = z.object({ + version: z.literal("spawnfile.leak-check.v1"), at: trainingInstantSchema, ok: z.boolean(), + checked: z.object({ files: count, gitObjects: count, memoryEvents: count, moltnetMessages: count, + moltnetEvents: count, snapshots: count }).strict(), findings: z.array(leakFindingSchema) +}).strict(); +export type LeakCheck = z.infer; +export type LeakFinding = z.infer; +export const taskClockSchema = z.object({ version: z.literal("spawnfile.task-clock.v1"), origin: trainingInstantSchema }).strict(); +export const materializedMomentJsonSchema = z.toJSONSchema(materializedMomentSchema); +export const leakCheckJsonSchema = z.toJSONSchema(leakCheckSchema); +export const taskClockJsonSchema = z.toJSONSchema(taskClockSchema); diff --git a/src/compiler/training/rewind/feed.test.ts b/src/compiler/training/rewind/feed.test.ts new file mode 100644 index 00000000..eda4760d --- /dev/null +++ b/src/compiler/training/rewind/feed.test.ts @@ -0,0 +1,157 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { mkdir, readFile, readdir, readlink, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { TeamWorkspaceResource } from "../../../manifest/index.js"; +import { digestDirectory, feedIdentityFindings, resolveFeedSource } from "../../../volume/index.js"; +import { feedLandingSchema, materializeFeed, selectFeedLanding, type FeedLanding } from "./feed.js"; +import { History } from "./history.js"; +import { commitTestGit, gitTestRoot, initTestGit, testGit, testMomentRecord } from "./git.test-helper.js"; +import type { InputContext } from "./types.js"; + +const AT = "2026-10-09T08:20:31Z", EARLY = "2026-10-09T08:00:00Z", LATE = "2026-10-09T09:00:00Z"; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const fixture = async () => { + const root = await gitTestRoot(); roots.push(root); + const directory = path.join(root, "history/feeds/agent%3Awriter/news"), repository = path.join(directory, "repo"); + await initTestGit(repository); + const past = await commitTestGit(repository, { "published/news.txt": "past\n", "ignored/private.txt": "not-fed\n" }, EARLY); + const future = await commitTestGit(repository, { "published/news.txt": "future\n" }, LATE); + const revision = resolveFeedSource({ kind: "git", fetch: false, repo: repository, ref: past, paths: ["published"] }).revision; + const landing: FeedLanding = { revision, tree: revision, commit: past, ref: "main", paths: ["published"], landedAt: EARLY }; + const resource: TeamWorkspaceResource = { id: "news", kind: "volume", mode: "mutable", mount: "./news", name: "org-news", + feed: { git: { repo: "https://never-fetch.invalid/source", ref: "main", paths: ["published"], fetch: true } } }; + const context: InputContext = { at: AT, destination: path.join(root, "output"), history: new History(path.join(root, "history")), record: null, + input: { section: "resources", declaredBy: "agent:writer", id: "news", declarationBase: root, overridden: false, declaration: resource } }; + const log = async (entries: FeedLanding[]) => writeFile(path.join(directory, "landings.jsonl"), `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`); + return { root, repository, directory, past, future, revision, landing, context, resource, log }; +}; + +describe("feed rewind", () => { + it("selects exact boundary landings and matches offset-equivalent records", async () => { + const { context, landing, log, revision } = await fixture(); + const selected = { ...landing, landedAt: AT }; + await log([selected, { ...landing, revision: "f".repeat(64), tree: "f".repeat(64), landedAt: "2026-10-09T08:20:31.001Z" }]); + context.record = { ...testMomentRecord(), feeds: [{ id: "news", revision, tree: revision, commit: landing.commit!, paths: ["published"], landedAt: "2026-10-09T10:20:31+02:00" }] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "landing-log", source: { commit: landing.commit, revision }, instant: AT }); + expect(await readlink(path.join(context.destination, "current"))).toBe(`trees/${revision}`); + expect(await readFile(path.join(context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); + const identity = JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")); + expect(feedIdentityFindings(identity)).toEqual([]); + expect(identity).toMatchObject({ landed_at: AT, files: 1, source: { commit: landing.commit, paths: ["published"], ref: "main" } }); + }); + + it("falls back to the feed rule at the trial instant with historical committer landing time", async () => { + const { context, past, revision } = await fixture(); + expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", basis: "feed-rule-at", source: { commit: past, revision }, instant: "2026-10-09T08:00:00.000Z" }); + const identity = JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")); + expect(Date.parse(identity.landed_at)).toBeLessThanOrEqual(Date.parse(AT)); + expect(await readdir(path.join(context.destination, "current"))).toEqual(["published"]); + }); + + it("expands the Berlin date in both ref and paths near UTC midnight", async () => { + const { context, resource, repository } = await fixture(); + const commit = await commitTestGit(repository, { "2026-10-10/news.txt": "Berlin tomorrow\n" }, "2026-10-09T22:00:00Z"); + testGit(repository, ["branch", "daily/2026-10-10", commit]); + resource.feed = { git: { repo: repository, ref: { template: "daily/${date:Europe/Berlin}" }, paths: ["${date:Europe/Berlin}"] } }; + context.at = "2026-10-09T22:30:00Z"; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", source: { commit } }); + expect(await readFile(path.join(context.destination, "current/2026-10-10/news.txt"), "utf8")).toBe("Berlin tomorrow\n"); + expect(JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")).source).toMatchObject({ ref: "daily/2026-10-10", paths: ["2026-10-10"] }); + }); + + it("uses fallback when the primary ref exists but has no commit at or before at", async () => { + const { context, resource, repository, past } = await fixture(); + testGit(repository, ["checkout", "--orphan", "daily/2026-10-09"]); + await commitTestGit(repository, { "published/news.txt": "future orphan\n" }, LATE); + resource.feed!.git!.ref = { template: "daily/${date}", fallback: "main" }; + expect(await materializeFeed(context)).toMatchObject({ source: { commit: past } }); + expect(JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")).source.ref).toBe("main"); + }); + + it("uses retained trees for prepared feeds without executing the recipe or ref command", async () => { + const { context, resource, directory, landing, log } = await fixture(); + resource.feed = { git: { repo: "never-used", ref: { command: ["never-run-ref"] } }, freeze: { timezone: "UTC", after: "08:00" }, + include: [{ from: "not-on-disk", to: "assets" }], prepare: { host: true, command: ["never-run-prepare"] } }; + const retained = path.join(directory, "trees", landing.revision); + await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "prepared.txt"), "retained\n"); + await log([{ ...landing, tree: `${landing.revision}.2` }]); + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "landing-log" }); + expect(await readFile(path.join(context.destination, "current/prepared.txt"), "utf8")).toBe("retained\n"); + }); + + it("uses retained directory feed landings and starts overridden feeds empty", async () => { + const { context, resource, directory, log } = await fixture(); + resource.feed = { directory: "never-read" }; + const retainedTemp = path.join(directory, "retained"); await mkdir(retainedTemp); await writeFile(path.join(retainedTemp, "data.txt"), "directory\n"); + const revision = digestDirectory(retainedTemp), retained = path.join(directory, "trees", revision); + await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "data.txt"), "directory\n"); + await log([{ revision, tree: revision, commit: null, ref: null, paths: null, landedAt: EARLY }]); + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", source: { revision } }); + expect(await readFile(path.join(context.destination, "current/data.txt"), "utf8")).toBe("directory\n"); + await rm(context.destination, { recursive: true }); context.input.overridden = true; context.history = undefined; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "overridden", basis: "override" }); + expect(await readdir(context.destination)).toEqual([]); + }); + + it("refuses revision mismatches and disagreements with the moment record", async () => { + const { context, landing, log } = await fixture(); + await log([{ ...landing, revision: "0".repeat(64), tree: "0".repeat(64) }]); + await expect(materializeFeed(context)).rejects.toThrow("Feed revision mismatch"); + await log([landing]); + context.record = { ...testMomentRecord(), feeds: [{ id: "news", revision: "f".repeat(64), tree: landing.tree, + commit: landing.commit!, paths: ["published"], landedAt: EARLY }] }; + await expect(materializeFeed(context)).rejects.toThrow("disagrees with its landing log"); + }); + + it("refuses nonhistorical feed rules and missing retained prepared bytes", async () => { + const { context, resource, landing, log } = await fixture(); + resource.feed = { directory: "never-read" }; + await expect(materializeFeed(context)).rejects.toThrow("Directory feed cannot be rewound"); + resource.feed = { git: { repo: "never-read", ref: { command: ["never-execute"] } } }; + await expect(materializeFeed(context)).rejects.toThrow("command refs cannot be evaluated"); + resource.feed = { git: { repo: "never-read" }, freeze: { after: "08:00", timezone: "UTC" } }; + await expect(materializeFeed(context)).rejects.toThrow("freeze/waiting"); + resource.feed = { git: { repo: "never-read" }, prepare: { host: true, command: ["never-execute"] } }; + await log([landing]); + await expect(materializeFeed(context)).rejects.toThrow("retained landed tree bytes"); + context.history = undefined; + await expect(materializeFeed(context)).rejects.toThrow("--history is required"); + }); + + it("validates every landing line and refuses no eligible landing", async () => { + const { context, directory, landing, log } = await fixture(); + await log([{ ...landing, landedAt: LATE }]); + await expect(materializeFeed(context)).rejects.toThrow("No feed landing"); + await writeFile(path.join(directory, "landings.jsonl"), `${JSON.stringify(landing)}\n${JSON.stringify({ ...landing, unknown: true })}\n`); + await expect(materializeFeed(context)).rejects.toThrow(/landings.jsonl:2/u); + await writeFile(path.join(directory, "landings.jsonl"), ""); + await expect(materializeFeed(context)).rejects.toThrow("No feed landing"); + }); + + it("applies committed archive attributes and PAX long names when reconstructing feed bytes", async () => { + const { context, repository, resource } = await fixture(); + const longName = `published/${"x".repeat(150)}.txt`; + const commit = await commitTestGit(repository, { ".gitattributes": "published/hidden.txt export-ignore\npublished/version.txt export-subst\n", + "published/hidden.txt": "omitted\n", "published/version.txt": "$Format:%H$\n", [longName]: "long\n" }, "2026-10-09T08:10:00Z"); + resource.feed!.git!.ref = commit; + await materializeFeed(context); + expect(await readFile(path.join(context.destination, "current/published/version.txt"), "utf8")).toBe(`${commit}\n`); + expect(await readFile(path.join(context.destination, "current", longName), "utf8")).toBe("long\n"); + await expect(readFile(path.join(context.destination, "current/published/hidden.txt"))).rejects.toThrow(); + }); + + it("refuses retained history symlinks outside the history root", async () => { + const { context, directory, landing, log } = await fixture(); + const retained = path.join(directory, "trees", landing.revision); + await mkdir(retained, { recursive: true }); await symlink("/etc/passwd", path.join(retained, "escape")); await log([landing]); + await expect(materializeFeed(context)).rejects.toThrow("History symlink escape"); + }); + + it("selects pure instant boundaries and refuses contradictory equal-instant landings", () => { + const revision = "a".repeat(64), landing = feedLandingSchema.parse({ revision, tree: revision, commit: "commit", ref: "main", paths: [], landedAt: AT }); + expect(selectFeedLanding([landing], "2026-10-09T10:20:31+02:00")).toBe(landing); + expect(selectFeedLanding([landing], "2026-10-09T08:20:30.999Z")).toBeNull(); + expect(() => selectFeedLanding([landing, { ...landing, commit: "other", landedAt: "2026-10-09T10:20:31+02:00" }], AT)).toThrow("Conflicting feed landings"); + }); +}); diff --git a/src/compiler/training/rewind/feed.ts b/src/compiler/training/rewind/feed.ts new file mode 100644 index 00000000..a86100b9 --- /dev/null +++ b/src/compiler/training/rewind/feed.ts @@ -0,0 +1,129 @@ +import { lstat, mkdir, readdir, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { z } from "zod"; +import { trainingInstantSchema, type TeamWorkspaceResource } from "../../../manifest/index.js"; +import { expandFeedPaths, expandFeedRefTemplate, FEED_IDENTITY_FILE, FEED_IDENTITY_VERSION, + resolveFeedSource, type FeedExec, type FeedIdentity } from "../../../volume/index.js"; +import { exportFeedGitTree } from "./gitArchive.js"; +import { assertLocalGitRepository } from "./gitHistory.js"; +import { assertCommitNotFuture, commitAt, rewindGitSync, type SelectedCommit } from "./gitProcess.js"; +import { historyKey } from "./history.js"; +import { compareInstants } from "./instant.js"; +import type { InputContext, InputResult } from "./types.js"; + +type VolumeFeed = NonNullable["feed"]>; + +const revision = z.string().regex(/^[a-f0-9]{64}$/u); +const plainPath = z.string().min(1).refine((value) => !value.startsWith("/") && !/[\\\0]/u.test(value) + && !value.split("/").some((segment) => !segment || segment === "." || segment === ".." || segment.toLowerCase() === ".git"), "Expected a plain tree-relative path"); +export const feedLandingSchema = z.object({ + revision, tree: z.string().regex(/^(?:trees\/)?[a-f0-9]{64}(?:\.[1-9][0-9]*)?$/u), + commit: z.string().min(1).nullable(), ref: z.string().min(1).nullable(), paths: z.array(plainPath).nullable(), + landedAt: trainingInstantSchema +}).strict().superRefine((value, context) => { + if (!value.tree.replace(/^trees\//u, "").startsWith(value.revision)) context.addIssue({ code: "custom", message: "Landing tree must belong to its revision" }); +}); +export type FeedLanding = z.infer; + +/** Equal-instant landings must agree; log order cannot break a contradictory timestamp tie. */ +export const selectFeedLanding = (landings: readonly FeedLanding[], at: string): FeedLanding | null => { + const eligible = landings.filter((landing) => compareInstants(landing.landedAt, at) <= 0); + const selected = eligible.reduce((latest, entry) => !latest || compareInstants(entry.landedAt, latest.landedAt) > 0 ? entry : latest, null); + if (selected && eligible.some((entry) => compareInstants(entry.landedAt, selected.landedAt) === 0 + && JSON.stringify({ ...entry, landedAt: "" }) !== JSON.stringify({ ...selected, landedAt: "" }))) throw new Error("Conflicting feed landings at the same instant"); + return selected; +}; + +const revisionExec: FeedExec = (command, args) => { + if (command !== "git" || args[0] !== "-C" || !args[1]) throw new Error("Feed revision may only read a local git repository"); + return rewindGitSync(args[1], args.slice(2)); +}; +const feedPaths = (paths: string[] | null): string[] | undefined => paths?.length ? paths : undefined; + +/** Same template/fallback ordering as chooseFeedRef, with historical eligibility replacing existence. */ +export const selectFeedCommit = async (feed: VolumeFeed, repository: string, at: string): Promise => { + if (!feed.git) throw new Error("Directory feed cannot be rewound without a landing log; provide an override"); + if (feed.freeze) throw new Error("Feed freeze/waiting state cannot be reconstructed without a landing log; provide an override"); + const rule = feed.git.ref; + if (typeof rule === "object" && rule.command) throw new Error("Feed command refs cannot be evaluated historically without a landing log; provide an override"); + const selectors = typeof rule === "object" ? [expandFeedRefTemplate(rule.template!, new Date(at)), rule.fallback] + : [rule ?? "HEAD"]; + for (const selector of selectors) { + if (!selector) continue; + const selected = await commitAt(repository, selector, at); + if (selected) return { ...selected, ref: selector }; + } + throw new Error(`No feed ref has a first-parent commit at or before ${at}`); +}; + +const fileCount = async (directory: string): Promise => { + let count = 0; + for (const name of await readdir(directory)) { + const file = path.join(directory, name), stats = await lstat(file); + count += stats.isDirectory() ? await fileCount(file) : 1; + } + return count; +}; + +export const materializeFeed = async (context: InputContext): Promise => { + const { input, history } = context; + if (input.section !== "resources" || input.declaration.kind !== "volume" || !input.declaration.feed) throw new Error("Feed rewind requires a fed volume"); + const resource = input.declaration, feed = resource.feed!; + if (input.overridden) { + await mkdir(context.destination, { recursive: true }); + return { fidelity: "overridden", basis: "override", source: {}, notes: ["Volume override starts empty; no feed refresh is run."], instant: context.at }; + } + if (!history) throw new Error(`Missing feed history for ${input.id}; --history is required`); + const key = `feeds/${historyKey(input)}`; + const log = await history.jsonl(`${key}/landings.jsonl`, feedLandingSchema); + let landing: FeedLanding, repository: string | null = null; + const openRepository = async (): Promise => { + repository ??= await history.repository(`${key}/repo`); + if (!repository) throw new Error(`Missing feed source repository for ${input.id}`); + await assertLocalGitRepository(repository, history); + const gitDirectory = (rewindGitSync(repository, ["rev-parse", "--absolute-git-dir"])).toString().trim(); + if (await history.text(path.relative(history.root, path.join(gitDirectory, "info/attributes")))) throw new Error("Feed archive cannot reproduce unrecorded info/attributes; retain the landed tree"); + return repository; + }; + if (log) { + const selected = selectFeedLanding(log, context.at); + if (!selected) throw new Error(`No feed landing exists at or before ${context.at}`); + landing = selected; + const record = context.record?.feeds.find((entry) => entry.id === input.id); + if (record && (record.revision !== landing.revision || record.tree !== landing.tree || record.commit !== landing.commit + || JSON.stringify(feedPaths(record.paths)) !== JSON.stringify(feedPaths(landing.paths)) || compareInstants(record.landedAt, landing.landedAt) !== 0)) { + throw new Error(`Moment feed ${input.id} disagrees with its landing log`); + } + } else { + // Refuse unknown historical behavior before touching any declared source or executing any command. + if (!feed.git || feed.freeze || typeof feed.git.ref === "object" && feed.git.ref.command) await selectFeedCommit(feed, "", context.at); + if (feed.include || feed.prepare) throw new Error("Prepared/included feed requires a landing log and retained tree bytes"); + const repo = await openRepository(), selected = await selectFeedCommit(feed, repo, context.at); + const paths = expandFeedPaths(feed.git!.paths, new Date(context.at)); + const source = resolveFeedSource({ kind: "git", fetch: false, repo, ref: selected.commit, label: selected.ref, paths }, { exec: revisionExec }); + landing = { revision: source.revision, tree: source.revision, commit: selected.commit, ref: selected.ref, paths: paths ?? null, landedAt: selected.instant }; + } + const tree = path.join(context.destination, "trees", landing.revision); + if (await history.path(`${key}/trees/${landing.revision}`)) await history.copyTree(`${key}/trees/${landing.revision}`, tree); + else { + if (!feed.git || feed.include || feed.prepare) throw new Error("Directory/prepared/included feed requires retained landed tree bytes"); + if (!landing.commit || !landing.ref) throw new Error("Git feed landing must record its commit and ref"); + const repo = await openRepository(); + const resolved = resolveFeedSource({ kind: "git", fetch: false, repo, ref: landing.commit, label: landing.ref, paths: feedPaths(landing.paths) }, { exec: revisionExec }); + if (resolved.revision !== landing.revision) throw new Error(`Feed revision mismatch: recorded ${landing.revision}, reconstructed ${resolved.revision}`); + const commit = resolved.provenance.kind === "git" ? resolved.provenance.commit : landing.commit; + await assertCommitNotFuture(repo, commit, context.at); + await exportFeedGitTree(repo, commit, tree, feedPaths(landing.paths)); + } + const files = await fileCount(tree); + if (!files) throw new Error("Retained feed tree is empty; a fed volume never serves nothing"); + const identity: FeedIdentity = { version: FEED_IDENTITY_VERSION, resource: input.id, volume: resource.name!, revision: landing.revision, + tree: `trees/${landing.revision}`, files, landed_at: landing.landedAt, + source: feed.git ? { kind: "git", commit: landing.commit!, ref: landing.ref!, paths: landing.paths } : { kind: "directory" } }; + if (feed.git && (!landing.commit || !landing.ref)) throw new Error("Git feed landing must record its commit and ref"); + await symlink(identity.tree, path.join(context.destination, "current")); + await writeFile(path.join(context.destination, FEED_IDENTITY_FILE), `${JSON.stringify(identity, null, 2)}\n`); + return { fidelity: log ? "exact" : "approximate", basis: log ? "landing-log" : "feed-rule-at", + source: { revision: landing.revision, ...(landing.commit ? { commit: landing.commit } : {}) }, instant: landing.landedAt, + notes: log ? [] : ["Commit dates do not establish when a ref advanced; historical freeze holds cannot be reconstructed."] }; +}; diff --git a/src/compiler/training/rewind/fixtures.test-helper.ts b/src/compiler/training/rewind/fixtures.test-helper.ts new file mode 100644 index 00000000..ac6d3a1f --- /dev/null +++ b/src/compiler/training/rewind/fixtures.test-helper.ts @@ -0,0 +1,68 @@ +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { trainingProject, writeTrainingFile, teamFiles, teamManifest, volume, example, network } from "../plan/fixtures.test-helper.js"; +import { loadTrainingProject, resolveTrainingPlan } from "../plan/index.js"; +import { rewindGitEnvironment } from "./gitProcess.js"; +import { resolveFeedSource } from "../../../volume/feedSource.js"; +import type { MomentRecord } from "./contract.js"; +import type { MemoryLedgerEvent } from "./memorySchemas.js"; + +export const AT = "2026-10-09T08:20:31Z"; +export const BEFORE = "2026-10-09T07:00:00Z"; +export const FUTURE = "2026-10-09T08:20:31.001Z"; +export const memoryEvent = (seq: number, createdAt = BEFORE): MemoryLedgerEvent => ({ + id: `event-${seq}`, seq, createdAt, type: "memory.observed", principal: { agentId: "a", scope: "global" }, + scope: "global", visibility: "global", source: "rewind-test", content: { kind: "text", text: `Fact ${seq}` }, + tags: [], entities: [], sensitivity: "normal", parentEventIds: [], checksum: `checksum-${seq}` +}); +export const emptyRecord = (agent = "agent:a", at = AT): MomentRecord => ({ + version: "spawnfile.moment-record.v1", agent, wake: { at, surface: "moltnet", room: "assignment", from: "brass" }, + recordedAt: FUTURE, resources: [], feeds: [], memory: [], moltnet: [], volumes: [] +}); +export const jsonl = async (root: string, relative: string, values: unknown[]): Promise => { + const file = path.join(root, relative); + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, values.map((value) => JSON.stringify(value)).join("\n") + (values.length ? "\n" : "")); +}; +export const testGit = (repo: string, args: string[], date = BEFORE): string => execFileSync("git", ["-C", repo, ...args], { + encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env: { ...rewindGitEnvironment(), GIT_AUTHOR_NAME: "Rewind", GIT_AUTHOR_EMAIL: "rewind@example.invalid", + GIT_COMMITTER_NAME: "Rewind", GIT_COMMITTER_EMAIL: "rewind@example.invalid", GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date } +}).trim(); + +/** Real compiled team with all five storage paths, including a historical git feed. */ +export const rewindFixture = async () => { + const root = await trainingProject(teamFiles()); + const historyDir = path.join(root, "history"), gitRepo = path.join(historyDir, "git/team%3Apublication/code"); + await mkdir(gitRepo, { recursive: true }); + testGit(gitRepo, ["init", "--quiet", "--initial-branch=main"]); + await writeFile(path.join(gitRepo, "news.txt"), "Before the wake\n"); + testGit(gitRepo, ["add", "."]); testGit(gitRepo, ["commit", "--quiet", "-m", "before"]); + const commit = testGit(gitRepo, ["rev-parse", "HEAD"]); + const feedRepo = path.join(historyDir, "feeds/team%3Apublication/news/repo"); + await mkdir(path.dirname(feedRepo), { recursive: true }); + testGit(root, ["clone", "--quiet", "--no-local", gitRepo, feedRepo]); + const feedSource = resolveFeedSource({ kind: "git", repo: feedRepo, ref: commit, fetch: false }); + await jsonl(historyDir, "feeds/team%3Apublication/news/landings.jsonl", [{ revision: feedSource.revision, + tree: feedSource.revision, commit, ref: "main", paths: null, landedAt: BEFORE }]); + await writeFile(path.join(gitRepo, "future.txt"), "secret after wake\n"); + testGit(gitRepo, ["add", "."]); testGit(gitRepo, ["commit", "--quiet", "-m", "future"], "2026-10-10T00:00:00Z"); + const futureCommit = testGit(gitRepo, ["rev-parse", "HEAD"]); + const resources = [volume, { id: "code", kind: "git", mount: "./code", mode: "readonly", url: gitRepo, branch: "main" }, + { id: "news", kind: "volume", name: "news", mount: "./news", mode: "mutable", feed: { git: { repo: feedRepo } } }]; + const manifest = teamManifest({ memory: [] }); + manifest.shared.workspace.resources = resources as typeof manifest.shared.workspace.resources; + await writeTrainingFile(root, "Spawnfile", manifest); + await writeTrainingFile(root, "training/train.yaml", { examples: [{ ...example, networks: [network] }] }); + await jsonl(historyDir, "memory/agent%3Aa/self/events.jsonl", [memoryEvent(1), memoryEvent(2, FUTURE)]); + const data = Buffer.from("snapshot bytes\n"), digest = createHash("sha256").update(data).digest("hex"); + await jsonl(historyDir, "volumes/team%3Apublication/edition-state/snapshots.jsonl", [ + { snapshot: "snapshot-1", takenAt: BEFORE, parent: null, files: { "saved.txt": digest } } + ]); + const objectDir = path.join(historyDir, "volumes/team%3Apublication/edition-state/objects"); + await mkdir(objectDir, { recursive: true }); await writeFile(path.join(objectDir, digest), data); + const project = await loadTrainingProject(root), plan = resolveTrainingPlan(project, "agent:a", "test"); + return { root, historyDir, project, plan, gitRepo, futureCommit, commit, feedRepo, + options: { project, plan, example: example.id, historyDir, out: path.join(root, "out") } }; +}; diff --git a/src/compiler/training/rewind/git.test-helper.ts b/src/compiler/training/rewind/git.test-helper.ts new file mode 100644 index 00000000..7857ff66 --- /dev/null +++ b/src/compiler/training/rewind/git.test-helper.ts @@ -0,0 +1,30 @@ +import { execFileSync } from "node:child_process"; +import { mkdir, mkdtemp, realpath, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { MomentRecord } from "./contract.js"; +import { rewindGitEnvironment } from "./gitProcess.js"; + +export const gitTestRoot = async (): Promise => realpath(await mkdtemp(path.join(os.tmpdir(), "spawnfile-rewind-git-"))); +export const testGit = (repository: string, args: string[], at = "2026-10-09T08:00:00Z"): string => execFileSync("git", [ + "-c", "core.hooksPath=/dev/null", "-c", "core.autocrlf=false", "-C", repository, ...args +], { encoding: "utf8", env: { ...rewindGitEnvironment(), GIT_AUTHOR_NAME: "Rewind Test", GIT_AUTHOR_EMAIL: "rewind@example.invalid", + GIT_COMMITTER_NAME: "Rewind Test", GIT_COMMITTER_EMAIL: "rewind@example.invalid", GIT_AUTHOR_DATE: at, GIT_COMMITTER_DATE: at }, stdio: ["ignore", "pipe", "pipe"] }).trim(); +export const initTestGit = async (repository: string): Promise => { + await mkdir(repository, { recursive: true }); + testGit(repository, ["init", "--quiet", "--initial-branch=main", "--template="]); +}; +export const commitTestGit = async (repository: string, files: Record, at: string): Promise => { + for (const [name, contents] of Object.entries(files)) { + const file = path.join(repository, name); + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, contents); + } + testGit(repository, ["add", "--all"]); + testGit(repository, ["commit", "--quiet", "--allow-empty", "-m", `at ${at}`], at); + return testGit(repository, ["rev-parse", "HEAD"]); +}; +export const testMomentRecord = (at = "2026-10-09T08:20:31Z"): MomentRecord => ({ + version: "spawnfile.moment-record.v1", agent: "agent:writer", wake: { at, surface: "moltnet", room: "assignment", from: "brass" }, + recordedAt: at, resources: [], feeds: [], memory: [], moltnet: [], volumes: [] +}); diff --git a/src/compiler/training/rewind/git.test.ts b/src/compiler/training/rewind/git.test.ts new file mode 100644 index 00000000..addecc47 --- /dev/null +++ b/src/compiler/training/rewind/git.test.ts @@ -0,0 +1,187 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { mkdir, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { TeamWorkspaceResource } from "../../../manifest/index.js"; +import { BundleTarWriter } from "../../workspaceBundleTar.js"; +import { materializeGit, localRepositoryPath, selectCommitAt } from "./git.js"; +import { History } from "./history.js"; +import { commitTestGit, gitTestRoot, initTestGit, testGit, testMomentRecord } from "./git.test-helper.js"; +import type { InputContext } from "./types.js"; + +const AT = "2026-10-09T08:20:31Z", EARLY = "2026-10-09T08:00:00Z", LATE = "2026-10-09T09:00:00Z"; +const roots: string[] = []; +afterEach(async () => { vi.unstubAllEnvs(); await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const fixture = async () => { + const root = await gitTestRoot(); roots.push(root); + const repository = path.join(root, "history/git/agent%3Awriter/archive"); + await initTestGit(repository); + const past = await commitTestGit(repository, { "readme.txt": "past\n", "package/package.json": "{}\n", "package/secret.txt": "omit\n" }, EARLY); + const future = await commitTestGit(repository, { "readme.txt": "future\n", "future.txt": "tomorrow\n" }, LATE); + testGit(repository, ["tag", "future-tag", future]); + testGit(repository, ["remote", "add", "origin", "https://network-must-not-run.invalid/repo.git"]); + const resource: TeamWorkspaceResource = { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: repository }; + const context: InputContext = { at: AT, destination: path.join(root, "output"), history: new History(path.join(root, "history")), record: null, + input: { section: "resources", declaredBy: "agent:writer", id: "archive", declarationBase: repository, overridden: false, declaration: resource } }; + return { root, repository, past, future, context }; +}; + +describe("git rewind", () => { + it("records exactly one commit's reachable closure and removes future objects, refs and configuration", async () => { + const { repository, past, future, context } = await fixture(); + context.record = { ...testMomentRecord(), resources: [{ id: "archive", kind: "git", commit: past }] }; + const result = await materializeGit(context); + expect(result).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: past } }); + expect(await readFile(path.join(context.destination, "readme.txt"), "utf8")).toBe("past\n"); + const objects = testGit(context.destination, ["cat-file", "--batch-all-objects", "--batch-check=%(objectname)"]).split("\n").sort(); + expect(objects).toEqual(testGit(repository, ["rev-list", "--objects", past]).split("\n").map((line) => line.split(" ")[0]).sort()); + expect(objects).not.toContain(future); + expect(testGit(context.destination, ["for-each-ref", "--format=%(refname)"])).toBe("refs/heads/moment"); + expect(testGit(context.destination, ["remote"])).toBe(""); + expect(testGit(context.destination, ["status", "--porcelain"])).toBe(""); + for (const name of ["logs", "FETCH_HEAD", "ORIG_HEAD", "packed-refs"]) await expect(readFile(path.join(context.destination, ".git", name))).rejects.toThrow(); + expect(testGit(repository, ["rev-parse", "HEAD"])).toBe(future); + expect(testGit(repository, ["tag"])).toBe("future-tag"); + }); + + it("can materialize a local full-object pin without history and ignores injected git environment", async () => { + const { repository, past, context } = await fixture(); + context.history = undefined; + context.input.declaration = { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: repository, ref: past.toUpperCase() }; + context.record = { ...testMomentRecord(), resources: [{ id: "archive", kind: "bundle", commit: "not-the-pin" }] }; + vi.stubEnv("GIT_DIR", "/does/not/exist"); + vi.stubEnv("GIT_CONFIG_COUNT", "1"); vi.stubEnv("GIT_CONFIG_KEY_0", "core.worktree"); vi.stubEnv("GIT_CONFIG_VALUE_0", "/elsewhere"); + expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "pinned", source: { commit: past } }); + expect(await readFile(path.join(context.destination, "readme.txt"), "utf8")).toBe("past\n"); + }); + + it("uses selector-at first-parent fallback and labels its approximation", async () => { + const { past, context } = await fixture(); + expect(await materializeGit(context)).toMatchObject({ fidelity: "approximate", basis: "selector-at", source: { commit: past }, notes: [expect.stringContaining("dates")] }); + }); + + it("does not visit a later merged side branch when selecting the first-parent date", async () => { + const { repository, past, context } = await fixture(); + testGit(repository, ["checkout", "-b", "side", past]); + await commitTestGit(repository, { "side.txt": "side\n" }, "2026-10-09T08:19:00Z"); + testGit(repository, ["checkout", "main"]); + testGit(repository, ["merge", "--no-ff", "--no-edit", "side"], "2026-10-09T10:00:00Z"); + expect((await materializeGit(context)).source.commit).toBe(past); + }); + + it("uses override declaration content and marks it overridden", async () => { + const { past, repository, context } = await fixture(); + context.history = undefined; context.input.overridden = true; + context.input.declaration = { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: repository, ref: past }; + expect(await materializeGit(context)).toMatchObject({ fidelity: "overridden", basis: "override", source: { commit: past } }); + }); + + it("refuses absent history, prebuilt bundles without records and network-only pins", async () => { + const { past, context } = await fixture(); context.history = undefined; + await expect(materializeGit(context)).rejects.toThrow("unpinned resources require --history"); + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", source: "today.tar" }; + await expect(materializeGit(context)).rejects.toThrow("Prebuilt bundle has no recorded commit"); + context.input.declaration = { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: "https://example.invalid/repo", ref: past }; + await expect(materializeGit(context)).rejects.toThrow("network access is forbidden"); + }); + + it("exports files bundle roots and exclusions at their pinned commit", async () => { + const { past, context } = await fixture(); + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { files: { root: "package", ref: past, exclude: ["secret.txt"] } } }; + expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "pinned", source: { commit: past } }); + expect(await readdir(context.destination)).toEqual(["package.json"]); + }); + + it("refuses future pinned bundle metadata before exporting a plain tree", async () => { + const { repository, past, future, context } = await fixture(); + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { files: { root: "package", ref: future } } }; + await expect(materializeGit(context)).rejects.toThrow("author or committer date after"); + // The author clock is independently checked: an old committer date alone is insufficient. + testGit(repository, ["checkout", "--detach", past]); + testGit(repository, ["commit", "--amend", "--no-edit", `--date=${LATE}`], EARLY); + context.input.declaration.build!.files!.ref = testGit(repository, ["rev-parse", "HEAD"]); + await expect(materializeGit(context)).rejects.toThrow("author or committer date after"); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it("exports a prebuilt bundle's recorded git tree instead of today's tar", async () => { + const { past, context } = await fixture(); + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", source: "does-not-exist.tar" }; + context.record = { ...testMomentRecord(), resources: [{ id: "archive", kind: "bundle", commit: past }] }; + expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "record" }); + expect(await readFile(path.join(context.destination, "readme.txt"), "utf8")).toBe("past\n"); + await expect(readFile(path.join(context.destination, ".git"))).rejects.toThrow(); + }); + + it("defers dependency and generated recipes while keeping all declared historical inputs", async () => { + const { context } = await fixture(); + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { dependencies: { directory: "package", image: `node@sha256:${"0".repeat(64)}` } } }; + expect(await materializeGit(context)).toMatchObject({ build: "deferred", fidelity: "approximate" }); + expect(await readdir(context.destination)).toEqual(["package.json", "secret.txt"]); + await rm(context.destination, { recursive: true }); + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { generated: { + command: ["never-run"], cwd: ".", inputs: [{ root: ".", exclude: ["package/secret.txt"] }, { root: "package" }] + } } }; + expect(await materializeGit(context)).toMatchObject({ build: "deferred" }); + expect(await readFile(path.join(context.destination, "package/secret.txt"), "utf8")).toBe("omit\n"); + expect(await readFile(path.join(context.destination, "readme.txt"), "utf8")).toBe("past\n"); + }); + + it("refuses generated inputs spanning repositories instead of substituting the wrong tree", async () => { + const { root, context } = await fixture(), other = path.join(root, "other"); + await initTestGit(other); await commitTestGit(other, { "other.txt": "other repository\n" }, EARLY); + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { generated: { + command: ["never-run"], inputs: [{ root: "." }, { root: other }] + } } }; + await expect(materializeGit(context)).rejects.toThrow("multiple git repositories"); + }); + + it("preserves the compiler's bundle symlink refusal after applying exclusions", async () => { + const { repository, context } = await fixture(); + await symlink("package.json", path.join(repository, "package/link")); + const ref = await commitTestGit(repository, {}, "2026-10-09T08:10:00Z"); + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { files: { root: "package", ref } } }; + await expect(materializeGit(context)).rejects.toThrow("Bundle input is a symlink"); + await rm(context.destination, { recursive: true, force: true }); + context.input.declaration.build!.files!.exclude = ["link"]; + expect(await materializeGit(context)).toMatchObject({ fidelity: "exact" }); + }); + + it("extracts a validated local prebuilt override", async () => { + const { root, context } = await fixture(), tar = path.join(root, "override.tar"); + const writer = await BundleTarWriter.create(tar); + await writer.begin("override.txt", 6, 0o644); await writer.data(Buffer.from("fresh\n")); await writer.end(); await writer.finish(); + context.input.overridden = true; + context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", source: tar }; + expect(await materializeGit(context)).toMatchObject({ fidelity: "overridden", source: {} }); + expect(await readFile(path.join(context.destination, "override.txt"), "utf8")).toBe("fresh\n"); + }); + + it("rejects textual history indirections before following them", async () => { + const { repository, context } = await fixture(); + await writeFile(path.join(repository, ".git/objects/info/alternates"), "/outside/object-store\n"); + await expect(materializeGit(context)).rejects.toThrow("alternates"); + await rm(path.join(repository, ".git/objects/info/alternates")); + await writeFile(path.join(repository, ".git/config"), "[include]\npath = /outside/config\n"); + await expect(materializeGit(context)).rejects.toThrow("configuration indirection"); + await writeFile(path.join(repository, ".git/commondir"), "/outside/common\n"); + await expect(materializeGit(context)).rejects.toThrow("commondir"); + }); + + it("selects date boundaries by instant, rejects unsafe local locations", () => { + const at = "2026-10-09T10:20:31+02:00"; + expect(selectCommitAt([{ commit: "equal", instant: AT }, { commit: "later", instant: "2026-10-09T08:20:31.001Z" }], at)?.commit).toBe("equal"); + expect(selectCommitAt([{ commit: "later", instant: LATE }], AT)).toBeNull(); + expect(() => localRepositoryPath("git@host:repo", "/tmp")).toThrow("network"); + expect(() => localRepositoryPath("file://remote/tmp/repo", "/tmp")).toThrow("remote file hosts"); + }); + + it("supports standalone bare repositories", async () => { + const { root, repository, past, context } = await fixture(); + const bare = path.join(root, "bare"); await mkdir(bare); + testGit(root, ["clone", "--quiet", "--bare", repository, bare]); + await rm(repository, { recursive: true }); + await mkdir(path.dirname(repository), { recursive: true }); + testGit(root, ["clone", "--quiet", "--bare", bare, repository]); + expect((await materializeGit(context)).source.commit).toBe(past); + }); +}); diff --git a/src/compiler/training/rewind/git.ts b/src/compiler/training/rewind/git.ts new file mode 100644 index 00000000..ba36f3e1 --- /dev/null +++ b/src/compiler/training/rewind/git.ts @@ -0,0 +1,120 @@ +import { mkdir, rm } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import type { TeamWorkspaceResource } from "../../../manifest/index.js"; +import { historyKey } from "./history.js"; +import { assertLocalGitRepository } from "./gitHistory.js"; +import { assertCommitNotFuture, commitAt, commitInstant, resolveRewindCommit, rewindGit } from "./gitProcess.js"; +import { exportGitTree, extractOverrideBundle } from "./gitTree.js"; +import type { InputContext, InputResult } from "./types.js"; + +export { commitAt, commitInstant, resolveRewindCommit, rewindGit, rewindGitEnvironment, rewindGitSync, selectCommitAt } from "./gitProcess.js"; +export { exportGitTree } from "./gitTree.js"; +export { assertLocalGitRepository } from "./gitHistory.js"; + +type GitResource = Extract; +type BundleResource = Extract; +const FULL_OBJECT = /^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/u; + +export const localRepositoryPath = (location: string, base: string): string => { + if (location.startsWith("file://")) { + const url = new URL(location); + if (url.hostname && url.hostname !== "localhost") throw new Error("Rewind only reads local repositories; remote file hosts are forbidden"); + return fileURLToPath(url); + } + if (/^[A-Za-z][A-Za-z0-9+.-]*:/u.test(location) || /^[^/]+@[^:]+:/u.test(location)) throw new Error("Rewind only reads local repositories; network access is forbidden"); + return path.resolve(base, location); +}; + +/** Import only the transitive object set of the selected commit; no source refs or config survive. */ +export const materializeCleanRepository = async (repository: string, commit: string, destination: string): Promise => { + const format = (await rewindGit(repository, ["rev-parse", "--show-object-format"])).toString().trim(); + const pack = await rewindGit(repository, ["pack-objects", "--revs", "--stdout"], Buffer.from(`${commit}\n`)); + await mkdir(destination, { recursive: true }); + await rewindGit(destination, ["init", "--quiet", "--template=", "--initial-branch=moment", `--object-format=${format}`]); + await rewindGit(destination, ["config", "core.logAllRefUpdates", "false"]); + await rewindGit(destination, ["index-pack", "--stdin"], pack); + await rewindGit(destination, ["update-ref", "refs/heads/moment", commit]); + await rewindGit(destination, ["read-tree", commit]); + await rewindGit(destination, ["checkout-index", "--all", "--force"]); + for (const name of ["FETCH_HEAD", "ORIG_HEAD", "logs", "packed-refs"]) await rm(path.join(destination, ".git", name), { recursive: true, force: true }); +}; + +const selectorOf = (resource: GitResource | BundleResource): string => resource.kind === "git" + ? resource.ref || (resource.branch ? `refs/heads/${resource.branch}` : resource.tag ? `refs/tags/${resource.tag}` : "HEAD") + : resource.build?.files?.ref ?? "HEAD"; + +const localRoot = (resource: GitResource | BundleResource): string | undefined => resource.kind === "git" ? resource.url + : resource.build?.files?.root ?? resource.build?.dependencies?.directory ?? resource.build?.generated?.cwd ?? resource.build?.generated?.inputs[0]?.root; + +/** Preserve the authored root's prefix when its checkout exists; otherwise use a plain manifest-relative root. */ +const bundlePrefix = async (root: string, base: string): Promise => { + const absolute = path.resolve(base, root); + try { return (await rewindGit(absolute, ["rev-parse", "--show-prefix"])).toString().trim().replace(/\/$/u, ""); } + catch { + if (path.isAbsolute(root) || root.split(/[\\/]/u).includes("..")) throw new Error(`Cannot determine historical repository prefix for bundle root ${root}; keep its local checkout or use a repository-relative root`); + return root.replace(/^\.\//u, "").replace(/\/$/u, "").replace(/^\.$/u, ""); + } +}; + +const materializeBundle = async (resource: BundleResource, context: InputContext, repository: string, commit: string): Promise => { + const build = resource.build; + if (!build) { await exportGitTree(repository, commit, context.destination, { refuseSymlinks: true }); return false; } + if (build.files) { + await exportGitTree(repository, commit, context.destination, { root: await bundlePrefix(build.files.root, context.input.declarationBase), exclude: build.files.exclude, refuseSymlinks: true }); + return false; + } + if (build.dependencies) { + await exportGitTree(repository, commit, context.destination, { root: await bundlePrefix(build.dependencies.directory, context.input.declarationBase), refuseSymlinks: true }); + } else { + const origins = await Promise.all(build.generated!.inputs.map(async (input) => { + try { return (await rewindGit(path.resolve(context.input.declarationBase, input.root), ["rev-parse", "--show-toplevel"])).toString().trim(); } + catch { return null; } // Missing relative roots are mapped against the retained repository below. + })); + if (new Set(origins.filter((origin) => origin !== null)).size > 1) { + throw new Error("Generated bundle inputs span multiple git repositories; one recorded repository/commit cannot rewind them correctly"); + } + // Keep repository-relative paths: trial preparation can restore the declared cwd/input relationship. + for (const input of build.generated!.inputs) { + if (input.ref !== undefined) throw new Error("Generated bundle inputs cannot pin a ref: the deferred command reads the work tree"); + const root = await bundlePrefix(input.root, context.input.declarationBase); + await exportGitTree(repository, commit, context.destination, { root, prefix: root, exclude: input.exclude, mergeIdentical: true, refuseSymlinks: true }); + } + } + return true; +}; + +export const materializeGit = async (context: InputContext): Promise => { + const { input, history } = context; + if (input.section !== "resources" || input.declaration.kind === "volume") throw new Error("Git rewind requires a git or bundle resource"); + const resource = input.declaration, selector = selectorOf(resource), pinned = FULL_OBJECT.test(selector); + const record = input.overridden || pinned ? undefined : context.record?.resources.find((entry) => entry.id === input.id); + if (record && record.kind !== resource.kind) throw new Error(`Moment resource ${input.id} kind disagrees with its declaration`); + if (input.overridden && resource.kind === "bundle" && resource.source) { + await extractOverrideBundle(localRepositoryPath(resource.source, input.declarationBase), context.destination, resource.sha256); + return { fidelity: "overridden", basis: "override", source: {}, notes: [], instant: context.at }; + } + if (resource.kind === "bundle" && resource.source && !record) throw new Error("Prebuilt bundle has no recorded commit and cannot be rewound; provide an override"); + let repository = input.overridden ? null : await history?.repository(`git/${historyKey(input)}`); + if (!repository && (pinned || input.overridden)) { + const declared = localRoot(resource); + if (declared) repository = localRepositoryPath(declared, input.declarationBase); + } + if (!repository) throw new Error(`Missing git history for ${input.id}; unpinned resources require --history`); + await assertLocalGitRepository(repository, !input.overridden && history && repository.startsWith(`${history.root}${path.sep}`) ? history : undefined); + const selected = pinned || record || input.overridden + ? await resolveRewindCommit(repository, pinned ? selector : record?.commit ?? selector).then(async (commit) => ({ commit, instant: await commitInstant(repository!, commit) })) + : await commitAt(repository, selector, context.at); + if (!selected) throw new Error(`No first-parent commit for ${selector} exists at or before ${context.at}`); + if (resource.kind === "bundle") await assertCommitNotFuture(repository, selected.commit, context.at); + const deferred = resource.kind === "bundle" ? await materializeBundle(resource, context, repository, selected.commit) : false; + if (resource.kind === "git") await materializeCleanRepository(repository, selected.commit, context.destination); + return { + fidelity: input.overridden ? "overridden" : pinned || record ? "exact" : "approximate", + basis: input.overridden ? "override" : pinned ? "pinned" : record ? "record" : "selector-at", + source: { commit: selected.commit }, instant: selected.instant, + notes: pinned || record || input.overridden ? [] : ["Commit dates do not establish when a commit reached the selected ref."], + ...(deferred ? { build: "deferred" } : {}) + }; +}; diff --git a/src/compiler/training/rewind/gitArchive.ts b/src/compiler/training/rewind/gitArchive.ts new file mode 100644 index 00000000..5c5ebd3b --- /dev/null +++ b/src/compiler/training/rewind/gitArchive.ts @@ -0,0 +1,81 @@ +import { lstat, mkdir, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { rewindGit } from "./gitProcess.js"; +import { safeCommittedPath } from "./gitTree.js"; + +const BLOCK = 512; +const field = (bytes: Buffer): string => bytes.toString("utf8").replace(/\0.*$/su, ""); +const octal = (bytes: Buffer): number => { + const value = field(bytes).trim(); + if (!/^[0-7]+$/u.test(value)) throw new Error("Invalid git archive numeric field"); + const number = Number.parseInt(value, 8); + if (!Number.isSafeInteger(number) || number < 0) throw new Error("Invalid git archive size"); + return number; +}; + +const pax = (bytes: Buffer): Record => { + const result: Record = {}; + for (let offset = 0; offset < bytes.length;) { + const space = bytes.indexOf(32, offset), length = Number(bytes.subarray(offset, space).toString("ascii")); + if (space < offset || !Number.isSafeInteger(length) || length <= space - offset + 1 || offset + length > bytes.length || bytes[offset + length - 1] !== 10) { + throw new Error("Invalid git archive PAX field"); + } + const value = bytes.subarray(space + 1, offset + length - 1).toString("utf8"), equal = value.indexOf("="); + if (equal <= 0) throw new Error("Invalid git archive PAX value"); + result[value.slice(0, equal)] = value.slice(equal + 1); + offset += length; + } + return result; +}; + +/** Never follow an archive symlink while creating a later entry beneath it. */ +const parentDirectory = async (root: string, relative: string): Promise => { + let directory = root; + for (const segment of relative.split("/").slice(0, -1)) { + directory = path.join(directory, segment); + const existing = await lstat(directory).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); + if (existing && !existing.isDirectory()) throw new Error(`Git archive entry traverses a non-directory: ${relative}`); + if (!existing) await mkdir(directory); + } +}; + +/** Git feeds use git archive, including committed export-ignore/export-subst attributes. */ +export const exportFeedGitTree = async (repository: string, commit: string, destination: string, paths?: string[]): Promise => { + for (const selected of paths ?? []) safeCommittedPath(selected); + const archive = await rewindGit(repository, ["archive", "--format=tar", commit, ...(paths ? ["--", ...paths] : [])]); + await mkdir(destination, { recursive: true }); + let extended: Record = {}, entries = 0, terminated = false; + for (let offset = 0; offset + BLOCK <= archive.length;) { + const header = archive.subarray(offset, offset + BLOCK); + if (header.every((byte) => byte === 0)) { + if (offset + 2 * BLOCK > archive.length || !archive.subarray(offset).every((byte) => byte === 0)) throw new Error("Invalid git archive termination"); + terminated = true; + break; + } + const checksum = header.reduce((sum, byte, index) => sum + (index >= 148 && index < 156 ? 32 : byte), 0); + if (field(header.subarray(257, 263)) !== "ustar" || checksum !== octal(header.subarray(148, 156))) throw new Error("Invalid git archive header"); + const size = octal(header.subarray(124, 136)), mode = octal(header.subarray(100, 108)), type = String.fromCharCode(header[156] || 48); + const data = archive.subarray(offset + BLOCK, offset + BLOCK + size); + offset += BLOCK + Math.ceil(size / BLOCK) * BLOCK; + if (data.length !== size || offset > archive.length) throw new Error("Truncated git archive"); + if (type === "g" || type === "x") { + const values = pax(data); + if (type === "x") extended = values; + else if (values.path || values.linkpath || values.size) throw new Error("Unsupported global git archive path override"); + continue; + } + const prefix = field(header.subarray(345, 500)); + const name = safeCommittedPath((extended.path ?? `${prefix ? `${prefix}/` : ""}${field(header.subarray(0, 100))}`).replace(/\/$/u, "")); + const link = extended.linkpath ?? field(header.subarray(157, 257)); + if (extended.size && Number(extended.size) !== size) throw new Error("Unsupported git archive size override"); + extended = {}; + if (!["0", "2", "5"].includes(type) || (type !== "0" && size)) throw new Error(`Unsupported git archive entry ${name}`); + await parentDirectory(destination, name); + const target = path.join(destination, name); + if (type === "5") await mkdir(target); + else if (type === "2") await symlink(link, target); + else await writeFile(target, data, { flag: "wx", mode: mode & 0o111 ? 0o755 : 0o644 }); + entries += 1; + } + if (!terminated || !entries) throw new Error("Git feed archive is empty or lacks valid termination"); +}; diff --git a/src/compiler/training/rewind/gitHistory.ts b/src/compiler/training/rewind/gitHistory.ts new file mode 100644 index 00000000..dd0fe497 --- /dev/null +++ b/src/compiler/training/rewind/gitHistory.ts @@ -0,0 +1,49 @@ +import { lstat, readFile, readdir } from "node:fs/promises"; +import path from "node:path"; +import type { History } from "./history.js"; +import { rewindGit } from "./gitProcess.js"; + +/** Git's textual indirections are not covered by ordinary realpath containment. */ +export const assertLocalGitRepository = async (repository: string, history?: History): Promise => { + const exists = (file: string) => history ? history.path(path.relative(history.root, file)) : lstat(file).then(() => file, (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + const text = (file: string) => history ? history.text(path.relative(history.root, file)) : readFile(file, "utf8").catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + const dotGit = path.join(repository, ".git"); + if (history && await exists(dotGit) && !(await lstat(dotGit)).isDirectory()) throw new Error("History repository .git indirection files are not supported; retain a standalone repository"); + // `git -C` would discover an ancestor repository if this history directory is not itself one. + if (history && !await exists(dotGit) && (!await exists(path.join(repository, "HEAD")) || !await exists(path.join(repository, "objects")))) { + throw new Error("History entry is not a standalone git repository"); + } + // Check config includes before asking git anything: even rev-parse reads included config files. + if (history) { + const directory = await exists(dotGit) ? dotGit : repository; + // `commondir` is another textual redirect git follows during discovery, before any command runs. + for (const name of ["commondir", "objects/info/alternates", "objects/info/http-alternates", "shallow", "info/grafts"]) { + if (await text(path.join(directory, name))) throw new Error(`Rewind requires complete standalone git history; ${name} is unsupported`); + } + const configFile = path.join(directory, "config"); + const configText = await text(configFile); + if (configText && /^\s*\[\s*include(?:if)?\b/imu.test(configText)) throw new Error("Rewind refuses git configuration indirection include"); + } + // Resolve only after refusing textual worktree links that could leave the history root. + const gitDirectory = (await rewindGit(repository, ["rev-parse", "--absolute-git-dir"])).toString().trim(); + if (history) await history.path(path.relative(history.root, gitDirectory)); + for (const name of ["commondir", "objects/info/alternates", "objects/info/http-alternates", "shallow", "info/grafts"]) { + if (await text(path.join(gitDirectory, name))) throw new Error(`Rewind requires complete standalone git history; ${name} is unsupported`); + } + const config = (await rewindGit(repository, ["config", "--local", "--no-includes", "--list", "--null"])) + .toString().split("\0").filter(Boolean); + for (const entry of config) { + const key = entry.split("\n")[0]!.toLowerCase(); + if (key === "include.path" || /^includeif\..*\.path$/u.test(key) || key === "extensions.partialclone" || /^remote\..*\.promisor$/u.test(key)) { + throw new Error(`Rewind refuses git configuration indirection ${key}`); + } + } + const packs = path.join(gitDirectory, "objects/pack"); + if (await exists(packs) && (await readdir(packs)).some((name) => name.endsWith(".promisor"))) throw new Error("Rewind refuses incomplete promisor git history"); +}; diff --git a/src/compiler/training/rewind/gitProcess.ts b/src/compiler/training/rewind/gitProcess.ts new file mode 100644 index 00000000..8e55ea34 --- /dev/null +++ b/src/compiler/training/rewind/gitProcess.ts @@ -0,0 +1,77 @@ +import { execFile, execFileSync } from "node:child_process"; +import { compareInstants } from "./instant.js"; + +const LIMIT = 268_435_456; + +/** Do not inherit GIT_DIR, injected -c options, credentials or user/system configuration. */ +export const rewindGitEnvironment = (): NodeJS.ProcessEnv => ({ + PATH: process.env.PATH, + ...(process.env.SystemRoot ? { SystemRoot: process.env.SystemRoot } : {}), + LC_ALL: "C", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_SYSTEM: "/dev/null", GIT_CONFIG_GLOBAL: "/dev/null", + GIT_NO_REPLACE_OBJECTS: "1", GIT_OPTIONAL_LOCKS: "0", GIT_TERMINAL_PROMPT: "0", GIT_NO_LAZY_FETCH: "1", + GIT_ATTR_NOSYSTEM: "1", GIT_CONFIG_COUNT: "0" +}); + +const argumentsFor = (repo: string, args: readonly string[]): string[] => [ + "-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", "-c", "core.autocrlf=false", + "-c", "core.logAllRefUpdates=false", "-c", "core.attributesFile=/dev/null", "-c", "protocol.allow=never", "-c", "protocol.file.allow=always", + "-c", "maintenance.auto=false", "-c", "gc.auto=0", "-c", "safe.directory=*", "-C", repo, ...args +]; + +/** Bounded argv-only git invocation. Local transport is the only enabled protocol. */ +export const rewindGit = (repo: string, args: readonly string[], input?: Buffer): Promise => + new Promise((resolve, reject) => { + const child = execFile("git", argumentsFor(repo, args), { + encoding: "buffer", env: rewindGitEnvironment(), maxBuffer: LIMIT, timeout: 60_000 + }, (error, stdout, stderr) => { + if (error) reject(new Error(`Rewind git ${args[0]} failed: ${stderr.toString("utf8").trim() || error.message}`)); + else resolve(stdout); + }); + child.stdin?.on("error", () => undefined); + child.stdin?.end(input); + }); + +/** Adapter for the existing synchronous feed revision function; it never fetches. */ +export const rewindGitSync = (repo: string, args: readonly string[]): Buffer => + execFileSync("git", argumentsFor(repo, args), { + env: rewindGitEnvironment(), maxBuffer: LIMIT, stdio: ["ignore", "pipe", "pipe"], timeout: 60_000 + }); + +export interface SelectedCommit { commit: string; instant: string } + +export const commitInstant = async (repo: string, commit: string): Promise => { + const timestamp = Number((await rewindGit(repo, ["show", "-s", "--format=%ct", commit, "--"])).toString().trim()); + if (!Number.isSafeInteger(timestamp)) throw new Error(`Commit ${commit} has an invalid committer date`); + return new Date(timestamp * 1000).toISOString(); +}; + +/** Plain trees discard git metadata, so reject a future selected commit before that evidence disappears. */ +export const assertCommitNotFuture = async (repo: string, commit: string, at: string): Promise => { + const dates = (await rewindGit(repo, ["show", "-s", "--format=%at %ct", commit, "--"])).toString().trim().split(" "); + if (dates.length !== 2 || dates.some((value) => !Number.isSafeInteger(Number(value)) || compareInstants(new Date(Number(value) * 1000).toISOString(), at) > 0)) { + throw new Error(`Selected commit ${commit} has an author or committer date after ${at}`); + } +}; + +export const resolveRewindCommit = async (repo: string, selector: string): Promise => { + if (!selector || selector.startsWith("-") || /[\0\r\n]/u.test(selector)) throw new Error("Invalid git selector"); + const ref = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/iu.test(selector) ? selector.toLowerCase() : selector; + return (await rewindGit(repo, ["rev-parse", "--verify", "--end-of-options", `${ref}^{commit}`])).toString().trim(); +}; + +/** Dates approximate ref history; they cannot establish when a commit reached a moving ref. */ +export const commitAt = async (repo: string, selector: string, at: string): Promise => { + let head: string; + try { head = await resolveRewindCommit(repo, selector); } catch { return null; } + const lines = (await rewindGit(repo, ["log", "--first-parent", "--format=%H %ct", head, "--"])) + .toString().trim().split("\n").filter(Boolean); + return selectCommitAt(lines.map((line) => { + const [commit, seconds] = line.split(" "); + return { commit: commit!, instant: new Date(Number(seconds) * 1000).toISOString() }; + }), at); +}; + +/** Stable ties preserve the first-parent order returned by git. */ +export const selectCommitAt = (commits: readonly SelectedCommit[], at: string): SelectedCommit | null => + commits.filter((entry) => compareInstants(entry.instant, at) <= 0) + .reduce((latest, entry) => latest === null || compareInstants(entry.instant, latest.instant) > 0 ? entry : latest, null); diff --git a/src/compiler/training/rewind/gitTree.ts b/src/compiler/training/rewind/gitTree.ts new file mode 100644 index 00000000..7c60404c --- /dev/null +++ b/src/compiler/training/rewind/gitTree.ts @@ -0,0 +1,69 @@ +import { lstat, mkdir, readFile, readlink, stat, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { createHash } from "node:crypto"; + +import { compileExcludePatterns } from "../../workspaceBundleFiles.js"; +import { validateWorkspaceBundleTar } from "../../workspaceBundleArtifacts.js"; +import { rewindGit } from "./gitProcess.js"; + +export const safeCommittedPath = (value: string): string => { + if (!value || value.startsWith("/") || value.includes("\\") || value.includes("\0") || value.split("/").some((part) => !part || part === "." || part === ".." || part.toLowerCase() === ".git")) { + throw new Error(`Unsafe committed path: ${value}`); + } + return value; +}; + +export interface ExportTreeOptions { root?: string; exclude?: readonly string[]; paths?: readonly string[]; prefix?: string; mergeIdentical?: boolean; refuseSymlinks?: boolean } + +/** Materialize blobs by object id, without invoking archive attributes, filters or extraction tools. */ +export const exportGitTree = async (repo: string, commit: string, destination: string, options: ExportTreeOptions = {}): Promise => { + const root = options.root && options.root !== "." ? safeCommittedPath(options.root.replace(/\/$/u, "")) : ""; + const tree = root ? `${commit}:${root}` : `${commit}^{tree}`; + if ((await rewindGit(repo, ["cat-file", "-t", tree])).toString().trim() !== "tree") throw new Error(`Bundle root ${root || "."} is not a tree at ${commit}`); + const entries = (await rewindGit(repo, ["ls-tree", "-rz", tree])).toString("utf8").split("\0").filter(Boolean); + const excluded = compileExcludePatterns(options.exclude); + const prefix = options.prefix ? safeCommittedPath(options.prefix) : ""; + await mkdir(destination, { recursive: true }); + for (const entry of entries) { + const tab = entry.indexOf("\t"), name = safeCommittedPath(entry.slice(tab + 1)); + const [mode, kind, oid] = entry.slice(0, tab).split(" "); + if (excluded(name) || (options.paths && !options.paths.some((selected) => name === selected || name.startsWith(`${selected}/`)))) continue; + if (options.refuseSymlinks && mode === "120000") throw new Error(`Bundle input is a symlink; exclude it or replace it with a file: ${name}`); + if (kind !== "blob" || !["100644", "100755", "120000"].includes(mode!)) throw new Error(`Committed path ${name} is not a supported file or symlink (submodules need their own history)`); + const target = path.join(destination, prefix, name), bytes = await rewindGit(repo, ["cat-file", "blob", oid!]); + await mkdir(path.dirname(target), { recursive: true }); + if (options.mergeIdentical) { + const present = await lstat(target).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); + if (present) { + const identical = mode === "120000" ? present.isSymbolicLink() && await readlink(target) === bytes.toString("utf8") + : present.isFile() && (present.mode & 0o111 ? "100755" : "100644") === mode && (await readFile(target)).equals(bytes); + if (!identical) throw new Error(`Conflicting generated bundle inputs at ${name}`); + continue; + } + } + if (mode === "120000") await symlink(bytes.toString("utf8"), target); + else await writeFile(target, bytes, { flag: "wx", mode: mode === "100755" ? 0o755 : 0o644 }); + } +}; + +/** Overrides use validated local ustar bytes, never tar commands or the network. */ +export const extractOverrideBundle = async (file: string, destination: string, expected?: string): Promise => { + const info = await stat(file); + if (!info.isFile() || info.size > 536_870_912) throw new Error("Override bundle must be a bounded local tar file"); + const bytes = await readFile(file); + if (expected && `sha256:${createHash("sha256").update(bytes).digest("hex")}` !== expected) throw new Error("Override bundle checksum mismatch"); + validateWorkspaceBundleTar(bytes); + const text = (buffer: Buffer) => buffer.toString("utf8").replace(/\0.*$/su, ""); + for (let offset = 0; offset < bytes.length && !bytes.subarray(offset, offset + 512).every((byte) => byte === 0);) { + const header = bytes.subarray(offset, offset + 512), prefix = text(header.subarray(345, 500)), name = text(header.subarray(0, 100)); + const target = path.join(destination, safeCommittedPath(`${prefix ? `${prefix}/` : ""}${name}`.replace(/\/$/u, ""))); + const size = Number.parseInt(text(header.subarray(124, 136)).trim(), 8); + if (header[156] === 53) await mkdir(target, { recursive: true }); + else { + await mkdir(path.dirname(target), { recursive: true }); + const executable = Number.parseInt(text(header.subarray(100, 108)).trim(), 8) & 0o111; + await writeFile(target, bytes.subarray(offset + 512, offset + 512 + size), { flag: "wx", mode: executable ? 0o755 : 0o644 }); + } + offset += 512 + Math.ceil(size / 512) * 512; + } +}; diff --git a/src/compiler/training/rewind/history.test.ts b/src/compiler/training/rewind/history.test.ts new file mode 100644 index 00000000..eeb4ce55 --- /dev/null +++ b/src/compiler/training/rewind/history.test.ts @@ -0,0 +1,117 @@ +import { chmod, mkdir, mkdtemp, readFile, readlink, realpath, rm, stat, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { z } from "zod"; +import { momentRecordSchema } from "./contract.js"; +import { History, historyKey, openHistory, safeSegment } from "./history.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const fixture = async () => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "spawnfile-rewind-history-"))); + roots.push(root); + const directory = path.join(root, "history"), outside = path.join(root, "outside"); + await mkdir(directory); await mkdir(outside); + await writeFile(path.join(outside, "secret"), "outside bytes"); + return { root, directory, outside, history: (await openHistory(directory))! }; +}; +const record = { version: "spawnfile.moment-record.v1", agent: "agent:writer", + wake: { at: "2026-10-09T08:20:31Z", surface: "moltnet", room: "assignment", from: "brass" }, + recordedAt: "2026-10-09T08:20:31Z", resources: [], feeds: [], memory: [], moltnet: [], volumes: [] }; + +describe("history containment and strict reads", () => { + it("allows missing history and entries but rejects non-directory roots and non-file reads", async () => { + const { directory, history } = await fixture(); + expect(await openHistory()).toBeUndefined(); + expect(await history.path("missing/file")).toBeNull(); + expect(await history.bytes("missing")).toBeNull(); + expect(await history.jsonl("missing", z.object({}).strict())).toBeNull(); + await writeFile(path.join(directory, "file"), "bytes"); + await expect(openHistory(path.join(directory, "file"))).rejects.toThrow(/must be a directory/u); + await expect(history.text(".")).rejects.toThrow(/regular file/u); + }); + + it.each(["../outside/secret", "child/../../outside/secret", "/etc/passwd", "bad\\path", "bad\0path"])( + "refuses lexical escape or invalid history path %s", async (name) => { + const { history } = await fixture(); + await expect(history.text(name)).rejects.toThrow(/escapes root|Unsafe history path/u); + } + ); + + it("rejects outside symlinks, including missing descendants and outside links back inside", async () => { + const { root, directory, outside, history } = await fixture(); + await writeFile(path.join(directory, "inside"), "retained"); + await symlink(path.join(outside, "secret"), path.join(directory, "escape")); + await symlink(outside, path.join(directory, "external-directory")); + await symlink(path.join(directory, "inside"), path.join(root, "back-inside")); + for (const name of ["escape", "external-directory/secret", "external-directory/missing"]) { + await expect(history.bytes(name)).rejects.toThrow(/History symlink escape/u); + } + await expect(history.bytes("../back-inside")).rejects.toThrow(/History path escapes root/u); + }); + + it("distinguishes dangling links from missing data and permits contained links", async () => { + const { directory, history } = await fixture(); + await writeFile(path.join(directory, "real"), "retained"); + await symlink("real", path.join(directory, "alias")); + await symlink("missing", path.join(directory, "dangling")); + expect(await history.text("alias")).toBe("retained"); + await expect(history.text("dangling")).rejects.toThrow(/Dangling history symlink/u); + }); + + it.each(["{broken", '{"id":"second","unknown":true}', ""])( + "names the exact malformed JSONL line for %j", async (second) => { + const { directory, history } = await fixture(); + await writeFile(path.join(directory, "records.jsonl"), `{"id":"first"}\n${second}\n`); + await expect(history.jsonl("records.jsonl", z.object({ id: z.string() }).strict())) + .rejects.toThrow(/Invalid history records.jsonl:2:/u); + } + ); + + it("loads every moment JSONL filename and ignores other files", async () => { + const { directory, history } = await fixture(); + await mkdir(path.join(directory, "moments")); + await writeFile(path.join(directory, "moments/z-any-name.jsonl"), `${JSON.stringify(record)}\n`); + await writeFile(path.join(directory, "moments/a-capture.jsonl"), `${JSON.stringify({ ...record, agent: "agent:other" })}\n`); + await writeFile(path.join(directory, "moments/ignore.txt"), "not JSON"); + expect((await history.records()).map((entry) => entry.agent)).toEqual(["agent:other", "agent:writer"]); + }); + + it.each([ + { ...record, unexpected: true }, + { ...record, wake: { ...record.wake, text: "not in the record format" } }, + { ...record, resources: [{ id: "same", kind: "git", commit: "one" }, { id: "same", kind: "git", commit: "two" }] } + ])("strictly validates record fields and identities", async (invalid) => { + const { directory, history } = await fixture(); + await writeFile(path.join(directory, "invalid.jsonl"), `${JSON.stringify(invalid)}\n`); + await expect(history.jsonl("invalid.jsonl", momentRecordSchema)).rejects.toThrow(/invalid.jsonl:1:/u); + }); + + it("checks nested repository entries before granting a repository path", async () => { + const { directory, outside, history } = await fixture(); + await mkdir(path.join(directory, "repo", "objects"), { recursive: true }); + await symlink(path.join(outside, "secret"), path.join(directory, "repo", "objects", "escape")); + await expect(history.repository("repo")).rejects.toThrow(/History symlink escape/u); + }); + + it("copies retained bytes, executable mode, and internal relative links through the boundary", async () => { + const { root, directory, history } = await fixture(); + await mkdir(path.join(directory, "tree")); + await writeFile(path.join(directory, "tree", "script"), "retained bytes\n"); + await chmod(path.join(directory, "tree", "script"), 0o755); + await symlink("script", path.join(directory, "tree", "alias")); + const destination = path.join(root, "copy"); + await history.copyTree("tree", destination); + expect(await readFile(path.join(destination, "script"), "utf8")).toBe("retained bytes\n"); + expect((await stat(path.join(destination, "script"))).mode & 0o777).toBe(0o755); + expect(await readlink(path.join(destination, "alias"))).toBe("script"); + }); + + it("encodes declaring node IDs and rejects unsafe input segments", () => { + expect(historyKey({ declaredBy: "agent:writer", id: "archive" })).toBe("agent%3Awriter/archive"); + for (const value of ["", ".", "..", "a/b", "a\\b", "bad\0id"]) expect(() => safeSegment(value)).toThrow(/Unsafe history identity/u); + expect(safeSegment("archive-1")).toBe("archive-1"); + expect(new History("/fixture").root).toBe("/fixture"); + }); +}); diff --git a/src/compiler/training/rewind/history.ts b/src/compiler/training/rewind/history.ts new file mode 100644 index 00000000..3f558ba7 --- /dev/null +++ b/src/compiler/training/rewind/history.ts @@ -0,0 +1,120 @@ +import { chmod, lstat, mkdir, readFile, readdir, readlink, realpath, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { z } from "zod"; +import { momentRecordSchema, type MomentRecord } from "./contract.js"; + +export const inside = (root: string, target: string): boolean => { + const relative = path.relative(root, target); + return relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative); +}; +export const safeSegment = (value: string): string => { + if (!value || value === "." || value === ".." || /[/\\\0]/u.test(value)) throw new Error(`Unsafe history identity ${JSON.stringify(value)}`); + return value; +}; +export const historyKey = (identity: { declaredBy: string; id: string }): string => + `${safeSegment(encodeURIComponent(identity.declaredBy))}/${safeSegment(identity.id)}`; +const reason = (error: unknown): string => error instanceof Error ? error.message : String(error); + +/** All reads of the consumer history layout pass through this containment boundary. */ +export class History { + readonly root: string; + constructor(root: string) { this.root = root; } + async path(relative: string): Promise { + if (path.isAbsolute(relative) || relative.includes("\\") || relative.includes("\0")) throw new Error(`Unsafe history path ${relative}`); + const target = path.resolve(this.root, relative); + if (!inside(this.root, target)) throw new Error(`History path escapes root: ${relative}`); + let ancestor = target; + while (true) { + try { + const physical = await realpath(ancestor); + if (!inside(this.root, physical)) throw new Error(`History symlink escape: ${relative}`); + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if (ancestor === this.root) throw new Error(`History root no longer exists: ${this.root}`); + ancestor = path.dirname(ancestor); + } + } + try { + await lstat(target); + // A dangling symlink is not a missing history entry. + const physical = await realpath(target); + if (!inside(this.root, physical)) throw new Error(`History symlink escape: ${relative}`); + return target; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + const entry = await lstat(target).catch(() => null); + if (entry?.isSymbolicLink()) throw new Error(`Dangling history symlink: ${relative}`); + return null; + } + } + async bytes(relative: string): Promise { + const file = await this.path(relative); + if (file === null) return null; + if (!(await lstat(await realpath(file))).isFile()) throw new Error(`History entry must be a regular file: ${relative}`); + return readFile(file); + } + async text(relative: string): Promise { return (await this.bytes(relative))?.toString("utf8") ?? null; } + async jsonl(relative: string, schema: z.ZodType): Promise { + const payload = await this.text(relative); + if (payload === null) return null; + const lines = payload.split("\n"); + if (lines.at(-1) === "") lines.pop(); + return lines.map((line, index) => { + try { return schema.parse(JSON.parse(line)); } + catch (error) { throw new Error(`Invalid history ${relative}:${index + 1}: ${reason(error)}`); } + }); + } + async records(): Promise { + const directory = await this.path("moments"); + if (!directory) return []; + const records: MomentRecord[] = []; + for (const name of (await readdir(directory)).sort()) { + if (name.endsWith(".jsonl")) records.push(...(await this.jsonl(`moments/${name}`, momentRecordSchema) ?? [])); + } + return records; + } + /** Validate all tree entries before git or another local parser receives the directory. */ + async repository(relative: string): Promise { + const root = await this.path(relative); + if (!root) return null; + const seen = new Set(); + const walk = async (entry: string): Promise => { + const file = await this.path(entry); + if (!file) throw new Error(`History entry vanished: ${entry}`); + const physical = await realpath(file), stats = await lstat(physical); + if (seen.has(physical)) return; + seen.add(physical); + if (stats.isDirectory()) for (const name of await readdir(file)) await walk(`${entry}/${name}`); + else if (!stats.isFile()) throw new Error(`Special history file: ${entry}`); + }; + await walk(relative); + return root; + } + async copyTree(relative: string, destination: string): Promise { + const source = await this.path(relative); + if (!source) throw new Error(`Missing retained tree: ${relative}`); + const stats = await lstat(source); + if (!stats.isDirectory()) throw new Error(`Retained tree must be a real directory: ${relative}`); + const copy = async (entry: string, target: string): Promise => { + const file = await this.path(entry); + if (!file) throw new Error(`History entry vanished: ${entry}`); + const stat = await lstat(file); + if (stat.isSymbolicLink()) { await symlink(await readlink(file), target); return; } + if (stat.isDirectory()) { + await mkdir(target, { recursive: true }); + for (const name of (await readdir(file)).sort()) await copy(`${entry}/${name}`, path.join(target, name)); + } else if (stat.isFile()) { + await writeFile(target, (await this.bytes(entry))!); + await chmod(target, stat.mode & 0o777); + } else throw new Error(`Special history file: ${entry}`); + }; + await copy(relative, destination); + } +} +export const openHistory = async (directory?: string): Promise => { + if (directory === undefined) return undefined; + const root = await realpath(path.resolve(directory)); + if (!(await lstat(root)).isDirectory()) throw new Error(`History must be a directory: ${directory}`); + return new History(root); +}; diff --git a/src/compiler/training/rewind/index.ts b/src/compiler/training/rewind/index.ts new file mode 100644 index 00000000..b2b233db --- /dev/null +++ b/src/compiler/training/rewind/index.ts @@ -0,0 +1,14 @@ +export * from "./contract.js"; +export * from "./history.js"; +export * from "./inputs.js"; +export * from "./instant.js"; +export * from "./leakCheck.js"; +export * from "./materialize.js"; +export * from "./select.js"; +export * from "./taskClock.js"; +export { selectFeedLanding, selectFeedCommit } from "./feed.js"; +export { selectCommitAt } from "./gitProcess.js"; +export { selectMemoryPrefix, selectMemoryGeneration, parseMemoryLedger } from "./memory.js"; +export { rewindMoltnetJson, moltnetCursorExact } from "./moltnetJson.js"; +export { selectVolumeSnapshot, replayVolumeSnapshot } from "./volumes.js"; +export type * from "./types.js"; diff --git a/src/compiler/training/rewind/inputs.test.ts b/src/compiler/training/rewind/inputs.test.ts new file mode 100644 index 00000000..20585d4d --- /dev/null +++ b/src/compiler/training/rewind/inputs.test.ts @@ -0,0 +1,122 @@ +import { rm } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { loadTrainingProject, resolveTrainingPlan } from "../plan/index.js"; +import { + agentManifest, cleanTrainingProjects, dataFiles, example, memory, network, surfaces, target, + teamFiles, teamManifest, training, trainingProject, volume, writeTrainingFile +} from "../plan/fixtures.test-helper.js"; +import { resolveRewindInputs } from "./inputs.js"; + +afterEach(cleanTrainingProjects); +afterEach(() => vi.restoreAllMocks()); +const resolve = async (root: string, agent = "agent:a") => { + const project = await loadTrainingProject(root); + return { project, plan: resolveTrainingPlan(project, agent, "test") }; +}; +const identity = ({ section, id, declaredBy }: { section: string; id: string; declaredBy: string }) => ({ section, id, declaredBy }); + +describe("effective rewind inputs", () => { + it("includes every resource kind, accessible bank and attached network with P2 declaring identities", async () => { + const root = await trainingProject(teamFiles()); + await writeTrainingFile(root, "a/Spawnfile", agentManifest({ name: "a", training: undefined, surfaces, memory: [memory], + workspace: { docs: { system: "AGENTS.md" }, resources: [ + { id: "source", kind: "git", url: "./local-repo", mount: "./source", mode: "readonly" }, + { id: "bundle", kind: "bundle", source: "./local.tar", mount: "./bundle", mode: "readonly" } + ] } + })); + const { plan, project } = await resolve(root), before = structuredClone(project); + // All source files are gone: this is a pure projection of already loaded facts. + await rm(root, { recursive: true, force: true }); + const clock = vi.spyOn(Date, "now").mockImplementation(() => { throw Error("clock read"); }); + const first = resolveRewindInputs(plan, project, example.id); + expect(first.at).toBe("2026-10-09T08:20:31Z"); + expect(first.inputs.map(identity)).toEqual([ + { section: "memory", id: "self", declaredBy: "agent:a" }, + { section: "memory", id: "team-memory", declaredBy: "team:publication" }, + { section: "networks", id: "news", declaredBy: "team:publication" }, + { section: "resources", id: "bundle", declaredBy: "agent:a" }, + { section: "resources", id: "source", declaredBy: "agent:a" }, + { section: "resources", id: "edition-state", declaredBy: "team:publication" } + ]); + expect(first.inputs.every((input) => !input.overridden)).toBe(true); + expect(resolveRewindInputs(plan, project, `train:${example.id}`)).toEqual(first); + expect(project).toEqual(before); expect(clock).not.toHaveBeenCalled(); + }); + + it("replaces whole declarations from the loaded dataset and retains inherited identity", async () => { + const root = await trainingProject(teamFiles()); + const resourceOverride = { ...volume, mount: "./fresh" }; + const memoryOverride = { ...memory, id: "team-memory" }; + const networkOverride = { id: network.id, provider: "moltnet", rooms: [{ id: "assignment", members: ["a", "b"] }] }; + await writeTrainingFile(root, "training/train.yaml", { examples: [{ ...example, + resources: [resourceOverride], memory: [memoryOverride], networks: [networkOverride] + }] }); + const { plan, project } = await resolve(root), before = structuredClone(project); + const result = resolveRewindInputs(plan, project, example.id); + const replacements = result.inputs.filter((input) => input.overridden); + expect(replacements).toHaveLength(3); + for (const [section, declaration] of [["resources", resourceOverride], ["memory", memoryOverride], ["networks", networkOverride]] as const) { + expect(replacements.find((input) => input.section === section)).toEqual({ section, id: declaration.id, + declaredBy: "team:publication", declaration, overridden: true, declarationBase: path.join(root, "training") }); + } + expect(result.inputs.find((input) => input.id === "self")).toMatchObject({ overridden: false, declaredBy: "agent:a" }); + expect(project).toEqual(before); + }); + + it("deduplicates multiple access slots by bank source, without collapsing distinct same-id banks", async () => { + const root = await trainingProject(teamFiles()); + await writeTrainingFile(root, "Spawnfile", teamManifest({ networks: undefined, memory: [memory], + shared: { training: { ...training, targets: [] } }, members: [{ id: "a", ref: "a" }, { id: "alias", ref: "a" }] })); + await writeTrainingFile(root, "a/Spawnfile", agentManifest({ name: "a", memory: [memory], training: { + targets: [{ ...target, datasets: { train: "../training/train.yaml", test: "../training/test.yaml" } }] + } })); + const { plan, project } = await resolve(root); + const agent = project.compilePlan.nodes.find((node) => node.id === "agent:a")!; + expect(agent.value.kind).toBe("agent"); + if (agent.value.kind !== "agent") throw Error("expected agent"); + expect(agent.value.memoryAccess!.filter((entry) => entry.declaringKind === "team")).toHaveLength(2); + expect(resolveRewindInputs(plan, project, example.id).inputs.map(identity)).toEqual([ + { section: "memory", id: "self", declaredBy: "agent:a" }, + { section: "memory", id: "self", declaredBy: "team:publication" } + ]); + }); + + it("includes representative attachments synthesized from a parent team's network", async () => { + const root = await trainingProject({ ...dataFiles, + Spawnfile: teamManifest({ shared: undefined, memory: undefined, + networks: [{ ...network, rooms: [{ id: "assignment", members: ["child"] }] }], members: [{ id: "child", ref: "child" }] }), + "child/Spawnfile": teamManifest({ name: "child", shared: undefined, networks: undefined, memory: undefined, + members: [{ id: "writer", ref: "writer" }] }), + ...Object.fromEntries(Object.entries(dataFiles).map(([name, value]) => [`child/writer/${name}`, value])), + "child/writer/Spawnfile": agentManifest() + }); + const { plan, project } = await resolve(root, "agent:writer"); + expect(resolveRewindInputs(plan, project, example.id).inputs.map(identity)).toEqual([ + { section: "networks", id: "news", declaredBy: "team:publication" } + ]); + }); + + it("excludes inaccessible banks and unattached networks", async () => { + const root = await trainingProject(teamFiles()); + await writeTrainingFile(root, "Spawnfile", teamManifest({ memory: [{ ...memory, id: "private", access: { members: ["b"] } }] })); + await writeTrainingFile(root, "a/Spawnfile", agentManifest({ name: "a", training: undefined, memory: [memory] })); + const { plan, project } = await resolve(root); + expect(resolveRewindInputs(plan, project, example.id).inputs.map(identity)).toEqual([ + { section: "memory", id: "self", declaredBy: "agent:a" }, + { section: "resources", id: "edition-state", declaredBy: "team:publication" } + ]); + }); + + it("accepts equivalent loaded instants and refuses stale or missing example data", async () => { + const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest() }); + const { plan, project } = await resolve(root, "agent:writer"); + const raw = project.datasets[path.join(root, "training/train.yaml")]!.value.examples[0]!; + raw.at = "2026-10-09T08:20:31Z"; + expect(resolveRewindInputs(plan, project, example.id).at).toBe(raw.at); + raw.at = "2026-10-09T08:20:31.001Z"; + expect(() => resolveRewindInputs(plan, project, example.id)).toThrow(/Loaded example disagrees with plan/u); + delete project.datasets[path.join(root, "training/train.yaml")]; + expect(() => resolveRewindInputs(plan, project, example.id)).toThrow(/Loaded example disagrees with plan/u); + }); +}); diff --git a/src/compiler/training/rewind/inputs.ts b/src/compiler/training/rewind/inputs.ts new file mode 100644 index 00000000..e66d3653 --- /dev/null +++ b/src/compiler/training/rewind/inputs.ts @@ -0,0 +1,64 @@ +import path from "node:path"; +import type { LoadedTrainingProject, TrainingPlan, TrainingAgentNode } from "../plan/index.js"; +import { resolveTrainingOverride, resolveTrainingResource } from "../plan/references.js"; +import { selectTrainingExample } from "./select.js"; +import { normalizeInstant } from "./instant.js"; +import type { RewindInput } from "./types.js"; + +/** The same compiled resources, accessible bank sources and surface attachments P2 resolves. */ +export const resolveRewindInputs = ( + plan: TrainingPlan, project: LoadedTrainingProject, selector: string +) => { + const selected = selectTrainingExample(plan, selector); + const agent = project.compilePlan.nodes.find((node): node is TrainingAgentNode => + node.value.kind === "agent" && node.id === plan.agent.id); + if (!agent) throw new Error(`No compiled training agent ${plan.agent.id}`); + const raw = project.datasets[selected.sourcePath]?.value.examples.find((example) => example.id === selected.id); + if (!raw || normalizeInstant(raw.at) !== normalizeInstant(selected.example.at)) throw new Error(`Loaded example disagrees with plan: ${selector}`); + const inputs: RewindInput[] = []; + const declarationBase = (id: string): string => { + const node = project.compilePlan.nodes.find((candidate) => candidate.id === id); + if (!node) throw new Error(`Missing declaring node ${id}`); + return path.dirname(node.value.kind === "agent" ? node.value.sourcePath ?? node.value.source : node.value.source); + }; + for (const resource of agent.value.workspaceResources ?? []) { + const { declaredBy } = resolveTrainingResource(project.compilePlan, agent, resource.id, "Rewind resource"); + inputs.push({ section: "resources", id: resource.id, declaredBy, declarationBase: declarationBase(declaredBy), + declaration: resource, overridden: false }); + } + const banks = new Set(); + for (const entry of agent.value.memoryAccess ?? []) { + const key = JSON.stringify([entry.source, entry.bank.id]); + if (banks.has(key)) continue; + banks.add(key); + const node = project.compilePlan.nodes.find((candidate) => candidate.value.source === entry.source); + if (!node) throw new Error(`Memory bank ${entry.bank.id} has no declaring node`); + inputs.push({ section: "memory", id: entry.bank.id, declaredBy: node.id, + declarationBase: declarationBase(node.id), declaration: entry.bank, overridden: false }); + } + const networks = new Set(); + for (const attachment of agent.value.surfaces?.moltnet ?? []) { + const sources = new Set([attachment.teamSource, ...Object.keys(attachment.contextRooms ?? {})]); + for (const node of project.compilePlan.nodes) { + if (node.value.kind !== "team" || !sources.has(node.value.source)) continue; + for (const network of node.value.networks ?? []) { + if (network.id !== attachment.network) continue; + const key = JSON.stringify([node.id, network.id]); + if (networks.has(key)) continue; + networks.add(key); + inputs.push({ section: "networks", id: network.id, declaredBy: node.id, + declarationBase: declarationBase(node.id), declaration: network, overridden: false }); + } + } + } + for (const section of ["resources", "memory", "networks"] as const) { + for (const declaration of raw[section] ?? []) { + const identity = resolveTrainingOverride(project.compilePlan, agent, section, declaration.id, `Rewind override ${selector}`); + const index = inputs.findIndex((input) => input.section === section && input.id === identity.id && input.declaredBy === identity.declaredBy); + if (index === -1) throw new Error(`Missing effective override ${section}:${identity.id}`); + inputs[index] = { ...inputs[index], declaration, declarationBase: path.dirname(selected.sourcePath), overridden: true } as RewindInput; + } + } + inputs.sort((a, b) => a.section.localeCompare(b.section) || a.declaredBy.localeCompare(b.declaredBy) || a.id.localeCompare(b.id)); + return { inputs, selected, at: normalizeInstant(raw.at) }; +}; diff --git a/src/compiler/training/rewind/instant.test.ts b/src/compiler/training/rewind/instant.test.ts new file mode 100644 index 00000000..1efc70e1 --- /dev/null +++ b/src/compiler/training/rewind/instant.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import { compareInstants, instantNanos, normalizeInstant } from "./instant.js"; + +describe("rewind instant comparisons", () => { + it("compares RFC3339Nano fractions numerically when string order is reversed", () => { + const earlier = "2026-10-09T08:20:31.1Z", later = "2026-10-09T08:20:31.11Z"; + expect(earlier > later).toBe(true); + expect(compareInstants(earlier, later)).toBe(-1); + expect(compareInstants(later, earlier)).toBe(1); + expect(compareInstants("2026-10-09T08:20:31.100000000Z", earlier)).toBe(0); + }); + + it("preserves nanosecond boundaries and offset-equivalent instants", () => { + const before = "2026-10-09T08:20:31.123456789Z", after = "2026-10-09T08:20:31.123456790Z"; + expect(instantNanos(after) - instantNanos(before)).toBe(1n); + expect(compareInstants(before, "2026-10-09T10:20:31.123456789+02:00")).toBe(0); + expect(compareInstants(after, before)).toBe(1); + }); + + it.each([ + ["2026-10-09T10:20:31.120000000+02:00", "2026-10-09T08:20:31.12Z"], + ["2026-10-09T08:20Z", "2026-10-09T08:20:00Z"], + ["2026-10-09T00:01:00+14:00", "2026-10-08T10:01:00Z"], + ["1969-12-31T23:59:59.999999999Z", "1969-12-31T23:59:59.999999999Z"] + ])("normalizes %s to UTC without losing precision", (input, expected) => { + expect(normalizeInstant(input)).toBe(expected); + expect(compareInstants(input, expected)).toBe(0); + }); + + it.each(["invalid", "2026-02-30T00:00:00Z", "2026-10-09T08:20:31", "2026-10-09T08:20:31.1234567891Z"])( + "refuses unsupported or invalid instant %s", (value) => { + expect(() => instantNanos(value)).toThrow(); + } + ); +}); diff --git a/src/compiler/training/rewind/instant.ts b/src/compiler/training/rewind/instant.ts new file mode 100644 index 00000000..44b77e06 --- /dev/null +++ b/src/compiler/training/rewind/instant.ts @@ -0,0 +1,23 @@ +import { trainingInstantSchema } from "../../../manifest/index.js"; + +/** RFC3339Nano ordering, including offsets and fractions Date.parse would truncate. */ +export const instantNanos = (value: string): bigint => { + trainingInstantSchema.parse(value); + const match = /^(.*T\d{2}:\d{2})(?::(\d{2})(?:\.(\d{1,9}))?)?(Z|[+-]\d{2}:\d{2})$/u.exec(value); + if (!match) throw new Error(`Unsupported instant ${value}: at most nanosecond precision is required`); + const seconds = Date.parse(`${match[1]}:${match[2] ?? "00"}${match[4]}`); + if (!Number.isFinite(seconds)) throw new Error(`Invalid instant ${value}`); + return BigInt(seconds) * 1_000_000n + BigInt((match[3] ?? "").padEnd(9, "0")); +}; +export const compareInstants = (left: string, right: string): number => { + const a = instantNanos(left), b = instantNanos(right); + return a < b ? -1 : a > b ? 1 : 0; +}; +export const normalizeInstant = (value: string): string => { + const ns = instantNanos(value), seconds = ns / 1_000_000_000n; + const fraction = (ns % 1_000_000_000n + 1_000_000_000n) % 1_000_000_000n; + const floorSeconds = ns < 0n && fraction !== 0n ? seconds - 1n : seconds; + const base = new Date(Number(floorSeconds * 1000n)).toISOString().replace(/\.000Z$/u, ""); + const tail = fraction.toString().padStart(9, "0").replace(/0+$/u, ""); + return `${base}${tail ? `.${tail}` : ""}Z`; +}; diff --git a/src/compiler/training/rewind/leakCheck.test.ts b/src/compiler/training/rewind/leakCheck.test.ts new file mode 100644 index 00000000..09eb5005 --- /dev/null +++ b/src/compiler/training/rewind/leakCheck.test.ts @@ -0,0 +1,96 @@ +import { appendFile, mkdir, readFile, readdir, rm, symlink, utimes, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { cleanTrainingProjects } from "../plan/fixtures.test-helper.js"; +import type { MaterializedMoment, LeakFinding } from "./contract.js"; +import { AT, FUTURE, memoryEvent, rewindFixture, testGit } from "./fixtures.test-helper.js"; +import { leakCheckTrainingMoment } from "./leakCheck.js"; +import { finalizeTrainingMoment, materializeTrainingMoment } from "./materialize.js"; +import { rewindGit } from "./gitProcess.js"; +import { stampTree } from "./timestamps.js"; + +afterEach(cleanTrainingProjects); +type Fixture = Awaited>; +type Mutation = (fixture: Fixture, moment: MaterializedMoment) => Promise; +const inputPath = (fixture: Fixture, moment: MaterializedMoment, kind: string) => + path.join(fixture.options.out, moment.inputs.find((input) => input.kind === kind)!.path!); +const patchNetwork = async (fixture: Fixture, moment: MaterializedMoment, patch: (state: Record) => void) => { + const file = path.join(inputPath(fixture, moment, "moltnet"), "state.json"); + const state = JSON.parse(await readFile(file, "utf8")); patch(state); await writeFile(file, JSON.stringify(state)); +}; +const cases: Array<[string, LeakFinding["class"], Mutation]> = [ + ["future file mtime", "file-time", async (fixture) => { await writeFile(path.join(fixture.options.out, "future"), "future"); }], + ["future directory mtime", "file-time", async (fixture) => { await mkdir(path.join(fixture.options.out, "new-directory")); }], + ["future symlink mtime", "file-time", async (fixture) => { await symlink("moment.json", path.join(fixture.options.out, "new-link")); }], + ["extra unreachable future commit", "git-object", async (fixture, moment) => { + const code = inputPath(fixture, moment, "git"); + const pack = await rewindGit(fixture.gitRepo, ["pack-objects", "--revs", "--stdout"], Buffer.from(`${fixture.futureCommit}\n`)); + await rewindGit(code, ["index-pack", "--stdin"], pack); + }], + ["extra tag", "git-object", async (fixture, moment) => { testGit(inputPath(fixture, moment, "git"), ["tag", "leaked", fixture.commit]); }], + ["extra remote", "git-object", async (fixture, moment) => { testGit(inputPath(fixture, moment, "git"), ["remote", "add", "leaked", fixture.gitRepo]); }], + ["reachable commit future author only", "git-object", async (fixture, moment) => { + const code = inputPath(fixture, moment, "git"); + testGit(code, ["commit", "--quiet", "--allow-empty", "--date=2026-10-10T00:00:00Z", "-m", "future author"]); + moment.inputs.find((input) => input.kind === "git")!.source.commit = testGit(code, ["rev-parse", "HEAD"]); + await writeFile(path.join(fixture.options.out, "moment.json"), JSON.stringify(moment)); + }], + ["reachable commit future committer only", "git-object", async (fixture, moment) => { + const code = inputPath(fixture, moment, "git"); + testGit(code, ["commit", "--quiet", "--allow-empty", "--date=2026-10-09T07:00:00Z", "-m", "future committer"], "2026-10-10T00:00:00Z"); + moment.inputs.find((input) => input.kind === "git")!.source.commit = testGit(code, ["rev-parse", "HEAD"]); + await writeFile(path.join(fixture.options.out, "moment.json"), JSON.stringify(moment)); + }], + ["future memory event", "memory-event", async (fixture, moment) => { + await appendFile(path.join(inputPath(fixture, moment, "memory"), "memory/events.jsonl"), `${JSON.stringify(memoryEvent(2, FUTURE))}\n`); + }], + ["memory seq gap", "memory-event", async (fixture, moment) => { + await appendFile(path.join(inputPath(fixture, moment, "memory"), "memory/events.jsonl"), `${JSON.stringify(memoryEvent(3))}\n`); + }], + ["memory audit state", "memory-event", async (fixture, moment) => { + await writeFile(path.join(inputPath(fixture, moment, "memory"), "memory/index.sqlite"), "future derived state"); + }], + ["future Moltnet message", "moltnet-message", async (fixture, moment) => patchNetwork(fixture, moment, (state) => { + state.room_messages.assignment = [{ id: "future", network_id: "news", target: { kind: "room", room_id: "assignment" }, + from: { type: "agent", id: "writer" }, parts: [{ kind: "text", text: "future message" }], created_at: FUTURE }]; + })], + ["future Moltnet event", "moltnet-event", async (fixture, moment) => patchNetwork(fixture, moment, (state) => { + state.attachment_delivery_events = [{ id: "future", type: "test", network_id: "news", created_at: FUTURE }]; + })], + ["future Moltnet room", "moltnet-room", async (fixture, moment) => patchNetwork(fixture, moment, (state) => { + state.rooms.future = { id: "future", name: "Future", created_at: FUTURE }; + })], + ["escaping symlink", "symlink-escape", async (fixture) => { await symlink("../outside", path.join(fixture.options.out, "escape")); }], + ["absolute symlink", "symlink-escape", async (fixture) => { await symlink(fixture.root, path.join(fixture.options.out, "escape")); }], + ["future dated path", "dated-path", async (fixture) => { await mkdir(path.join(fixture.options.out, "edition-2026-10-11")); }], + ["tampered snapshot", "snapshot", async (fixture, moment) => { await writeFile(path.join(inputPath(fixture, moment, "volume"), "saved.txt"), "tampered"); }], + ["future feed landing", "feed-identity", async (fixture, moment) => { + const file = path.join(inputPath(fixture, moment, "feed"), ".spawnfile-feed.json"); + const identity = JSON.parse(await readFile(file, "utf8")); identity.landed_at = FUTURE; await writeFile(file, JSON.stringify(identity)); + }], + ["feed current outside its volume", "feed-identity", async (fixture, moment) => { + const link = path.join(inputPath(fixture, moment, "feed"), "current"); + await rm(link); await symlink("../code", link); + }] +]; + +describe("independent disk leak check mutation guarantees", () => { + it.each(cases)("refuses %s with class %s", async (_name, expectedClass, mutate) => { + const fixture = await rewindFixture(); + const moment = await materializeTrainingMoment(fixture.options); + expect(moment.status).toBe("materialized"); + await mutate(fixture, moment); + // Non-time violations must be detected from content even when their filesystem timestamps look old. + if (expectedClass !== "file-time") await stampTree(fixture.options.out, AT, AT); + else await utimes(path.join(fixture.options.out, "moment.json"), new Date(AT), new Date(AT)); + const check = await leakCheckTrainingMoment(fixture.options.out, { historyDir: fixture.historyDir }); + expect(check.ok).toBe(false); + expect(check.findings.map((finding) => finding.class)).toContain(expectedClass); + const out = path.join(fixture.root, "rejected"); + const refused = await finalizeTrainingMoment(fixture.options.out, out, { historyDir: fixture.historyDir }); + expect(refused.status).toBe("refused"); + expect(refused.leakCheck.ok).toBe(false); + expect(refused.inputs.every((input) => input.path === null)).toBe(true); + expect(await readdir(out)).toEqual(["leak-check.json", "moment.json"]); + }); +}); diff --git a/src/compiler/training/rewind/leakCheck.ts b/src/compiler/training/rewind/leakCheck.ts new file mode 100644 index 00000000..a20de5c1 --- /dev/null +++ b/src/compiler/training/rewind/leakCheck.ts @@ -0,0 +1,39 @@ +import { readFile, realpath } from "node:fs/promises"; +import path from "node:path"; +import { materializedMomentSchema, leakCheckSchema, type LeakCheck, type LeakFinding } from "./contract.js"; +import { openHistory } from "./history.js"; +import { diskPath, inspectBundleFiles } from "./leakFiles.js"; +import { inspectGitBundle } from "./leakGit.js"; +import { addInputFinding, inspectFeedBundle, inspectMemoryBundle, inspectMoltnetBundle, inspectVolumeBundle } from "./leakInputs.js"; + +export interface LeakCheckOptions { historyDir?: string } + +/** Independent acceptance boundary: only disk reports, disk payloads and disk history are trusted. */ +export const leakCheckTrainingMoment = async (bundle: string, options: LeakCheckOptions = {}): Promise => { + const root = await realpath(path.resolve(bundle)); + const moment = materializedMomentSchema.parse(JSON.parse(await readFile(await diskPath(root, "moment.json"), "utf8"))); + const report: LeakCheck = { version: "spawnfile.leak-check.v1", at: moment.at, ok: false, + checked: { files: 0, gitObjects: 0, memoryEvents: 0, moltnetMessages: 0, moltnetEvents: 0, snapshots: 0 }, findings: [] }; + await inspectBundleFiles(root, moment.inputs, report); + for (const input of moment.inputs) { + const kind: LeakFinding["class"] = input.kind === "memory" ? "memory-event" + : input.kind === "moltnet" ? "moltnet-message" : input.kind === "feed" ? "feed-identity" + : input.kind === "volume" ? "snapshot" : "git-object"; + try { + if (!input.path || input.refusal || !input.fidelity) throw new Error("Input is not materialized"); + const directory = await diskPath(root, input.path); + // Do not hand git/SQLite a payload with escaping paths, even for a failed check. + if (report.findings.some((finding) => finding.class === "symlink-escape" + && (finding.path === input.path || finding.path.startsWith(`${input.path}/`)))) continue; + if (input.kind === "git") await inspectGitBundle(directory, input, report); + if (input.kind === "memory") await inspectMemoryBundle(root, input, report); + if (input.kind === "feed") await inspectFeedBundle(root, input, report); + if (input.kind === "moltnet") await inspectMoltnetBundle(root, input, report); + if (input.kind === "volume") await inspectVolumeBundle(root, input, report, await openHistory(options.historyDir)); + } catch (error) { + addInputFinding(report, input, kind, error instanceof Error ? error.message : String(error), input.path ?? "."); + } + } + report.ok = report.findings.length === 0; + return leakCheckSchema.parse(report); +}; diff --git a/src/compiler/training/rewind/leakFiles.ts b/src/compiler/training/rewind/leakFiles.ts new file mode 100644 index 00000000..044c99ac --- /dev/null +++ b/src/compiler/training/rewind/leakFiles.ts @@ -0,0 +1,62 @@ +import { lstat, readdir, readlink, realpath } from "node:fs/promises"; +import path from "node:path"; +import { inside } from "./history.js"; +import { instantNanos } from "./instant.js"; +import type { LeakCheck, MaterializedInput } from "./contract.js"; + +export const inputAtPath = (inputs: MaterializedInput[], relative: string): MaterializedInput | undefined => + inputs.find((input) => input.path !== null && (relative === input.path || relative.startsWith(`${input.path}/`))); + +export const diskPath = async (root: string, relative: string): Promise => { + const candidate = path.resolve(root, relative); + if (!inside(root, candidate) || path.isAbsolute(relative)) throw new Error(`Bundle path escapes root: ${relative}`); + if (!inside(root, await realpath(candidate))) throw new Error(`Bundle symlink escapes root: ${relative}`); + return candidate; +}; + +/** No symlinks are traversed. Inspection never reads a target outside the bundle. */ +export const inspectBundleFiles = async (root: string, inputs: MaterializedInput[], report: LeakCheck): Promise => { + const at = instantNanos(report.at); + const latestDate = new Date(Date.parse(report.at) + 14 * 3600_000).toISOString().slice(0, 10); + const visit = async (file: string): Promise => { + const relative = path.relative(root, file).split(path.sep).join("/") || "."; + const owner = inputAtPath(inputs, relative); + const input = owner ? { section: owner.section, id: owner.id, declaredBy: owner.declaredBy } : null; + const add = (kind: "file-time" | "symlink-escape" | "dated-path", detail: string) => { + report.findings.push({ input, class: kind, path: relative, detail }); + }; + const stats = await lstat(file, { bigint: true }); + report.checked.files++; + if (stats.mtimeNs > at) add("file-time", "Modification time is later than the moment"); + for (const match of path.basename(file).matchAll(/\d{4}-\d{2}-\d{2}/gu)) { + const date = match[0]; + const parsed = Date.parse(`${date}T00:00:00Z`); + if (Number.isFinite(parsed) && new Date(parsed).toISOString().startsWith(date) && date > latestDate) { + add("dated-path", `Calendar date ${date} is later than the latest possible local date ${latestDate}`); + } + } + if (stats.isSymbolicLink()) { + const link = await readlink(file); + if (path.isAbsolute(link) || !inside(root, path.resolve(path.dirname(file), link))) { + add("symlink-escape", `Unsafe symlink target ${JSON.stringify(link)}`); return; + } + // Even a dangling leaf can cross an existing symlink in a parent component. + let target = path.resolve(path.dirname(file), link); + for (;;) { + try { + if (!inside(root, await realpath(target))) add("symlink-escape", "Symlink resolves outside the bundle"); + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + add("symlink-escape", `Cannot resolve symlink: ${String(error)}`); break; + } + if (target === root) break; + target = path.dirname(target); + } + } + } else if (stats.isDirectory()) { + for (const name of (await readdir(file)).sort()) await visit(path.join(file, name)); + } else if (!stats.isFile()) add("symlink-escape", "Special files are not allowed in a moment bundle"); + }; + await visit(root); +}; diff --git a/src/compiler/training/rewind/leakGit.ts b/src/compiler/training/rewind/leakGit.ts new file mode 100644 index 00000000..fdff98fc --- /dev/null +++ b/src/compiler/training/rewind/leakGit.ts @@ -0,0 +1,57 @@ +import { lstat, readFile, readdir } from "node:fs/promises"; +import path from "node:path"; +import { rewindGit } from "./gitProcess.js"; +import { instantNanos } from "./instant.js"; +import type { LeakCheck, MaterializedInput } from "./contract.js"; + +/** Enumerate every stored object, independently of the materializer's chosen-object list. */ +export const inspectGitBundle = async (repository: string, input: MaterializedInput, report: LeakCheck): Promise => { + const add = (detail: string) => report.findings.push({ + input: { section: input.section, id: input.id, declaredBy: input.declaredBy }, + class: "git-object", path: input.path!, detail + }); + const gitDir = path.join(repository, ".git"); + if (!(await lstat(gitDir)).isDirectory()) throw new Error("The materialized repository must have its own .git directory"); + const names = new Set(await readdir(gitDir)); + for (const name of ["FETCH_HEAD", "ORIG_HEAD", "logs", "packed-refs", "commondir", "shallow"]) { + if (names.has(name)) add(`Forbidden git metadata ${name}`); + } + const config = await readFile(path.join(gitDir, "config"), "utf8"); + if (/^\s*\[\s*(?:include|includeif)\b/imu.test(config)) throw new Error("Git configuration includes are forbidden"); + for (const name of ["objects/info/alternates", "objects/info/http-alternates", "info/grafts"]) { + const exists = await lstat(path.join(gitDir, name)).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (exists) throw new Error(`Git object indirection ${name} is forbidden`); + } + const localConfig = (await rewindGit(repository, ["config", "--local", "--no-includes", "--list", "--null"])) + .toString().split("\0").filter(Boolean); + for (const entry of localConfig) { + const key = entry.split("\n")[0]!.toLowerCase(); + if (key.startsWith("remote.") || key === "core.worktree" || key === "extensions.partialclone") add(`Forbidden git configuration ${key}`); + } + const refs = (await rewindGit(repository, ["for-each-ref", "--format=%(refname) %(objectname)"])).toString().trim().split("\n").filter(Boolean); + const head = (await readFile(path.join(gitDir, "HEAD"), "utf8")).trim(); + const commit = input.source.commit; + if (!commit || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/u.test(commit)) throw new Error("Missing materialized commit identity"); + if (refs.length !== 1 || !refs[0]!.startsWith("refs/heads/") || refs[0]!.split(" ")[1] !== commit + || head !== `ref: ${refs[0]?.split(" ")[0]}`) add("Exactly one branch and HEAD must name the materialized commit"); + const reachable = new Set((await rewindGit(repository, ["rev-list", "--objects", commit, "--"])) + .toString().split("\n").filter(Boolean).map((line) => line.split(" ")[0])); + const objects = (await rewindGit(repository, ["cat-file", "--batch-all-objects", "--batch-check=%(objectname) %(objecttype)"])) + .toString().trim().split("\n").filter(Boolean); + report.checked.gitObjects += objects.length; + for (const row of objects) { + const [oid, kind] = row.split(" "); + if (!reachable.has(oid)) add(`Object ${oid} is not reachable from ${commit}`); + if (kind !== "commit") continue; + const object = (await rewindGit(repository, ["cat-file", "commit", oid!])).toString(); + const headers = object.split("\n\n", 1)[0]!; + for (const role of ["author", "committer"]) { + const match = new RegExp(`^${role} .* (-?\\d+) [+-]\\d{4}$`, "mu").exec(headers); + if (!match) { add(`Commit ${oid} has no valid ${role} date`); continue; } + if (BigInt(match[1]!) * 1_000_000_000n > instantNanos(report.at)) add(`Commit ${oid} has a future ${role} date`); + } + } +}; diff --git a/src/compiler/training/rewind/leakInputs.ts b/src/compiler/training/rewind/leakInputs.ts new file mode 100644 index 00000000..edd239ae --- /dev/null +++ b/src/compiler/training/rewind/leakInputs.ts @@ -0,0 +1,112 @@ +import { createHash } from "node:crypto"; +import { lstat, readFile, readdir, readlink, realpath } from "node:fs/promises"; +import path from "node:path"; +import { feedIdentityFindings, type FeedIdentity } from "../../../volume/feedRecord.js"; +import type { LeakCheck, LeakFinding, MaterializedInput } from "./contract.js"; +import { historyKey, inside, type History } from "./history.js"; +import { compareInstants } from "./instant.js"; +import { diskPath } from "./leakFiles.js"; +import { parseMemoryLedger } from "./memory.js"; +import { inspectMoltnetJson } from "./moltnetJson.js"; +import { inspectMoltnetSqlite } from "./moltnetSqlite.js"; +import { replayVolumeSnapshot, volumeSnapshotSchema } from "./volumes.js"; + +export const addInputFinding = (report: LeakCheck, input: MaterializedInput, kind: LeakFinding["class"], detail: string, file = input.path!): void => { + report.findings.push({ input: { section: input.section, id: input.id, declaredBy: input.declaredBy }, class: kind, path: file, detail }); +}; + +export const inspectMemoryBundle = async (root: string, input: MaterializedInput, report: LeakCheck): Promise => { + const home = await diskPath(root, input.path!); + const visit = async (directory: string): Promise => { + for (const entry of await readdir(directory, { withFileTypes: true })) { + if (/^(?:index\.sqlite(?:$|[-.])|causal|authority-receipts)/u.test(entry.name)) { + addInputFinding(report, input, "memory-event", `Derived or audit memory state ${entry.name} is forbidden`); + } + if (entry.isDirectory()) await visit(path.join(directory, entry.name)); + } + }; + await visit(home); + const ledger = parseMemoryLedger(await readFile(await diskPath(root, `${input.path}/memory/events.jsonl`), "utf8")); + report.checked.memoryEvents += ledger.events.length; + let native = false; + for (const [index, event] of ledger.events.entries()) { + if (event.seq === undefined ? native : event.seq !== index + 1) { + addInputFinding(report, input, "memory-event", `Noncontiguous memory sequence at line ${index + 1}`); + } + native ||= event.seq !== undefined; + if (compareInstants(event.createdAt, report.at) > 0) addInputFinding(report, input, "memory-event", `Future memory event at line ${index + 1}`); + if (index > 0 && compareInstants(event.createdAt, ledger.events[index - 1]!.createdAt) < 0) { + addInputFinding(report, input, "memory-event", `Memory clock regression at line ${index + 1}`); + } + } + if (input.source.seq !== ledger.events.length) addInputFinding(report, input, "memory-event", "Ledger length disagrees with the materialized sequence"); +}; + +export const inspectFeedBundle = async (root: string, input: MaterializedInput, report: LeakCheck): Promise => { + const directory = await diskPath(root, input.path!); + if (input.fidelity === "overridden") { + if ((await readdir(directory)).length) addInputFinding(report, input, "feed-identity", "Overridden feed volume must be empty"); + return; + } + const identity: FeedIdentity = JSON.parse(await readFile(await diskPath(root, `${input.path}/.spawnfile-feed.json`), "utf8")); + for (const issue of feedIdentityFindings(identity)) addInputFinding(report, input, "feed-identity", issue); + if (compareInstants(identity.landed_at, report.at) > 0) addInputFinding(report, input, "feed-identity", "Feed landed_at is later than the moment"); + if (identity.resource !== input.id || identity.revision !== input.source.revision) { + addInputFinding(report, input, "feed-identity", "Feed identity disagrees with the moment input"); + } + const current = path.join(directory, "current"); + if (!(await lstat(current)).isSymbolicLink()) throw new Error("Feed current must be a relative symlink"); + const link = await readlink(current); + if (link !== identity.tree || !/^trees\/[a-f0-9]{64}(?:\.[1-9][0-9]*)?$/u.test(link) + || !inside(path.join(directory, "trees"), await realpath(current))) { + addInputFinding(report, input, "feed-identity", "Feed current does not point to its tree inside the volume"); + } else if (!(await lstat(await realpath(current))).isDirectory()) addInputFinding(report, input, "feed-identity", "Feed tree must be a directory"); +}; + +export const inspectMoltnetBundle = async (root: string, input: MaterializedInput, report: LeakCheck): Promise => { + const directory = await diskPath(root, input.path!); + const stores = (await readdir(directory)).filter((name) => name === "state.json" || name === "moltnet.sqlite"); + if (stores.length !== 1) throw new Error("Expected exactly one Moltnet store"); + const store = await diskPath(root, `${input.path}/${stores[0]}`); + const result = stores[0] === "state.json" ? await inspectMoltnetJson(store, report.at) : await inspectMoltnetSqlite(store, report.at); + report.checked.moltnetMessages += result.messages; + report.checked.moltnetEvents += result.events; + for (const finding of result.findings) addInputFinding(report, input, finding.class, finding.detail, path.relative(root, finding.path)); +}; + +export const inspectVolumeBundle = async (root: string, input: MaterializedInput, report: LeakCheck, history?: History): Promise => { + let expected: Record = {}; + if (input.fidelity !== "overridden") { + if (!history || !input.source.snapshot) throw new Error("Snapshot verification requires its history and selected snapshot"); + const snapshots = await history.jsonl(`volumes/${historyKey(input)}/snapshots.jsonl`, volumeSnapshotSchema); + if (!snapshots) throw new Error("Snapshot history is missing"); + const replay = replayVolumeSnapshot(snapshots, input.source.snapshot); + if (compareInstants(replay.takenAt, report.at) > 0) throw new Error("Selected snapshot is later than the moment"); + expected = replay.files; + report.checked.snapshots += replay.chain.length; + } + const found = new Set(); + const directories = new Set(); + for (const name of Object.keys(expected)) { + for (let parent = path.posix.dirname(name); parent !== "."; parent = path.posix.dirname(parent)) directories.add(parent); + } + const visit = async (relative: string): Promise => { + const folder = await diskPath(root, `${input.path}${relative ? `/${relative}` : ""}`); + for (const entry of await readdir(folder, { withFileTypes: true })) { + const name = relative ? `${relative}/${entry.name}` : entry.name; + if (entry.isDirectory()) { + if (!directories.has(name)) addInputFinding(report, input, "snapshot", `Unexpected snapshot directory ${name}`); + await visit(name); + } else { + found.add(name); + if (!entry.isFile() || !Object.hasOwn(expected, name)) addInputFinding(report, input, "snapshot", `Unexpected snapshot entry ${name}`); + else { + const digest = createHash("sha256").update(await readFile(await diskPath(root, `${input.path}/${name}`))).digest("hex"); + if (digest !== expected[name]) addInputFinding(report, input, "snapshot", `Snapshot digest mismatch for ${name}`); + } + } + } + }; + await visit(""); + for (const name of Object.keys(expected)) if (!found.has(name)) addInputFinding(report, input, "snapshot", `Missing snapshot file ${name}`); +}; diff --git a/src/compiler/training/rewind/materialize.test.ts b/src/compiler/training/rewind/materialize.test.ts new file mode 100644 index 00000000..9286951e --- /dev/null +++ b/src/compiler/training/rewind/materialize.test.ts @@ -0,0 +1,107 @@ +import { cp, lstat, mkdir, readFile, readdir, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { loadTrainingProject, resolveTrainingPlan } from "../plan/index.js"; +import { agentManifest, cleanTrainingProjects, dataFiles, example, trainingProject } from "../plan/fixtures.test-helper.js"; +import { materializeTrainingMoment, resolveTrainingMoment } from "./materialize.js"; +import { leakCheckTrainingMoment } from "./leakCheck.js"; +import * as gitMaterializer from "./git.js"; +import { AT, BEFORE, emptyRecord, jsonl, rewindFixture } from "./fixtures.test-helper.js"; + +afterEach(cleanTrainingProjects); +afterEach(() => vi.restoreAllMocks()); + +describe("materializeTrainingMoment", () => { + it("stages every effective input, verifies disk, and preserves source times", async () => { + const fixture = await rewindFixture(); + const resolved = resolveTrainingMoment(fixture.plan, fixture.project, example.id); + expect(resolved.inputs.map((input) => `${input.section}:${input.id}`)).toEqual([ + "memory:self", "networks:news", "resources:code", "resources:edition-state", "resources:news" + ]); + const result = await materializeTrainingMoment(fixture.options); + expect(result.status).toBe("materialized"); + expect(result.leakCheck).toEqual({ ok: true, findings: 0 }); + expect(result.inputs.map((input) => input.fidelity)).toEqual(["approximate", "overridden", "approximate", "approximate", "exact"]); + expect((await leakCheckTrainingMoment(fixture.options.out, { historyDir: fixture.historyDir })).ok).toBe(true); + for (const input of result.inputs.filter((item) => item.kind !== "moltnet")) { + expect((await lstat(path.join(fixture.options.out, input.path!))).mtime.toISOString()).toBe(new Date(BEFORE).toISOString()); + } + expect((await readdir(fixture.root)).filter((name) => name.includes(".rewind-"))).toEqual([]); + expect(await readFile(path.join(fixture.gitRepo, "future.txt"), "utf8")).toContain("after wake"); + }); + it("uses equal-instant records and accepts an existing empty output directory", async () => { + const fixture = await rewindFixture(); + await mkdir(fixture.options.out); + const record = { ...emptyRecord("agent:a", "2026-10-09T10:20:31+02:00"), + resources: [{ id: "code", kind: "git", commit: fixture.commit }], + memory: [{ id: "self", declaredBy: "agent:a", seq: 1, generation: 0 }], + volumes: [{ id: "edition-state", snapshot: "snapshot-1" }] }; + await jsonl(fixture.historyDir, "moments/anything.jsonl", [record]); + const result = await materializeTrainingMoment(fixture.options); + expect(result.status).toBe("materialized"); + expect(result.record.matched).toBe(true); + expect(result.inputs.filter((input) => input.fidelity === "approximate")).toEqual([]); + }); + it("collects all missing-history refusals and leaves only moment.json", async () => { + const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest({ + memory: [{ id: "self", store: { kind: "memory" } }], + workspace: { docs: { system: "AGENTS.md" }, resources: [ + { id: "code", kind: "git", mode: "readonly", mount: "./code", url: "https://invalid.example/code.git" }, + { id: "prebuilt", kind: "bundle", mode: "readonly", mount: "./prebuilt", source: "archive.tar" }, + { id: "news", kind: "volume", name: "news", mode: "mutable", mount: "./news", feed: { directory: "." } } + ] } + }) }); + const project = await loadTrainingProject(root), plan = resolveTrainingPlan(project, "agent:writer", "test"); + const out = path.join(root, "out"), result = await materializeTrainingMoment({ project, plan, example: example.id, out }); + expect(result.status).toBe("refused"); + expect(result.inputs).toHaveLength(4); + expect(result.inputs.every((input) => input.refusal && input.path === null)).toBe(true); + expect(result.inputs.map((input) => input.refusal).join(" ")).toMatch(/Prebuilt bundle/); + expect(await readdir(out)).toEqual(["moment.json"]); + expect((await readdir(root)).filter((name) => name.includes(".rewind-"))).toEqual([]); + const missing = await materializeTrainingMoment({ project, plan, example: example.id, + out: path.join(root, "missing-history-out"), historyDir: path.join(root, "does-not-exist") }); + expect(missing.status).toBe("refused"); + expect(missing.inputs.every((input) => input.refusal?.startsWith("Cannot open history:"))).toBe(true); + }); + it("refuses contradictory matching records and malformed history lines without a usable payload", async () => { + const fixture = await rewindFixture(); + await jsonl(fixture.historyDir, "moments/conflict.jsonl", [emptyRecord(), { + ...emptyRecord(), memory: [{ id: "self", declaredBy: "agent:a", seq: 1, generation: 0 }] + }]); + const refused = await materializeTrainingMoment(fixture.options); + expect(refused.inputs.filter((input) => input.fidelity !== "overridden") + .every((input) => input.refusal?.includes("Conflicting moment records"))).toBe(true); + expect(refused.inputs.find((input) => input.section === "networks")!.fidelity).toBe("overridden"); + expect(await readdir(fixture.options.out)).toEqual(["moment.json"]); + await jsonl(fixture.historyDir, "moments/conflict.jsonl", [{ invalid: true }]); + const malformed = await materializeTrainingMoment({ ...fixture.options, out: path.join(fixture.root, "out-2") }); + expect(malformed.inputs[0]!.refusal).toContain("moments/conflict.jsonl:1"); + }); + it("does not overwrite nonempty output or output symlinks", async () => { + const fixture = await rewindFixture(); + await mkdir(fixture.options.out); await writeFile(path.join(fixture.options.out, "keep"), "keep"); + await expect(materializeTrainingMoment(fixture.options)).rejects.toThrow(/empty directory/); + expect(await readFile(path.join(fixture.options.out, "keep"), "utf8")).toBe("keep"); + const link = path.join(fixture.root, "out-link"); await symlink(fixture.options.out, link); + await expect(materializeTrainingMoment({ ...fixture.options, out: link })).rejects.toThrow(/empty directory/); + const alias = path.join(fixture.root, "history-alias"); await symlink(fixture.historyDir, alias); + await expect(materializeTrainingMoment({ ...fixture.options, out: path.join(alias, "output") })).rejects.toThrow(/must not overlap/); + expect(await readdir(fixture.historyDir)).not.toContain("output"); + }); + it("catches a deliberately broken git producer that copies full history", async () => { + const fixture = await rewindFixture(), correct = gitMaterializer.materializeGit; + vi.spyOn(gitMaterializer, "materializeGit").mockImplementation(async (context) => { + const result = await correct(context); + await cp(path.join(fixture.gitRepo, ".git"), path.join(context.destination, ".git"), { recursive: true }); + return result; + }); + const result = await materializeTrainingMoment(fixture.options); + expect(result.status).toBe("refused"); + expect(result.inputs.find((input) => input.id === "code")!.refusal).toContain("git-object"); + expect(await readdir(fixture.options.out)).toEqual(["leak-check.json", "moment.json"]); + expect(JSON.parse(await readFile(path.join(fixture.options.out, "leak-check.json"), "utf8")).findings) + .toEqual(expect.arrayContaining([expect.objectContaining({ class: "git-object" })])); + expect(result.at).toBe(AT); + }); +}); diff --git a/src/compiler/training/rewind/materialize.ts b/src/compiler/training/rewind/materialize.ts new file mode 100644 index 00000000..4397ad02 --- /dev/null +++ b/src/compiler/training/rewind/materialize.ts @@ -0,0 +1,149 @@ +import { lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, utimes, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { LoadedTrainingProject, TrainingPlan } from "../plan/index.js"; +import { materializedMomentSchema, type MaterializedInput, type MaterializedMoment, type MomentRecord } from "./contract.js"; +import { materializeFeed } from "./feed.js"; +import { materializeGit } from "./git.js"; +import { historyKey, inside, openHistory, type History } from "./history.js"; +import { resolveRewindInputs } from "./inputs.js"; +import { leakCheckTrainingMoment, type LeakCheckOptions } from "./leakCheck.js"; +import { materializeMemory } from "./memory.js"; +import { materializeMoltnet } from "./moltnet.js"; +import { selectMomentRecord } from "./select.js"; +import { createTaskClock } from "./taskClock.js"; +import { restoreAccessTimes, stampTree } from "./timestamps.js"; +import type { InputContext, InputResult, RewindInput } from "./types.js"; +import { materializeVolume } from "./volumes.js"; + +export interface MaterializeTrainingMomentOptions { + plan: TrainingPlan; + project: LoadedTrainingProject; + example: string; + historyDir?: string; + out: string; +} + +/** Pure core for a per-trial caller: effective declarations, example identity and matching record. */ +export const resolveTrainingMoment = ( + plan: TrainingPlan, project: LoadedTrainingProject, example: string, records: MomentRecord[] = [] +) => { + const resolved = resolveRewindInputs(plan, project, example); + return { ...resolved, record: selectMomentRecord(records, plan.agent.id, resolved.at, + resolved.inputs.filter((input) => !input.overridden)) }; +}; + +const reason = (error: unknown): string => error instanceof Error ? error.message : String(error); +const serialize = (value: unknown): string => `${JSON.stringify(value, null, 2)}\n`; +const kindOf = (input: RewindInput): MaterializedInput["kind"] => input.section === "memory" ? "memory" + : input.section === "networks" ? "moltnet" : input.declaration.kind === "volume" && input.declaration.feed ? "feed" : input.declaration.kind; +const reportInput = (input: RewindInput): MaterializedInput => ({ + section: input.section, id: input.id, declaredBy: input.declaredBy, kind: kindOf(input), fidelity: null, + basis: "refused", source: {}, path: null, notes: [], refusal: null +}); +const dispatch = (context: InputContext): Promise => { + const { input } = context; + if (input.section === "memory") return materializeMemory(context); + if (input.section === "networks") return materializeMoltnet(context); + if (input.declaration.kind !== "volume") return materializeGit(context); + return input.declaration.feed ? materializeFeed(context) : materializeVolume(context); +}; + +const assertEmptyOutput = async (out: string): Promise => { + const existing = await lstat(out).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (existing && (!existing.isDirectory() || (await readdir(out)).length)) throw new Error("Rewind --out must not exist or must be an empty directory"); +}; + +const discardPayload = async (staging: string, moment: MaterializedMoment): Promise => { + moment.status = "refused"; + for (const input of moment.inputs) input.path = null; + for (const name of await readdir(staging)) if (name !== "moment.json" && name !== "leak-check.json") { + await rm(path.join(staging, name), { recursive: true, force: true }); + } +}; + +/** A separately callable disk acceptance gate, also used to test deliberately broken producers. */ +export const finalizeTrainingMoment = async (staging: string, out: string, options: LeakCheckOptions = {}): Promise => { + const moment = materializedMomentSchema.parse(JSON.parse(await readFile(path.join(staging, "moment.json"), "utf8"))); + if (moment.status === "materialized") { + const check = await leakCheckTrainingMoment(staging, options); + moment.leakCheck = { ok: check.ok, findings: check.findings.length }; + if (!check.ok) { + for (const finding of check.findings) { + const affected = finding.input ? moment.inputs.filter((input) => input.section === finding.input!.section + && input.id === finding.input!.id && input.declaredBy === finding.input!.declaredBy) : moment.inputs; + for (const input of affected) input.refusal = [input.refusal, `${finding.class}: ${finding.path}: ${finding.detail}`].filter(Boolean).join("; "); + } + await discardPayload(staging, moment); + } + await writeFile(path.join(staging, "leak-check.json"), serialize(check)); + await stampTree(path.join(staging, "leak-check.json"), moment.at, moment.at); + } else await discardPayload(staging, moment); + await writeFile(path.join(staging, "moment.json"), serialize(materializedMomentSchema.parse(moment))); + await stampTree(path.join(staging, "moment.json"), moment.at, moment.at); + await utimes(staging, new Date(moment.at), new Date(moment.at)); + await restoreAccessTimes(staging, moment.at); + await assertEmptyOutput(out); + await rename(staging, out); + return moment; +}; + +/** Local-only rewind. No compilation artifacts, model invocation, auth, shell or network access. */ +export const materializeTrainingMoment = async (options: MaterializeTrainingMomentOptions): Promise => { + const { plan, project, example } = options; + const resolved = resolveRewindInputs(plan, project, example); + let out = path.resolve(options.out); + await assertEmptyOutput(out); + let history: History | undefined, historyError: string | undefined; + try { history = await openHistory(options.historyDir); } + catch (error) { historyError = `Cannot open history: ${reason(error)}`; } + await mkdir(path.dirname(out), { recursive: true }); + out = path.join(await realpath(path.dirname(out)), path.basename(out)); + if (history && (inside(history.root, out) || inside(out, history.root))) throw new Error("Rewind output and history must not overlap"); + const staging = await mkdtemp(path.join(path.dirname(out), `.${path.basename(out)}.rewind-`)); + try { + let record: MomentRecord | null = null, recordError: string | undefined; + try { + record = selectMomentRecord(await history?.records() ?? [], plan.agent.id, resolved.at, + resolved.inputs.filter((input) => !input.overridden)); + } catch (error) { recordError = reason(error); } + const moment: MaterializedMoment = { + version: "spawnfile.materialized-moment.v1", status: "materialized", agent: plan.agent.id, + example: { id: resolved.selected.id, role: resolved.selected.role, dataset: resolved.selected.dataset }, + at: resolved.at, inputs: [], record: { matched: record !== null }, leakCheck: { ok: false, findings: 0 } + }; + for (const input of resolved.inputs) { + const entry = reportInput(input); + moment.inputs.push(entry); + try { + if (!input.overridden && (historyError || recordError)) throw new Error(historyError ?? recordError); + const relative = `${input.section === "networks" ? "moltnet" : input.section}/${historyKey(input)}`; + const destination = path.join(staging, relative); + const result = await dispatch({ input, at: resolved.at, record, history, destination }); + await stampTree(destination, result.instant, resolved.at); + Object.assign(entry, { fidelity: result.fidelity, basis: result.basis, source: result.source, + notes: result.notes, path: relative, ...(result.build ? { build: result.build } : {}) }); + } catch (error) { + entry.refusal = reason(error); + moment.status = "refused"; + } + } + if (recordError && resolved.inputs.length === 0) throw new Error(recordError); + await writeFile(path.join(staging, "task-clock.json"), serialize(createTaskClock(resolved.at))); + await writeFile(path.join(staging, "moment.json"), serialize(moment)); + // Set only unstamped wrapper directories; per-input source times remain intact. + const wrappers = async (directory: string): Promise => { + const relative = path.relative(staging, directory).split(path.sep).join("/"); + if (moment.inputs.some((input) => input.path === relative)) return; + for (const entry of await readdir(directory, { withFileTypes: true })) { + if (entry.isDirectory()) await wrappers(path.join(directory, entry.name)); + else if (directory === staging) await stampTree(path.join(directory, entry.name), resolved.at, resolved.at); + } + await utimes(directory, new Date(resolved.at), new Date(resolved.at)); + }; + await wrappers(staging); + return await finalizeTrainingMoment(staging, out, { historyDir: options.historyDir }); + } finally { await rm(staging, { recursive: true, force: true }); } +}; diff --git a/src/compiler/training/rewind/memory.test.ts b/src/compiler/training/rewind/memory.test.ts new file mode 100644 index 00000000..e9ffdb1d --- /dev/null +++ b/src/compiler/training/rewind/memory.test.ts @@ -0,0 +1,77 @@ +import { mkdtemp, readFile, readdir, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { openHistory } from "./history.js"; +import { materializeMemory, parseMemoryLedger, selectMemoryGeneration, selectMemoryPrefix } from "./memory.js"; +import { AT, BEFORE, FUTURE, emptyRecord, jsonl, memoryEvent } from "./fixtures.test-helper.js"; +import type { InputContext, RewindInput } from "./types.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const setup = async (): Promise => { + const root = await mkdtemp(path.join(os.tmpdir(), "rewind-memory-")); roots.push(root); + const input: RewindInput = { section: "memory", id: "self", declaredBy: "agent:a", declarationBase: root, + declaration: { id: "self", store: { kind: "memory" } }, overridden: false }; + return { input, at: AT, record: null, history: await openHistory(root), destination: path.join(root, "out") }; +}; +const base = "memory/agent%3Aa/self"; +const ledger = (...events: unknown[]) => parseMemoryLedger(events.map((e) => JSON.stringify(e)).join("\n") + "\n"); + +describe("memory rewind", () => { + it("cuts at the first future event and never filters a clock regression back into the bank", () => { + const source = ledger(memoryEvent(1), memoryEvent(2, FUTURE), memoryEvent(3)); + expect(selectMemoryPrefix(source, AT).events.map((event) => event.seq)).toEqual([1]); + expect(() => selectMemoryPrefix(source, AT, 3)).toThrow(/later than/); + }); + it("refuses seq gaps, prefix time regressions, invalid framing, strict unknown fields and resumed legacy runs", () => { + expect(() => selectMemoryPrefix(ledger(memoryEvent(2)), AT)).toThrow(/contiguous/); + expect(() => selectMemoryPrefix(ledger(memoryEvent(1, AT), memoryEvent(2)), AT)).toThrow(/regresses/); + expect(() => parseMemoryLedger(JSON.stringify(memoryEvent(1)))).toThrow(/LF/); + expect(() => ledger({ ...memoryEvent(1), extra: true })).toThrow(/invalid Mneme event/); + expect(() => selectMemoryPrefix(ledger(memoryEvent(1), { ...memoryEvent(2), seq: undefined }), AT)).toThrow(/leading run/); + expect(selectMemoryPrefix(ledger({ ...memoryEvent(1), seq: undefined }, memoryEvent(2)), AT).events).toHaveLength(2); + }); + it("materializes an exact record prefix using only the public Mneme rebuild API", async () => { + const context = await setup(); + await jsonl(context.history!.root, `${base}/events.jsonl`, [memoryEvent(1), memoryEvent(2, AT)]); + context.record = { ...emptyRecord(), memory: [{ id: "self", declaredBy: "agent:a", seq: 1, generation: 0 }] }; + const result = await materializeMemory(context); + expect(result).toMatchObject({ fidelity: "exact", basis: "record", source: { seq: 1, generation: 0 } }); + expect(await readdir(path.join(context.destination, "memory"))).toEqual(["events.jsonl"]); + expect(await readFile(path.join(context.destination, "memory/events.jsonl"), "utf8")).toBe(`${JSON.stringify(memoryEvent(1))}\n`); + expect(result.notes.join(" ")).toContain("public index rebuild"); + }); + it("includes the boundary and labels the timestamp prefix approximate", async () => { + const context = await setup(); + await jsonl(context.history!.root, `${base}/events.jsonl`, [memoryEvent(1, AT), memoryEvent(2, FUTURE)]); + expect(await materializeMemory(context)).toMatchObject({ fidelity: "approximate", basis: "created-at-prefix", source: { seq: 1 } }); + }); + it("selects an archived generation before a switch and refuses a missing archive", async () => { + const context = await setup(); + await jsonl(context.history!.root, `${base}/generations.jsonl`, [ + { generation: 3, switchedAt: BEFORE, ledger: "archive/g3.jsonl" }, + { generation: 4, switchedAt: FUTURE, ledger: "events.jsonl" } + ]); + await jsonl(context.history!.root, `${base}/events.jsonl`, [memoryEvent(1, FUTURE)]); + await expect(materializeMemory(context)).rejects.toThrow(/Missing memory ledger.*g3/); + await jsonl(context.history!.root, `${base}/archive/g3.jsonl`, [memoryEvent(1)]); + expect(await materializeMemory(context)).toMatchObject({ source: { generation: 3, seq: 1 } }); + expect(() => selectMemoryGeneration([{ generation: 4, switchedAt: FUTURE, ledger: "events.jsonl" }], AT)).toThrow(/archived ledger/); + expect(() => selectMemoryGeneration(null, AT, 3)).toThrow(/disagrees/); + }); + it("creates an empty override without history and refuses a missing non-overridden bank", async () => { + const context = await setup(); + await expect(materializeMemory(context)).rejects.toThrow(/Missing memory ledger/); + context.history = undefined; + await expect(materializeMemory(context)).rejects.toThrow(/requires --history/); + context.input.overridden = true; + expect(await materializeMemory(context)).toMatchObject({ fidelity: "overridden", source: { seq: 0 } }); + expect(await readFile(path.join(context.destination, "memory/events.jsonl"), "utf8")).toBe(""); + }); + it("validates malformed suffix history even when the selected prefix would be empty", async () => { + const context = await setup(); + await jsonl(context.history!.root, `${base}/events.jsonl`, [memoryEvent(1, FUTURE), { bad: true }]); + await expect(materializeMemory(context)).rejects.toThrow(/events.jsonl:2/); + }); +}); diff --git a/src/compiler/training/rewind/memory.ts b/src/compiler/training/rewind/memory.ts new file mode 100644 index 00000000..4d3ec77a --- /dev/null +++ b/src/compiler/training/rewind/memory.ts @@ -0,0 +1,122 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { historyKey } from "./history.js"; +import { compareInstants } from "./instant.js"; +import { memoryEventSchema, memoryGenerationSchema, type MemoryGeneration, type MemoryLedgerEvent } from "./memorySchemas.js"; +import type { InputContext, InputResult } from "./types.js"; + +export interface MemoryLedger { lines: string[]; events: MemoryLedgerEvent[] } +const reason = (error: unknown): string => error instanceof Error ? error.message : String(error); + +export const parseMemoryLedger = (payload: string, label = "memory/events.jsonl"): MemoryLedger => { + if (payload === "") return { lines: [], events: [] }; + if (!payload.endsWith("\n")) throw new Error(`${label}: ledger must end with LF`); + const lines = payload.slice(0, -1).split("\n"); + const events = lines.map((line, index) => { + try { return memoryEventSchema.parse(JSON.parse(line)); } + catch (error) { throw new Error(`${label}:${index + 1}: invalid Mneme event: ${reason(error)}`); } + }); + return { lines, events }; +}; + +/** The first future event is a boundary; a clock regression after it cannot reopen the cut. */ +export const selectMemoryPrefix = (ledger: MemoryLedger, at: string, seq?: number): MemoryLedger => { + if (seq !== undefined && (!Number.isSafeInteger(seq) || seq < 0 || seq > ledger.events.length)) { + throw new Error(`Memory record seq ${seq} exceeds or does not identify a ledger prefix`); + } + const firstFuture = ledger.events.findIndex((event) => compareInstants(event.createdAt, at) > 0); + const length = seq ?? (firstFuture < 0 ? ledger.events.length : firstFuture); + const events = ledger.events.slice(0, length); + let sawNative = false; + events.forEach((event, index) => { + if (event.seq !== undefined) { + if (event.seq !== index + 1) throw new Error(`Memory ledger line ${index + 1}: seq ${event.seq} is not contiguous`); + sawNative = true; + } else if (sawNative) throw new Error(`Memory ledger line ${index + 1}: seq-less legacy events must form a leading run`); + if (compareInstants(event.createdAt, at) > 0) throw new Error(`Memory ledger line ${index + 1}: createdAt is later than the moment`); + if (index > 0 && compareInstants(event.createdAt, events[index - 1]!.createdAt) < 0) { + throw new Error(`Memory ledger line ${index + 1}: createdAt regresses within the selected prefix`); + } + }); + return { events, lines: ledger.lines.slice(0, length) }; +}; + +export const selectMemoryGeneration = ( + generations: MemoryGeneration[] | null, at: string, recordGeneration?: number +): { generation: number; ledger: string; switchedAt: string | null } => { + // With no curation log, only the original generation can be established. + let selected: { generation: number; ledger: string; switchedAt: string | null } = { + generation: 0, ledger: "events.jsonl", switchedAt: null + }; + if (generations !== null) { + const known = new Map(); + for (const generation of generations) { + const prior = known.get(generation.generation); + if (prior && JSON.stringify(prior) !== JSON.stringify(generation)) { + throw new Error(`Conflicting memory generation ${generation.generation}`); + } + known.set(generation.generation, generation); + } + const eligible = [...known.values()].filter((entry) => compareInstants(entry.switchedAt, at) <= 0) + .sort((a, b) => compareInstants(b.switchedAt, a.switchedAt)); + if (!eligible.length) throw new Error("No memory generation switched at or before the moment; an archived ledger is required"); + if (eligible[1] && compareInstants(eligible[0]!.switchedAt, eligible[1].switchedAt) === 0) { + throw new Error("Ambiguous memory generations switched at the same instant"); + } + selected = eligible[0]!; + } + if (recordGeneration !== undefined && recordGeneration !== selected.generation) { + throw new Error(`Memory record generation ${recordGeneration} disagrees with selected generation ${selected.generation}`); + } + return selected; +}; + +/** Public rebuild validates the cut without shipping copied or derived index/audit bytes. */ +const validateWithMneme = async (home: string): Promise => { + let mneme: typeof import("@noopolis/mneme"); + try { mneme = await import("@noopolis/mneme"); } + catch (error) { throw new Error(`Mneme public index rebuild is unavailable; install the optional @noopolis/mneme package: ${reason(error)}`); } + if (typeof mneme.createMemoryIndex !== "function") throw new Error("Mneme root does not export createMemoryIndex; cannot validate the rewind index"); + const temporary = await mkdtemp(path.join(os.tmpdir(), "spawnfile-memory-index-")); + try { + const events = await new mneme.JsonlMemoryStore(home).read(); + const index = mneme.createMemoryIndex({ runtimeHomePath: home, indexFilePath: path.join(temporary, "index.sqlite") }); + try { + await index.rebuildFromEvents(events); + const indexed = await index.query({ limit: Math.max(events.length, 1) }); + // query() is a positive completion signal, including the empty-ledger case. + if (!Array.isArray(indexed)) throw new Error("Mneme rebuild did not produce a readable index"); + } finally { index.close(); } + } finally { await rm(temporary, { recursive: true, force: true }); } +}; + +export const materializeMemory = async (context: InputContext): Promise => { + const { input, at, record, history, destination } = context; + if (input.section !== "memory") throw new Error("Memory materialization requires a memory input"); + const directory = path.join(destination, "memory"); + if (input.overridden) { + await mkdir(directory, { recursive: true }); + await writeFile(path.join(directory, "events.jsonl"), ""); + return { fidelity: "overridden", basis: "override", source: { seq: 0, generation: 0 }, notes: [], instant: at }; + } + if (!history) throw new Error(`Memory bank ${input.id} requires --history or an override`); + const base = `memory/${historyKey(input)}`; + const pin = record?.memory.find((entry) => entry.id === input.id && entry.declaredBy === input.declaredBy); + const generation = selectMemoryGeneration(await history.jsonl(`${base}/generations.jsonl`, memoryGenerationSchema), at, pin?.generation); + const ledgerPath = `${base}/${generation.ledger}`; + const payload = await history.text(ledgerPath); + if (payload === null) throw new Error(`Missing memory ledger ${ledgerPath}; an archived ledger is required for this generation`); + const prefix = selectMemoryPrefix(parseMemoryLedger(payload, ledgerPath), at, pin?.seq); + await mkdir(directory, { recursive: true }); + await writeFile(path.join(directory, "events.jsonl"), prefix.lines.length ? `${prefix.lines.join("\n")}\n` : ""); + await validateWithMneme(destination); + return { + fidelity: pin ? "exact" : "approximate", basis: pin ? "record" : "created-at-prefix", + source: { seq: prefix.events.length, generation: generation.generation }, + notes: ["The selected ledger was validated by Mneme's public index rebuild. The temporary index was discarded; no derived or authority audit state is included.", + "Mneme 0.1.1 rebuilds its index before kernel searches, not merely when a runtime opens.", + ...(generation.switchedAt === null ? ["No generation history: the current ledger is treated as original generation 0."] : [])], + instant: prefix.events.at(-1)?.createdAt ?? generation.switchedAt ?? at + }; +}; diff --git a/src/compiler/training/rewind/memorySchemas.ts b/src/compiler/training/rewind/memorySchemas.ts new file mode 100644 index 00000000..5bb899a2 --- /dev/null +++ b/src/compiler/training/rewind/memorySchemas.ts @@ -0,0 +1,37 @@ +import { z } from "zod"; +import { trainingInstantSchema } from "../../../manifest/index.js"; + +const safeSequence = z.number().int().nonnegative().refine(Number.isSafeInteger); +const relativeLedger = z.string().min(1).refine((value) => !value.includes("\\") + && !value.includes("\0") && value.split("/").every((part) => part !== "" && part !== "." && part !== ".."), +"ledger must be a relative path inside its bank directory"); +export const memoryGenerationSchema = z.object({ + generation: safeSequence, switchedAt: trainingInstantSchema, ledger: relativeLedger +}).strict(); +export type MemoryGeneration = z.infer; + +/** Mirrors the public MemoryEvent contract and Mneme 0.1.1's strict ledger reader. */ +export const memoryEventSchema = z.object({ + id: z.string().min(1), + type: z.enum(["memory.observed", "memory.claimed", "memory.registered", "memory.summarized", + "memory.recalled", "memory.located", "memory.denied", "memory.forgotten", "memory.promoted", "memory.consolidated"]), + createdAt: trainingInstantSchema, + principal: z.object({ agentId: z.string().min(1), + scope: z.enum(["global", "team", "room", "pair", "task", "role", "artifact"]), qualifier: z.string().optional() + }).strict(), + scope: z.string(), visibility: z.enum(["private", "pair", "team", "room", "global", "public", "sealed"]), + source: z.string(), + content: z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("text"), text: z.string() }).strict(), + z.object({ kind: z.literal("claim"), subject: z.string(), predicate: z.string(), object: z.string() }).strict(), + z.object({ kind: z.literal("decision"), decision: z.string(), rationale: z.string().optional() }).strict(), + z.object({ kind: z.literal("artifact"), description: z.string(), path: z.string().optional(), uri: z.string().optional() }).strict(), + z.object({ kind: z.literal("relationship"), from: z.string(), relation: z.string(), to: z.string() }).strict() + ]), + tags: z.array(z.string()), entities: z.array(z.string()), + sensitivity: z.enum(["normal", "sensitive", "secret"]), confidence: z.number().finite().optional(), + ttl: z.string().optional(), parentEventIds: z.array(z.string()), checksum: z.string().min(1), + seq: safeSequence.refine((value) => value > 0).optional(), memoryId: z.string().optional(), + origin: z.enum(["awake", "dream", "system"]).optional(), highWaterSeq: safeSequence.optional() +}).strict(); +export type MemoryLedgerEvent = z.infer; diff --git a/src/compiler/training/rewind/moltnet.ts b/src/compiler/training/rewind/moltnet.ts new file mode 100644 index 00000000..bc0da50e --- /dev/null +++ b/src/compiler/training/rewind/moltnet.ts @@ -0,0 +1,44 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { historyKey } from "./history.js"; +import { emptyMoltnetSnapshot, moltnetCursorExact, moltnetMessages, parseMoltnetSnapshot, rewindMoltnetJson } from "./moltnetJson.js"; +import { rewindMoltnetSqlite } from "./moltnetSqlite.js"; +import type { InputContext, InputResult } from "./types.js"; + +export const materializeMoltnet = async (context: InputContext): Promise => { + const { input, at, record, history, destination } = context; + if (input.section !== "networks") throw new Error("Moltnet materialization requires a network input"); + if (input.overridden) { + await mkdir(destination, { recursive: true }); + await writeFile(path.join(destination, "state.json"), `${JSON.stringify(emptyMoltnetSnapshot(input.declaration, at), null, 2)}\n`); + return { fidelity: "overridden", basis: "override", source: {}, notes: ["Fresh network state is stored as a JSON snapshot with the declared rooms."], instant: at }; + } + if (!history) throw new Error(`Moltnet network ${input.id} requires --history or an override`); + const base = `moltnet/${historyKey(input)}`; + const json = await history.bytes(`${base}/state.json`), sqlite = await history.bytes(`${base}/moltnet.sqlite`); + if (!json && !sqlite) throw new Error(`Missing Moltnet store ${base}/state.json or moltnet.sqlite`); + if (json && sqlite) throw new Error(`Ambiguous Moltnet history: both state.json and moltnet.sqlite exist for ${input.id}`); + await mkdir(destination, { recursive: true }); + let messages: Array<{ id: string; created_at: string }>; + if (json) { + const cut = rewindMoltnetJson(parseMoltnetSnapshot(JSON.parse(json.toString("utf8"))), at); + messages = moltnetMessages(cut); + await writeFile(path.join(destination, "state.json"), `${JSON.stringify(cut, null, 2)}\n`); + } else { + const file = path.join(destination, "moltnet.sqlite"); + await writeFile(file, sqlite!); + // History readers never open the source database: SQLite recovery is confined to these copies. + for (const suffix of ["-wal", "-journal"]) { + const bytes = await history.bytes(`${base}/moltnet.sqlite${suffix}`); + if (bytes) await writeFile(`${file}${suffix}`, bytes); + } + messages = await rewindMoltnetSqlite(file, at); + } + const cursor = record?.moltnet.find((entry) => entry.network === input.id)?.cursor; + const exact = moltnetCursorExact(messages, cursor); + return { fidelity: exact ? "exact" : "approximate", basis: exact ? "record-cursor" : "created-at-cut", + source: cursor === undefined ? {} : { cursor }, instant: at, + notes: ["Agent registrations and memberships have no historical log and are kept as now (best effort).", + "Room removals without a recorded removal instant retain the current removal state; current JSON Moltnet uses a zero-time marker.", + ...(cursor !== undefined && !exact ? ["The recorded cursor is absent after the cut or is earlier than a retained message; exact cursor fidelity cannot be established."] : [])] }; +}; diff --git a/src/compiler/training/rewind/moltnetJson.test.ts b/src/compiler/training/rewind/moltnetJson.test.ts new file mode 100644 index 00000000..a37004d7 --- /dev/null +++ b/src/compiler/training/rewind/moltnetJson.test.ts @@ -0,0 +1,95 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { afterEach, describe, expect, it } from "vitest"; +import { materializeMoltnet } from "./moltnet.js"; +import { momentRecordSchema } from "./contract.js"; +import { openHistory } from "./history.js"; +import { emptyMoltnetSnapshot, inspectMoltnetJson, moltnetCursorExact, moltnetMessages, parseMoltnetSnapshot, rewindMoltnetJson } from "./moltnetJson.js"; +import type { MoltnetMessage } from "./moltnetSchemas.js"; +import type { InputContext } from "./types.js"; + +const at = "2026-10-09T10:00:00.1Z", before = "2026-10-09T10:00:00Z", future = "2026-10-09T10:00:00.11Z"; +const network = { id: "news", provider: "moltnet" as const, rooms: [{ id: "assignment", members: ["writer"] }] }; +const message = (id: string, created_at: string, target: MoltnetMessage["target"] = { kind: "room", room_id: "assignment" }): MoltnetMessage => + ({ id, network_id: "news", from: { type: "agent", id: "writer" }, target, parts: [{ kind: "text", text: id }], created_at }); +const temps: string[] = []; +afterEach(async () => { await Promise.all(temps.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); }); + +describe("Moltnet JSON rewind", () => { + it("cuts RFC3339Nano times chronologically, removes future topology, recomputes counters and remaps delivery cursors", () => { + const source = emptyMoltnetSnapshot(network, before); + source.room_messages.assignment = [message("old", before), message("boundary", at), message("future", future)]; + source.rooms.later = { id: "later", name: "later", created_at: future }; + source.room_messages.later = [message("regressed-room", before, { kind: "room", room_id: "later" })]; + for (const [id, parent] of [["kept", "old"], ["dropped", "future"]]) { + source.threads[id] = { id, room_id: "assignment", parent_message_id: parent, message_count: 99, last_message_at: future }; + source.thread_messages[id] = [message(`${id}-old`, before, { kind: "thread", room_id: "assignment", thread_id: id }), + message(`${id}-future`, future, { kind: "thread", room_id: "assignment", thread_id: id })]; + } + source.direct_messages.dm = [message("dm-root-future", future, { kind: "dm", dm_id: "dm" }), + message("dm-regression", before, { kind: "dm", dm_id: "dm" })]; + source.direct_members.dm = { writer: {}, brass: {} }; + source.attachment_delivery_events = [ + { id: "e1", type: "message.created", network_id: "news", created_at: before, message: message("old", before) }, + { id: "e2", type: "message.created", network_id: "news", created_at: future, message: message("future", future) }, + { id: "e3", type: "message.created", network_id: "news", created_at: at, message: message("boundary", at) } + ]; + source.attachment_delivery_cursors = { writer: 3, brass: 2 }; + const cut = rewindMoltnetJson(source, at); + expect(Object.keys(cut.rooms)).toEqual(["assignment"]); + expect(cut.threads.kept).toMatchObject({ message_count: 1, last_message_at: before }); + expect(cut.threads.dropped).toBeUndefined(); expect(cut.direct_messages).toEqual({}); expect(cut.direct_members).toEqual({}); + expect(cut.attachment_delivery_events.map((event) => event.id)).toEqual(["e1", "e3"]); + expect(cut.attachment_delivery_cursors).toEqual({ writer: 2, brass: 1 }); + expect(moltnetMessages(cut).map((entry) => entry.id)).toEqual(["old", "boundary", "kept-old"]); + expect(source.threads.kept?.message_count).toBe(99); + expect(moltnetCursorExact(moltnetMessages(cut), "boundary")).toBe(true); + expect(moltnetCursorExact(moltnetMessages(cut), "old")).toBe(false); + expect(moltnetCursorExact(moltnetMessages(cut), "missing")).toBe(false); + }); + + it("restores known later removals, keeps zero-time unknown removals and refuses missing restoration metadata", () => { + const source = emptyMoltnetSnapshot(network, before); + source.removed_rooms.assignment = future; + const room = source.rooms.assignment!; delete source.rooms.assignment; + expect(() => rewindMoltnetJson(source, at)).toThrow("room metadata was not retained"); + source.attachment_delivery_events = [{ id: "room-created", type: "room.created", network_id: "news", created_at: before, room }]; + expect(rewindMoltnetJson(source, at).rooms.assignment).toEqual(room); + source.removed_rooms.assignment = "0001-01-01T00:00:00Z"; + expect(rewindMoltnetJson(source, at).removed_rooms.assignment).toBe("0001-01-01T00:00:00Z"); + }); + + it("materializes exact cursor and approximate cuts, keeps history unchanged and makes fresh overridden rooms", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "rewind-moltnet-")); temps.push(root); + const bank = path.join(root, "moltnet/team%3Aorg/news"); await mkdir(bank, { recursive: true }); + const source = emptyMoltnetSnapshot(network, before); source.room_messages.assignment = [message("cursor", at), message("future", future)]; + const bytes = JSON.stringify(source); await writeFile(path.join(bank, "state.json"), bytes); + const context: InputContext = { at, record: null, history: await openHistory(root), destination: path.join(root, "out"), + input: { section: "networks", id: "news", declaredBy: "team:org", overridden: false, declarationBase: root, declaration: network } }; + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "approximate", basis: "created-at-cut" }); + expect(await readFile(path.join(bank, "state.json"), "utf8")).toBe(bytes); + expect(await inspectMoltnetJson(path.join(context.destination, "state.json"), at)).toMatchObject({ messages: 1, findings: [] }); + context.record = momentRecordSchema.parse({ version: "spawnfile.moment-record.v1", agent: "agent:writer", + wake: { at, surface: "moltnet", room: "assignment", from: "brass" }, recordedAt: at, + resources: [], feeds: [], memory: [], moltnet: [{ network: "news", cursor: "cursor" }], volumes: [] }); + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "exact", basis: "record-cursor" }); + context.record.moltnet[0]!.cursor = "missing"; + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "approximate" }); + context.input.overridden = true; context.history = undefined; + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "overridden" }); + const output = parseMoltnetSnapshot(JSON.parse(await readFile(path.join(context.destination, "state.json"), "utf8"))); + expect(output.rooms.assignment?.members).toEqual(["writer"]); expect(moltnetMessages(output)).toEqual([]); + }); + + it("independently detects mutated messages, events and rooms on disk", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "rewind-moltnet-leak-")); temps.push(root); + const file = path.join(root, "state.json"), snapshot = emptyMoltnetSnapshot(network, before); + await writeFile(file, JSON.stringify(snapshot)); expect((await inspectMoltnetJson(file, at)).findings).toEqual([]); + snapshot.room_messages.assignment = [message("future", future)]; + snapshot.rooms.future = { id: "future", name: "future", created_at: future }; + snapshot.attachment_delivery_events = [{ id: "future", type: "message.created", network_id: "news", created_at: future }]; + await writeFile(file, JSON.stringify(snapshot)); + expect((await inspectMoltnetJson(file, at)).findings.map((finding) => finding.class)).toEqual(["moltnet-message", "moltnet-room", "moltnet-event"]); + }); +}); diff --git a/src/compiler/training/rewind/moltnetJson.ts b/src/compiler/training/rewind/moltnetJson.ts new file mode 100644 index 00000000..e2309e1d --- /dev/null +++ b/src/compiler/training/rewind/moltnetJson.ts @@ -0,0 +1,158 @@ +import { readFile } from "node:fs/promises"; +import type { TeamNetwork } from "../../../manifest/index.js"; +import { compareInstants } from "./instant.js"; +import { moltnetSnapshotSchema, type MoltnetSnapshot, type MoltnetEvent, + type MoltnetMessage, type MoltnetInspection, type MoltnetFinding } from "./moltnetSchemas.js"; + +export const moltnetMessages = (snapshot: MoltnetSnapshot): MoltnetMessage[] => + [snapshot.room_messages, snapshot.thread_messages, snapshot.direct_messages] + .flatMap((group) => Object.values(group).flat()); + +export const parseMoltnetSnapshot = (value: unknown): MoltnetSnapshot => { + const snapshot = moltnetSnapshotSchema.parse(value); + const ids = new Set(); + for (const message of moltnetMessages(snapshot)) { + if (ids.has(message.id)) throw new Error(`Moltnet snapshot has duplicate message ${message.id}`); + ids.add(message.id); + } + for (const group of [snapshot.rooms, snapshot.threads]) { + for (const [id, value] of Object.entries(group)) { + if (id !== value.id) throw new Error(`Moltnet snapshot key ${id} disagrees with id ${value.id}`); + } + } + return snapshot; +}; + +export const emptyMoltnetSnapshot = (network: TeamNetwork, at: string): MoltnetSnapshot => + parseMoltnetSnapshot({ + rooms: Object.fromEntries(network.rooms.map((room) => [room.id, { + ...room, name: room.name ?? room.id, network_id: network.id, + fqid: `molt://${network.id}/rooms/${room.id}`, created_at: at + }])), removed_rooms: {}, room_messages: {}, threads: {}, thread_messages: {}, + direct_messages: {}, direct_members: {}, agents: {}, attachment_delivery_events: [], + attachment_delivery_cursors: {} + }); + +export const latestMoltnetMessage = (messages: T[]): T | undefined => + messages.reduce((latest, item) => !latest + || compareInstants(item.created_at, latest.created_at) > 0 ? item : latest, undefined); + +const eventFuture = (event: MoltnetEvent, at: string): boolean => [event.created_at, + event.message?.created_at, event.room?.created_at, event.thread?.last_message_at, + event.dm?.last_message_at, event.pairing?.diagnostics?.checked_at] + .some((instant) => instant !== undefined && compareInstants(instant, at) > 0); + +/** Pure cut. Delivery offsets must be remapped after removing interior future events. */ +export const rewindMoltnetJson = (source: MoltnetSnapshot, at: string): MoltnetSnapshot => { + const snapshot = structuredClone(source); + const futureRooms = new Set(Object.values(source.rooms) + .filter((room) => compareInstants(room.created_at, at) > 0).map((room) => room.id)); + for (const id of futureRooms) { + delete snapshot.rooms[id]; delete snapshot.room_messages[id]; delete snapshot.removed_rooms[id]; + } + for (const [id, removedAt] of Object.entries(snapshot.removed_rooms)) { + if (compareInstants(removedAt, at) <= 0) continue; + if (!snapshot.rooms[id]) { + const candidates = source.attachment_delivery_events.filter((event) => event.room?.id === id + && compareInstants(event.room.created_at, at) <= 0) + .sort((a, b) => compareInstants(b.created_at, a.created_at)); + const room = candidates.find((event) => compareInstants(event.created_at, at) <= 0)?.room + ?? candidates[0]?.room; + if (!room) throw new Error(`Cannot restore Moltnet room ${id}: removal is after the moment but room metadata was not retained`); + snapshot.rooms[id] = structuredClone(room); + } + delete snapshot.removed_rooms[id]; + } + for (const group of [snapshot.room_messages, snapshot.thread_messages, snapshot.direct_messages]) { + for (const [id, messages] of Object.entries(group)) { + group[id] = messages.filter((message) => compareInstants(message.created_at, at) <= 0 + && !futureRooms.has(message.target.room_id ?? "")); + } + } + // A removed root can itself belong to a removed thread; resolve until no dangling root remains. + let changed = true; + while (changed) { + changed = false; + const retainedIds = new Set(moltnetMessages(snapshot).map((message) => message.id)); + for (const [id, thread] of Object.entries(snapshot.threads)) { + const messages = snapshot.thread_messages[id] ?? []; + const root = source.thread_messages[id]?.[0]; + if (!messages.length || futureRooms.has(thread.room_id) + || (thread.parent_message_id && !retainedIds.has(thread.parent_message_id)) + || (!thread.parent_message_id && root && compareInstants(root.created_at, at) > 0)) { + delete snapshot.threads[id]; delete snapshot.thread_messages[id]; changed = true; + } else { + thread.message_count = messages.length; + thread.last_message_at = latestMoltnetMessage(messages)!.created_at; + } + } + } + for (const id of Object.keys(snapshot.thread_messages)) { + if (!snapshot.threads[id]) delete snapshot.thread_messages[id]; + } + for (const [id, messages] of Object.entries(snapshot.direct_messages)) { + const root = source.direct_messages[id]?.[0]; + if (!messages.length || (root && compareInstants(root.created_at, at) > 0)) { + delete snapshot.direct_messages[id]; delete snapshot.direct_members[id]; + } + } + for (const id of Object.keys(snapshot.direct_members)) { + if (!snapshot.direct_messages[id]) delete snapshot.direct_members[id]; + } + const finalIds = new Set(moltnetMessages(snapshot).map((message) => message.id)); + const retainedEvents = source.attachment_delivery_events.map((event, index) => ({ event, index })) + .filter(({ event }) => !eventFuture(event, at) + && (!event.message || finalIds.has(event.message.id)) + && (!event.room || !futureRooms.has(event.room.id)) + && (!event.thread || Boolean(snapshot.threads[event.thread.id])) + && (!event.dm || Boolean(snapshot.direct_messages[event.dm.id]))); + snapshot.attachment_delivery_events = retainedEvents.map(({ event }) => event); + for (const [agent, cursor] of Object.entries(snapshot.attachment_delivery_cursors)) { + snapshot.attachment_delivery_cursors[agent] = retainedEvents.filter(({ index }) => index < cursor).length; + } + return snapshot; +}; + +/** UUID cursors have no ordering: exactness is established only from the retained message instant. */ +export const moltnetCursorExact = (messages: Array<{ id: string; created_at: string }>, cursor?: string): boolean => { + const message = cursor === undefined ? undefined : messages.find((entry) => entry.id === cursor); + return Boolean(message && messages.every((entry) => compareInstants(entry.created_at, message.created_at) <= 0)); +}; + +/** Independent timestamp inspection of event payloads, including embedded messages and rooms. */ +export const inspectMoltnetEvent = (event: MoltnetEvent, at: string, path: string): MoltnetFinding[] => { + const findings: MoltnetFinding[] = []; + const check = (instant: string | undefined, kind: MoltnetFinding["class"], detail: string) => { + if (instant !== undefined && compareInstants(instant, at) > 0) findings.push({ class: kind, path, detail }); + }; + check(event.created_at, "moltnet-event", `Attachment event ${event.id} is later than the moment`); + check(event.message?.created_at, "moltnet-message", `Event ${event.id} contains a future message`); + check(event.room?.created_at, "moltnet-room", `Event ${event.id} contains a future room`); + check(event.thread?.last_message_at, "moltnet-event", `Event ${event.id} contains a future thread counter`); + check(event.dm?.last_message_at, "moltnet-event", `Event ${event.id} contains a future DM counter`); + check(event.pairing?.diagnostics?.checked_at, "moltnet-event", `Event ${event.id} contains future pairing diagnostics`); + return findings; +}; + +export const inspectMoltnetJson = async (path: string, at: string): Promise => { + const snapshot = parseMoltnetSnapshot(JSON.parse(await readFile(path, "utf8"))); + const findings: MoltnetFinding[] = []; + const messages = moltnetMessages(snapshot); + for (const message of messages) if (compareInstants(message.created_at, at) > 0) { + findings.push({ class: "moltnet-message", path, detail: `Message ${message.id} is later than the moment` }); + } + for (const room of Object.values(snapshot.rooms)) if (compareInstants(room.created_at, at) > 0) { + findings.push({ class: "moltnet-room", path, detail: `Room ${room.id} is later than the moment` }); + } + for (const thread of Object.values(snapshot.threads)) { + if (thread.last_message_at && compareInstants(thread.last_message_at, at) > 0) { + findings.push({ class: "moltnet-message", path, detail: `Thread ${thread.id} has a future counter` }); + } + } + for (const [id, instant] of Object.entries(snapshot.removed_rooms)) { + if (compareInstants(instant, at) > 0) findings.push({ class: "moltnet-room", path, + detail: `Room ${id} has a future removal marker` }); + } + for (const event of snapshot.attachment_delivery_events) findings.push(...inspectMoltnetEvent(event, at, path)); + return { messages: messages.length, events: snapshot.attachment_delivery_events.length, findings }; +}; diff --git a/src/compiler/training/rewind/moltnetSchemas.ts b/src/compiler/training/rewind/moltnetSchemas.ts new file mode 100644 index 00000000..c83b74df --- /dev/null +++ b/src/compiler/training/rewind/moltnetSchemas.ts @@ -0,0 +1,78 @@ +import { z } from "zod"; +import { instantNanos } from "./instant.js"; + +const text = z.string(); +const id = text.min(1); +export const moltnetInstantSchema = text.refine((value) => { + try { instantNanos(value); return true; } catch { return false; } +}, "Expected an RFC3339 instant with at most nanosecond precision"); +const instant = moltnetInstantSchema; +const strings = z.array(text); +const actor = z.object({ type: id, id, name: text.optional(), network_id: text.optional(), + fqid: text.optional(), credential_bound: z.boolean().optional() }).strict(); +const target = z.object({ kind: z.enum(["room", "thread", "dm"]), room_id: text.optional(), + thread_id: text.optional(), parent_message_id: text.optional(), dm_id: text.optional(), + participant_ids: strings.optional() }).strict(); +const part = z.object({ kind: id, text: text.optional(), media_type: text.optional(), + url: text.optional(), filename: text.optional(), data: z.record(text, z.unknown()).optional() }).strict(); + +export const moltnetMessageSchema = z.object({ + id, network_id: text, origin: z.object({ network_id: text, message_id: text, + received_via: text.optional() }).strict().optional(), target, from: actor, + parts: z.array(part).nullable(), mentions: strings.nullable().optional(), created_at: instant +}).strict(); +export const moltnetRoomSchema = z.object({ + id, network_id: text.optional(), fqid: text.optional(), name: text, + members: strings.nullable().optional(), visibility: text.optional(), write_policy: text.optional(), + federation: z.union([z.enum(["all", "none"]), strings]).nullable().optional(), + access: z.object({ can_read: z.boolean(), can_write: z.boolean(), reason: text.optional() }).strict().optional(), + created_at: instant +}).strict(); +export const moltnetThreadSchema = z.object({ + id, network_id: text.optional(), fqid: text.optional(), room_id: id, + parent_message_id: text.optional(), message_count: z.number().int().nonnegative(), + last_message_at: instant.optional() +}).strict(); +const direct = z.object({ id, network_id: text.optional(), fqid: text.optional(), + participant_ids: strings.optional(), message_count: z.number().int().nonnegative(), + last_message_at: instant.optional() }).strict(); +const agentEvent = z.object({ agent_id: id, network_id: text.optional(), fqid: text.optional(), + name: text.optional(), message_id: text.optional(), reason: text.optional(), target: target.optional(), + error: text.optional(), attempts: z.number().int().optional(), classification: text.optional() }).strict(); +const pairing = z.object({ id, remote_network_id: text, remote_network_name: text.optional(), + remote_base_url: text.optional(), status: text.optional(), token: text.optional(), + relay: z.object({ url: text, room: text.optional(), token: text.optional() }).strict().optional(), + diagnostics: z.object({ checked_at: instant.optional(), remote_version: text.optional(), + remote_network_id: text.optional(), remote_protocols: z.object({ + http: strings.optional(), attach: strings.optional(), pair: strings.optional() + }).strict().optional(), reason: text.optional(), message: text.optional() }).strict().optional() +}).strict(); +export const moltnetEventSchema = z.object({ id, type: id, network_id: text, + agent: agentEvent.optional(), message: moltnetMessageSchema.optional(), room: moltnetRoomSchema.optional(), + thread: moltnetThreadSchema.optional(), dm: direct.optional(), pairing: pairing.optional(), + replay_gap: z.object({ requested_event_id: text.optional(), oldest_event_id: text.optional(), + newest_event_id: text.optional() }).strict().optional(), created_at: instant +}).strict(); +const registration = z.object({ network_id: text, agent_id: id, actor_uid: text, actor_uri: text, + display_name: text.optional(), credential_key: text.optional(), created_at: instant.optional(), + updated_at: instant.optional() }).strict(); +const map = (value: T) => z.record(text, value).nullable().transform((v) => v ?? {}); +const messages = map(z.array(moltnetMessageSchema).nullable().transform((v) => v ?? [])); +export const moltnetSnapshotSchema = z.object({ + rooms: map(moltnetRoomSchema), removed_rooms: map(instant).optional().default({}), + room_messages: messages, threads: map(moltnetThreadSchema), thread_messages: messages, + direct_messages: messages, direct_members: map(map(z.object({}).strict())), + agents: map(registration).optional().default({}), + attachment_delivery_events: z.array(moltnetEventSchema).nullable().optional().transform((v) => v ?? []), + attachment_delivery_cursors: map(z.number().int().nonnegative()).optional().default({}) +}).strict(); + +export type MoltnetSnapshot = z.infer; +export type MoltnetMessage = z.infer; +export type MoltnetEvent = z.infer; +export interface MoltnetFinding { + class: "moltnet-message" | "moltnet-event" | "moltnet-room"; + path: string; + detail: string; +} +export interface MoltnetInspection { messages: number; events: number; findings: MoltnetFinding[] } diff --git a/src/compiler/training/rewind/moltnetSqlite.test.ts b/src/compiler/training/rewind/moltnetSqlite.test.ts new file mode 100644 index 00000000..b57c70dd --- /dev/null +++ b/src/compiler/training/rewind/moltnetSqlite.test.ts @@ -0,0 +1,139 @@ +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { DatabaseSync } from "node:sqlite"; +import path from "node:path"; +import os from "node:os"; +import { afterEach, describe, expect, it } from "vitest"; +import { materializeMoltnet } from "./moltnet.js"; +import { momentRecordSchema } from "./contract.js"; +import { openHistory } from "./history.js"; +import { inspectMoltnetSqlite } from "./moltnetSqlite.js"; +import type { InputContext } from "./types.js"; + +const at = "2026-10-09T10:00:00.1Z", before = "2026-10-09T10:00:00Z", future = "2026-10-09T10:00:00.11Z"; +const temps: string[] = []; +afterEach(async () => { await Promise.all(temps.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); }); + +const schema = ` +CREATE TABLE rooms (id TEXT PRIMARY KEY, network_id TEXT, fqid TEXT, name TEXT, created_at TEXT, deleted_at TEXT, + visibility TEXT DEFAULT 'private', write_policy TEXT DEFAULT 'members', federation TEXT DEFAULT '"none"'); +CREATE TABLE room_members (room_id TEXT, member_id TEXT); +CREATE TABLE threads (id TEXT PRIMARY KEY, network_id TEXT, fqid TEXT, room_id TEXT, parent_message_id TEXT, + message_count INTEGER, last_message_at TEXT); +CREATE TABLE dm_conversations (dm_id TEXT PRIMARY KEY, network_id TEXT, fqid TEXT, message_count INTEGER, last_message_at TEXT); +CREATE TABLE dm_participants (dm_id TEXT, participant_id TEXT); +CREATE TABLE messages (id TEXT PRIMARY KEY, network_id TEXT, target_kind TEXT, room_id TEXT, thread_id TEXT, + parent_message_id TEXT, dm_id TEXT, target_json TEXT, from_json TEXT, parts_json TEXT, mentions_json TEXT, + created_at TEXT, origin_json TEXT DEFAULT '{}'); +CREATE TABLE artifacts (id TEXT PRIMARY KEY, message_id TEXT, created_at TEXT); +CREATE TABLE agents (agent_id TEXT PRIMARY KEY, network_id TEXT, actor_uid TEXT, actor_uri TEXT, + display_name TEXT, credential_key TEXT, created_at TEXT, updated_at TEXT); +CREATE TABLE attachment_delivery_events (seq INTEGER PRIMARY KEY AUTOINCREMENT, event_id TEXT UNIQUE, event_json TEXT, created_at TEXT); +CREATE TABLE attachment_delivery_cursors (agent_id TEXT PRIMARY KEY, event_seq INTEGER, updated_at TEXT);`; +const room = (database: DatabaseSync, id: string, time = before, deleted: string | null = null) => { + database.prepare("INSERT INTO rooms (id, network_id, fqid, name, created_at, deleted_at) VALUES (?, 'news', ?, ?, ?, ?)") + .run(id, `molt://news/rooms/${id}`, id, time, deleted); + database.prepare("INSERT INTO room_members VALUES (?, 'writer')").run(id); +}; +const message = (database: DatabaseSync, id: string, time: string, target: Record = { kind: "room", room_id: "assignment" }) => { + database.prepare(`INSERT INTO messages (id, network_id, target_kind, room_id, thread_id, parent_message_id, dm_id, + target_json, from_json, parts_json, mentions_json, created_at) VALUES (?, 'news', ?, ?, ?, ?, ?, ?, ?, ?, '[]', ?)`) + .run(id, target.kind, target.room_id ?? null, target.thread_id ?? null, target.parent_message_id ?? null, + target.dm_id ?? null, JSON.stringify(target), JSON.stringify({ type: "agent", id: "writer" }), + JSON.stringify([{ kind: "text", text: id.startsWith("future") ? "SECRET_FUTURE_PAYLOAD" : id }]), time); +}; +const thread = (database: DatabaseSync, id: string, root: string) => database.prepare("INSERT INTO threads VALUES (?, 'news', ?, 'assignment', ?, 99, ?)") + .run(id, `molt://news/threads/${id}`, root, future); +const dm = (database: DatabaseSync, id: string) => { + database.prepare("INSERT INTO dm_conversations VALUES (?, 'news', ?, 99, ?)").run(id, `molt://news/dms/${id}`, future); + database.prepare("INSERT INTO dm_participants VALUES (?, 'writer')").run(id); +}; +const event = (database: DatabaseSync, id: string, time: string) => database.prepare("INSERT INTO attachment_delivery_events (event_id, event_json, created_at) VALUES (?, ?, ?)") + .run(id, JSON.stringify({ id, type: "agent.connected", network_id: "news", created_at: time }), time); +const fixture = async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "rewind-moltnet-sqlite-")); temps.push(root); + const bank = path.join(root, "moltnet/team%3Aorg/news"); await mkdir(bank, { recursive: true }); + const file = path.join(bank, "moltnet.sqlite"), database = new DatabaseSync(file); database.exec(schema); + const context: InputContext = { at, record: null, history: await openHistory(root), destination: path.join(root, "out"), + input: { section: "networks", id: "news", declaredBy: "team:org", overridden: false, declarationBase: root, + declaration: { id: "news", provider: "moltnet", rooms: [{ id: "assignment", members: ["writer"] }] } } }; + return { root, bank, file, database, context }; +}; + +describe("Moltnet SQLite rewind", () => { + it("cuts a copy, preserves nanos ordering, prunes topology/artifacts, restores rooms, and erases deleted payload bytes", async () => { + const { file, database, context } = await fixture(); + room(database, "assignment"); room(database, "future-room", future); room(database, "restored", before, future); room(database, "removed", before, before); + message(database, "old", before); message(database, "boundary", at); message(database, "future-root", future); + message(database, "regressed-room", before, { kind: "room", room_id: "future-room" }); + thread(database, "kept", "old"); thread(database, "dropped", "future-root"); + message(database, "thread-old", before, { kind: "thread", room_id: "assignment", thread_id: "kept" }); + message(database, "future-thread", future, { kind: "thread", room_id: "assignment", thread_id: "kept" }); + message(database, "thread-regressed", before, { kind: "thread", room_id: "assignment", thread_id: "dropped" }); + dm(database, "dm-kept"); dm(database, "dm-dropped"); + message(database, "dm-old", before, { kind: "dm", dm_id: "dm-kept" }); + message(database, "future-dm", future, { kind: "dm", dm_id: "dm-kept" }); + message(database, "future-dm-root", future, { kind: "dm", dm_id: "dm-dropped" }); + message(database, "dm-regressed", before, { kind: "dm", dm_id: "dm-dropped" }); + database.prepare("INSERT INTO artifacts VALUES (?, ?, ?)").run("old-artifact", "old", before); + database.prepare("INSERT INTO artifacts VALUES (?, ?, ?)").run("future-artifact", "future-root", future); + for (const agent of ["writer", "brass"]) database.prepare("INSERT INTO agents VALUES (?, 'news', 'uid', 'uri', NULL, 'key', ?, ?)").run(agent, future, future); + event(database, "event-old", before); event(database, "event-future", future); event(database, "event-boundary", at); + database.prepare("INSERT INTO attachment_delivery_cursors VALUES (?, ?, ?)").run("writer", 2, future); + database.prepare("INSERT INTO attachment_delivery_cursors VALUES (?, ?, ?)").run("brass", 3, future); + database.close(); + const sourceBytes = await readFile(file); + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "approximate", basis: "created-at-cut" }); + expect(await readFile(file)).toEqual(sourceBytes); + const output = path.join(context.destination, "moltnet.sqlite"), result = new DatabaseSync(output, { readOnly: true }); + try { + expect(result.prepare("SELECT id FROM messages ORDER BY id").all().map((row) => row.id)).toEqual(["boundary", "dm-old", "old", "thread-old"]); + expect(result.prepare("SELECT id, message_count, last_message_at FROM threads").all()).toEqual([{ id: "kept", message_count: 1, last_message_at: before }]); + expect(result.prepare("SELECT dm_id, message_count, last_message_at FROM dm_conversations").all()).toEqual([{ dm_id: "dm-kept", message_count: 1, last_message_at: before }]); + expect(result.prepare("SELECT id, deleted_at FROM rooms WHERE id IN ('restored', 'removed') ORDER BY id").all()) + .toEqual([{ id: "removed", deleted_at: before }, { id: "restored", deleted_at: null }]); + expect(result.prepare("SELECT id FROM rooms WHERE id = 'future-room'").get()).toBeUndefined(); + expect(result.prepare("SELECT * FROM artifacts").all()).toEqual([{ id: "old-artifact", message_id: "old", created_at: before }]); + expect(result.prepare("SELECT agent_id, event_seq, updated_at FROM attachment_delivery_cursors ORDER BY agent_id").all()) + .toEqual([{ agent_id: "brass", event_seq: 3, updated_at: at }, { agent_id: "writer", event_seq: 1, updated_at: at }]); + expect(result.prepare("SELECT count(*) AS total FROM agents").get()?.total).toBe(2); + } finally { result.close(); } + expect((await readFile(output)).includes(Buffer.from("SECRET_FUTURE_PAYLOAD"))).toBe(false); + expect(await readdir(context.destination)).toEqual(["moltnet.sqlite"]); + expect(await inspectMoltnetSqlite(output, at)).toMatchObject({ messages: 4, events: 2, findings: [] }); + context.record = momentRecordSchema.parse({ version: "spawnfile.moment-record.v1", agent: "agent:writer", + wake: { at, surface: "moltnet", room: "assignment", from: "brass" }, recordedAt: at, + resources: [], feeds: [], memory: [], moltnet: [{ network: "news", cursor: "boundary" }], volumes: [] }); + context.destination += "-exact"; + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "exact", basis: "record-cursor" }); + }); + + it("copies WAL state and independently detects message, room and attachment-event mutations without changing the file", async () => { + const { database, context } = await fixture(); + database.exec("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0"); + room(database, "assignment"); message(database, "cursor", at); + try { expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "approximate" }); } + finally { database.close(); } + const output = path.join(context.destination, "moltnet.sqlite"); + expect(await inspectMoltnetSqlite(output, at)).toMatchObject({ messages: 1, findings: [] }); + const mutate = new DatabaseSync(output); + room(mutate, "future", future); message(mutate, "future-message", future); event(mutate, "future-event", future); mutate.close(); + const bytes = await readFile(output), findings = (await inspectMoltnetSqlite(output, at)).findings; + expect(new Set(findings.map((finding) => finding.class))).toEqual(new Set(["moltnet-message", "moltnet-room", "moltnet-event"])); + expect(await readFile(output)).toEqual(bytes); expect(await readdir(context.destination)).toEqual(["moltnet.sqlite"]); + await writeFile(`${output}-wal`, "uninspected bytes"); + expect((await inspectMoltnetSqlite(output, at)).findings).toMatchObject([{ class: "moltnet-event", detail: expect.stringContaining("sidecar") }]); + expect(await readFile(`${output}-wal`, "utf8")).toBe("uninspected bytes"); + }); + + it("refuses unknown persistent state rather than silently retaining it", async () => { + const { database, context } = await fixture(); + database.exec("CREATE TABLE unknown_future_state (payload TEXT)"); database.close(); + await expect(materializeMoltnet(context)).rejects.toThrow("Unsupported Moltnet SQLite table unknown_future_state"); + }); + + it("refuses unrecognized table columns rather than silently retaining added state", async () => { + const { database, context } = await fixture(); + database.exec("ALTER TABLE messages ADD COLUMN later_edits TEXT"); database.close(); + await expect(materializeMoltnet(context)).rejects.toThrow("Unsupported Moltnet SQLite column messages.later_edits"); + }); +}); diff --git a/src/compiler/training/rewind/moltnetSqlite.ts b/src/compiler/training/rewind/moltnetSqlite.ts new file mode 100644 index 00000000..d025b4ad --- /dev/null +++ b/src/compiler/training/rewind/moltnetSqlite.ts @@ -0,0 +1,116 @@ +import { lstat, rm } from "node:fs/promises"; +import type { DatabaseSync } from "node:sqlite"; +import { compareInstants } from "./instant.js"; +import { inspectMoltnetEvent, latestMoltnetMessage, moltnetMessages, rewindMoltnetJson } from "./moltnetJson.js"; +import { readMoltnetSqlite, sqliteInteger, sqliteText, sqliteJson } from "./moltnetSqliteData.js"; +import { moltnetEventSchema, type MoltnetFinding, type MoltnetInspection, type MoltnetMessage } from "./moltnetSchemas.js"; + +const openSqlite = async (file: string, readOnly = false): Promise => { + let sqlite: typeof import("node:sqlite"); + try { sqlite = await import("node:sqlite"); } + catch { throw new Error("Moltnet SQLite rewind requires node:sqlite (supported Node >=22.19); this Node runtime does not expose it"); } + return new sqlite.DatabaseSync(file, { readOnly, enableForeignKeyConstraints: false, allowExtension: false }); +}; + +/** Mutates only a caller-created COPY. secure_delete and VACUUM remove future content from free pages. */ +export const rewindMoltnetSqlite = async (file: string, at: string): Promise => { + const database = await openSqlite(file); + try { + const source = readMoltnetSqlite(database), cut = rewindMoltnetJson(source, at); + const messages = moltnetMessages(cut), keepMessages = new Set(messages.map((message) => message.id)); + const eventRows = database.prepare("SELECT * FROM attachment_delivery_events ORDER BY seq").all(); + const keepEvents = new Set(cut.attachment_delivery_events.map((event) => event.id)); + // The database column and the embedded event each carry an instant; both must pass the cut. + const remainingEvents = eventRows.filter((row) => keepEvents.has(sqliteText(row, "event_id")) + && compareInstants(sqliteText(row, "created_at"), at) <= 0); + const remainingSeq = new Set(remainingEvents.map((row) => sqliteInteger(row, "seq"))); + database.exec("PRAGMA secure_delete=ON; PRAGMA foreign_keys=OFF; BEGIN IMMEDIATE"); + try { + const dropMessage = database.prepare("DELETE FROM messages WHERE id = ?"); + for (const message of moltnetMessages(source)) if (!keepMessages.has(message.id)) dropMessage.run(message.id); + const dropRoom = database.prepare("DELETE FROM rooms WHERE id = ?"), restoreRoom = database.prepare("UPDATE rooms SET deleted_at = NULL WHERE id = ?"); + for (const id of Object.keys(source.rooms)) { + if (!cut.rooms[id]) dropRoom.run(id); + else if (source.removed_rooms[id] && !cut.removed_rooms[id]) restoreRoom.run(id); + } + database.exec("DELETE FROM room_members WHERE room_id NOT IN (SELECT id FROM rooms)"); + const dropThread = database.prepare("DELETE FROM threads WHERE id = ?"); + const updateThread = database.prepare("UPDATE threads SET message_count = ?, last_message_at = ? WHERE id = ?"); + for (const id of Object.keys(source.threads)) { + const thread = cut.threads[id]; + if (!thread) dropThread.run(id); else updateThread.run(thread.message_count, thread.last_message_at!, id); + } + const dropDm = database.prepare("DELETE FROM dm_conversations WHERE dm_id = ?"); + const updateDm = database.prepare("UPDATE dm_conversations SET message_count = ?, last_message_at = ? WHERE dm_id = ?"); + for (const row of database.prepare("SELECT dm_id FROM dm_conversations").all()) { + const id = sqliteText(row, "dm_id"), retained = cut.direct_messages[id] ?? []; + if (!retained.length) dropDm.run(id); else updateDm.run(retained.length, latestMoltnetMessage(retained)!.created_at, id); + } + database.exec("DELETE FROM dm_participants WHERE dm_id NOT IN (SELECT dm_id FROM dm_conversations)"); + database.exec("DELETE FROM artifacts WHERE message_id NOT IN (SELECT id FROM messages)"); + const dropArtifact = database.prepare("DELETE FROM artifacts WHERE id = ?"); + for (const row of database.prepare("SELECT id, created_at FROM artifacts").all()) { + if (compareInstants(sqliteText(row, "created_at"), at) > 0) dropArtifact.run(sqliteText(row, "id")); + } + const dropEvent = database.prepare("DELETE FROM attachment_delivery_events WHERE seq = ?"); + for (const row of eventRows) if (!remainingSeq.has(sqliteInteger(row, "seq"))) dropEvent.run(sqliteInteger(row, "seq")); + const updateCursor = database.prepare("UPDATE attachment_delivery_cursors SET event_seq = ?, updated_at = ? WHERE agent_id = ?"); + for (const row of database.prepare("SELECT * FROM attachment_delivery_cursors").all()) { + const cursor = sqliteInteger(row, "event_seq"); + const clamped = remainingEvents.reduce((value, event) => { + const seq = sqliteInteger(event, "seq"); return seq <= cursor ? Math.max(value, seq) : value; + }, 0); + const updated = sqliteText(row, "updated_at"); + updateCursor.run(clamped, compareInstants(updated, at) > 0 ? at : updated, sqliteText(row, "agent_id")); + } + const hasSequence = database.prepare("SELECT name FROM sqlite_schema WHERE name = 'sqlite_sequence'").get(); + if (hasSequence) database.prepare("UPDATE sqlite_sequence SET seq = ? WHERE name = 'attachment_delivery_events'") + .run(remainingEvents.reduce((value, row) => Math.max(value, sqliteInteger(row, "seq")), 0)); + database.exec("COMMIT"); + } catch (error) { database.exec("ROLLBACK"); throw error; } + database.exec("PRAGMA wal_checkpoint(TRUNCATE); PRAGMA journal_mode=DELETE; VACUUM"); + if (database.prepare("PRAGMA integrity_check").get()?.integrity_check !== "ok") throw new Error("Rewound Moltnet SQLite integrity check failed"); + return messages; + } finally { + database.close(); + for (const suffix of ["-wal", "-shm", "-journal"]) await rm(`${file}${suffix}`, { force: true }); + } +}; + +/** Reopens on-disk SQLite read-only; no materializer state or cut helper supplies the verdict. */ +export const inspectMoltnetSqlite = async (file: string, at: string): Promise => { + for (const suffix of ["-wal", "-shm", "-journal"]) { + const entry = await lstat(`${file}${suffix}`).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + // The materialized database is checkpointed. Refuse sidecars before SQLite can recover or touch them. + if (entry) return { messages: 0, events: 0, findings: [{ class: "moltnet-event", path: `${file}${suffix}`, + detail: "Materialized Moltnet SQLite has an uninspected journal/WAL sidecar" }] }; + } + const database = await openSqlite(file, true); + try { + const snapshot = readMoltnetSqlite(database), messages = moltnetMessages(snapshot); + const findings: MoltnetFinding[] = []; + const check = (instant: string, kind: MoltnetFinding["class"], detail: string) => { + if (compareInstants(instant, at) > 0) findings.push({ class: kind, path: file, detail }); + }; + for (const message of messages) check(message.created_at, "moltnet-message", `Message ${message.id} is later than the moment`); + for (const room of Object.values(snapshot.rooms)) check(room.created_at, "moltnet-room", `Room ${room.id} is later than the moment`); + for (const [id, instant] of Object.entries(snapshot.removed_rooms)) check(instant, "moltnet-room", `Room ${id} has a future removal marker`); + for (const row of database.prepare("SELECT id, created_at FROM artifacts").all()) { + check(sqliteText(row, "created_at"), "moltnet-message", `Artifact ${row.id} is later than the moment`); + } + for (const [table, id] of [["threads", "id"], ["dm_conversations", "dm_id"]]) { + for (const row of database.prepare(`SELECT ${id}, last_message_at FROM ${table}`).all()) { + check(sqliteText(row, "last_message_at"), "moltnet-message", `${table} ${row[id]} has a future counter`); + } + } + const events = database.prepare("SELECT * FROM attachment_delivery_events").all(); + for (const row of events) { + check(sqliteText(row, "created_at"), "moltnet-event", `Attachment event ${row.event_id} is later than the moment`); + findings.push(...inspectMoltnetEvent(moltnetEventSchema.parse(sqliteJson(row, "event_json")), at, file)); + } + return { messages: messages.length, events: events.length, findings }; + } finally { database.close(); } +}; diff --git a/src/compiler/training/rewind/moltnetSqliteData.ts b/src/compiler/training/rewind/moltnetSqliteData.ts new file mode 100644 index 00000000..d9b50434 --- /dev/null +++ b/src/compiler/training/rewind/moltnetSqliteData.ts @@ -0,0 +1,105 @@ +import type { DatabaseSync, SQLOutputValue } from "node:sqlite"; +import { parseMoltnetSnapshot } from "./moltnetJson.js"; +import { moltnetEventSchema, type MoltnetSnapshot } from "./moltnetSchemas.js"; + +export type SqliteRow = Record; +export const sqliteText = (row: SqliteRow, key: string): string => { + if (typeof row[key] !== "string") throw new Error(`Invalid Moltnet SQLite ${key}: expected text`); + return row[key]; +}; +export const sqliteInteger = (row: SqliteRow, key: string): number => { + const value = row[key]; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`Invalid Moltnet SQLite ${key}: expected nonnegative safe integer`); + } + return value; +}; +export const sqliteJson = (row: SqliteRow, key: string): unknown => { + try { return JSON.parse(sqliteText(row, key)); } + catch (error) { throw new Error(`Invalid Moltnet SQLite ${key}: ${(error as Error).message}`); } +}; +const allowedColumns: Record = { + rooms: ["id", "network_id", "fqid", "name", "created_at", "deleted_at", "visibility", "write_policy", "federation"], + room_members: ["room_id", "member_id"], + threads: ["id", "network_id", "fqid", "room_id", "parent_message_id", "message_count", "last_message_at"], + messages: ["id", "network_id", "target_kind", "room_id", "thread_id", "parent_message_id", "dm_id", "target_json", + "from_json", "parts_json", "mentions_json", "created_at", "origin_json"], + dm_conversations: ["dm_id", "network_id", "fqid", "message_count", "last_message_at"], + dm_participants: ["dm_id", "participant_id"], + artifacts: ["id", "network_id", "fqid", "message_id", "target_kind", "room_id", "thread_id", "dm_id", "target_json", + "part_index", "kind", "media_type", "filename", "url", "created_at"], + agents: ["agent_id", "network_id", "actor_uid", "actor_uri", "display_name", "credential_key", "created_at", "updated_at"], + attachment_delivery_events: ["seq", "event_id", "event_json", "created_at"], + attachment_delivery_cursors: ["agent_id", "event_seq", "updated_at"], + schema_migrations: ["version", "name", "applied_at"], sqlite_sequence: ["name", "seq"] +}; + +/** A snapshot is an offline store: unknown tables, views or triggers cannot be safely rewound. */ +export const validateSqliteLayout = (database: DatabaseSync): void => { + for (const row of database.prepare("SELECT type, name FROM sqlite_schema WHERE type IN ('table', 'view', 'trigger')").all()) { + const table = sqliteText(row, "name"); + if (row.type !== "table" || !Object.hasOwn(allowedColumns, table)) { + throw new Error(`Unsupported Moltnet SQLite ${row.type} ${row.name}; cannot establish its historical contents`); + } + for (const column of database.prepare(`PRAGMA table_info(${table})`).all()) { + if (!allowedColumns[table]!.includes(sqliteText(column, "name"))) { + throw new Error(`Unsupported Moltnet SQLite column ${table}.${column.name}; cannot establish its historical contents`); + } + } + } +}; + +export const readMoltnetSqlite = (database: DatabaseSync): MoltnetSnapshot => { + validateSqliteLayout(database); + const rooms: Record = {}, removed: Record = {}, threads: Record = {}; + const roomMessages: Record = {}, threadMessages: Record = {}, directMessages: Record = {}; + const directMembers: Record> = {}, agents: Record = {}; + const roomMembers = database.prepare("SELECT room_id, member_id FROM room_members").all(); + for (const row of database.prepare("SELECT * FROM rooms").all()) { + const id = sqliteText(row, "id"); + rooms[id] = { id, network_id: row.network_id, fqid: row.fqid, name: row.name, created_at: row.created_at, + members: roomMembers.filter((member) => member.room_id === id).map((member) => member.member_id), + ...(row.visibility !== undefined ? { visibility: row.visibility } : {}), + ...(row.write_policy !== undefined ? { write_policy: row.write_policy } : {}), + ...(row.federation !== undefined ? { federation: sqliteJson(row, "federation") } : {}) }; + if (row.deleted_at !== null && row.deleted_at !== undefined && row.deleted_at !== "") removed[id] = row.deleted_at; + } + for (const row of database.prepare("SELECT * FROM threads").all()) threads[sqliteText(row, "id")] = { + id: row.id, network_id: row.network_id, fqid: row.fqid, room_id: row.room_id, + parent_message_id: row.parent_message_id, message_count: row.message_count, last_message_at: row.last_message_at + }; + // rowid preserves append order for a DM's first/root message; random message IDs do not. + for (const row of database.prepare("SELECT * FROM messages ORDER BY rowid").all()) { + const target = sqliteJson(row, "target_json") as Record; + for (const [column, key] of [["target_kind", "kind"], ["room_id", "room_id"], ["thread_id", "thread_id"], + ["parent_message_id", "parent_message_id"], ["dm_id", "dm_id"]]) { + if ((row[column] ?? "") !== (target?.[key] ?? "")) throw new Error(`Moltnet SQLite message ${row.id}: ${column} disagrees with target_json`); + } + const message = { id: row.id, network_id: row.network_id, target, + from: sqliteJson(row, "from_json"), parts: sqliteJson(row, "parts_json"), mentions: sqliteJson(row, "mentions_json"), + created_at: row.created_at, ...(row.origin_json !== undefined && row.origin_json !== "{}" + ? { origin: sqliteJson(row, "origin_json") } : {}) }; + const group = row.target_kind === "room" ? roomMessages : row.target_kind === "thread" ? threadMessages + : row.target_kind === "dm" ? directMessages : undefined; + if (!group) throw new Error(`Unknown Moltnet SQLite message target kind ${row.target_kind}`); + const key = sqliteText(row, row.target_kind === "room" ? "room_id" : row.target_kind === "thread" ? "thread_id" : "dm_id"); + (group[key] ??= []).push(message); + } + for (const row of database.prepare("SELECT dm_id, participant_id FROM dm_participants").all()) { + (directMembers[sqliteText(row, "dm_id")] ??= {})[sqliteText(row, "participant_id")] = {}; + } + for (const row of database.prepare("SELECT * FROM agents").all()) agents[sqliteText(row, "agent_id")] = { + ...row, ...(row.display_name === null ? { display_name: "" } : {}) + }; + const eventRows = database.prepare("SELECT * FROM attachment_delivery_events ORDER BY seq").all(); + const events = eventRows.map((row) => { + const event = moltnetEventSchema.parse(sqliteJson(row, "event_json")); + if (event.id !== row.event_id) throw new Error(`Moltnet SQLite event ${row.event_id} disagrees with event_json`); + return event; + }); + const cursors = Object.fromEntries(database.prepare("SELECT * FROM attachment_delivery_cursors").all() + .map((row) => [sqliteText(row, "agent_id"), eventRows.filter((event) => sqliteInteger(event, "seq") <= sqliteInteger(row, "event_seq")).length])); + return parseMoltnetSnapshot({ rooms, removed_rooms: removed, threads, room_messages: roomMessages, + thread_messages: threadMessages, direct_messages: directMessages, direct_members: directMembers, + agents, attachment_delivery_events: events, attachment_delivery_cursors: cursors }); +}; diff --git a/src/compiler/training/rewind/select.test.ts b/src/compiler/training/rewind/select.test.ts new file mode 100644 index 00000000..fc70c354 --- /dev/null +++ b/src/compiler/training/rewind/select.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "vitest"; +import type { TrainingPlan } from "../plan/index.js"; +import type { MomentRecord } from "./contract.js"; +import { selectMomentRecord, selectTrainingExample } from "./select.js"; + +const at = "2026-10-09T08:20:31Z"; +const record = (): MomentRecord => ({ + version: "spawnfile.moment-record.v1", agent: "agent:writer", + wake: { at, surface: "moltnet", room: "assignment", from: "brass" }, recordedAt: at, + resources: [{ id: "repo", kind: "git", commit: "a".repeat(40) }], + feeds: [{ id: "news", revision: "rev", tree: "tree", commit: "b".repeat(40), paths: ["published"], landedAt: at }], + memory: [{ id: "self", declaredBy: "agent:writer", seq: 2, generation: 0 }], + moltnet: [{ network: "network", cursor: "cursor" }], volumes: [{ id: "state", snapshot: "snapshot" }] +}); +const dataset = (name: string, id: string) => ({ sourcePath: `/project/${name}.yaml`, path: `${name}.yaml`, + examples: [{ id, at, wake: { surface: "moltnet" as const, room: "assignment", from: "brass", text: "write" }, overrides: [] }] }); +const plan = (): TrainingPlan => ({ + version: "spawnfile.training-plan.v1", producer: { package: "spawnfile", version: "test" }, + project: { root: "/project", manifest: "/project/Spawnfile" }, agent: { id: "agent:writer", name: "writer" }, + record: null, optimizer: null, graders: [], targets: [{ declaredBy: "agent:writer", grader: "writing", + files: [{ sourcePath: "/project/AGENTS.md", path: "AGENTS.md", consumers: ["agent:writer"] }], + datasets: { train: dataset("train", "example"), validation: null, test: dataset("test", "held-out") } + }] +}); + +describe("pure rewind record selection", () => { + it("matches equal instants across offsets and ignores capture time and unused wake payload", () => { + const first = record(), second = structuredClone(first); + second.wake = { ...second.wake, at: "2026-10-09T10:20:31+02:00", room: "other", messageId: "later-capture" }; + second.recordedAt = "2026-10-11T00:00:00Z"; + second.feeds[0]!.landedAt = "2026-10-09T10:20:31.000000000+02:00"; + expect(selectMomentRecord([first, second], "agent:writer", "2026-10-09T03:20:31-05:00")).toBe(first); + }); + + it("requires exact agent and wake instant, including sub-millisecond differences", () => { + expect(selectMomentRecord([record()], "writer", at)).toBeNull(); + expect(selectMomentRecord([record()], "agent:writer", "2026-10-09T08:20:31.000000001Z")).toBeNull(); + expect(selectMomentRecord([], "agent:writer", at)).toBeNull(); + }); + + const changes: Array<[string, (value: MomentRecord) => void]> = [ + ["resource commit", (value) => { value.resources[0]!.commit = "c".repeat(40); }], + ["feed revision", (value) => { value.feeds[0]!.revision = "different"; }], + ["feed paths", (value) => { value.feeds[0]!.paths = ["private"]; }], + ["memory sequence", (value) => { value.memory[0]!.seq = 3; }], + ["memory generation", (value) => { value.memory[0]!.generation = 1; }], + ["network cursor", (value) => { value.moltnet[0]!.cursor = "different"; }], + ["volume snapshot", (value) => { value.volumes[0]!.snapshot = "different"; }] + ]; + it.each(changes)("refuses conflicting %s fields", (_name, change) => { + const first = record(), second = structuredClone(first); + change(second); + expect(() => selectMomentRecord([first, second], "agent:writer", at)).toThrow(/Conflicting moment records.*agent:writer/u); + }); + + it("compares only inputs being used and retains memory declaring-node identity", () => { + const first = record(), second = structuredClone(first); + second.resources[0]!.commit = "c".repeat(40); + second.memory.push({ id: "self", declaredBy: "team:other", generation: 9, seq: 88 }); + const selected = [{ section: "memory" as const, id: "self", declaredBy: "agent:writer" }]; + expect(selectMomentRecord([first, second], "agent:writer", at, selected)).toBe(first); + second.memory[0]!.seq = 3; + expect(() => selectMomentRecord([first, second], "agent:writer", at, selected)).toThrow(/Conflicting/u); + }); + + it("accepts reordered record entries without mutating its inputs", () => { + const first = record(); + first.resources.push({ id: "another", kind: "bundle", commit: "d".repeat(40) }); + const second = structuredClone(first); + second.resources.reverse(); + const before = structuredClone([first, second]); + expect(selectMomentRecord([first, second], "agent:writer", at)).toBe(first); + expect([first, second]).toEqual(before); + }); + + it("does not compare record fields superseded by a full commit pin or another resource kind", () => { + const first = record(), second = structuredClone(first); + second.resources[0]!.commit = "c".repeat(40); + second.feeds[0]!.revision = "other"; + expect(selectMomentRecord([first, second], "agent:writer", at, [{ + section: "resources", id: "repo", declaredBy: "agent:writer", overridden: false, declarationBase: "/project", + declaration: { id: "repo", kind: "git", mode: "readonly", mount: "./repo", url: "/repo", ref: "a".repeat(40) } + }, { + section: "resources", id: "news", declaredBy: "agent:writer", overridden: false, declarationBase: "/project", + declaration: { id: "news", kind: "volume", mode: "mutable", mount: "./news" } + }])).toBe(first); + }); +}); + +describe("pure rewind example selection", () => { + it.each(["example", "train:example"])("selects %s and deduplicates repeated target references", (selector) => { + const source = plan(); + source.targets.push(structuredClone(source.targets[0]!)); + const before = structuredClone(source); + expect(selectTrainingExample(source, selector)).toMatchObject({ id: "example", role: "train", dataset: "train.yaml" }); + expect(source).toEqual(before); + }); + + it("lists ambiguous roles and permits an exact role qualifier", () => { + const source = plan(); + source.targets[0]!.datasets.test = dataset("test", "example"); + expect(() => selectTrainingExample(source, "example")).toThrow( + "Ambiguous training example example; candidates: test:example (test.yaml), train:example (train.yaml)" + ); + expect(selectTrainingExample(source, "test:example").dataset).toBe("test.yaml"); + }); + + it("still refuses a qualified id shared by distinct datasets and lists both candidates", () => { + const source = plan(), other = structuredClone(source.targets[0]!); + other.datasets.train = dataset("another-train", "example"); + source.targets.push(other); + expect(() => selectTrainingExample(source, "train:example")).toThrow( + "candidates: train:example (another-train.yaml), train:example (train.yaml)" + ); + }); + + it.each(["missing", "validation:example", "train:", "other:example"])("refuses unknown selector %s", (selector) => { + expect(() => selectTrainingExample(plan(), selector)).toThrow(`No training example matches ${selector}`); + }); +}); diff --git a/src/compiler/training/rewind/select.ts b/src/compiler/training/rewind/select.ts new file mode 100644 index 00000000..61dbdced --- /dev/null +++ b/src/compiler/training/rewind/select.ts @@ -0,0 +1,66 @@ +import type { TrainingPlan } from "../plan/index.js"; +import type { MomentRecord } from "./contract.js"; +import { compareInstants, normalizeInstant } from "./instant.js"; +import type { InputIdentity, RewindInput } from "./types.js"; + +export type DatasetRole = "train" | "validation" | "test"; +export interface SelectedExample { + id: string; role: DatasetRole; dataset: string; sourcePath: string; + example: TrainingPlan["targets"][number]["datasets"]["train"]["examples"][number]; +} +/** Shared targets referencing the same role/dataset/example are one trial. */ +export const selectTrainingExample = (plan: TrainingPlan, selector: string): SelectedExample => { + const match = /^(train|validation|test):(.*)$/u.exec(selector); + const role = match?.[1], id = match?.[2] ?? selector; + const candidates = new Map(); + for (const target of plan.targets) for (const candidateRole of ["train", "validation", "test"] as const) { + const dataset = target.datasets[candidateRole]; + if (!dataset || (role && candidateRole !== role)) continue; + for (const example of dataset.examples) { + if (example.id !== id) continue; + const candidate = { id, role: candidateRole, dataset: dataset.path, sourcePath: dataset.sourcePath, example }; + candidates.set(JSON.stringify([candidateRole, dataset.sourcePath, id]), candidate); + } + } + const found = [...candidates.values()]; + if (found.length !== 1) throw new Error(found.length === 0 + ? `No training example matches ${selector}` + : `Ambiguous training example ${selector}; candidates: ${found.map((item) => `${item.role}:${id} (${item.dataset})`).sort().join(", ")}`); + return found[0]!; +}; +const canonical = (value: unknown): string => JSON.stringify(value, (_key, item: unknown) => + item && typeof item === "object" && !Array.isArray(item) + ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a.localeCompare(b))) : item); + +/** Capture time and wake payload are not used to materialize data. Normalize actual used instants. */ +export const selectMomentRecord = ( + records: MomentRecord[], agent: string, at: string, inputs?: Array +): MomentRecord | null => { + const matches = records.filter((record) => record.agent === agent && compareInstants(record.wake.at, at) === 0); + if (!matches.length) return null; + const includes = (section: InputIdentity["section"], id: string, declaredBy?: string, field?: "resources" | "feeds" | "volumes"): boolean => + !inputs || inputs.some((input) => { + if (input.section !== section || input.id !== id || (declaredBy && input.declaredBy !== declaredBy)) return false; + if (!("declaration" in input)) return true; + if (input.overridden) return false; + if (input.section !== "resources" || !field) return true; + const resource = input.declaration; + if (field === "feeds") return resource.kind === "volume" && Boolean(resource.feed); + if (field === "volumes") return resource.kind === "volume" && !resource.feed; + if (resource.kind === "volume") return false; + const ref = resource.kind === "git" ? resource.ref : resource.build?.files?.ref; + return !ref || !/^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/u.test(ref); + }); + const sort = (entries: T[]): T[] => entries.sort((a, b) => canonical(a).localeCompare(canonical(b))); + const used = (record: MomentRecord): string => canonical({ + resources: sort(record.resources.filter((item) => includes("resources", item.id, undefined, "resources"))), + feeds: sort(record.feeds.filter((item) => includes("resources", item.id, undefined, "feeds")).map((item) => ({ + ...item, landedAt: normalizeInstant(item.landedAt) + }))), + memory: sort(record.memory.filter((item) => includes("memory", item.id, item.declaredBy))), + moltnet: sort(record.moltnet.filter((item) => includes("networks", item.network))), + volumes: sort(record.volumes.filter((item) => includes("resources", item.id, undefined, "volumes"))) + }); + if (matches.some((record) => used(record) !== used(matches[0]!))) throw new Error(`Conflicting moment records for ${agent} at ${normalizeInstant(at)}`); + return matches[0]!; +}; diff --git a/src/compiler/training/rewind/taskClock.test.ts b/src/compiler/training/rewind/taskClock.test.ts new file mode 100644 index 00000000..9ec1c80d --- /dev/null +++ b/src/compiler/training/rewind/taskClock.test.ts @@ -0,0 +1,32 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createTaskClock, taskClockEnvironment } from "./taskClock.js"; + +afterEach(() => vi.restoreAllMocks()); + +describe("training task clock contract", () => { + it("normalizes the bundle origin and returns exact cross-repository environment values", () => { + const origin = "2026-10-09T10:20:31.120000000+02:00", anchor = 1_790_000_000_123; + expect(createTaskClock(origin)).toEqual({ version: "spawnfile.task-clock.v1", origin: "2026-10-09T08:20:31.12Z" }); + expect(taskClockEnvironment(origin, anchor)).toEqual({ + NOOPOLIS_TASK_CLOCK: '{"version":"noopolis.task-clock.v1","origin":"2026-10-09T08:20:31.12Z","anchorEpochMs":1790000000123}', + MNEME_CLOCK_ORIGIN: "2026-10-09T08:20:31.12Z", MNEME_CLOCK_ANCHOR_MS: "1790000000123", + FAKETIME: "@2026-10-09 08:20:31", FAKETIME_DONT_FAKE_MONOTONIC: "1" + }); + }); + + it("uses only its supplied anchor without wall-clock reads, sleeping, or process environment mutation", () => { + const environment = { ...process.env }; + const now = vi.spyOn(Date, "now").mockImplementation(() => { throw Error("read live clock"); }); + const timeout = vi.spyOn(globalThis, "setTimeout").mockImplementation(() => { throw Error("started timeout"); }); + const interval = vi.spyOn(globalThis, "setInterval").mockImplementation(() => { throw Error("started interval"); }); + const first = taskClockEnvironment("2026-10-09T08:20:31Z", 0); + expect(taskClockEnvironment("2026-10-09T10:20:31+02:00", 0)).toEqual(first); + expect(JSON.parse(first.NOOPOLIS_TASK_CLOCK!).anchorEpochMs).toBe(0); + expect(now).not.toHaveBeenCalled(); expect(timeout).not.toHaveBeenCalled(); expect(interval).not.toHaveBeenCalled(); + expect(process.env).toEqual(environment); + }); + + it.each([NaN, Infinity, 1.5, Number.MAX_SAFE_INTEGER + 1])("refuses non-integer or unsafe anchor %s", (anchor) => { + expect(() => taskClockEnvironment("2026-10-09T08:20:31Z", anchor)).toThrow(/anchorEpochMs must be a safe integer/u); + }); +}); diff --git a/src/compiler/training/rewind/taskClock.ts b/src/compiler/training/rewind/taskClock.ts new file mode 100644 index 00000000..9bef899f --- /dev/null +++ b/src/compiler/training/rewind/taskClock.ts @@ -0,0 +1,17 @@ +import { normalizeInstant } from "./instant.js"; +import { taskClockSchema } from "./contract.js"; + +export const createTaskClock = (origin: string) => taskClockSchema.parse({ + version: "spawnfile.task-clock.v1", origin: normalizeInstant(origin) +}); +/** Offsets reported wall time only. Callers supply the real anchor at trial launch. */ +export const taskClockEnvironment = (origin: string, anchorEpochMs: number): Record => { + if (!Number.isSafeInteger(anchorEpochMs)) throw new Error("Task clock anchorEpochMs must be a safe integer"); + const normalized = normalizeInstant(origin); + return { + NOOPOLIS_TASK_CLOCK: JSON.stringify({ version: "noopolis.task-clock.v1", origin: normalized, anchorEpochMs }), + MNEME_CLOCK_ORIGIN: normalized, MNEME_CLOCK_ANCHOR_MS: String(anchorEpochMs), + FAKETIME: `@${new Date(normalized).toISOString().slice(0, 19).replace("T", " ")}`, + FAKETIME_DONT_FAKE_MONOTONIC: "1" + }; +}; diff --git a/src/compiler/training/rewind/timestamps.ts b/src/compiler/training/rewind/timestamps.ts new file mode 100644 index 00000000..a83972a8 --- /dev/null +++ b/src/compiler/training/rewind/timestamps.ts @@ -0,0 +1,24 @@ +import { lstat, lutimes, readdir, utimes } from "node:fs/promises"; +import path from "node:path"; +import { compareInstants } from "./instant.js"; + +/** Round down to milliseconds so filesystem precision cannot put a stamp after the instant. */ +export const stampTree = async (root: string, instant: string, at: string): Promise => { + const stamp = new Date(compareInstants(instant, at) > 0 ? at : instant); + const visit = async (file: string): Promise => { + const stats = await lstat(file); + if (stats.isDirectory()) for (const name of await readdir(file)) await visit(path.join(file, name)); + await (stats.isSymbolicLink() ? lutimes : utimes)(file, stamp, stamp); + }; + await visit(root); +}; + +/** Disk verification reads may update atime. Restore access time without changing verified mtimes. */ +export const restoreAccessTimes = async (root: string, at: string): Promise => { + const visit = async (file: string): Promise => { + const stats = await lstat(file); + if (stats.isDirectory()) for (const name of await readdir(file)) await visit(path.join(file, name)); + await (stats.isSymbolicLink() ? lutimes : utimes)(file, Math.min(stats.mtimeMs, Date.parse(at)) / 1000, stats.mtimeMs / 1000); + }; + await visit(root); +}; diff --git a/src/compiler/training/rewind/types.ts b/src/compiler/training/rewind/types.ts new file mode 100644 index 00000000..7dd09741 --- /dev/null +++ b/src/compiler/training/rewind/types.ts @@ -0,0 +1,27 @@ +import type { MemoryBank, TeamNetwork, TeamWorkspaceResource } from "../../../manifest/index.js"; +import type { ResolvedMemoryBank } from "../../types.js"; +import type { MomentRecord, MaterializedInput } from "./contract.js"; +import type { History } from "./history.js"; + +export interface InputIdentity { section: "resources" | "memory" | "networks"; id: string; declaredBy: string } +interface InputBase extends InputIdentity { overridden: boolean; declarationBase: string } +export type RewindInput = InputBase & ( + | { section: "resources"; declaration: TeamWorkspaceResource } + | { section: "memory"; declaration: MemoryBank | ResolvedMemoryBank } + | { section: "networks"; declaration: TeamNetwork } +); +export interface InputContext { + input: RewindInput; + at: string; + record: MomentRecord | null; + history: History | undefined; + destination: string; +} +export interface InputResult { + fidelity: "exact" | "approximate" | "overridden"; + basis: string; + source: MaterializedInput["source"]; + notes: string[]; + instant: string; + build?: "deferred"; +} diff --git a/src/compiler/training/rewind/volumes.test.ts b/src/compiler/training/rewind/volumes.test.ts new file mode 100644 index 00000000..cd22c2f2 --- /dev/null +++ b/src/compiler/training/rewind/volumes.test.ts @@ -0,0 +1,93 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, mkdir, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { openHistory } from "./history.js"; +import { momentRecordSchema } from "./contract.js"; +import type { InputContext } from "./types.js"; +import { materializeVolume, replayVolumeSnapshot, selectVolumeSnapshot, type VolumeSnapshot } from "./volumes.js"; + +const at = "2026-10-09T10:00:00Z", before = "2026-10-09T09:00:00Z", future = "2026-10-09T10:00:00.001Z"; +const digest = (bytes: string): string => createHash("sha256").update(bytes).digest("hex"); +const temps: string[] = []; +afterEach(async () => { await Promise.all(temps.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); }); +const snapshot = (id: string, time: string, parent: string | null, files: VolumeSnapshot["files"]): VolumeSnapshot => + ({ snapshot: id, takenAt: time, parent, files }); +const fixtures = async (snapshots?: VolumeSnapshot[]) => { + const root = await mkdtemp(path.join(os.tmpdir(), "rewind-volumes-")); temps.push(root); + const bank = path.join(root, "volumes/team%3Aorg/state"); await mkdir(path.join(bank, "objects"), { recursive: true }); + if (snapshots) await writeFile(path.join(bank, "snapshots.jsonl"), snapshots.map((item) => JSON.stringify(item)).join("\n") + "\n"); + const context: InputContext = { at, record: null, history: await openHistory(root), destination: path.join(root, "out"), + input: { section: "resources", id: "state", declaredBy: "team:org", declarationBase: root, overridden: false, + declaration: { id: "state", kind: "volume", mount: "./state", mode: "mutable" } } }; + return { root, bank, context, object: async (bytes: string) => writeFile(path.join(bank, "objects", digest(bytes)), bytes) }; +}; + +describe("unfed volume replay", () => { + it("includes equal instants, excludes one millisecond later, and selects an ancestor record exactly", async () => { + const rows = [snapshot("first", before, null, { old: digest("old") }), + snapshot("next", "2026-10-09T12:00:00+02:00", "first", { old: null, "nested/new": digest("new") }), + snapshot("future", future, "next", { later: digest("future") })]; + expect(selectVolumeSnapshot(rows, at).snapshot).toBe("next"); + expect(selectVolumeSnapshot(rows, at, "first").snapshot).toBe("first"); + expect(replayVolumeSnapshot(rows, "next").files).toEqual({ "nested/new": digest("new") }); + const { context, object } = await fixtures(rows); await object("old"); await object("new"); + const result = await materializeVolume(context); + expect(result).toMatchObject({ fidelity: "approximate", basis: "snapshot-at", source: { snapshot: "next" } }); + expect(await readdir(context.destination)).toEqual(["nested"]); + expect(await readFile(path.join(context.destination, "nested/new"), "utf8")).toBe("new"); + context.destination += "-record"; + context.record = momentRecordSchema.parse({ version: "spawnfile.moment-record.v1", agent: "agent:writer", + wake: { at, surface: "moltnet", room: "assignment", from: "brass" }, recordedAt: at, + resources: [], feeds: [], memory: [], moltnet: [], volumes: [{ id: "state", snapshot: "first" }] }); + expect(await materializeVolume(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { snapshot: "first" } }); + expect(await readFile(path.join(context.destination, "old"), "utf8")).toBe("old"); + }); + + it("creates an empty overridden volume without history", async () => { + const { context } = await fixtures(); context.history = undefined; context.input.overridden = true; + expect(await materializeVolume(context)).toMatchObject({ fidelity: "overridden", basis: "override" }); + expect(await readdir(context.destination)).toEqual([]); + }); + + it("refuses unavailable history, future-only snapshots, divergent pins, cycles and missing ancestors", async () => { + const { context } = await fixtures(); + await expect(materializeVolume(context)).rejects.toThrow("Missing volume snapshot history"); + context.history = undefined; + await expect(materializeVolume(context)).rejects.toThrow("requires --history"); + expect(() => selectVolumeSnapshot([snapshot("late", future, null, {})], at)).toThrow("No volume snapshot"); + const rows = [snapshot("one", before, null, {}), snapshot("other", at, null, {})]; + expect(() => selectVolumeSnapshot(rows, at, "one")).toThrow("not the latest eligible snapshot or its ancestor"); + expect(() => replayVolumeSnapshot([snapshot("one", at, "missing", {})], "one")).toThrow("Missing volume snapshot ancestor"); + expect(() => replayVolumeSnapshot([snapshot("one", at, "one", {})], "one")).toThrow("Cycle"); + expect(() => replayVolumeSnapshot([snapshot("one", at, null, {}), snapshot("two", before, "one", {})], "two")).toThrow("later than child"); + }); + + it("breaks same-instant ties only by ancestry and refuses malformed records and unsafe paths", async () => { + const rows = [snapshot("one", at, null, {}), snapshot("two", at, "one", {})]; + expect(selectVolumeSnapshot(rows, at).snapshot).toBe("two"); + expect(() => selectVolumeSnapshot([...rows, snapshot("three", at, null, {})], at)).toThrow("Ambiguous"); + expect(() => replayVolumeSnapshot([rows[0]!, rows[0]!], "one")).toThrow("Duplicate"); + expect(() => replayVolumeSnapshot([snapshot("one", at, null, { a: digest("a"), "a/b": digest("b") })], "one")).toThrow("collision"); + const { context, bank } = await fixtures(); + await writeFile(path.join(bank, "snapshots.jsonl"), JSON.stringify({ ...rows[0], unexpected: true }) + "\n"); + await expect(materializeVolume(context)).rejects.toThrow("snapshots.jsonl:1"); + await writeFile(path.join(bank, "snapshots.jsonl"), JSON.stringify(snapshot("one", at, null, { "../escape": digest("x") })) + "\n"); + await expect(materializeVolume(context)).rejects.toThrow("normalized relative POSIX"); + }); + + it("verifies every chain object, including deleted bytes, and refuses symlink escape", async () => { + const { context, bank, root, object } = await fixtures([ + snapshot("one", before, null, { old: digest("old") }), snapshot("two", at, "one", { old: null })]); + await expect(materializeVolume(context)).rejects.toThrow("Missing volume snapshot object"); + await object("old"); await writeFile(path.join(bank, "objects", digest("old")), "tampered"); + await expect(materializeVolume(context)).rejects.toThrow("SHA-256 mismatch"); + const outside = await mkdtemp(path.join(os.tmpdir(), "rewind-volume-outside-")); temps.push(outside); + await writeFile(path.join(outside, "object"), "old"); + await rm(path.join(bank, "objects", digest("old"))); + await symlink(path.join(outside, "object"), path.join(bank, "objects", digest("old"))); + await expect(materializeVolume(context)).rejects.toThrow("symlink escape"); + expect(await readdir(root)).not.toContain("out"); + }); +}); diff --git a/src/compiler/training/rewind/volumes.ts b/src/compiler/training/rewind/volumes.ts new file mode 100644 index 00000000..d0d6ba04 --- /dev/null +++ b/src/compiler/training/rewind/volumes.ts @@ -0,0 +1,122 @@ +import { createHash } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { z } from "zod"; +import { trainingInstantSchema } from "../../../manifest/index.js"; +import { historyKey } from "./history.js"; +import { compareInstants } from "./instant.js"; +import type { InputContext, InputResult } from "./types.js"; + +const snapshotId = z.string().min(1); +const relativeFile = z.string().min(1).refine((value) => !value.includes("\\") && !value.includes("\0") + && !path.posix.isAbsolute(value) && !value.split("/").some((part) => !part || part === "." || part === ".."), +"Snapshot file paths must be normalized relative POSIX paths"); +export const volumeSnapshotSchema = z.object({ + snapshot: snapshotId, takenAt: trainingInstantSchema, parent: snapshotId.nullable(), + files: z.record(relativeFile, z.string().regex(/^[a-f0-9]{64}$/u).nullable()) +}).strict(); +export type VolumeSnapshot = z.infer; +export interface ReplayedVolumeSnapshot { + snapshot: string; takenAt: string; files: Record; chain: VolumeSnapshot[]; +} + +const catalog = (snapshots: VolumeSnapshot[]): Map => { + const result = new Map(); + for (const item of snapshots) { + volumeSnapshotSchema.parse(item); + if (result.has(item.snapshot)) throw new Error(`Duplicate volume snapshot ${item.snapshot}`); + result.set(item.snapshot, item); + } + return result; +}; + +/** Replay metadata only. Both materialization and the independent disk checker can call this. */ +export const replayVolumeSnapshot = (snapshots: VolumeSnapshot[], snapshot: string): ReplayedVolumeSnapshot => { + const entries = catalog(snapshots), seen = new Set(), chain: VolumeSnapshot[] = []; + let next: string | null = snapshot; + while (next !== null) { + if (seen.has(next)) throw new Error(`Cycle in volume snapshot chain at ${next}`); + seen.add(next); + const current = entries.get(next); + if (!current) throw new Error(`Missing volume snapshot ancestor ${next}`); + const descendant = chain.at(-1); + if (descendant && compareInstants(current.takenAt, descendant.takenAt) > 0) { + throw new Error(`Volume snapshot parent ${next} is later than child ${descendant.snapshot}`); + } + chain.push(current); next = current.parent; + } + chain.reverse(); + const files = new Map(); + for (const item of chain) { + for (const [file, digest] of Object.entries(item.files)) { + if (digest === null) files.delete(file); else files.set(file, digest); + } + for (const file of files.keys()) { + const components = file.split("/"); + for (let length = 1; length < components.length; length++) { + const parent = components.slice(0, length).join("/"); + if (files.has(parent)) throw new Error(`Snapshot ${item.snapshot} has a file/directory collision at ${parent}`); + } + } + } + return { snapshot, takenAt: chain.at(-1)!.takenAt, + files: Object.fromEntries([...files].sort(([a], [b]) => a.localeCompare(b))), chain }; +}; + +export const selectVolumeSnapshot = (snapshots: VolumeSnapshot[], at: string, recordId?: string): VolumeSnapshot => { + const entries = catalog(snapshots); + const candidates = [...entries.values()].filter((item) => compareInstants(item.takenAt, at) <= 0) + .sort((a, b) => compareInstants(b.takenAt, a.takenAt)); + if (!candidates.length) throw new Error("No volume snapshot was taken at or before the moment"); + let selected = candidates[0]!; + for (const candidate of candidates.slice(1)) { + if (compareInstants(candidate.takenAt, selected.takenAt) !== 0) break; + if (replayVolumeSnapshot(snapshots, candidate.snapshot).chain.some((item) => item.snapshot === selected.snapshot)) { + selected = candidate; + } else if (!replayVolumeSnapshot(snapshots, selected.snapshot).chain.some((item) => item.snapshot === candidate.snapshot)) { + throw new Error("Ambiguous volume snapshots taken at the same instant on separate branches"); + } + } + const chain = replayVolumeSnapshot(snapshots, selected.snapshot).chain; + if (recordId !== undefined) { + const pinned = chain.find((item) => item.snapshot === recordId); + if (!pinned) throw new Error(`Recorded volume snapshot ${recordId} is not the latest eligible snapshot or its ancestor`); + return pinned; + } + return selected; +}; + +export const materializeVolume = async (context: InputContext): Promise => { + const { input, at, record, history, destination } = context; + if (input.section !== "resources" || input.declaration.kind !== "volume" || input.declaration.feed) { + throw new Error("Volume materialization requires an unfed volume input"); + } + if (input.overridden) { + await mkdir(destination, { recursive: true }); + return { fidelity: "overridden", basis: "override", source: {}, notes: [], instant: at }; + } + if (!history) throw new Error(`Volume ${input.id} requires --history or an override`); + const base = `volumes/${historyKey(input)}`; + const snapshots = await history.jsonl(`${base}/snapshots.jsonl`, volumeSnapshotSchema); + if (!snapshots) throw new Error(`Missing volume snapshot history ${base}/snapshots.jsonl`); + const pin = record?.volumes.find((entry) => entry.id === input.id); + const selected = selectVolumeSnapshot(snapshots, at, pin?.snapshot); + const replayed = replayVolumeSnapshot(snapshots, selected.snapshot); + const objects = new Map(); + // Verify every object referenced by the replay, including content later overwritten or deleted. + for (const item of replayed.chain) for (const digest of Object.values(item.files)) { + if (digest === null || objects.has(digest)) continue; + const objectPath = `${base}/objects/${digest}`, bytes = await history.bytes(objectPath); + if (bytes === null) throw new Error(`Missing volume snapshot object ${objectPath}`); + if (createHash("sha256").update(bytes).digest("hex") !== digest) throw new Error(`Volume snapshot object SHA-256 mismatch: ${objectPath}`); + objects.set(digest, bytes); + } + await mkdir(destination, { recursive: true }); + for (const [file, digest] of Object.entries(replayed.files)) { + const target = path.join(destination, file); + await mkdir(path.dirname(target), { recursive: true }); + await writeFile(target, objects.get(digest)!); + } + return { fidelity: pin ? "exact" : "approximate", basis: pin ? "record" : "snapshot-at", + source: { snapshot: selected.snapshot }, notes: [], instant: selected.takenAt }; +}; From bb9d73ab66b8cb54acca8258702fa09a82094c3f Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sun, 11 Oct 2026 02:23:52 +0200 Subject: [PATCH 2/9] feat: add train --materialize and specify rewind, task clock and leak check --- specs/TRAINING.md | 732 ++++++++++++++++++++++++++++++++--- src/cli/AGENTS.md | 2 +- src/cli/runCli.ts | 10 +- src/cli/trainCommand.test.ts | 118 ++++++ src/cli/trainCommand.ts | 29 +- 5 files changed, 821 insertions(+), 70 deletions(-) diff --git a/specs/TRAINING.md b/specs/TRAINING.md index a4e39a7f..aa523959 100644 --- a/specs/TRAINING.md +++ b/specs/TRAINING.md @@ -155,68 +155,9 @@ appears between the `training-plan-schema` markers below. ## Moment record -`spawnfile.moment-record.v1` records one turn attempt of one compiled agent. Recording MUST -be off without an effective `record`, with no recording I/O or artifact changes when -off, except reconciliation owed by an earlier recorded deployment or landing. Recording requires a Daimon runtime whose receipt-bound contract manifest declares -`wakeMoments` version `noopolis.daimon.wake-moment.v1` and consumes `agents[].recording`; -other runtimes or a missing capability fail compile. Enabling it requires a capable -Daimon release and a `runtimes.yaml` / contract-manifest re-pin, never a version guess. - -The compiler emits per-agent `recording: {directory, keepMs, snapshots:[{id,path}]}`. -The directory is `/var/lib/spawnfile/recording/`, a runtime-owned `0700` -exclusive-reattach volume; snapshot paths are the resolved unfed volume backing roots, -not workspace symlinks. Recording stores receive the same deletion guard as memory. - -`spawnfile record sync [path] [--deployment ]` selects a deployment like `usage`, -reads wake rows and memory ledgers in bounded exec chunks, and joins host histories using -the compiled agent facts from the activation entry in effect at each row's `started_at`. -Missing activation entries or agent facts MUST set `exact:false` with a reason, including -when the agent has no resource pins. Today's deployment facts MUST NOT fill historical gaps. -It writes private `0600` records via temporary file, fsync and rename to -`/moments//--.json`. -For project records the state directory is `/deployments//`; for home -records it is `~/.spawnfile/deployments//`, scoped to the same identity as the -sibling resource log. The row hash is the first 12 lowercase hex characters of SHA-256 -over canonical wake-row JSON (object keys sorted recursively, array order preserved). -Existing records are never rewritten. Replays may share both `executionId` and `startedAt`; -different row contents produce distinct attempt filenames, while resync skips the same row. Moment -retention deletes attempts older than `now - keep`, preserving the newest per agent -(including ties). Snapshots remain in the deployment's recording volume; records name -them, and P4 reads them. Non-exact records do not fail sync; operational failures do. - -All instants MUST use one host clock, shared by its containers, and include an offset -or `Z`. Every pin MUST describe state in effect at `wake.startedAt`, not capture time. -A later `recordedAt` MUST NOT advance a pin. - -- `agent` MUST be the exact compiled node ID; `recordedAt` is the capture instant. -- `wake` MUST contain `at`, `startedAt`, `executionId`, and nonempty `deliveries` of - `{deliveryId, kind, occurredAt}`. `at` is the first delivery's `occurred_at` (real wake - time); `startedAt` is when the turn began. Queued deliveries can precede that instant. - `surface`, `room`, `from`, and `messageId` are OPTIONAL, derived from a - `moltnet:` delivery and its Moltnet message when available; other deliveries - MUST NOT invent Moltnet metadata. -- `resources` MUST use `{id, kind:"git", commit, sha256?}` or - `{id, kind:"bundle", sha256}` from the deploy resource entry in effect at - `startedAt`, selected by deployment record identity and `agent_node_ids` containing - that exact agent node ID. Bundles have no commit. An inexact activation boundary - MUST set the moment's `exact:false`; an overlapping predecessor can still run old pins. -- `feeds` MUST use `{id, revision, tree, commit?, ref?, paths?, landedAt, period?}` - from the landing entry in effect at `startedAt` for each consumed fed volume. - A wake inside the inclusive `[switching_at, landed_at]` transition window MUST - set `exact:false`; the helper returns that landing with `ambiguous:true`. -- `memory` MUST use `{id, declaredBy, seq}` for each accessible bank. `seq` is the - highest Mneme ledger sequence whose `createdAt <= startedAt`; 0 means empty. - `generation` is OPTIONAL and MUST be omitted while Mneme has no generations. - Mneme's append-only `events.jsonl` is the source of truth; there are no curation - archives or generations to pin today. -- `moltnet` MUST use `{network, cursor}` with `cursor = startedAt`. Visible history - contains messages with `created_at <= cursor`: server timestamps order history, - while message IDs do not order messages across rooms. OPTIONAL `messageIds` lists - wake message IDs, not a history cursor. -- `volumes` MUST use `{id, snapshot}` for each successful pre-turn snapshot of an - effective `record.volumes` root. A failed root MUST be omitted and named in OPTIONAL - `errors: [string]`. REQUIRED `exact` MUST be `false` on any snapshot or pin failure; - otherwise it MUST be `true`. Missing or invalid history MUST NOT be guessed. +`spawnfile.moment-record.v1` specifies one record per wake; rewind consumes this format, +while the recording phase produces it. Recording MUST be off unless an effective `record` exists, +with zero recording cost when off. `record.keep` is snapshot retention. ```json { @@ -368,6 +309,265 @@ Mneme stays independent and its ledger MUST NOT be truncated by recording retent Rewind MUST use records when present; otherwise it is best effort and the trial MUST be labelled as such. +## Rewind + +```sh +spawnfile train ./Spawnfile --agent agent:writer --materialize train:ice-shooting --history ./history --out ./moment +``` + +`--materialize ` MUST require the existing `--out` option and follows +the plan's exact-agent selection (inferred only for a single-agent project). +It requires neither `--train` nor `--training-config`. It is mutually exclusive +with `--plan`. `--history` is OPTIONAL and applies only to materialization. The +command resolves the authoring plan and selected example using local files. It +MUST NOT invoke cognition, Paideia, Docker, authentication, network access, or +compiled-artifact emission. `materializeTrainingMoment({plan, project, example, +historyDir?, out})` exposes the same operation to trial preparation. Selection, +input enumeration, and historical cuts are available as pure library functions. + +The example selector is an ID from the agent's plan targets. If that ID occurs +in several dataset/role pairs it MUST be qualified as `train:`, +`validation:`, or `test:`. Repeated references to the same dataset, role, +and example are one candidate. A qualified selector that still matches several +datasets MUST refuse. Ambiguity errors MUST list role, ID, and dataset for every +candidate. + +### Inputs and history + +Inputs are EVERY effective item in the agent's compiled `workspaceResources`, +accessible `memoryAccess` banks deduplicated by bank source, and Moltnet networks +in its resolved surface attachments, including representative attachments. Each +input is identified by `{section, id, declaredBy}`, using the same declaring-node +resolution as authoring-plan overrides. Sections are `resources`, `memory`, and +`networks`. A dataset example's full declaration replaces the effective item +with the same section and ID; the plan guarantees exactly one match. Relative +paths in replacement declarations resolve from the dataset file's directory. +Every input MUST materialize successfully; present-day data MUST NOT substitute +for unavailable history. + +The history consumer layout is normative: + +```text +DIR/ + moments/*.jsonl + git/// + feeds///landings.jsonl + feeds///repo/ + feeds///trees// + memory///events.jsonl + memory///generations.jsonl + moltnet///state.json | moltnet.sqlite + volumes///snapshots.jsonl + volumes///objects/ +``` + +`` MUST be `encodeURIComponent` of the node ID: `agent:writer` +becomes `agent%3Awriter`. IDs and content-addressed names MUST be safe path +segments. Every read MUST remain inside `DIR` both lexically and after realpath +resolution; symlink escapes MUST refuse. History repositories MAY be bare or +ordinary repositories and MUST remain unmodified. Every JSONL line MUST be +validated strictly; an invalid line MUST refuse affected inputs and identify +the history file, line, and reason. History writers use these line shapes: + +- `moments/*.jsonl`: one `spawnfile.moment-record.v1` object per line, any filename. +- `landings.jsonl`: `{revision, tree, commit, ref, paths, landedAt}`. + `revision` is the lowercase 64-hex feed digest; `tree` names that revision, + optionally with its re-land generation or `trees/` prefix. Directory sources + use `null` commit/ref/paths; git sources require commit/ref, with `null` or `[]` + paths denoting the whole tree. +- OPTIONAL `generations.jsonl`: `{generation, switchedAt, ledger}`, with `ledger` + relative to that bank's directory and naming its current or archived ledger. +- `snapshots.jsonl`: `{snapshot, takenAt, parent, files}`, where `parent` is a + snapshot ID or `null`; `files` maps changed relative paths to SHA-256 digests or + `null` for deletion. `objects/` contains each referenced file's bytes. + +Moment records match when `agent` equals the plan agent ID and `wake.at` is the +same parsed instant as example `at`; different offset spellings are equivalent. +Several matching records MUST agree in every field used for the selected +inputs, otherwise the moment refuses. No matching record triggers each input's +fallback rule. Fallbacks are labelled `approximate`; independent immutable pins +and landing logs retain the exact fidelity specified below. All timestamp +comparisons MUST compare instants, preserving RFC3339 nanoseconds where present, +never timestamp strings or UUID order. Boundary instants equal to `at` are included. + +### Per-input reconstruction and fidelity + +Each successful input reports `fidelity: exact | approximate | overridden`, a +`basis`, and the selected source identifiers. Explicit overrides report +`overridden` and `basis: override`. A replacement git/bundle declaration must +still supply locally available historical bytes. A replacement memory bank, +network, or volume starts fresh: an empty bank/volume, or an empty network store +with its declared rooms. A network override writes a fresh `state.json` snapshot +regardless of its declared storage backend; trial preparation consumes or imports +that snapshot. With no `--history`, only locally materializable pins +and explicit overrides can succeed; everything else refuses. + +Git resources and source-backed bundles select a full 40- or 64-hex declared +commit first (`exact`, `pinned`), otherwise the matching record's resource +commit (`exact`, `record`). Otherwise they walk the declared selector's +first-parent history and choose its newest commit with committer time `<= at` +(`approximate`, `selector-at`); an absent selector means `HEAD`. Commit dates +do not establish when a commit reached a branch or tag, so this fallback cannot +prove historical ref membership. A missing eligible commit refuses. + +Git output MUST be a clean new repository containing only objects reachable +from the selected commit, with one checked-out branch, no remotes, tags, source +packed refs, reflogs, `FETCH_HEAD`, or `ORIG_HEAD`. Git commands use argv without +a shell and sanitized configuration. History MUST NOT be mutated. The leak +check also rejects any reachable ancestor with author or committer time after +`at`. Bundle `build.files` extracts `files.root` at the selected commit and honors +`exclude`. `build.dependencies` and `build.generated` retain their source +directory or declared inputs at that commit and report `build: deferred`; +trial preparation owns builds. Bundle output MAY be a plain tree. A prebuilt +`source` tar without a record or explicit override is un-rewindable and refuses. +Generated inputs spanning multiple repositories or independently pinned input +refs cannot be represented by this one-repository/one-commit history contract +and MUST refuse. Bundle roots whose historical repository prefix cannot be +established also refuse. Bundle file trees preserve the existing compiler's +symlink refusal; excluded symlinks do not need materialization. + +Fed volumes select the newest landing with `landedAt <= at` when a landing log +exists (`exact`, `landing-log`); no eligible landing refuses. A matching record +feed entry MUST agree with that landing. Without a log, a plain git feed expands +its ref template and paths using `at`, including `${date}` in UTC and +`${date:}` in that zone. The primary ref is used only if it has an +eligible first-parent commit; otherwise the declared fallback is tried. Its +newest eligible commit is `approximate`, `feed-rule-at`. Commands, directory +feeds, and freeze/waiting behavior cannot reconstruct historical landings +without a log and MUST refuse. Ref dates alone do not capture earlier freeze +holds or when a producer made a ref available. + +Feed bytes come from OPTIONAL retained `trees//` when available. +Otherwise a plain git feed re-extracts the landing's expanded paths at its commit +and MUST verify the revision with the existing feed revision function. A mismatch +refuses. `include` or `prepare` requires retained bytes; rewind MUST NOT run them. +Unrecorded repository-local `info/attributes` also requires retained bytes; +reconstruction otherwise honors the commit's archive attributes. +Output preserves the agent-visible `trees//`, relative `current` +symlink, and `.spawnfile-feed.json` (`spawnfile.volume-feed.v1`). Its `landed_at` +is the selected landing time, or the selected commit's committer time for an +approximation, always `<= at`. + +Memory selects the generation with greatest `switchedAt <= at`, refusing if +the generation log contains none or its archived ledger is unavailable. Without +a generation log, the sole current ledger is treated as original generation 0; +a record naming another generation refuses. A matching record MUST name the +selected generation. Its `seq` selects the first `seq` ledger lines (`exact`, +`record`). Otherwise selection stops before the FIRST event after `at` +(`approximate`, `created-at-prefix`): subsequent clock regression never reopens +the cut. The prefix MUST end in LF, contain contiguous sequences from 1, and +contain only nondecreasing event times `<= at`. Legacy seq-less events are +allowed only as a leading run. Invalid sequence or time ordering refuses. + +Memory output is `memory///memory/events.jsonl`. It MUST exclude +`index.sqlite`, `causal*`, `authority-receipts.jsonl`, and `authority-receipts/`. +The installed Mneme 0.1.1 package publicly exports `createMemoryIndex`; rewind +rebuilds a temporary index from the selected ledger through this root API, +verifies it can be queried, then discards it. No deep imports are permitted. +Mneme rebuilds from its ledger before kernel searches; opening an index alone +does NOT rebuild it. Trial preparation must retain that search/rebuild behavior. +Mneme remains a development dependency: if its public rebuild API is unavailable, +historical memory materialization refuses explicitly without auto-installing it. +Missing bank history also refuses unless the bank is overridden. + +Moltnet JSON and SQLite stores remove messages, rooms, and attachment-delivery +events after `at`. Empty threads/DMs and threads/DMs whose root is after the cut +are dropped. Thread/DM message counts and last-message times are recomputed; +attachment cursors are clamped/remapped to retained events. Room removals after +`at` are undone when removal time and restorable room metadata exist; a missing +historical removal time retains current removal state. Agents and memberships +have no historical log and are retained as now, explicitly disclosed in input +`notes`: even a cursor-exact content cut cannot prove historical membership. +SQLite changes apply only to a copy through `node:sqlite` on the supported +Node >=22.19 line; if that API is unavailable, SQLite rewind refuses. RFC3339Nano +fractions with omitted trailing zeros MUST be compared as parsed instants. +A record cursor yields `exact`, `record-cursor` only if it identifies a retained +message and no retained message is later than that cursor's message. Otherwise +fidelity is `approximate`, `created-at-cut`. Missing stores refuse unless overridden. + +Unfed volumes choose the newest snapshot with `takenAt <= at`. A record MUST +name that snapshot or an ancestor-consistent exact snapshot; a valid record is +`exact`, otherwise selection is `approximate`. The full parent chain is replayed +from its root, including deletions, and every copied object is SHA-256 verified. +Missing history, missing parents/objects, or no eligible snapshot refuses. +Materialized content must equal the replayed snapshot, checked independently. + +All materialized files and directories MUST have atime/mtime `<= at`, using the +source commit, landing, event, or snapshot instant capped at `at`. Symlink times +MUST be set with `lutimes`; following a symlink to set times is insufficient. + +### Task clock + +`task-clock.json` is strict `spawnfile.task-clock.v1` with `{version, origin}`; +`origin` is the example instant normalized to ISO-8601 with `Z`. The pure helper +`taskClockEnvironment(origin, anchorEpochMs)` accepts a caller-supplied safe +integer real launch anchor in milliseconds and returns these string variables: + +| Variable | Value | +| --- | --- | +| `NOOPOLIS_TASK_CLOCK` | Compact JSON `{"version":"noopolis.task-clock.v1","origin":"","anchorEpochMs":}` | +| `MNEME_CLOCK_ORIGIN` | The same normalized origin | +| `MNEME_CLOCK_ANCHOR_MS` | Decimal anchor milliseconds | +| `FAKETIME` | `@YYYY-MM-DD HH:MM:SS`, expressed in UTC | +| `FAKETIME_DONT_FAKE_MONOTONIC` | `1` | + +Daimon consumes the cross-repository `noopolis.task-clock.v1` contract: reported +agent wall time is `origin + (realNow - anchorEpochMs)`. Mneme consumes its generic +clock variables. The libfaketime values take effect only in capable images with +`LD_PRELOAD` configured by trial preparation. The clock offsets reported wall +time; real timeouts and monotonic elapsed time MUST remain real. Rewind neither +sleeps on nor advances a synthetic scheduler. Trial preparation owns the anchor +and runtime environment; the immutable bundle carries only the origin. + +### Independent leak check and output + +The leak checker MUST re-read the bundle from disk independently of materializer +state. Its strict `spawnfile.leak-check.v1` result contains `{version, at, ok, +checked, findings}`; counters cover files, git objects, memory events, Moltnet +messages/events, and snapshots. Each finding carries `{input, class, path, +detail}`; `input` is an input identity or `null` for bundle-level problems. +Every finding refuses the moment: + +| Class | Refusal condition | +| --- | --- | +| `file-time` | A file, directory, or symlink mtime is after `at`. | +| `symlink-escape` | An absolute link or a lexically/physically escaping link resolves outside the bundle. | +| `git-object` | Any commit author/committer time is after `at`, any object is unreachable from the selected commit, or refs/remotes/reflogs/fetch metadata exceed the single branch. | +| `memory-event` | Future ledger events, sequence gaps, invalid prefix ordering, or derived/audit files are present. | +| `moltnet-message` / `moltnet-event` / `moltnet-room` | Retained messages, delivery events, or rooms are after `at`. | +| `feed-identity` | `landed_at` is after `at` or `current` does not target a tree inside the volume. | +| `dated-path` | A path segment contains a calendar date `YYYY-MM-DD` later than the UTC date of `at + 14h`, the latest possible local date. | +| `snapshot` | An unfed volume differs from its selected replayed snapshot. | + +`OUT` MUST be absent or an empty directory. The materializer builds in a sibling +staging directory and renames into place only after the disk leak check passes: + +```text +OUT/ + moment.json + leak-check.json + task-clock.json + resources///... + memory///memory/events.jsonl + moltnet///... +``` + +`moment.json` is strict `spawnfile.materialized-moment.v1`: status, agent, +`example: {id, role, dataset}`, normalized `at`, sorted inputs, `record: {matched}`, +and `leakCheck: {ok, findings}`. Inputs sort by section, declaring node, then ID; +each reports kind, fidelity (or `null` on refusal), basis, source identifiers, +relative output path (or `null`), notes, and refusal reason (or `null`). Deferred +bundle builds additionally report `build: deferred`. + +On any input refusal or leak, staging is removed and `OUT` contains ONLY +`moment.json` with `status: refused`, all refusal reasons, and `leak-check.json` +if the check ran. Partial input data and task-clock files MUST NOT remain usable. +Success prints `moment.json` to stdout as JSON and exits 0. Refusal emits a +one-line stderr summary and exits 1. Invalid flags, projects, or example +selections can fail before a moment exists and need not create a report. +Generated schemas for the three bundle documents appear below; schema-sync +tests MUST keep them identical to the exported runtime contracts. + ## Public handoff The child invocation is `paideia train --spawnfile-context FILE` followed by the @@ -1318,3 +1518,411 @@ The child deadline is Paideia's declared deadline plus five seconds for cleanup. } ``` + +## Materialized moment JSON Schema + + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "spawnfile.materialized-moment.v1" + }, + "status": { + "type": "string", + "enum": [ + "materialized", + "refused" + ] + }, + "agent": { + "type": "string", + "minLength": 1 + }, + "example": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "role": { + "type": "string", + "enum": [ + "train", + "validation", + "test" + ] + }, + "dataset": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "id", + "role", + "dataset" + ], + "additionalProperties": false + }, + "at": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "inputs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "section": { + "type": "string", + "enum": [ + "resources", + "memory", + "networks" + ] + }, + "id": { + "type": "string", + "minLength": 1 + }, + "declaredBy": { + "type": "string", + "minLength": 1 + }, + "kind": { + "type": "string", + "enum": [ + "git", + "bundle", + "feed", + "volume", + "memory", + "moltnet" + ] + }, + "fidelity": { + "anyOf": [ + { + "type": "string", + "enum": [ + "exact", + "approximate", + "overridden" + ] + }, + { + "type": "null" + } + ] + }, + "basis": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "object", + "properties": { + "commit": { + "type": "string", + "minLength": 1 + }, + "revision": { + "type": "string", + "minLength": 1 + }, + "seq": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "generation": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "snapshot": { + "type": "string", + "minLength": 1 + }, + "cursor": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "path": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "null" + } + ] + }, + "notes": { + "type": "array", + "items": { + "type": "string" + } + }, + "refusal": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "null" + } + ] + }, + "build": { + "type": "string", + "const": "deferred" + } + }, + "required": [ + "section", + "id", + "declaredBy", + "kind", + "fidelity", + "basis", + "source", + "path", + "notes", + "refusal" + ], + "additionalProperties": false + } + }, + "record": { + "type": "object", + "properties": { + "matched": { + "type": "boolean" + } + }, + "required": [ + "matched" + ], + "additionalProperties": false + }, + "leakCheck": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "findings": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "ok", + "findings" + ], + "additionalProperties": false + } + }, + "required": [ + "version", + "status", + "agent", + "example", + "at", + "inputs", + "record", + "leakCheck" + ], + "additionalProperties": false +} +``` + + +## Leak check JSON Schema + + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "spawnfile.leak-check.v1" + }, + "at": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "ok": { + "type": "boolean" + }, + "checked": { + "type": "object", + "properties": { + "files": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "gitObjects": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "memoryEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "moltnetMessages": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "moltnetEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "snapshots": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "files", + "gitObjects", + "memoryEvents", + "moltnetMessages", + "moltnetEvents", + "snapshots" + ], + "additionalProperties": false + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "input": { + "anyOf": [ + { + "type": "object", + "properties": { + "section": { + "type": "string", + "enum": [ + "resources", + "memory", + "networks" + ] + }, + "id": { + "type": "string", + "minLength": 1 + }, + "declaredBy": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "section", + "id", + "declaredBy" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "class": { + "type": "string", + "enum": [ + "file-time", + "symlink-escape", + "git-object", + "memory-event", + "moltnet-message", + "moltnet-event", + "moltnet-room", + "feed-identity", + "dated-path", + "snapshot" + ] + }, + "path": { + "type": "string", + "minLength": 1 + }, + "detail": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "input", + "class", + "path", + "detail" + ], + "additionalProperties": false + } + } + }, + "required": [ + "version", + "at", + "ok", + "checked", + "findings" + ], + "additionalProperties": false +} +``` + + +## Task clock JSON Schema + + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "spawnfile.task-clock.v1" + }, + "origin": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "version", + "origin" + ], + "additionalProperties": false +} +``` + diff --git a/src/cli/AGENTS.md b/src/cli/AGENTS.md index d280c66c..43e96d65 100644 --- a/src/cli/AGENTS.md +++ b/src/cli/AGENTS.md @@ -13,7 +13,7 @@ src/cli/ ├── composedLifecycleContractSet.ts # Closed machine command/contract inventory ├── evidenceExportHelperCommand.ts # Local helper construction command ├── compileBuildCommands.ts # `compile` and `build` command registration -├── trainCommand.ts # Canonical agent selection and Paideia CLI option forwarding +├── trainCommand.ts # Canonical plan/rewind selection and Paideia CLI option forwarding ├── paideiaDelegation.ts # Private versioned context handoff, child lifecycle and completion receipts ├── paideiaSupervisor.ts # Packaged group leader retaining identity until native child/group cleanup ├── lifecycleCommands.ts # Thin lifecycle/compile/build/run/publish/up/down registration composition diff --git a/src/cli/runCli.ts b/src/cli/runCli.ts index fd97c30e..9f96f3fa 100644 --- a/src/cli/runCli.ts +++ b/src/cli/runCli.ts @@ -24,7 +24,7 @@ import { clearProjectModelFallbacks, compileProject, createTrainingContext, - createTrainingPlan, + createTrainingPlan, loadTrainingProject, resolveTrainingPlan, materializeTrainingMoment, initProject, listInitTemplates, preflightCompilePlan, @@ -77,9 +77,7 @@ import { delegatePaideiaTraining } from "./paideiaDelegation.js"; const packageJsonPath = new URL("../../package.json", import.meta.url); const readPackageVersion = (): string => { - const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8")) as { - version?: unknown; - }; + const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8")) as { version?: unknown }; return typeof packageJson.version === "string" ? packageJson.version : "0.0.0"; }; @@ -105,6 +103,8 @@ const createDefaultRenderEnvironment = (): CliRenderEnvironment => ({ export interface CliHandlers { createTrainingContext: typeof createTrainingContext; createTrainingPlan: typeof createTrainingPlan; + loadTrainingProject: typeof loadTrainingProject; resolveTrainingPlan: typeof resolveTrainingPlan; + materializeTrainingMoment: typeof materializeTrainingMoment; delegatePaideiaTraining: typeof delegatePaideiaTraining; buildCompilePlan: typeof buildCompilePlan; buildOrganizationView: typeof buildOrganizationView; buildProject: typeof buildProject; compileProject: typeof compileProject; @@ -139,7 +139,7 @@ export interface CliHandlers { } const createDefaultHandlers = (): CliHandlers => ({ - createTrainingContext, createTrainingPlan, delegatePaideiaTraining, + createTrainingContext, createTrainingPlan, loadTrainingProject, resolveTrainingPlan, materializeTrainingMoment, delegatePaideiaTraining, buildCompilePlan, buildOrganizationView, buildProject, compileProject, publishProject, addAgentProject, addProjectModelFallback, addProjectSurface, addSubagentProject, addTeamProject, clearProjectModelFallbacks, diff --git a/src/cli/trainCommand.test.ts b/src/cli/trainCommand.test.ts index e402a640..26942102 100644 --- a/src/cli/trainCommand.test.ts +++ b/src/cli/trainCommand.test.ts @@ -5,9 +5,13 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { runCli, type CliHandlers } from "./runCli.js"; import type { DelegatePaideiaTrainingOptions } from "./paideiaDelegation.js"; +import { + agentManifest, cleanTrainingProjects, dataFiles, example, trainingProject, volume +} from "../compiler/training/plan/fixtures.test-helper.js"; const directories: string[] = []; afterEach(async () => { await Promise.all(directories.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); }); +afterEach(cleanTrainingProjects); async function project(): Promise { const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "spawnfile-train-command-"))); directories.push(root); @@ -24,6 +28,120 @@ const emptyPlan = (root: string): Awaited { + it.each([example.id, `train:${example.id}`])("materializes %s locally without delegated training flags", async (selector) => { + const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest() }); + const out = path.join(root, "rewound"), stdout: string[] = [], stderr: string[] = []; + const forbidden = vi.fn(async () => { throw Error("must not execute, authenticate, or compile output"); }); + const code = await runCli(["train", path.join(root, "Spawnfile"), "--agent", "agent:writer", + "--materialize", selector, "--out", out], { + handlers: { createTrainingContext: forbidden, delegatePaideiaTraining: forbidden, compileProject: forbidden, + buildProject: forbidden, runProject: forbidden, upProject: forbidden, publishProject: forbidden, + importClaudeCodeAuth: forbidden, importCodexAuth: forbidden, importGrokAuth: forbidden, + requireAuthProfile: forbidden, provisionCredentials: forbidden }, + streams: { stdout: (value) => stdout.push(value), stderr: (value) => stderr.push(value) } + }); + expect(code).toBe(0); + expect(stderr).toEqual([]); + expect(stdout).toHaveLength(1); + const moment = JSON.parse(stdout[0]!); + expect(moment).toMatchObject({ status: "materialized", agent: "agent:writer", + example: { id: example.id, role: "train", dataset: "training/train.yaml" }, inputs: [], + record: { matched: false }, leakCheck: { ok: true, findings: 0 } }); + expect(JSON.parse(await readFile(path.join(out, "moment.json"), "utf8"))).toEqual(moment); + expect(JSON.parse(await readFile(path.join(out, "task-clock.json"), "utf8"))).toEqual({ + version: "spawnfile.task-clock.v1", origin: "2026-10-09T08:20:31Z" + }); + expect(forbidden).not.toHaveBeenCalled(); + }); + + it("lists ambiguous roles and materializes a qualified selection", async () => { + const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest(), + "training/test.yaml": { examples: [example] } }); + const out = path.join(root, "rewound"), stderr: string[] = []; + const options = { streams: { stdout() {}, stderr: (value: string) => stderr.push(value) } }; + const args = ["train", root, "--agent", "agent:writer", "--out", out, "--materialize"]; + expect(await runCli([...args, example.id], options)).toBe(1); + expect(stderr).toHaveLength(1); + expect(stderr[0]).toContain(`train:${example.id} (training/train.yaml)`); + expect(stderr[0]).toContain(`test:${example.id} (training/test.yaml)`); + expect(await readdir(root)).not.toContain("rewound"); + expect(await runCli([...args, `test:${example.id}`], options)).toBe(0); + expect(JSON.parse(await readFile(path.join(out, "moment.json"), "utf8")).example.role).toBe("test"); + }); + + it("retains the plan's single-agent inference when --agent is omitted", async () => { + const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest() }); + const stdout: string[] = []; + expect(await runCli(["train", root, "--materialize", example.id, "--out", path.join(root, "out")], { + streams: { stdout: (value) => stdout.push(value), stderr() {} } + })).toBe(0); + expect(JSON.parse(stdout[0]!).agent).toBe("agent:writer"); + }); + + it("forwards a loaded project, resolved plan, history and the existing --out option", async () => { + const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest() }); + const materializeTrainingMoment = vi.fn(async ({ plan }) => ({ + version: "spawnfile.materialized-moment.v1", status: "materialized", agent: plan.agent.id, + example: { id: example.id, role: "train", dataset: "training/train.yaml" }, at: example.at, + inputs: [], record: { matched: false }, leakCheck: { ok: true, findings: 0 } + })); + const stdout: string[] = []; + expect(await runCli(["train", root, "--agent", "agent:writer", "--materialize", example.id, + "--history", "history with spaces", "--out", "out with spaces"], { + handlers: { materializeTrainingMoment }, streams: { stdout: (value) => stdout.push(value), stderr() {} } + })).toBe(0); + expect(materializeTrainingMoment).toHaveBeenCalledOnce(); + expect(materializeTrainingMoment.mock.calls[0]![0]).toMatchObject({ + project: { root, manifest: path.join(root, "Spawnfile") }, + plan: { agent: { id: "agent:writer" }, producer: { version: packageVersion } }, + example: example.id, historyDir: "history with spaces", out: "out with spaces" + }); + expect(JSON.parse(stdout[0]!).status).toBe("materialized"); + }); + + it("returns one-line stderr and exit 1 for a refused moment with report-only output", async () => { + const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest({ + workspace: { docs: { system: "AGENTS.md" }, resources: [volume] } + }) }); + const out = path.join(root, "refused"), stdout: string[] = [], stderr: string[] = []; + expect(await runCli(["train", root, "--agent", "agent:writer", "--materialize", example.id, "--out", out], { + streams: { stdout: (value) => stdout.push(value), stderr: (value) => stderr.push(value) } + })).toBe(1); + expect(stdout).toEqual([]); + expect(stderr).toHaveLength(1); + expect(stderr[0]).toMatch(/^error: Moment refused: 1 input refusal\(s\), 0 leak finding\(s\)/u); + expect(stderr[0]).not.toContain("\n"); + expect(await readdir(out)).toEqual(["moment.json"]); + expect(JSON.parse(await readFile(path.join(out, "moment.json"), "utf8"))).toMatchObject({ + status: "refused", inputs: [{ id: volume.id, refusal: expect.any(String) }] + }); + expect((await readdir(root)).filter((name) => name.includes("staging"))).toEqual([]); + }); + + it.each([ + ["--materialize", example.id, "--agent", "agent:writer"], + ["--materialize", example.id, "--agent", "agent:writer", "--out", "output", "--plan"], + ["--history", "history", "--plan"] + ])("rejects incomplete or conflicting local mode options %j before reads", async (...flags) => { + const forbidden = vi.fn(async () => { throw Error("must not read project"); }); + expect(await runCli(["train", "/missing", ...flags], { + handlers: { loadTrainingProject: forbidden, createTrainingPlan: forbidden, materializeTrainingMoment: forbidden, + createTrainingContext: forbidden, delegatePaideiaTraining: forbidden }, streams: { stdout() {}, stderr() {} } + })).toBe(2); + expect(forbidden).not.toHaveBeenCalled(); + }); + + it("registers --out exactly once alongside --materialize and --history", async () => { + const stdout: string[] = []; + expect(await runCli(["train", "--help"], { streams: { stdout: (value) => stdout.push(value), stderr() {} } })).toBe(0); + const help = stdout.join("\n"); + expect(help.match(/--out /gu)).toHaveLength(1); + expect(help).toContain("--materialize "); + expect(help).toContain("--history "); + }); +}); + describe("spawnfile train --plan", () => { it.each([false, true])("prints the real authoring plan without execution with dryRun=%s", async (dryRun) => { const flags = dryRun ? ["--dry-run"] : []; diff --git a/src/cli/trainCommand.ts b/src/cli/trainCommand.ts index 7e5fb989..df9cc8b8 100644 --- a/src/cli/trainCommand.ts +++ b/src/cli/trainCommand.ts @@ -17,13 +17,15 @@ export const registerTrainCommand = ( signal?: AbortSignal ): void => { const command = program.command("train") - .description("Train one canonical agent through Paideia's isolated native integration") + .description("Plan, rewind, or train one canonical agent through its declared training inputs") .argument("[path]", "Canonical project directory or Spawnfile path", process.cwd()) .option("--agent ", "Exact canonical agent node id (inferred only for a single-agent project)") .option("--paideia-command ", "Installed Paideia executable; no shell or automatic install", "paideia") .option("--training-image ", "Pinned image containing the complete training environment") .option("--training-config ", "V2 image recipe and declared inputs, or advanced v1 local bindings") .option("--plan", "Print the resolved training authoring plan without executing training") + .option("--materialize ", "Rewind an authored example into --out without running training") + .option("--history ", "Local recorded history for --materialize") .option("--dry-run", "Validate and estimate without compiling, authenticating or starting models") .option("--repair-measurements ", "Fork captured work into a fresh output for authorized measurement repair") .option("--repair-witness ", "Explicit verified image witness for a legacy parent") @@ -31,9 +33,32 @@ export const registerTrainCommand = ( for (const name of forwarded) { const flag = `--${name} `; if (repeated.has(name)) command.option(flag, `Paideia ${name}; repeatable`, (value: string, previous: string[]) => [...previous, value], []); - else command.option(flag, `Paideia ${name}`); + else command.option(flag, name === "out" ? "Output directory for training or moment materialization" : `Paideia ${name}`); } command.action(async (inputPath: string, options: Record) => { + if (typeof options.materialize === "string") { + if (options.plan === true) { + throw new SpawnfileError("validation_error", "--materialize and --plan are mutually exclusive"); + } + if (typeof options.out !== "string") { + throw new SpawnfileError("validation_error", "--materialize requires --out"); + } + const project = await handlers.loadTrainingProject(inputPath); + const plan = handlers.resolveTrainingPlan(project, options.agent as string | undefined, packageVersion); + const moment = await handlers.materializeTrainingMoment({ plan, project, example: options.materialize, + historyDir: options.history as string | undefined, out: options.out }); + if (moment.status === "refused") { + streams.stderr(`error: Moment refused: ${moment.inputs.filter((input) => input.refusal !== null).length} input refusal(s), ${moment.leakCheck.findings} leak finding(s); see moment.json in --out`); + setExitCode(1); + } else { + streams.stdout(JSON.stringify(moment, null, 2)); + setExitCode(0); + } + return; + } + if (options.history !== undefined) { + throw new SpawnfileError("validation_error", "--history requires --materialize"); + } if (options.plan === true) { const plan = await handlers.createTrainingPlan(inputPath, { agent: options.agent as string | undefined, packageVersion From cecf3ae7142fd409a5934cdc681aa8fdee575743 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sun, 11 Oct 2026 03:02:42 +0200 Subject: [PATCH 3/9] fix: extract raw git blobs, refuse stray repositories and moving recorded commits, drop the mneme runtime import --- specs/TRAINING.md | 212 +++++------------ src/compiler/training/rewind/AGENTS.md | 10 +- src/compiler/training/rewind/contract.ts | 7 +- src/compiler/training/rewind/feed.test.ts | 10 +- src/compiler/training/rewind/feed.ts | 16 +- src/compiler/training/rewind/git.test.ts | 10 +- src/compiler/training/rewind/git.ts | 20 +- .../training/rewind/gitArchive.test.ts | 70 ++++++ src/compiler/training/rewind/gitArchive.ts | 80 +------ src/compiler/training/rewind/gitTree.ts | 16 +- src/compiler/training/rewind/history.test.ts | 2 +- .../training/rewind/leakCheck.test.ts | 59 ++++- src/compiler/training/rewind/leakCheck.ts | 63 ++--- src/compiler/training/rewind/leakFiles.ts | 47 +++- .../training/rewind/leakRepositories.test.ts | 59 +++++ .../training/rewind/materialize.test.ts | 52 +++- src/compiler/training/rewind/materialize.ts | 21 +- src/compiler/training/rewind/memory.test.ts | 15 +- src/compiler/training/rewind/memory.ts | 32 +-- .../training/rewind/memoryPackage.test.ts | 84 +++++++ src/compiler/training/rewind/memorySchemas.ts | 18 ++ .../training/rewind/recordedCommit.test.ts | 223 ++++++++++++++++++ .../training/rewind/recordedCommit.ts | 10 + src/compiler/training/rewind/select.ts | 5 +- src/compiler/training/rewind/types.ts | 1 + 25 files changed, 797 insertions(+), 345 deletions(-) create mode 100644 src/compiler/training/rewind/gitArchive.test.ts create mode 100644 src/compiler/training/rewind/leakRepositories.test.ts create mode 100644 src/compiler/training/rewind/memoryPackage.test.ts create mode 100644 src/compiler/training/rewind/recordedCommit.test.ts create mode 100644 src/compiler/training/rewind/recordedCommit.ts diff --git a/specs/TRAINING.md b/specs/TRAINING.md index aa523959..76d1ac70 100644 --- a/specs/TRAINING.md +++ b/specs/TRAINING.md @@ -163,151 +163,32 @@ with zero recording cost when off. `record.keep` is snapshot retention. { "version": "spawnfile.moment-record.v1", "agent": "agent:writer", - "wake": { - "at": "2026-10-09T08:20:31.000Z", - "startedAt": "2026-10-09T08:20:31.250Z", - "executionId": "execution-17", - "deliveries": [{ "deliveryId": "moltnet:42", "kind": "message", "occurredAt": "2026-10-09T08:20:31.000Z" }], - "surface": "moltnet", "messageId": "42" - }, - "recordedAt": "2026-10-09T08:20:32.000Z", - "exact": true, - "resources": [ - { "id": "archive", "kind": "git", "commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, - { "id": "tools", "kind": "bundle", "sha256": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" } - ], - "feeds": [{ "id": "news", "revision": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "tree": "trees/cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc.1", "commit": "dddddddddddddddddddddddddddddddddddddddd", "ref": "data/2026-10-09", "paths": ["published"], "landedAt": "2026-10-09T08:00:00.000Z", "period": "2026-10-09" }], - "memory": [{ "id": "self", "declaredBy": "agent:writer", "seq": 42 }], - "moltnet": [{ "network": "news", "cursor": "2026-10-09T08:20:31.250Z", "messageIds": ["42"] }], - "volumes": [{ "id": "edition-state", "snapshot": "incremental-snapshot-17" }] + "wake": { "at": "2026-10-09T08:20:31Z", "surface": "moltnet", "room": "assignment", "from": "brass", "messageId": "42" }, + "recordedAt": "2026-10-09T08:20:32Z", + "resources": [{ "id": "archive", "kind": "git", "commit": "0123456789abcdef0123456789abcdef01234567" }], + "feeds": [{ "id": "news", "revision": "revision-id", "tree": "tree-id", "commit": "0123456789abcdef0123456789abcdef01234567", "paths": ["published"], "landedAt": "2026-10-09T08:00:00Z" }], + "memory": [{ "id": "self", "declaredBy": "agent:writer", "seq": 42, "generation": 3 }], + "moltnet": [{ "network": "news", "cursor": "wake-cursor" }], + "volumes": [{ "id": "edition-state", "snapshot": "incremental-snapshot-id" }] } ``` -### Host histories and retention - -`record.keep` bounds landing logs, deploy logs, snapshots and moment records; omission -means **30d**. A shared history MUST use the maximum effective keep among its recording -consumers (all recording agents for the deploy log). History-log pruning MUST keep entries newer -than `now - keep` AND the newest entry at or before that cutoff, even across a long gap. -Lookups are inclusive; equal timestamps use the last appended entry. Before the first -entry, lookup returns no state. Invalid lines MUST be reported alongside valid entries. - -For each fed volume consumed by a recording agent, the host MUST append -`landings.jsonl` beside `landed.json` after each successful land, generation re-land, -retained-tree re-point or link repair. Provenance/period adoption MUST also be logged. -Before ANY replacement of `landed.json` (including revision advances, re-points, -repairs and adoption), the publisher MUST reconcile its previous recording snapshot -against the last log entry and append it if missing, preserving the original instant. -A failed reconciliation MUST prevent replacement. A clean refresh also seeds history -when recording is enabled later. Recorded host state retains the original source and -keep window so a changed source declaration cannot rewrite pending provenance. -The entry MUST follow a successful `landed.json` write and contain: -`{version:"spawnfile.volume-feed-landing.v1", resource, volume, revision, tree, -source, commit?, ref?, paths?, switching_at, landed_at, period?}`. `switching_at` -MUST be sampled immediately BEFORE the `current` rename, including link repair; -`landed_at` remains the post-switch instant and MUST equal -`identity.landed_at`; `period` is the host record's period. `source` MUST identify -`{kind:"git", repo}` or `{kind:"directory", directory}` and preserve any `prepared` -provenance. Git entries MUST carry the resolved commit and expanded requested ref; -`paths`, when present, MUST be the expanded git paths. Commit, paths and revision -permit refetch and verification without retaining trees. Directory feeds are verifiable -by revision digest but **not refetchable**. Prepared feeds also require their recorded -include digests and recipe inputs; a revision does not promise reproducible prepare output. -Lookup inside either endpoint of `[switching_at, landed_at]` MUST return the incoming -entry plus `ambiguous:true`. Outside transition windows, select the most recent -`landed_at <= instant`. Adoption without a content switch uses `switching_at = landed_at`; -old nonrecorded identities seeded on opt-in use the same degenerate window. Recording -history missing a required transition boundary is invalid, never assumed exact. - -When any compiled agent records, project `up` and home deployment publication -(including image `up` and release) MUST append a resource log beside their record: -`/deployments/.resource-log.jsonl` for project records, and -`~/.spawnfile/deployments//resource-log.jsonl` for home records. `deployment_id` -is the absolute canonical record path; lookup MUST require that identity. Independent -outputs with the same deployment name MUST NOT share history or retention. - -Each `{version:"spawnfile.deploy-resources.v1", deployment, deployment_id, activated_at, -activation_exact, activation_note?, deployed_at, report_sha256, resources, recording_agents}` entry MUST -reuse `workspace_git_resources` and `workspace_bundles`, without resolving sources again. -`recording_agents` MUST copy the validated compiled recording facts for every recording -agent: node ID, slug, recording directory, keep window, snapshot IDs, consumed feed IDs -and volume names, git/bundle resource IDs, memory bank IDs with declaring node and ledger -path, and attached Moltnet networks. These facts survive releases that remove or change -inputs. Legacy entries without them remain readable but cannot produce exact moments. -Every resource MUST retain its workspace-visible `id`, compiled `agent_node_ids`, and -`declared_by` node ID when known. Git adds `{kind:"git", commit, sha256, url?}`; bundles -add `{kind:"bundle", sha256, origin?}`. Ownership MUST survive compile and distribution -projection, including equal IDs/digests with different commits. Image pins omit host -paths and source URLs. Unowned legacy history MUST be reported as invalid, never guessed. -With no effective recording agent, compile reports MUST omit `agent_node_ids` and -`declared_by` and preserve the original resource deduplication and serialized shape. - -Lookups and retention MUST use `activated_at`; `deployed_at` is informational record -creation time. Activation ordering is path-specific: - -- Image `up` renames/stops the previous container, verifies it stopped, then samples - `activated_at` immediately before `docker run`. It waits for a later host-clock tick - than the stopped observation, so no old turn can have an equal timestamp. -- Drained release first closes admission and waits for old turns, then calls image `up` - under the home lock; the same stop/verify/activation/start sequence applies. -- Project `up` samples immediately before invoking its launch runner. First deployment - has no recorded predecessor. Replacement does not stop/drain its predecessor and MAY - overlap it: entries MUST set `activation_exact:false` with `activation_note`. Records - without a captured launch boundary (including recovered launches) are likewise inexact. - -If a started candidate is removed on readiness or publication failure, image `up`, -release and project `up` MUST append its pins with `{activated_at, rolled_back_at, -exact:false, activation_exact:false}`. `rolled_back_at` is sampled after candidate -removal, before restoration where applicable. This interval MUST NOT replace the -predecessor's authoritative record. Lookup inside the inclusive interval returns the -candidate with `exact:false`, even if it previously published a successful-looking -snapshot; after the interval, that snapshot is excluded and the predecessor applies -until another deployment activates. Retention MUST preserve intervals crossing the -cutoff and the predecessor needed after rollback. No candidate history is written when -recording is off. A failed candidate cleanup MUST NOT invent a completed rollback. -These interval entries describe execution and removal, not successful publication; -their append follows removal without requiring a new authoritative deployment record. - -The host clock MUST remain monotonic across these operations. Containers MUST share -that clock; remote clock skew invalidates exact reconstruction. A moment arriving before -publication MUST wait for its authoritative deployment record/history, or be inexact. - -The deployment record MUST persist `recording: {keep_ms, memory_mount_ids}` and its -complete `resource_snapshot`, including the hash of the exact persisted UTF-8 report -bytes. Before replacing a deployment record OR its cached report, the publisher MUST -reconcile that snapshot against the last log entry. Reconciliation failure MUST abort -replacement. Recompilation MUST preserve deployment records/history, including clean -compiles; it cannot erase a pending snapshot. Retried publication of the same snapshot -MUST NOT append a duplicate. Turning recording off still reconciles an owed predecessor. -Writers MUST receive predecessor metadata from the lifecycle's existing record read; -only effective recording or a recorded predecessor permits additional record reads. - -Host records and cached deployment reports MUST be published by file fsync, rename, -and parent-directory fsync BEFORE log append. Both logs MUST append each complete JSON -line in one write followed by file and directory fsync. Pruning MUST use an exclusive -0600 temporary file, file fsync, rename and directory fsync under a writer lock. -With recording off and no recorded predecessor, neither log nor its lock may be touched. -Host-record/report writes in that case MUST retain their original write/rename sequence, -permissions and bytes, without added reads, durable writes or fsyncs. A matching last -log line MUST still be file- and directory-fsynced before reconciliation succeeds: -visibility alone does not prove a previous failed append became durable. -A deploy-log failure after publication MUST be reported without rolling back the -container named by the published deployment record. -If record rename succeeds but directory fsync fails, the named container MUST also -remain; no log entry may be appended until the authoritative record is re-synced. - -Spawnfile MUST enforce memory protection from the DEPLOYED record's `recording` -metadata, independent of later compile settings. For recorded state, teardown with volume deletion MUST -refuse a changed report fingerprint, missing protected mount, reduced keep, or dropped -recording/mount protection unless explicitly overridden. `down --volumes --discard-recorded-memory` -is that override; `--force` alone is insufficient. Author-declared shared volume names -remain retained under the existing teardown rule. Internal fixture cleanup MUST apply -the same guard. Spawnfile has no memory-ledger reset/prune command; release prunes only -images. This guarantee covers Spawnfile's deletion paths, not external Docker commands. -Mneme stays independent and its ledger MUST NOT be truncated by recording retention. - -Rewind MUST use records when present; otherwise it is best effort and the trial MUST -be labelled as such. +`agent` is the exact node ID. `wake.at` MUST be the real wake time (not the dataset's +authored trial time); `recordedAt` is capture time. Both are instants with an offset or +`Z`; `messageId` is OPTIONAL. Resources identify the git/bundle resolved commit per +build/deploy. Every recorded `resources[].commit`, `feeds[].commit`, and non-null +landing-log `commit` MUST be a full 40- or 64-hex object ID, normalized to lowercase. +Moving selectors and abbreviated IDs refuse. For every non-overridden input, +rewind MUST verify matching moment-record IDs and every non-null landing-log ID +as commit objects in that input's history repository; tree/blob IDs and annotated +tag objects refuse without peeling. This validation also applies to pinned +resources, feed-rule fallback, and retained feed trees; it does not change which +valid commit takes selection precedence. +Feeds identify the fed-volume landing-log entry in effect at the wake. +Memory pins the Mneme ledger sequence and active curation generation for each declared +bank; Moltnet pins each network's per-wake message cursor. Volumes identify an incremental +snapshot for every effective `record.volumes` entry. Rewind MUST use records when present; +otherwise it is best effort and the trial MUST be labelled as such. ## Rewind @@ -384,7 +265,10 @@ the history file, line, and reason. History writers use these line shapes: Moment records match when `agent` equals the plan agent ID and `wake.at` is the same parsed instant as example `at`; different offset spellings are equivalent. Several matching records MUST agree in every field used for the selected -inputs, otherwise the moment refuses. No matching record triggers each input's +inputs, otherwise the moment refuses. Every matching record's commit IDs for each +non-overridden input MUST pass history-object validation, including fields omitted +from selection comparisons for pinned resources. An invalid candidate refuses +its affected input. No matching record triggers each input's fallback rule. Fallbacks are labelled `approximate`; independent immutable pins and landing logs retain the exact fidelity specified below. All timestamp comparisons MUST compare instants, preserving RFC3339 nanoseconds where present, @@ -423,8 +307,11 @@ trial preparation owns builds. Bundle output MAY be a plain tree. A prebuilt Generated inputs spanning multiple repositories or independently pinned input refs cannot be represented by this one-repository/one-commit history contract and MUST refuse. Bundle roots whose historical repository prefix cannot be -established also refuse. Bundle file trees preserve the existing compiler's -symlink refusal; excluded symlinks do not need materialization. +established also refuse. Feed and bundle extraction MUST use raw tree/blob objects +(`git ls-tree -r -z` and `git cat-file`, with `GIT_ATTR_NOSYSTEM=1`), without +attribute-driven transforms, `export-subst`, or `export-ignore` semantics. +Symlinks MUST be recreated as symlinks and pass the independent escape check; +gitlink/submodule entries MUST refuse with an explicit unsupported-history error. Fed volumes select the newest landing with `landedAt <= at` when a landing log exists (`exact`, `landing-log`); no eligible landing refuses. A matching record @@ -441,8 +328,8 @@ Feed bytes come from OPTIONAL retained `trees//` when available. Otherwise a plain git feed re-extracts the landing's expanded paths at its commit and MUST verify the revision with the existing feed revision function. A mismatch refuses. `include` or `prepare` requires retained bytes; rewind MUST NOT run them. -Unrecorded repository-local `info/attributes` also requires retained bytes; -reconstruction otherwise honors the commit's archive attributes. +Raw blob extraction ignores repository-local `info/attributes` as well as committed +archive attributes; later tags cannot change historical file bytes. Output preserves the agent-visible `trees//`, relative `current` symlink, and `.spawnfile-feed.json` (`spawnfile.volume-feed.v1`). Its `landed_at` is the selected landing time, or the selected commit's committer time for an @@ -461,13 +348,24 @@ allowed only as a leading run. Invalid sequence or time ordering refuses. Memory output is `memory///memory/events.jsonl`. It MUST exclude `index.sqlite`, `causal*`, `authority-receipts.jsonl`, and `authority-receipts/`. -The installed Mneme 0.1.1 package publicly exports `createMemoryIndex`; rewind -rebuilds a temporary index from the selected ledger through this root API, -verifies it can be queried, then discards it. No deep imports are permitted. -Mneme rebuilds from its ledger before kernel searches; opening an index alone -does NOT rebuild it. Trial preparation must retain that search/rebuild behavior. -Mneme remains a development dependency: if its public rebuild API is unavailable, -historical memory materialization refuses explicitly without auto-installing it. +Spawnfile validates the ledger structurally without importing Mneme: every line +MUST be a JSON event with an instant `createdAt`, and duplicate JSON keys refuse. +Sequence, leading legacy run, LF termination, and prefix time ordering are checked +locally. Event checksums are retained but not verified: Mneme 0.1.1 hashes +`JSON.stringify(event)` (`node_modules/@noopolis/mneme/dist/identity/ids.js:15-17`) +before the persisted event normalizes/defaults fields and injects its sequence +(`node_modules/@noopolis/mneme/dist/store/store.js:249-274`); the original input +and its property order cannot be recovered from the ledger alone. + +In installed Mneme 0.1.1, the index is rebuilt from the ledger before every index +read: embedding recall at `node_modules/@noopolis/mneme/dist/runtime/runtime.js:86-87`, +`memory.search` at `node_modules/@noopolis/mneme/dist/kernel/kernel.js:152-153`, +and `memory.locate` at `node_modules/@noopolis/mneme/dist/kernel/kernel.js:207-208`. +These sites read the store then call `this.index.rebuildFromEvents(allEvents)`. +Opening an index alone does NOT rebuild it. A materialized bank without +`index.sqlite` is complete because this index is derived state; trial preparation +MUST preserve those rebuild-before-read consumers. Rewind requires no Mneme runtime +dependency and does not create even a temporary index. Missing bank history also refuses unless the bank is overridden. Moltnet JSON and SQLite stores remove messages, rooms, and attachment-delivery @@ -526,13 +424,15 @@ state. Its strict `spawnfile.leak-check.v1` result contains `{version, at, ok, checked, findings}`; counters cover files, git objects, memory events, Moltnet messages/events, and snapshots. Each finding carries `{input, class, path, detail}`; `input` is an input identity or `null` for bundle-level problems. +Metadata is captured before payload reads; the checker restores the captured access +times after inspection, preserving pre-existing future atimes for refusal. Every finding refuses the moment: | Class | Refusal condition | | --- | --- | -| `file-time` | A file, directory, or symlink mtime is after `at`. | +| `file-time` | A file, directory, or symlink atime or mtime is after `at`. | | `symlink-escape` | An absolute link or a lexically/physically escaping link resolves outside the bundle. | -| `git-object` | Any commit author/committer time is after `at`, any object is unreachable from the selected commit, or refs/remotes/reflogs/fetch metadata exceed the single branch. | +| `git-object` | Any commit author/committer time is after `at`, any object is unreachable from the selected commit, or refs/remotes/reflogs/fetch metadata exceed the single branch. Any `.git` directory/gitfile or bare repository (`HEAD` + `objects/` + `refs/`) anywhere outside the materialized git resource repository also refuses, including inside feed trees and bundles. | | `memory-event` | Future ledger events, sequence gaps, invalid prefix ordering, or derived/audit files are present. | | `moltnet-message` / `moltnet-event` / `moltnet-room` | Retained messages, delivery events, or rooms are after `at`. | | `feed-identity` | `landed_at` is after `at` or `current` does not target a tree inside the volume. | diff --git a/src/compiler/training/rewind/AGENTS.md b/src/compiler/training/rewind/AGENTS.md index e9d1429b..3cf95171 100644 --- a/src/compiler/training/rewind/AGENTS.md +++ b/src/compiler/training/rewind/AGENTS.md @@ -4,17 +4,21 @@ - `history.ts` is the only boundary for reading history files, logs and retained bytes; every path must be lexically and physically contained. - `select.ts` and `inputs.ts` resolve records, examples and effective inputs without I/O. - `types.ts` defines input identities, declarations and per-kind contexts/results; `index.ts` exports the public materializer and pure selection/cut helpers. -- `git.ts` owns resources; `gitProcess.ts`, `gitHistory.ts`, `gitTree.ts`, and `gitArchive.ts` isolate argv execution, containment, blob trees, and feed archives. +- `git.ts` owns resources; `gitProcess.ts`, `gitHistory.ts`, `gitTree.ts`, and `gitArchive.ts` isolate argv execution, containment, blob trees, and raw feed trees; `recordedCommit.ts` verifies full recorded IDs name commit objects without peeling. - `feed.ts`, `memory.ts`, and `volumes.ts` own their cuts and history-log schemas; `memorySchemas.ts` validates native ledger events. - `moltnet.ts` dispatches stores; `moltnetJson.ts`, `moltnetSqlite.ts`, `moltnetSqliteData.ts`, and `moltnetSchemas.ts` own snapshots, copied databases, SQL decoding, and strict native shapes. - `materialize.ts` stages all inputs and publishes only after the independent disk reader in `leakCheck.ts` accepts them. -- `leakFiles.ts`, `leakGit.ts`, and `leakInputs.ts` inspect disk metadata, every git object, and native payloads independently of materialization decisions. +- `leakFiles.ts`, `leakGit.ts`, and `leakInputs.ts` inspect disk metadata, every git object, and native payloads independently of materialization decisions. Discover unexpected `.git` directories/gitfiles and bare repositories throughout every input; only the materialized git resource repository is allowed. - `instant.ts` compares parsed nanosecond instants; `timestamps.ts` stamps metadata without following symlinks. - `taskClock.ts` defines offset wall-clock environment only; timeouts remain real. libfaketime requires trial-owned LD_PRELOAD. - Adjacent tests use local temporary repositories and fixtures. Never use Docker, production hosts or network fetches. - `fixtures.test-helper.ts` and `git.test-helper.ts` build disposable real projects and repositories; package emission excludes them. - Selector-at git history is approximate: committer dates cannot prove when a commit reached a ref. Feed-rule approximation cannot reconstruct freeze holds. - Reuse P2 effective input identity resolution. No input may silently use present-day data. -- Mneme's root API validates the cut through a temporary rebuilt index, which is discarded: output contains neither derived indexes nor authority/audit state. Missing optional Mneme support refuses explicitly. +- Feed/bundle extraction reads raw blobs, never `git archive` or attribute-driven transformations. Preserve symlinks for the escape check; refuse gitlinks. Recorded commits must be full lowercase 40/64-hex IDs verified as commit objects, including retained feeds. +- Validate each non-overridden input's recorded commit IDs in every same-agent/same-wake candidate before accepting that input, even when pin precedence omits those fields from selection comparisons. +- Validate memory JSONL structure locally; rewind never imports Mneme or ships an index/audit state. Checksum verification is skipped because persisted normalized events cannot recover the original JSON checksum input (`node_modules/@noopolis/mneme/dist/identity/ids.js:15-17`, `dist/store/store.js:249-274`). +- Mneme 0.1.1's index is rebuilt from the ledger before every index read, not on open: `node_modules/@noopolis/mneme/dist/runtime/runtime.js:86-87` (embedding recall), `node_modules/@noopolis/mneme/dist/kernel/kernel.js:152-153` (search), and `:207-208` (locate). A ledger-only materialized bank is complete. +- Inspect atime and mtime from metadata captured before payload reads, then restore captured access times without hiding existing future atimes. - Generated inputs spanning repositories refuse: this history contract carries one commit per resource. Never claim that one repository proves another repository's bytes. - Keep modules under 400 lines, use named/barrel exports, and never import Mneme internals. diff --git a/src/compiler/training/rewind/contract.ts b/src/compiler/training/rewind/contract.ts index 6be30b3c..02f497c9 100644 --- a/src/compiler/training/rewind/contract.ts +++ b/src/compiler/training/rewind/contract.ts @@ -3,6 +3,9 @@ import { trainingInstantSchema } from "../../../manifest/index.js"; const text = z.string().min(1); const count = z.number().int().nonnegative(); +export const recordedCommitSchema = z.string() + .regex(/^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/u, "Expected a full 40- or 64-hex commit object id") + .toLowerCase(); export const inputIdentitySchema = z.object({ section: z.enum(["resources", "memory", "networks"]), id: text, declaredBy: text }).strict(); @@ -20,8 +23,8 @@ export const momentRecordSchema = z.object({ wake: z.object({ at: trainingInstantSchema, surface: z.literal("moltnet"), room: z.string(), from: z.string(), messageId: text.optional() }).strict(), recordedAt: trainingInstantSchema, - resources: unique(z.object({ id: text, kind: z.enum(["git", "bundle"]), commit: text }).strict(), (r) => r.id), - feeds: unique(z.object({ id: text, revision: text, tree: text, commit: text, + resources: unique(z.object({ id: text, kind: z.enum(["git", "bundle"]), commit: recordedCommitSchema }).strict(), (r) => r.id), + feeds: unique(z.object({ id: text, revision: text, tree: text, commit: recordedCommitSchema, paths: z.array(text), landedAt: trainingInstantSchema }).strict(), (r) => r.id), memory: unique(z.object({ id: text, declaredBy: text, seq: count, generation: count }).strict(), (r) => JSON.stringify([r.declaredBy, r.id])), diff --git a/src/compiler/training/rewind/feed.test.ts b/src/compiler/training/rewind/feed.test.ts index eda4760d..4d6dbcf2 100644 --- a/src/compiler/training/rewind/feed.test.ts +++ b/src/compiler/training/rewind/feed.test.ts @@ -129,16 +129,16 @@ describe("feed rewind", () => { await expect(materializeFeed(context)).rejects.toThrow("No feed landing"); }); - it("applies committed archive attributes and PAX long names when reconstructing feed bytes", async () => { + it("ignores archive attributes and preserves long names when reconstructing feed bytes", async () => { const { context, repository, resource } = await fixture(); const longName = `published/${"x".repeat(150)}.txt`; const commit = await commitTestGit(repository, { ".gitattributes": "published/hidden.txt export-ignore\npublished/version.txt export-subst\n", "published/hidden.txt": "omitted\n", "published/version.txt": "$Format:%H$\n", [longName]: "long\n" }, "2026-10-09T08:10:00Z"); resource.feed!.git!.ref = commit; await materializeFeed(context); - expect(await readFile(path.join(context.destination, "current/published/version.txt"), "utf8")).toBe(`${commit}\n`); + expect(await readFile(path.join(context.destination, "current/published/version.txt"), "utf8")).toBe("$Format:%H$\n"); expect(await readFile(path.join(context.destination, "current", longName), "utf8")).toBe("long\n"); - await expect(readFile(path.join(context.destination, "current/published/hidden.txt"))).rejects.toThrow(); + expect(await readFile(path.join(context.destination, "current/published/hidden.txt"), "utf8")).toBe("omitted\n"); }); it("refuses retained history symlinks outside the history root", async () => { @@ -149,9 +149,9 @@ describe("feed rewind", () => { }); it("selects pure instant boundaries and refuses contradictory equal-instant landings", () => { - const revision = "a".repeat(64), landing = feedLandingSchema.parse({ revision, tree: revision, commit: "commit", ref: "main", paths: [], landedAt: AT }); + const revision = "a".repeat(64), landing = feedLandingSchema.parse({ revision, tree: revision, commit: "a".repeat(40), ref: "main", paths: [], landedAt: AT }); expect(selectFeedLanding([landing], "2026-10-09T10:20:31+02:00")).toBe(landing); expect(selectFeedLanding([landing], "2026-10-09T08:20:30.999Z")).toBeNull(); - expect(() => selectFeedLanding([landing, { ...landing, commit: "other", landedAt: "2026-10-09T10:20:31+02:00" }], AT)).toThrow("Conflicting feed landings"); + expect(() => selectFeedLanding([landing, { ...landing, commit: "b".repeat(40), landedAt: "2026-10-09T10:20:31+02:00" }], AT)).toThrow("Conflicting feed landings"); }); }); diff --git a/src/compiler/training/rewind/feed.ts b/src/compiler/training/rewind/feed.ts index a86100b9..4850b3d9 100644 --- a/src/compiler/training/rewind/feed.ts +++ b/src/compiler/training/rewind/feed.ts @@ -4,6 +4,8 @@ import { z } from "zod"; import { trainingInstantSchema, type TeamWorkspaceResource } from "../../../manifest/index.js"; import { expandFeedPaths, expandFeedRefTemplate, FEED_IDENTITY_FILE, FEED_IDENTITY_VERSION, resolveFeedSource, type FeedExec, type FeedIdentity } from "../../../volume/index.js"; +import { recordedCommitSchema } from "./contract.js"; +import { resolveRecordedCommit } from "./recordedCommit.js"; import { exportFeedGitTree } from "./gitArchive.js"; import { assertLocalGitRepository } from "./gitHistory.js"; import { assertCommitNotFuture, commitAt, rewindGitSync, type SelectedCommit } from "./gitProcess.js"; @@ -18,7 +20,7 @@ const plainPath = z.string().min(1).refine((value) => !value.startsWith("/") && && !value.split("/").some((segment) => !segment || segment === "." || segment === ".." || segment.toLowerCase() === ".git"), "Expected a plain tree-relative path"); export const feedLandingSchema = z.object({ revision, tree: z.string().regex(/^(?:trees\/)?[a-f0-9]{64}(?:\.[1-9][0-9]*)?$/u), - commit: z.string().min(1).nullable(), ref: z.string().min(1).nullable(), paths: z.array(plainPath).nullable(), + commit: recordedCommitSchema.nullable(), ref: z.string().min(1).nullable(), paths: z.array(plainPath).nullable(), landedAt: trainingInstantSchema }).strict().superRefine((value, context) => { if (!value.tree.replace(/^trees\//u, "").startsWith(value.revision)) context.addIssue({ code: "custom", message: "Landing tree must belong to its revision" }); @@ -81,16 +83,20 @@ export const materializeFeed = async (context: InputContext): Promise record.id === input.id); + if (entry) await resolveRecordedCommit(await openRepository(), entry.commit); + } + const record = context.record?.feeds.find((entry) => entry.id === input.id); + const recordedCommit = record ? await resolveRecordedCommit(await openRepository(), record.commit) : null; if (log) { + for (const entry of log) if (entry.commit) await resolveRecordedCommit(await openRepository(), entry.commit); const selected = selectFeedLanding(log, context.at); if (!selected) throw new Error(`No feed landing exists at or before ${context.at}`); landing = selected; - const record = context.record?.feeds.find((entry) => entry.id === input.id); - if (record && (record.revision !== landing.revision || record.tree !== landing.tree || record.commit !== landing.commit + if (record && (record.revision !== landing.revision || record.tree !== landing.tree || recordedCommit !== landing.commit || JSON.stringify(feedPaths(record.paths)) !== JSON.stringify(feedPaths(landing.paths)) || compareInstants(record.landedAt, landing.landedAt) !== 0)) { throw new Error(`Moment feed ${input.id} disagrees with its landing log`); } diff --git a/src/compiler/training/rewind/git.test.ts b/src/compiler/training/rewind/git.test.ts index addecc47..eb1f7db4 100644 --- a/src/compiler/training/rewind/git.test.ts +++ b/src/compiler/training/rewind/git.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { mkdir, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; +import { mkdir, readFile, readdir, readlink, rm, symlink, writeFile } from "node:fs/promises"; import path from "node:path"; import type { TeamWorkspaceResource } from "../../../manifest/index.js"; import { BundleTarWriter } from "../../workspaceBundleTar.js"; @@ -47,7 +47,6 @@ describe("git rewind", () => { const { repository, past, context } = await fixture(); context.history = undefined; context.input.declaration = { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: repository, ref: past.toUpperCase() }; - context.record = { ...testMomentRecord(), resources: [{ id: "archive", kind: "bundle", commit: "not-the-pin" }] }; vi.stubEnv("GIT_DIR", "/does/not/exist"); vi.stubEnv("GIT_CONFIG_COUNT", "1"); vi.stubEnv("GIT_CONFIG_KEY_0", "core.worktree"); vi.stubEnv("GIT_CONFIG_VALUE_0", "/elsewhere"); expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "pinned", source: { commit: past } }); @@ -135,15 +134,18 @@ describe("git rewind", () => { await expect(materializeGit(context)).rejects.toThrow("multiple git repositories"); }); - it("preserves the compiler's bundle symlink refusal after applying exclusions", async () => { + it("recreates bundle symlinks and applies declared exclusions", async () => { const { repository, context } = await fixture(); await symlink("package.json", path.join(repository, "package/link")); const ref = await commitTestGit(repository, {}, "2026-10-09T08:10:00Z"); context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { files: { root: "package", ref } } }; - await expect(materializeGit(context)).rejects.toThrow("Bundle input is a symlink"); + expect(await materializeGit(context)).toMatchObject({ fidelity: "exact" }); + expect(await readlink(path.join(context.destination, "link"))).toBe("package.json"); + expect(await readFile(path.join(context.destination, "link"), "utf8")).toBe("{}\n"); await rm(context.destination, { recursive: true, force: true }); context.input.declaration.build!.files!.exclude = ["link"]; expect(await materializeGit(context)).toMatchObject({ fidelity: "exact" }); + expect(await readdir(context.destination)).not.toContain("link"); }); it("extracts a validated local prebuilt override", async () => { diff --git a/src/compiler/training/rewind/git.ts b/src/compiler/training/rewind/git.ts index ba36f3e1..8f291904 100644 --- a/src/compiler/training/rewind/git.ts +++ b/src/compiler/training/rewind/git.ts @@ -7,6 +7,7 @@ import { historyKey } from "./history.js"; import { assertLocalGitRepository } from "./gitHistory.js"; import { assertCommitNotFuture, commitAt, commitInstant, resolveRewindCommit, rewindGit } from "./gitProcess.js"; import { exportGitTree, extractOverrideBundle } from "./gitTree.js"; +import { resolveRecordedCommit } from "./recordedCommit.js"; import type { InputContext, InputResult } from "./types.js"; export { commitAt, commitInstant, resolveRewindCommit, rewindGit, rewindGitEnvironment, rewindGitSync, selectCommitAt } from "./gitProcess.js"; @@ -60,13 +61,13 @@ const bundlePrefix = async (root: string, base: string): Promise => { const materializeBundle = async (resource: BundleResource, context: InputContext, repository: string, commit: string): Promise => { const build = resource.build; - if (!build) { await exportGitTree(repository, commit, context.destination, { refuseSymlinks: true }); return false; } + if (!build) { await exportGitTree(repository, commit, context.destination); return false; } if (build.files) { - await exportGitTree(repository, commit, context.destination, { root: await bundlePrefix(build.files.root, context.input.declarationBase), exclude: build.files.exclude, refuseSymlinks: true }); + await exportGitTree(repository, commit, context.destination, { root: await bundlePrefix(build.files.root, context.input.declarationBase), exclude: build.files.exclude }); return false; } if (build.dependencies) { - await exportGitTree(repository, commit, context.destination, { root: await bundlePrefix(build.dependencies.directory, context.input.declarationBase), refuseSymlinks: true }); + await exportGitTree(repository, commit, context.destination, { root: await bundlePrefix(build.dependencies.directory, context.input.declarationBase) }); } else { const origins = await Promise.all(build.generated!.inputs.map(async (input) => { try { return (await rewindGit(path.resolve(context.input.declarationBase, input.root), ["rev-parse", "--show-toplevel"])).toString().trim(); } @@ -79,7 +80,7 @@ const materializeBundle = async (resource: BundleResource, context: InputContext for (const input of build.generated!.inputs) { if (input.ref !== undefined) throw new Error("Generated bundle inputs cannot pin a ref: the deferred command reads the work tree"); const root = await bundlePrefix(input.root, context.input.declarationBase); - await exportGitTree(repository, commit, context.destination, { root, prefix: root, exclude: input.exclude, mergeIdentical: true, refuseSymlinks: true }); + await exportGitTree(repository, commit, context.destination, { root, prefix: root, exclude: input.exclude, mergeIdentical: true }); } } return true; @@ -89,7 +90,10 @@ export const materializeGit = async (context: InputContext): Promise entry.id === input.id); + const matchingRecord = input.overridden ? undefined : context.record?.resources.find((entry) => entry.id === input.id); + const candidateCommits = input.overridden ? [] : context.candidateRecords?.flatMap((candidate) => + candidate.resources.filter((entry) => entry.id === input.id).map((entry) => entry.commit)) ?? []; + const record = pinned ? undefined : matchingRecord; if (record && record.kind !== resource.kind) throw new Error(`Moment resource ${input.id} kind disagrees with its declaration`); if (input.overridden && resource.kind === "bundle" && resource.source) { await extractOverrideBundle(localRepositoryPath(resource.source, input.declarationBase), context.destination, resource.sha256); @@ -97,14 +101,18 @@ export const materializeGit = async (context: InputContext): Promise ({ commit, instant: await commitInstant(repository!, commit) })) + ? await (record ? Promise.resolve(recordedCommit!) : resolveRewindCommit(repository, selector)) + .then(async (commit) => ({ commit, instant: await commitInstant(repository!, commit) })) : await commitAt(repository, selector, context.at); if (!selected) throw new Error(`No first-parent commit for ${selector} exists at or before ${context.at}`); if (resource.kind === "bundle") await assertCommitNotFuture(repository, selected.commit, context.at); diff --git a/src/compiler/training/rewind/gitArchive.test.ts b/src/compiler/training/rewind/gitArchive.test.ts new file mode 100644 index 00000000..2fde8fb2 --- /dev/null +++ b/src/compiler/training/rewind/gitArchive.test.ts @@ -0,0 +1,70 @@ +import { lstat, mkdir, readFile, readlink, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { resolveFeedSource } from "../../../volume/index.js"; +import { materializeFeed } from "./feed.js"; +import { materializeGit } from "./git.js"; +import { exportFeedGitTree } from "./gitArchive.js"; +import { commitTestGit, gitTestRoot, initTestGit, testGit, testMomentRecord } from "./git.test-helper.js"; +import { History } from "./history.js"; +import type { InputContext } from "./types.js"; + +const BEFORE = "2026-10-09T07:00:00Z", AT = "2026-10-09T08:20:31Z", AFTER = "2026-10-10T00:00:00Z"; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); + +describe("raw historical blob extraction", () => { + it.each(["feed", "bundle"] as const)("preserves literal export-subst and export-ignore bytes in a %s after a new tag", async (kind) => { + const root = await gitTestRoot(); roots.push(root); + const relative = kind === "feed" ? "feeds/agent%3Awriter/input/repo" : "git/agent%3Awriter/input"; + const repository = path.join(root, "history", relative); + await initTestGit(repository); + const commit = await commitTestGit(repository, { + ".gitattributes": "refs.txt export-subst\nhidden.txt export-ignore\n", + "refs.txt": "$Format:%D$\n", "hidden.txt": "raw retained bytes\n" + }, BEFORE); + // This ref did not exist at the historical moment; archive expansion would leak it. + testGit(repository, ["tag", "-a", "created-after-the-wake", "-m", "later tag", commit], AFTER); + await mkdir(path.join(repository, ".git/info"), { recursive: true }); + await writeFile(path.join(repository, ".git/info/attributes"), "hidden.txt export-ignore\n"); + const context: InputContext = { at: AT, destination: path.join(root, "out"), history: new History(path.join(root, "history")), + record: { ...testMomentRecord(), resources: kind === "bundle" ? [{ id: "input", kind: "bundle", commit }] : [] }, + input: { section: "resources", id: "input", declaredBy: "agent:writer", overridden: false, declarationBase: repository, + declaration: kind === "feed" + ? { id: "input", kind: "volume", mode: "mutable", mount: "./input", name: "input", feed: { git: { repo: repository } } } + : { id: "input", kind: "bundle", mode: "readonly", mount: "./input", build: { files: { root: "." } } } } }; + let output = context.destination; + if (kind === "feed") { + const revision = resolveFeedSource({ kind: "git", repo: repository, ref: commit, fetch: false }).revision; + await writeFile(path.join(repository, "../landings.jsonl"), `${JSON.stringify({ revision, tree: revision, commit, ref: "main", paths: null, landedAt: BEFORE })}\n`); + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", source: { revision, commit } }); + output = path.join(output, "current"); + } else expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", source: { commit } }); + expect(await readFile(path.join(output, "refs.txt"), "utf8")).toBe("$Format:%D$\n"); + expect(await readFile(path.join(output, "hidden.txt"), "utf8")).toBe("raw retained bytes\n"); + }); + + it("recreates symlinks and preserves binary blob bytes without conversion", async () => { + const root = await gitTestRoot(); roots.push(root); + const repository = path.join(root, "repo"); await initTestGit(repository); + const bytes = Buffer.from([0, 255, 10, 128]); + await writeFile(path.join(repository, "binary"), bytes); + await symlink("binary", path.join(repository, "link")); + const commit = await commitTestGit(repository, { ".gitattributes": "binary text eol=crlf\n" }, BEFORE); + const destination = path.join(root, "out"); + await exportFeedGitTree(repository, commit, destination); + expect(await readFile(path.join(destination, "binary"))).toEqual(bytes); + expect((await lstat(path.join(destination, "link"))).isSymbolicLink()).toBe(true); + expect(await readlink(path.join(destination, "link"))).toBe("binary"); + }); + + it("refuses gitlinks instead of silently dropping submodule contents", async () => { + const root = await gitTestRoot(); roots.push(root); + const repository = path.join(root, "repo"); await initTestGit(repository); + const parent = await commitTestGit(repository, { "file": "before\n" }, BEFORE); + testGit(repository, ["update-index", "--add", "--cacheinfo", `160000,${parent},nested`]); + testGit(repository, ["commit", "--quiet", "-m", "gitlink"], BEFORE); + await expect(exportFeedGitTree(repository, testGit(repository, ["rev-parse", "HEAD"]), path.join(root, "out"))) + .rejects.toThrow(/nested.*submodules need their own history/u); + }); +}); diff --git a/src/compiler/training/rewind/gitArchive.ts b/src/compiler/training/rewind/gitArchive.ts index 5c5ebd3b..0ae520d7 100644 --- a/src/compiler/training/rewind/gitArchive.ts +++ b/src/compiler/training/rewind/gitArchive.ts @@ -1,81 +1,7 @@ -import { lstat, mkdir, symlink, writeFile } from "node:fs/promises"; -import path from "node:path"; -import { rewindGit } from "./gitProcess.js"; -import { safeCommittedPath } from "./gitTree.js"; +import { exportGitTree, safeCommittedPath } from "./gitTree.js"; -const BLOCK = 512; -const field = (bytes: Buffer): string => bytes.toString("utf8").replace(/\0.*$/su, ""); -const octal = (bytes: Buffer): number => { - const value = field(bytes).trim(); - if (!/^[0-7]+$/u.test(value)) throw new Error("Invalid git archive numeric field"); - const number = Number.parseInt(value, 8); - if (!Number.isSafeInteger(number) || number < 0) throw new Error("Invalid git archive size"); - return number; -}; - -const pax = (bytes: Buffer): Record => { - const result: Record = {}; - for (let offset = 0; offset < bytes.length;) { - const space = bytes.indexOf(32, offset), length = Number(bytes.subarray(offset, space).toString("ascii")); - if (space < offset || !Number.isSafeInteger(length) || length <= space - offset + 1 || offset + length > bytes.length || bytes[offset + length - 1] !== 10) { - throw new Error("Invalid git archive PAX field"); - } - const value = bytes.subarray(space + 1, offset + length - 1).toString("utf8"), equal = value.indexOf("="); - if (equal <= 0) throw new Error("Invalid git archive PAX value"); - result[value.slice(0, equal)] = value.slice(equal + 1); - offset += length; - } - return result; -}; - -/** Never follow an archive symlink while creating a later entry beneath it. */ -const parentDirectory = async (root: string, relative: string): Promise => { - let directory = root; - for (const segment of relative.split("/").slice(0, -1)) { - directory = path.join(directory, segment); - const existing = await lstat(directory).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); - if (existing && !existing.isDirectory()) throw new Error(`Git archive entry traverses a non-directory: ${relative}`); - if (!existing) await mkdir(directory); - } -}; - -/** Git feeds use git archive, including committed export-ignore/export-subst attributes. */ +/** Retain raw blob bytes: archive attributes may disclose refs added after the moment. */ export const exportFeedGitTree = async (repository: string, commit: string, destination: string, paths?: string[]): Promise => { for (const selected of paths ?? []) safeCommittedPath(selected); - const archive = await rewindGit(repository, ["archive", "--format=tar", commit, ...(paths ? ["--", ...paths] : [])]); - await mkdir(destination, { recursive: true }); - let extended: Record = {}, entries = 0, terminated = false; - for (let offset = 0; offset + BLOCK <= archive.length;) { - const header = archive.subarray(offset, offset + BLOCK); - if (header.every((byte) => byte === 0)) { - if (offset + 2 * BLOCK > archive.length || !archive.subarray(offset).every((byte) => byte === 0)) throw new Error("Invalid git archive termination"); - terminated = true; - break; - } - const checksum = header.reduce((sum, byte, index) => sum + (index >= 148 && index < 156 ? 32 : byte), 0); - if (field(header.subarray(257, 263)) !== "ustar" || checksum !== octal(header.subarray(148, 156))) throw new Error("Invalid git archive header"); - const size = octal(header.subarray(124, 136)), mode = octal(header.subarray(100, 108)), type = String.fromCharCode(header[156] || 48); - const data = archive.subarray(offset + BLOCK, offset + BLOCK + size); - offset += BLOCK + Math.ceil(size / BLOCK) * BLOCK; - if (data.length !== size || offset > archive.length) throw new Error("Truncated git archive"); - if (type === "g" || type === "x") { - const values = pax(data); - if (type === "x") extended = values; - else if (values.path || values.linkpath || values.size) throw new Error("Unsupported global git archive path override"); - continue; - } - const prefix = field(header.subarray(345, 500)); - const name = safeCommittedPath((extended.path ?? `${prefix ? `${prefix}/` : ""}${field(header.subarray(0, 100))}`).replace(/\/$/u, "")); - const link = extended.linkpath ?? field(header.subarray(157, 257)); - if (extended.size && Number(extended.size) !== size) throw new Error("Unsupported git archive size override"); - extended = {}; - if (!["0", "2", "5"].includes(type) || (type !== "0" && size)) throw new Error(`Unsupported git archive entry ${name}`); - await parentDirectory(destination, name); - const target = path.join(destination, name); - if (type === "5") await mkdir(target); - else if (type === "2") await symlink(link, target); - else await writeFile(target, data, { flag: "wx", mode: mode & 0o111 ? 0o755 : 0o644 }); - entries += 1; - } - if (!terminated || !entries) throw new Error("Git feed archive is empty or lacks valid termination"); + await exportGitTree(repository, commit, destination, { paths }); }; diff --git a/src/compiler/training/rewind/gitTree.ts b/src/compiler/training/rewind/gitTree.ts index 7c60404c..99768890 100644 --- a/src/compiler/training/rewind/gitTree.ts +++ b/src/compiler/training/rewind/gitTree.ts @@ -13,7 +13,18 @@ export const safeCommittedPath = (value: string): string => { return value; }; -export interface ExportTreeOptions { root?: string; exclude?: readonly string[]; paths?: readonly string[]; prefix?: string; mergeIdentical?: boolean; refuseSymlinks?: boolean } +/** Never traverse a symlink left by an earlier generated input while merging trees. */ +const parentDirectory = async (root: string, relative: string): Promise => { + let directory = root; + for (const segment of relative.split("/").slice(0, -1)) { + directory = path.join(directory, segment); + const existing = await lstat(directory).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); + if (existing && !existing.isDirectory()) throw new Error(`Committed entry traverses a non-directory: ${relative}`); + if (!existing) await mkdir(directory); + } +}; + +export interface ExportTreeOptions { root?: string; exclude?: readonly string[]; paths?: readonly string[]; prefix?: string; mergeIdentical?: boolean } /** Materialize blobs by object id, without invoking archive attributes, filters or extraction tools. */ export const exportGitTree = async (repo: string, commit: string, destination: string, options: ExportTreeOptions = {}): Promise => { @@ -28,10 +39,9 @@ export const exportGitTree = async (repo: string, commit: string, destination: s const tab = entry.indexOf("\t"), name = safeCommittedPath(entry.slice(tab + 1)); const [mode, kind, oid] = entry.slice(0, tab).split(" "); if (excluded(name) || (options.paths && !options.paths.some((selected) => name === selected || name.startsWith(`${selected}/`)))) continue; - if (options.refuseSymlinks && mode === "120000") throw new Error(`Bundle input is a symlink; exclude it or replace it with a file: ${name}`); if (kind !== "blob" || !["100644", "100755", "120000"].includes(mode!)) throw new Error(`Committed path ${name} is not a supported file or symlink (submodules need their own history)`); const target = path.join(destination, prefix, name), bytes = await rewindGit(repo, ["cat-file", "blob", oid!]); - await mkdir(path.dirname(target), { recursive: true }); + await parentDirectory(destination, [prefix, name].filter(Boolean).join("/")); if (options.mergeIdentical) { const present = await lstat(target).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); if (present) { diff --git a/src/compiler/training/rewind/history.test.ts b/src/compiler/training/rewind/history.test.ts index eeb4ce55..fc853944 100644 --- a/src/compiler/training/rewind/history.test.ts +++ b/src/compiler/training/rewind/history.test.ts @@ -81,7 +81,7 @@ describe("history containment and strict reads", () => { it.each([ { ...record, unexpected: true }, { ...record, wake: { ...record.wake, text: "not in the record format" } }, - { ...record, resources: [{ id: "same", kind: "git", commit: "one" }, { id: "same", kind: "git", commit: "two" }] } + { ...record, resources: [{ id: "same", kind: "git", commit: "1".repeat(40) }, { id: "same", kind: "git", commit: "2".repeat(40) }] } ])("strictly validates record fields and identities", async (invalid) => { const { directory, history } = await fixture(); await writeFile(path.join(directory, "invalid.jsonl"), `${JSON.stringify(invalid)}\n`); diff --git a/src/compiler/training/rewind/leakCheck.test.ts b/src/compiler/training/rewind/leakCheck.test.ts index 09eb5005..b1ddf177 100644 --- a/src/compiler/training/rewind/leakCheck.test.ts +++ b/src/compiler/training/rewind/leakCheck.test.ts @@ -1,4 +1,4 @@ -import { appendFile, mkdir, readFile, readdir, rm, symlink, utimes, writeFile } from "node:fs/promises"; +import { lstat, lutimes, mkdir, readFile, readdir, rm, symlink, utimes, writeFile } from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { cleanTrainingProjects } from "../plan/fixtures.test-helper.js"; @@ -22,6 +22,11 @@ const cases: Array<[string, LeakFinding["class"], Mutation]> = [ ["future file mtime", "file-time", async (fixture) => { await writeFile(path.join(fixture.options.out, "future"), "future"); }], ["future directory mtime", "file-time", async (fixture) => { await mkdir(path.join(fixture.options.out, "new-directory")); }], ["future symlink mtime", "file-time", async (fixture) => { await symlink("moment.json", path.join(fixture.options.out, "new-link")); }], + ["future report atime", "file-time", async () => {}], + ["future report mtime", "file-time", async () => {}], + ["future file atime", "file-time", async (fixture) => { await writeFile(path.join(fixture.options.out, "future"), "future"); }], + ["future directory atime", "file-time", async (fixture) => { await mkdir(path.join(fixture.options.out, "new-directory")); }], + ["future symlink atime", "file-time", async (fixture) => { await symlink("moment.json", path.join(fixture.options.out, "new-link")); }], ["extra unreachable future commit", "git-object", async (fixture, moment) => { const code = inputPath(fixture, moment, "git"); const pack = await rewindGit(fixture.gitRepo, ["pack-objects", "--revs", "--stdout"], Buffer.from(`${fixture.futureCommit}\n`)); @@ -42,10 +47,12 @@ const cases: Array<[string, LeakFinding["class"], Mutation]> = [ await writeFile(path.join(fixture.options.out, "moment.json"), JSON.stringify(moment)); }], ["future memory event", "memory-event", async (fixture, moment) => { - await appendFile(path.join(inputPath(fixture, moment, "memory"), "memory/events.jsonl"), `${JSON.stringify(memoryEvent(2, FUTURE))}\n`); + const file = path.join(inputPath(fixture, moment, "memory"), "memory/events.jsonl"); + const event = JSON.parse((await readFile(file, "utf8")).trim()); + await writeFile(file, `${JSON.stringify({ ...event, createdAt: FUTURE })}\n`); }], ["memory seq gap", "memory-event", async (fixture, moment) => { - await appendFile(path.join(inputPath(fixture, moment, "memory"), "memory/events.jsonl"), `${JSON.stringify(memoryEvent(3))}\n`); + await writeFile(path.join(inputPath(fixture, moment, "memory"), "memory/events.jsonl"), `${JSON.stringify(memoryEvent(3))}\n`); }], ["memory audit state", "memory-event", async (fixture, moment) => { await writeFile(path.join(inputPath(fixture, moment, "memory"), "memory/index.sqlite"), "future derived state"); @@ -74,18 +81,54 @@ const cases: Array<[string, LeakFinding["class"], Mutation]> = [ }] ]; +const intendedFinding = (name: string, fixture: Fixture, moment: MaterializedMoment): { path: string; detail: string | RegExp } => { + const relative = (kind: string) => moment.inputs.find((input) => input.kind === kind)!.path!; + if (name.startsWith("future ") && /[am]time$/u.test(name)) return { + path: name.includes("report") ? "moment.json" : name.includes("directory") ? "new-directory" : name.includes("symlink") ? "new-link" : "future", + detail: `${name.endsWith("atime") ? "Access" : "Modification"} time is later than the moment` + }; + const findings: Record = { + "extra unreachable future commit": { path: relative("git"), detail: `Object ${fixture.futureCommit} is not reachable from ${fixture.commit}` }, + "extra tag": { path: relative("git"), detail: "Exactly one branch and HEAD must name the materialized commit" }, + "extra remote": { path: relative("git"), detail: "Forbidden git configuration remote.leaked.url" }, + "reachable commit future author only": { path: relative("git"), detail: /^Commit [a-f0-9]+ has a future author date$/u }, + "reachable commit future committer only": { path: relative("git"), detail: /^Commit [a-f0-9]+ has a future committer date$/u }, + "future memory event": { path: relative("memory"), detail: "Future memory event at line 1" }, + "memory seq gap": { path: relative("memory"), detail: "Noncontiguous memory sequence at line 1" }, + "memory audit state": { path: relative("memory"), detail: "Derived or audit memory state index.sqlite is forbidden" }, + "future Moltnet message": { path: `${relative("moltnet")}/state.json`, detail: "Message future is later than the moment" }, + "future Moltnet event": { path: `${relative("moltnet")}/state.json`, detail: "Attachment event future is later than the moment" }, + "future Moltnet room": { path: `${relative("moltnet")}/state.json`, detail: "Room future is later than the moment" }, + "escaping symlink": { path: "escape", detail: 'Unsafe symlink target "../outside"' }, + "absolute symlink": { path: "escape", detail: `Unsafe symlink target ${JSON.stringify(fixture.root)}` }, + "future dated path": { path: "edition-2026-10-11", detail: "Calendar date 2026-10-11 is later than the latest possible local date 2026-10-09" }, + "tampered snapshot": { path: relative("volume"), detail: "Snapshot digest mismatch for saved.txt" }, + "future feed landing": { path: relative("feed"), detail: "Feed landed_at is later than the moment" }, + "feed current outside its volume": { path: relative("feed"), detail: "Feed current does not point to its tree inside the volume" } + }; + return findings[name]!; +}; + describe("independent disk leak check mutation guarantees", () => { - it.each(cases)("refuses %s with class %s", async (_name, expectedClass, mutate) => { + it.each(cases)("refuses %s with class %s", async (name, expectedClass, mutate) => { const fixture = await rewindFixture(); const moment = await materializeTrainingMoment(fixture.options); expect(moment.status).toBe("materialized"); await mutate(fixture, moment); - // Non-time violations must be detected from content even when their filesystem timestamps look old. - if (expectedClass !== "file-time") await stampTree(fixture.options.out, AT, AT); - else await utimes(path.join(fixture.options.out, "moment.json"), new Date(AT), new Date(AT)); + const expected = intendedFinding(name, fixture, moment); + // Restore every parent and unrelated entry; time mutations change only the intended field on the target. + await stampTree(fixture.options.out, AT, AT); + if (expectedClass === "file-time") { + const target = path.join(fixture.options.out, expected.path); + const setTimes = (await lstat(target)).isSymbolicLink() ? lutimes : utimes; + await setTimes(target, new Date(name.endsWith("atime") ? FUTURE : AT), new Date(name.endsWith("mtime") ? FUTURE : AT)); + } const check = await leakCheckTrainingMoment(fixture.options.out, { historyDir: fixture.historyDir }); expect(check.ok).toBe(false); - expect(check.findings.map((finding) => finding.class)).toContain(expectedClass); + expect(check.findings).toContainEqual(expect.objectContaining({ class: expectedClass, path: expected.path, + detail: typeof expected.detail === "string" ? expected.detail : expect.stringMatching(expected.detail) })); + if (expectedClass === "file-time") expect(check.findings.filter((finding) => finding.class === "file-time")) + .toEqual([expect.objectContaining({ class: expectedClass, ...expected })]); const out = path.join(fixture.root, "rejected"); const refused = await finalizeTrainingMoment(fixture.options.out, out, { historyDir: fixture.historyDir }); expect(refused.status).toBe("refused"); diff --git a/src/compiler/training/rewind/leakCheck.ts b/src/compiler/training/rewind/leakCheck.ts index a20de5c1..5aea63eb 100644 --- a/src/compiler/training/rewind/leakCheck.ts +++ b/src/compiler/training/rewind/leakCheck.ts @@ -1,39 +1,48 @@ import { readFile, realpath } from "node:fs/promises"; import path from "node:path"; -import { materializedMomentSchema, leakCheckSchema, type LeakCheck, type LeakFinding } from "./contract.js"; +import { materializedMomentSchema, leakCheckSchema, type LeakCheck, type LeakFinding, type MaterializedMoment } from "./contract.js"; import { openHistory } from "./history.js"; -import { diskPath, inspectBundleFiles } from "./leakFiles.js"; +import { diskPath, inspectBundleFiles, restoreInspectedAccessTimes, snapshotBundleFiles } from "./leakFiles.js"; import { inspectGitBundle } from "./leakGit.js"; import { addInputFinding, inspectFeedBundle, inspectMemoryBundle, inspectMoltnetBundle, inspectVolumeBundle } from "./leakInputs.js"; export interface LeakCheckOptions { historyDir?: string } /** Independent acceptance boundary: only disk reports, disk payloads and disk history are trusted. */ -export const leakCheckTrainingMoment = async (bundle: string, options: LeakCheckOptions = {}): Promise => { +export const inspectTrainingMoment = async ( + bundle: string, options: LeakCheckOptions = {}, skipRefused = false +): Promise<{ moment: MaterializedMoment; check: LeakCheck | null }> => { const root = await realpath(path.resolve(bundle)); - const moment = materializedMomentSchema.parse(JSON.parse(await readFile(await diskPath(root, "moment.json"), "utf8"))); - const report: LeakCheck = { version: "spawnfile.leak-check.v1", at: moment.at, ok: false, - checked: { files: 0, gitObjects: 0, memoryEvents: 0, moltnetMessages: 0, moltnetEvents: 0, snapshots: 0 }, findings: [] }; - await inspectBundleFiles(root, moment.inputs, report); - for (const input of moment.inputs) { - const kind: LeakFinding["class"] = input.kind === "memory" ? "memory-event" - : input.kind === "moltnet" ? "moltnet-message" : input.kind === "feed" ? "feed-identity" - : input.kind === "volume" ? "snapshot" : "git-object"; - try { - if (!input.path || input.refusal || !input.fidelity) throw new Error("Input is not materialized"); - const directory = await diskPath(root, input.path); - // Do not hand git/SQLite a payload with escaping paths, even for a failed check. - if (report.findings.some((finding) => finding.class === "symlink-escape" - && (finding.path === input.path || finding.path.startsWith(`${input.path}/`)))) continue; - if (input.kind === "git") await inspectGitBundle(directory, input, report); - if (input.kind === "memory") await inspectMemoryBundle(root, input, report); - if (input.kind === "feed") await inspectFeedBundle(root, input, report); - if (input.kind === "moltnet") await inspectMoltnetBundle(root, input, report); - if (input.kind === "volume") await inspectVolumeBundle(root, input, report, await openHistory(options.historyDir)); - } catch (error) { - addInputFinding(report, input, kind, error instanceof Error ? error.message : String(error), input.path ?? "."); + const files = await snapshotBundleFiles(root); + try { + const moment = materializedMomentSchema.parse(JSON.parse(await readFile(await diskPath(root, "moment.json"), "utf8"))); + if (skipRefused && moment.status === "refused") return { moment, check: null }; + const report: LeakCheck = { version: "spawnfile.leak-check.v1", at: moment.at, ok: false, + checked: { files: 0, gitObjects: 0, memoryEvents: 0, moltnetMessages: 0, moltnetEvents: 0, snapshots: 0 }, findings: [] }; + await inspectBundleFiles(root, moment.inputs, report, files); + for (const input of moment.inputs) { + const kind: LeakFinding["class"] = input.kind === "memory" ? "memory-event" + : input.kind === "moltnet" ? "moltnet-message" : input.kind === "feed" ? "feed-identity" + : input.kind === "volume" ? "snapshot" : "git-object"; + try { + if (!input.path || input.refusal || !input.fidelity) throw new Error("Input is not materialized"); + const directory = await diskPath(root, input.path); + // Do not hand git/SQLite a payload with escaping paths, even for a failed check. + if (report.findings.some((finding) => finding.class === "symlink-escape" + && (finding.path === input.path || finding.path.startsWith(`${input.path}/`)))) continue; + if (input.kind === "git") await inspectGitBundle(directory, input, report); + if (input.kind === "memory") await inspectMemoryBundle(root, input, report); + if (input.kind === "feed") await inspectFeedBundle(root, input, report); + if (input.kind === "moltnet") await inspectMoltnetBundle(root, input, report); + if (input.kind === "volume") await inspectVolumeBundle(root, input, report, await openHistory(options.historyDir)); + } catch (error) { + addInputFinding(report, input, kind, error instanceof Error ? error.message : String(error), input.path ?? "."); + } } - } - report.ok = report.findings.length === 0; - return leakCheckSchema.parse(report); + report.ok = report.findings.length === 0; + return { moment, check: leakCheckSchema.parse(report) }; + } finally { await restoreInspectedAccessTimes(files); } }; + +export const leakCheckTrainingMoment = async (bundle: string, options: LeakCheckOptions = {}): Promise => + (await inspectTrainingMoment(bundle, options)).check!; diff --git a/src/compiler/training/rewind/leakFiles.ts b/src/compiler/training/rewind/leakFiles.ts index 044c99ac..9c33cd6e 100644 --- a/src/compiler/training/rewind/leakFiles.ts +++ b/src/compiler/training/rewind/leakFiles.ts @@ -1,4 +1,5 @@ -import { lstat, readdir, readlink, realpath } from "node:fs/promises"; +import type { BigIntStats } from "node:fs"; +import { lstat, lutimes, readdir, readlink, realpath, utimes } from "node:fs/promises"; import path from "node:path"; import { inside } from "./history.js"; import { instantNanos } from "./instant.js"; @@ -14,20 +15,52 @@ export const diskPath = async (root: string, relative: string): Promise return candidate; }; +/** Capture metadata before any inspection read can advance an entry's access time. */ +export const snapshotBundleFiles = async (root: string): Promise> => { + const files = new Map(); + const visit = async (file: string): Promise => { + const stats = await lstat(file, { bigint: true }); + files.set(file, stats); + if (stats.isDirectory()) for (const name of (await readdir(file)).sort()) await visit(path.join(file, name)); + }; + await visit(root); + return files; +}; + +/** Preserve the inspected access times, including future ones; never hide an existing violation. */ +export const restoreInspectedAccessTimes = async (files: Map): Promise => { + for (const [file, stats] of [...files].reverse()) { + await (stats.isSymbolicLink() ? lutimes : utimes)(file, Number(stats.atimeNs / 1_000n) / 1_000_000, Number(stats.mtimeNs / 1_000n) / 1_000_000); + } +}; + /** No symlinks are traversed. Inspection never reads a target outside the bundle. */ -export const inspectBundleFiles = async (root: string, inputs: MaterializedInput[], report: LeakCheck): Promise => { +export const inspectBundleFiles = async (root: string, inputs: MaterializedInput[], report: LeakCheck, files: Map): Promise => { const at = instantNanos(report.at); const latestDate = new Date(Date.parse(report.at) + 14 * 3600_000).toISOString().slice(0, 10); const visit = async (file: string): Promise => { const relative = path.relative(root, file).split(path.sep).join("/") || "."; const owner = inputAtPath(inputs, relative); const input = owner ? { section: owner.section, id: owner.id, declaredBy: owner.declaredBy } : null; - const add = (kind: "file-time" | "symlink-escape" | "dated-path", detail: string) => { + const add = (kind: "file-time" | "symlink-escape" | "dated-path" | "git-object", detail: string) => { report.findings.push({ input, class: kind, path: relative, detail }); }; - const stats = await lstat(file, { bigint: true }); + const stats = files.get(file)!; report.checked.files++; if (stats.mtimeNs > at) add("file-time", "Modification time is later than the moment"); + if (stats.atimeNs > at) add("file-time", "Access time is later than the moment"); + const allowedGitDirectory = owner?.kind === "git" && relative === `${owner.path}/.git` && stats.isDirectory(); + if (path.basename(file).toLowerCase() === ".git" && !allowedGitDirectory) { + add("git-object", "Unexpected .git repository metadata outside the materialized git resource repository"); + } + const head = files.get(path.join(file, "HEAD")); + const objects = files.get(path.join(file, "objects")); + const refs = files.get(path.join(file, "refs")); + if (stats.isDirectory() && head && (head.isFile() || head.isSymbolicLink()) + && objects && (objects.isDirectory() || objects.isSymbolicLink()) + && refs && (refs.isDirectory() || refs.isSymbolicLink()) && !allowedGitDirectory) { + add("git-object", "Unexpected bare repository (HEAD, objects/, refs/) outside the materialized git resource repository"); + } for (const match of path.basename(file).matchAll(/\d{4}-\d{2}-\d{2}/gu)) { const date = match[0]; const parsed = Date.parse(`${date}T00:00:00Z`); @@ -54,9 +87,7 @@ export const inspectBundleFiles = async (root: string, inputs: MaterializedInput target = path.dirname(target); } } - } else if (stats.isDirectory()) { - for (const name of (await readdir(file)).sort()) await visit(path.join(file, name)); - } else if (!stats.isFile()) add("symlink-escape", "Special files are not allowed in a moment bundle"); + } else if (!stats.isDirectory() && !stats.isFile()) add("symlink-escape", "Special files are not allowed in a moment bundle"); }; - await visit(root); + for (const file of files.keys()) await visit(file); }; diff --git a/src/compiler/training/rewind/leakRepositories.test.ts b/src/compiler/training/rewind/leakRepositories.test.ts new file mode 100644 index 00000000..689b0ae2 --- /dev/null +++ b/src/compiler/training/rewind/leakRepositories.test.ts @@ -0,0 +1,59 @@ +import { cp, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { cleanTrainingProjects } from "../plan/fixtures.test-helper.js"; +import { AT, rewindFixture, testGit } from "./fixtures.test-helper.js"; +import { leakCheckTrainingMoment } from "./leakCheck.js"; +import { finalizeTrainingMoment, materializeTrainingMoment } from "./materialize.js"; +import { stampTree } from "./timestamps.js"; + +afterEach(cleanTrainingProjects); +const unexpected = "Unexpected .git repository metadata outside the materialized git resource repository"; + +describe("repository discovery across every bundle input", () => { + it("refuses a future repository inside retained feed bytes after materializer timestamp stamping", async () => { + const fixture = await rewindFixture(); + const feedDirectory = path.dirname(fixture.feedRepo); + const landing = JSON.parse((await readFile(path.join(feedDirectory, "landings.jsonl"), "utf8")).trim()); + const retained = path.join(feedDirectory, "trees", landing.revision); + await mkdir(retained, { recursive: true }); + await cp(fixture.gitRepo, path.join(retained, "nested"), { recursive: true }); + await stampTree(retained, AT, AT); + const result = await materializeTrainingMoment(fixture.options); + expect(result.status).toBe("refused"); + const check = JSON.parse(await readFile(path.join(fixture.options.out, "leak-check.json"), "utf8")); + expect(check.findings).toContainEqual(expect.objectContaining({ class: "git-object", + path: `resources/team%3Apublication/news/trees/${landing.revision}/nested/.git`, detail: unexpected })); + expect(check.findings.filter((finding: { class: string }) => finding.class !== "git-object")).toEqual([]); + expect(await readdir(fixture.options.out)).toEqual(["leak-check.json", "moment.json"]); + }); + + it.each(["repository", "gitfile", "bare"] as const)("refuses an unexpected %s inside a bundle with all timestamps backdated", async (kind) => { + const fixture = await rewindFixture(), moment = await materializeTrainingMoment(fixture.options); + expect(moment.status).toBe("materialized"); + const input = moment.inputs.find((entry) => entry.kind === "git")!; + input.kind = "bundle"; + const directory = path.join(fixture.options.out, input.path!); + await rm(path.join(directory, ".git"), { recursive: true }); + const nested = path.join(directory, "nested"); + let relative = `${input.path}/nested/.git`, detail = unexpected; + if (kind === "repository") await cp(fixture.gitRepo, nested, { recursive: true }); + else if (kind === "gitfile") { + await mkdir(nested); + // Detection must refuse this metadata without following the textual pointer. + await writeFile(path.join(nested, ".git"), `gitdir: ${fixture.gitRepo}/.git\n`); + } else { + testGit(fixture.root, ["clone", "--quiet", "--bare", "--no-local", fixture.gitRepo, nested]); + relative = `${input.path}/nested`; + detail = "Unexpected bare repository (HEAD, objects/, refs/) outside the materialized git resource repository"; + } + await writeFile(path.join(fixture.options.out, "moment.json"), JSON.stringify(moment)); + await stampTree(fixture.options.out, AT, AT); + const check = await leakCheckTrainingMoment(fixture.options.out, { historyDir: fixture.historyDir }); + expect(check.findings).toContainEqual(expect.objectContaining({ class: "git-object", path: relative, detail })); + expect(check.findings.filter((finding) => finding.class !== "git-object")).toEqual([]); + const out = path.join(fixture.root, "rejected"); + expect((await finalizeTrainingMoment(fixture.options.out, out, { historyDir: fixture.historyDir })).status).toBe("refused"); + expect(await readdir(out)).toEqual(["leak-check.json", "moment.json"]); + }); +}); diff --git a/src/compiler/training/rewind/materialize.test.ts b/src/compiler/training/rewind/materialize.test.ts index 9286951e..5be116bb 100644 --- a/src/compiler/training/rewind/materialize.test.ts +++ b/src/compiler/training/rewind/materialize.test.ts @@ -6,7 +6,8 @@ import { agentManifest, cleanTrainingProjects, dataFiles, example, trainingProje import { materializeTrainingMoment, resolveTrainingMoment } from "./materialize.js"; import { leakCheckTrainingMoment } from "./leakCheck.js"; import * as gitMaterializer from "./git.js"; -import { AT, BEFORE, emptyRecord, jsonl, rewindFixture } from "./fixtures.test-helper.js"; +import * as volumeMaterializer from "./volumes.js"; +import { AT, BEFORE, FUTURE, emptyRecord, jsonl, rewindFixture, testGit } from "./fixtures.test-helper.js"; afterEach(cleanTrainingProjects); afterEach(() => vi.restoreAllMocks()); @@ -22,7 +23,14 @@ describe("materializeTrainingMoment", () => { expect(result.status).toBe("materialized"); expect(result.leakCheck).toEqual({ ok: true, findings: 0 }); expect(result.inputs.map((input) => input.fidelity)).toEqual(["approximate", "overridden", "approximate", "approximate", "exact"]); + const inspectTimes = async (file: string): Promise => { + const stats = await lstat(file); + expect(stats.mtimeMs, file).toBeLessThanOrEqual(Date.parse(AT)); + expect(stats.atimeMs, file).toBeLessThanOrEqual(Date.parse(AT)); + if (stats.isDirectory()) for (const name of await readdir(file)) await inspectTimes(path.join(file, name)); + }; expect((await leakCheckTrainingMoment(fixture.options.out, { historyDir: fixture.historyDir })).ok).toBe(true); + await inspectTimes(fixture.options.out); for (const input of result.inputs.filter((item) => item.kind !== "moltnet")) { expect((await lstat(path.join(fixture.options.out, input.path!))).mtime.toISOString()).toBe(new Date(BEFORE).toISOString()); } @@ -42,6 +50,45 @@ describe("materializeTrainingMoment", () => { expect(result.record.matched).toBe(true); expect(result.inputs.filter((input) => input.fidelity === "approximate")).toEqual([]); }); + it("refuses a suppressed candidate's noncommit only for its pinned input", async () => { + const fixture = await rewindFixture(); + const code = resolveTrainingMoment(fixture.plan, fixture.project, example.id).inputs.find((input) => input.id === "code")!; + if (code.section !== "resources" || code.declaration.kind !== "git") throw new Error("Expected git resource"); + code.declaration.ref = fixture.commit; delete code.declaration.branch; + const blob = testGit(fixture.gitRepo, ["rev-parse", `${fixture.commit}:news.txt`]); + const first = { ...emptyRecord(), resources: [{ id: "code", kind: "git", commit: fixture.commit }] }; + await jsonl(fixture.historyDir, "moments/candidates.jsonl", [first, + { ...first, resources: [{ id: "code", kind: "git", commit: blob }] }]); + let snapshotBeforeDiscard = ""; + const correct = volumeMaterializer.materializeVolume; + vi.spyOn(volumeMaterializer, "materializeVolume").mockImplementation(async (context) => { + const result = await correct(context); + snapshotBeforeDiscard = await readFile(path.join(context.destination, "saved.txt"), "utf8"); + return result; + }); + const result = await materializeTrainingMoment(fixture.options); + expect(result.status).toBe("refused"); + expect(result.inputs.find((input) => input.id === "code")!.refusal) + .toBe(`Recorded object ${blob} must be a commit object; found blob`); + expect(result.inputs.filter((input) => input.refusal !== null).map((input) => input.id)).toEqual(["code"]); + expect(result.inputs.find((input) => input.id === "edition-state")) + .toMatchObject({ fidelity: "approximate", source: { snapshot: "snapshot-1" }, refusal: null }); + expect(snapshotBeforeDiscard).toBe("snapshot bytes\n"); + expect(await readdir(fixture.options.out)).toEqual(["moment.json"]); + }); + it("excludes other agents, other wakes, and unrelated inputs from candidate commit validation", async () => { + const fixture = await rewindFixture(); + const code = resolveTrainingMoment(fixture.plan, fixture.project, example.id).inputs.find((input) => input.id === "code")!; + if (code.section !== "resources" || code.declaration.kind !== "git") throw new Error("Expected git resource"); + code.declaration.ref = fixture.commit; delete code.declaration.branch; + const blob = testGit(fixture.gitRepo, ["rev-parse", `${fixture.commit}:news.txt`]); + const first = { ...emptyRecord(), resources: [{ id: "code", kind: "git", commit: fixture.commit }] }; + await jsonl(fixture.historyDir, "moments/candidates.jsonl", [first, + { ...emptyRecord("agent:other"), resources: [{ id: "code", kind: "git", commit: blob }] }, + { ...emptyRecord("agent:a", FUTURE), resources: [{ id: "code", kind: "git", commit: blob }] }, + { ...first, resources: [...first.resources, { id: "unrelated", kind: "git", commit: blob }] }]); + expect((await materializeTrainingMoment(fixture.options)).status).toBe("materialized"); + }); it("collects all missing-history refusals and leaves only moment.json", async () => { const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest({ memory: [{ id: "self", store: { kind: "memory" } }], @@ -101,7 +148,8 @@ describe("materializeTrainingMoment", () => { expect(result.inputs.find((input) => input.id === "code")!.refusal).toContain("git-object"); expect(await readdir(fixture.options.out)).toEqual(["leak-check.json", "moment.json"]); expect(JSON.parse(await readFile(path.join(fixture.options.out, "leak-check.json"), "utf8")).findings) - .toEqual(expect.arrayContaining([expect.objectContaining({ class: "git-object" })])); + .toEqual(expect.arrayContaining([expect.objectContaining({ class: "git-object", path: "resources/team%3Apublication/code", + detail: `Object ${fixture.futureCommit} is not reachable from ${fixture.commit}` })])); expect(result.at).toBe(AT); }); }); diff --git a/src/compiler/training/rewind/materialize.ts b/src/compiler/training/rewind/materialize.ts index 4397ad02..1df637ad 100644 --- a/src/compiler/training/rewind/materialize.ts +++ b/src/compiler/training/rewind/materialize.ts @@ -1,4 +1,4 @@ -import { lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, utimes, writeFile } from "node:fs/promises"; +import { lstat, mkdir, mkdtemp, readdir, realpath, rename, rm, utimes, writeFile } from "node:fs/promises"; import path from "node:path"; import type { LoadedTrainingProject, TrainingPlan } from "../plan/index.js"; import { materializedMomentSchema, type MaterializedInput, type MaterializedMoment, type MomentRecord } from "./contract.js"; @@ -6,10 +6,10 @@ import { materializeFeed } from "./feed.js"; import { materializeGit } from "./git.js"; import { historyKey, inside, openHistory, type History } from "./history.js"; import { resolveRewindInputs } from "./inputs.js"; -import { leakCheckTrainingMoment, type LeakCheckOptions } from "./leakCheck.js"; +import { inspectTrainingMoment, type LeakCheckOptions } from "./leakCheck.js"; import { materializeMemory } from "./memory.js"; import { materializeMoltnet } from "./moltnet.js"; -import { selectMomentRecord } from "./select.js"; +import { matchingMomentRecords, selectMomentRecord } from "./select.js"; import { createTaskClock } from "./taskClock.js"; import { restoreAccessTimes, stampTree } from "./timestamps.js"; import type { InputContext, InputResult, RewindInput } from "./types.js"; @@ -28,7 +28,8 @@ export const resolveTrainingMoment = ( plan: TrainingPlan, project: LoadedTrainingProject, example: string, records: MomentRecord[] = [] ) => { const resolved = resolveRewindInputs(plan, project, example); - return { ...resolved, record: selectMomentRecord(records, plan.agent.id, resolved.at, + const candidateRecords = matchingMomentRecords(records, plan.agent.id, resolved.at); + return { ...resolved, candidateRecords, record: selectMomentRecord(candidateRecords, plan.agent.id, resolved.at, resolved.inputs.filter((input) => !input.overridden)) }; }; @@ -66,9 +67,9 @@ const discardPayload = async (staging: string, moment: MaterializedMoment): Prom /** A separately callable disk acceptance gate, also used to test deliberately broken producers. */ export const finalizeTrainingMoment = async (staging: string, out: string, options: LeakCheckOptions = {}): Promise => { - const moment = materializedMomentSchema.parse(JSON.parse(await readFile(path.join(staging, "moment.json"), "utf8"))); - if (moment.status === "materialized") { - const check = await leakCheckTrainingMoment(staging, options); + // Capture disk metadata before reading even moment.json; inspection preserves access times. + const { moment, check } = await inspectTrainingMoment(staging, options, true); + if (check) { moment.leakCheck = { ok: check.ok, findings: check.findings.length }; if (!check.ok) { for (const finding of check.findings) { @@ -105,8 +106,10 @@ export const materializeTrainingMoment = async (options: MaterializeTrainingMome const staging = await mkdtemp(path.join(path.dirname(out), `.${path.basename(out)}.rewind-`)); try { let record: MomentRecord | null = null, recordError: string | undefined; + let candidateRecords: MomentRecord[] = []; try { - record = selectMomentRecord(await history?.records() ?? [], plan.agent.id, resolved.at, + candidateRecords = matchingMomentRecords(await history?.records() ?? [], plan.agent.id, resolved.at); + record = selectMomentRecord(candidateRecords, plan.agent.id, resolved.at, resolved.inputs.filter((input) => !input.overridden)); } catch (error) { recordError = reason(error); } const moment: MaterializedMoment = { @@ -121,7 +124,7 @@ export const materializeTrainingMoment = async (options: MaterializeTrainingMome if (!input.overridden && (historyError || recordError)) throw new Error(historyError ?? recordError); const relative = `${input.section === "networks" ? "moltnet" : input.section}/${historyKey(input)}`; const destination = path.join(staging, relative); - const result = await dispatch({ input, at: resolved.at, record, history, destination }); + const result = await dispatch({ input, at: resolved.at, record, candidateRecords, history, destination }); await stampTree(destination, result.instant, resolved.at); Object.assign(entry, { fidelity: result.fidelity, basis: result.basis, source: result.source, notes: result.notes, path: relative, ...(result.build ? { build: result.build } : {}) }); diff --git a/src/compiler/training/rewind/memory.test.ts b/src/compiler/training/rewind/memory.test.ts index e9ffdb1d..238f54fa 100644 --- a/src/compiler/training/rewind/memory.test.ts +++ b/src/compiler/training/rewind/memory.test.ts @@ -32,7 +32,17 @@ describe("memory rewind", () => { expect(() => selectMemoryPrefix(ledger(memoryEvent(1), { ...memoryEvent(2), seq: undefined }), AT)).toThrow(/leading run/); expect(selectMemoryPrefix(ledger({ ...memoryEvent(1), seq: undefined }, memoryEvent(2)), AT).events).toHaveLength(2); }); - it("materializes an exact record prefix using only the public Mneme rebuild API", async () => { + it("rejects malformed JSON, invalid createdAt instants and duplicate JSON keys", () => { + expect(() => parseMemoryLedger("{broken}\n")).toThrow(/memory\/events.jsonl:1: invalid Mneme event/); + expect(() => ledger({ ...memoryEvent(1), createdAt: "not an instant" })).toThrow(/createdAt/); + expect(() => parseMemoryLedger(`${JSON.stringify(memoryEvent(1))}\n\n`)).toThrow(/memory\/events.jsonl:2/); + const payload = JSON.stringify(memoryEvent(1)); + for (const duplicate of [payload.replace('"seq":1', '"seq":2,"s\\u0065q":1'), + payload.replace('"agentId":"a"', '"agentId":"b","agentId":"a"')]) { + expect(() => parseMemoryLedger(`${duplicate}\n`)).toThrow(/duplicate JSON key/); + } + }); + it("materializes an exact record prefix without an index or a Mneme runtime dependency", async () => { const context = await setup(); await jsonl(context.history!.root, `${base}/events.jsonl`, [memoryEvent(1), memoryEvent(2, AT)]); context.record = { ...emptyRecord(), memory: [{ id: "self", declaredBy: "agent:a", seq: 1, generation: 0 }] }; @@ -40,7 +50,8 @@ describe("memory rewind", () => { expect(result).toMatchObject({ fidelity: "exact", basis: "record", source: { seq: 1, generation: 0 } }); expect(await readdir(path.join(context.destination, "memory"))).toEqual(["events.jsonl"]); expect(await readFile(path.join(context.destination, "memory/events.jsonl"), "utf8")).toBe(`${JSON.stringify(memoryEvent(1))}\n`); - expect(result.notes.join(" ")).toContain("public index rebuild"); + expect(result.notes.join(" ")).toContain("without a Mneme runtime dependency"); + expect(result.notes.join(" ")).toContain("Checksum verification is skipped"); }); it("includes the boundary and labels the timestamp prefix approximate", async () => { const context = await setup(); diff --git a/src/compiler/training/rewind/memory.ts b/src/compiler/training/rewind/memory.ts index 4d3ec77a..7e67c882 100644 --- a/src/compiler/training/rewind/memory.ts +++ b/src/compiler/training/rewind/memory.ts @@ -1,9 +1,8 @@ -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import os from "node:os"; +import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { historyKey } from "./history.js"; import { compareInstants } from "./instant.js"; -import { memoryEventSchema, memoryGenerationSchema, type MemoryGeneration, type MemoryLedgerEvent } from "./memorySchemas.js"; +import { parseMemoryLedgerEvent, memoryGenerationSchema, type MemoryGeneration, type MemoryLedgerEvent } from "./memorySchemas.js"; import type { InputContext, InputResult } from "./types.js"; export interface MemoryLedger { lines: string[]; events: MemoryLedgerEvent[] } @@ -14,7 +13,7 @@ export const parseMemoryLedger = (payload: string, label = "memory/events.jsonl" if (!payload.endsWith("\n")) throw new Error(`${label}: ledger must end with LF`); const lines = payload.slice(0, -1).split("\n"); const events = lines.map((line, index) => { - try { return memoryEventSchema.parse(JSON.parse(line)); } + try { return parseMemoryLedgerEvent(line); } catch (error) { throw new Error(`${label}:${index + 1}: invalid Mneme event: ${reason(error)}`); } }); return { lines, events }; @@ -72,25 +71,6 @@ export const selectMemoryGeneration = ( return selected; }; -/** Public rebuild validates the cut without shipping copied or derived index/audit bytes. */ -const validateWithMneme = async (home: string): Promise => { - let mneme: typeof import("@noopolis/mneme"); - try { mneme = await import("@noopolis/mneme"); } - catch (error) { throw new Error(`Mneme public index rebuild is unavailable; install the optional @noopolis/mneme package: ${reason(error)}`); } - if (typeof mneme.createMemoryIndex !== "function") throw new Error("Mneme root does not export createMemoryIndex; cannot validate the rewind index"); - const temporary = await mkdtemp(path.join(os.tmpdir(), "spawnfile-memory-index-")); - try { - const events = await new mneme.JsonlMemoryStore(home).read(); - const index = mneme.createMemoryIndex({ runtimeHomePath: home, indexFilePath: path.join(temporary, "index.sqlite") }); - try { - await index.rebuildFromEvents(events); - const indexed = await index.query({ limit: Math.max(events.length, 1) }); - // query() is a positive completion signal, including the empty-ledger case. - if (!Array.isArray(indexed)) throw new Error("Mneme rebuild did not produce a readable index"); - } finally { index.close(); } - } finally { await rm(temporary, { recursive: true, force: true }); } -}; - export const materializeMemory = async (context: InputContext): Promise => { const { input, at, record, history, destination } = context; if (input.section !== "memory") throw new Error("Memory materialization requires a memory input"); @@ -110,12 +90,12 @@ export const materializeMemory = async (context: InputContext): Promise { await Promise.all(roots.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); }); +const run = promisify(execFile); + +/** Emit only this runtime dependency closure; the child receives JS and production packages. */ +const emitMemoryPackage = async (directory: string): Promise => { + const seen = new Set(), sourceRoot = path.join(root, "src"); + const emit = async (source: string): Promise => { + if (seen.has(source)) return; + seen.add(source); + const relative = path.relative(sourceRoot, source); + if (relative.startsWith("..") || path.isAbsolute(relative)) throw new Error(`Unexpected package source ${source}`); + const output = path.join(directory, "dist", relative.replace(/\.ts$/u, ".js")); + const { outputText } = transpileModule(await readFile(source, "utf8"), { + fileName: source, compilerOptions: { module: ModuleKind.ESNext, target: ScriptTarget.ES2023 } + }); + await mkdir(path.dirname(output), { recursive: true }); + await writeFile(output, outputText); + for (const { fileName } of preProcessFile(outputText, true, true).importedFiles) { + if (fileName.startsWith(".")) await emit(path.resolve(path.dirname(source), fileName.replace(/\.js$/u, ".ts"))); + } + }; + await emit(path.join(sourceRoot, "compiler/training/rewind/memory.ts")); + const manifest = JSON.parse(await readFile(path.join(root, "package.json"), "utf8")) as { + dependencies: Record; + }; + await writeFile(path.join(directory, "package.json"), JSON.stringify(manifest)); + const copied = new Set(); + const copyDependency = async (name: string): Promise => { + if (copied.has(name)) return; + copied.add(name); + const source = await realpath(path.join(root, "node_modules", name)); + await cp(source, path.join(directory, "node_modules", name), { + recursive: true, dereference: true, + filter: (entry) => !path.relative(source, entry).split(path.sep).includes("node_modules") + }); + const dependency = JSON.parse(await readFile(path.join(source, "package.json"), "utf8")) as { + dependencies?: Record; + }; + for (const child of Object.keys(dependency.dependencies ?? {})) await copyDependency(child); + }; + for (const name of Object.keys(manifest.dependencies)) await copyDependency(name); +}; + +it("materializes nonempty historical memory in an isolated production package without development dependencies", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "rewind-memory-package-")); roots.push(directory); + await emitMemoryPackage(directory); + const history = path.join(directory, "history"), event = memoryEvent(1); + await jsonl(history, "memory/agent%3Aa/self/events.jsonl", [event]); + await writeFile(path.join(directory, "check.mjs"), ` +import assert from "node:assert/strict"; +import { readFile, readdir } from "node:fs/promises"; +import { materializeMemory } from "spawnfile/dist/compiler/training/rewind/memory.js"; +import { openHistory } from "spawnfile/dist/compiler/training/rewind/history.js"; +const manifest = JSON.parse(await readFile("package.json", "utf8")); +for (const name of Object.keys(manifest.devDependencies)) { + assert.throws(() => import.meta.resolve(name), { code: "ERR_MODULE_NOT_FOUND" }); +} +const result = await materializeMemory({ + input: { section: "memory", id: "self", declaredBy: "agent:a", declarationBase: process.cwd(), + declaration: { id: "self", store: { kind: "memory" } }, overridden: false }, + at: ${JSON.stringify(AT)}, record: null, history: await openHistory("history"), destination: "out" +}); +assert.equal(result.source.seq, 1); +assert.equal(await readFile("out/memory/events.jsonl", "utf8"), ${JSON.stringify(`${JSON.stringify(event)}\n`)}); +assert.deepEqual(await readdir("out/memory"), ["events.jsonl"]); +console.log("PRODUCTION_MEMORY_REWIND_OK"); +`); + const result = await run(process.execPath, ["check.mjs"], { + cwd: directory, env: { ...process.env, NODE_ENV: "production", NODE_PATH: "", NODE_OPTIONS: "" }, timeout: 15_000 + }); + expect(result.stdout.trim()).toBe("PRODUCTION_MEMORY_REWIND_OK"); +}); diff --git a/src/compiler/training/rewind/memorySchemas.ts b/src/compiler/training/rewind/memorySchemas.ts index 5bb899a2..bead69eb 100644 --- a/src/compiler/training/rewind/memorySchemas.ts +++ b/src/compiler/training/rewind/memorySchemas.ts @@ -35,3 +35,21 @@ export const memoryEventSchema = z.object({ origin: z.enum(["awake", "dream", "system"]).optional(), highWaterSeq: safeSequence.optional() }).strict(); export type MemoryLedgerEvent = z.infer; + +/** Mneme rejects duplicate JSON keys, including escaped and nested duplicates. */ +export const parseMemoryLedgerEvent = (line: string): MemoryLedgerEvent => { + const value: unknown = JSON.parse(line); + const objects: (Set | null)[] = []; + let string = ""; + for (const [token] of line.matchAll(/"(?:\\.|[^"\\])*"|[{}\[\]:]/gu)) { + if (token === "{") objects.push(new Set()); + else if (token === "[") objects.push(null); + else if (token === "}" || token === "]") objects.pop(); + else if (token === ":") { + const key = JSON.parse(string) as string, keys = objects.at(-1)!; + if (keys.has(key)) throw new Error(`duplicate JSON key ${JSON.stringify(key)}`); + keys.add(key); + } else string = token; + } + return memoryEventSchema.parse(value); +}; diff --git a/src/compiler/training/rewind/recordedCommit.test.ts b/src/compiler/training/rewind/recordedCommit.test.ts new file mode 100644 index 00000000..9e42180e --- /dev/null +++ b/src/compiler/training/rewind/recordedCommit.test.ts @@ -0,0 +1,223 @@ +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { momentRecordSchema } from "./contract.js"; +import { feedLandingSchema, materializeFeed } from "./feed.js"; +import { materializeGit } from "./git.js"; +import { commitTestGit, gitTestRoot, initTestGit, testGit, testMomentRecord } from "./git.test-helper.js"; +import { History } from "./history.js"; +import { resolveRecordedCommit } from "./recordedCommit.js"; +import type { InputContext } from "./types.js"; + +const AT = "2026-10-09T08:20:31Z", EARLY = "2026-10-09T08:00:00Z", REVISION = "e".repeat(64); +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const landing = (commit: string | null) => ({ revision: REVISION, tree: REVISION, commit, ref: "main", paths: [], landedAt: EARLY }); +const feedRecord = (commit: string) => ({ id: "news", revision: REVISION, tree: REVISION, commit, paths: [], landedAt: EARLY }); +const recordedValue = (field: "resources" | "feeds" | "landing", commit: string): string | null => { + if (field === "landing") return feedLandingSchema.parse(landing(commit)).commit; + const value = momentRecordSchema.parse({ ...testMomentRecord(), [field]: field === "resources" + ? [{ id: "archive", kind: "git", commit }] + : [{ id: "news", revision: REVISION, tree: REVISION, commit, paths: [], landedAt: EARLY }] }); + return value[field][0]!.commit; +}; +const fixture = async (kind: "git" | "bundle" = "git") => { + const root = await gitTestRoot(); roots.push(root); + const repository = path.join(root, "history/git/agent%3Awriter/archive"); + await initTestGit(repository); + const past = await commitTestGit(repository, { "readme.txt": "recorded bytes\n" }, EARLY); + testGit(repository, ["tag", "--annotate", "record-tag", "--message", "not a commit object", past]); + const objects = { blob: testGit(repository, ["rev-parse", `${past}:readme.txt`]), + tree: testGit(repository, ["rev-parse", `${past}^{tree}`]), tag: testGit(repository, ["rev-parse", "refs/tags/record-tag"]) }; + const context: InputContext = { at: AT, destination: path.join(root, "output"), history: new History(path.join(root, "history")), + record: { ...testMomentRecord(), resources: [{ id: "archive", kind, commit: past }] }, + input: { section: "resources", declaredBy: "agent:writer", id: "archive", declarationBase: root, overridden: false, + declaration: kind === "git" ? { id: "archive", kind, mode: "readonly", mount: "./archive", url: repository } + : { id: "archive", kind, mode: "readonly", mount: "./archive", source: "never-open-todays.tar" } } }; + return { root, repository, past, objects, context }; +}; +const retainedFeed = async () => { + const source = await fixture(); + const directory = path.join(source.root, "history/feeds/agent%3Awriter/news"), repository = path.join(directory, "repo"); + await mkdir(directory, { recursive: true }); + testGit(source.root, ["clone", "--quiet", "--bare", source.repository, repository]); + const retained = path.join(directory, "trees", REVISION); + await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "prepared.txt"), "retained bytes\n"); + const context: InputContext = { at: AT, destination: path.join(source.root, "feed-output"), history: source.context.history, record: null, + input: { section: "resources", declaredBy: "agent:writer", id: "news", declarationBase: source.root, overridden: false, + declaration: { id: "news", kind: "volume", mode: "mutable", mount: "./news", name: "org-news", + feed: { git: { repo: "never-fetch" }, prepare: { host: true, command: ["never-run"] } } } } }; + const log = async (commit: string) => writeFile(path.join(directory, "landings.jsonl"), `${JSON.stringify(landing(commit))}\n`); + return { ...source, repository, directory, context, log }; +}; + +describe("recorded commit contracts", () => { + it.each(["resources", "feeds", "landing"] as const)("normalizes full SHA-1 and SHA-256 %s commits to lowercase", (field) => { + for (const length of [40, 64]) expect(recordedValue(field, "AbCDef12".repeat(length / 8))).toBe("abcdef12".repeat(length / 8)); + }); + + it.each(["resources", "feeds", "landing"] as const)("rejects moving selectors, abbreviated and malformed %s commit IDs", (field) => { + for (const value of ["HEAD", "main", "refs/tags/old", "abcdef1", "a".repeat(39), "a".repeat(41), "a".repeat(63), "a".repeat(65), "g".repeat(40)]) { + expect(() => recordedValue(field, value), `${field}: ${value}`).toThrow("Expected a full 40- or 64-hex commit object id"); + } + }); + + it("retains null commit support for directory landing logs", () => { + expect(feedLandingSchema.parse(landing(null)).commit).toBeNull(); + }); +}); + +describe.each(["git", "bundle"] as const)("recorded %s commit objects", (kind) => { + it("materializes the full recorded ID case-insensitively", async () => { + const { past, context } = await fixture(kind); + context.record!.resources[0]!.commit = past.toUpperCase(); + expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: past } }); + expect(await readFile(path.join(context.destination, "readme.txt"), "utf8")).toBe("recorded bytes\n"); + }); + + it.each(["blob", "tree", "tag"] as const)("refuses a recorded %s object without peeling it", async (type) => { + const { objects, context } = await fixture(kind); + context.record!.resources[0]!.commit = objects[type]; + await expect(materializeGit(context)).rejects.toThrow(`Recorded object ${objects[type]} must be a commit object; found ${type}`); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it("refuses moving and abbreviated records even when their selectors resolve", async () => { + const { past, context } = await fixture(kind); + for (const commit of ["HEAD", past.slice(0, 12)]) { + context.record!.resources[0]!.commit = commit; + await expect(materializeGit(context)).rejects.toThrow("Expected a full 40- or 64-hex commit object id"); + } + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + const pin = (context: InputContext, commit: string) => { + if (context.input.section !== "resources" || context.input.declaration.kind === "volume") throw new Error("Expected git-backed resource"); + context.input.declaration = kind === "git" ? { ...context.input.declaration, kind, url: "never-read", ref: commit } + : { id: "archive", kind, mode: "readonly", mount: "./archive", build: { files: { root: ".", ref: commit } } }; + }; + + it.each(["blob", "tree", "tag"] as const)("refuses a matching %s record despite a valid declared pin", async (type) => { + const { context, past, objects } = await fixture(kind); + pin(context, past); context.record!.resources[0]!.commit = objects[type]; + await expect(materializeGit(context)).rejects.toThrow(`Recorded object ${objects[type]} must be a commit object; found ${type}`); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it("keeps pin precedence after verifying a different valid recorded commit", async () => { + const { context, repository, past } = await fixture(kind); + const next = await commitTestGit(repository, { "readme.txt": "other recorded commit\n" }, "2026-10-09T08:10:00Z"); + pin(context, past); context.record!.resources[0]!.commit = next.toUpperCase(); + expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "pinned", source: { commit: past } }); + expect(await readFile(path.join(context.destination, "readme.txt"), "utf8")).toBe("recorded bytes\n"); + }); + + it("does not verify unrelated recorded inputs", async () => { + const { context, past, objects } = await fixture(kind); + pin(context, past); context.record!.resources[0] = { id: "other", kind, commit: objects.blob }; + expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "pinned", source: { commit: past } }); + }); +}); + +describe("recorded commit validation authority", () => { + it("requires history for a matching pin record even if the local declared repository has the commit", async () => { + const { context, repository, past } = await fixture(); + context.history = undefined; + context.input.declaration = { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: repository, ref: past }; + await expect(materializeGit(context)).rejects.toThrow("Missing git history for recorded resource archive"); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it("does not verify the replaced input's recorded commit for an override", async () => { + const { context, repository, past, objects } = await fixture(); + context.input.overridden = true; context.history = undefined; context.record!.resources[0]!.commit = objects.blob; + context.candidateRecords = [context.record!]; + context.input.declaration = { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: repository, ref: past }; + expect(await materializeGit(context)).toMatchObject({ fidelity: "overridden", source: { commit: past } }); + }); +}); + +describe("retained feed recorded commit objects", () => { + it("normalizes landing and moment IDs before comparing and reports the commit object", async () => { + const { context, past, log } = await retainedFeed(); + await log(past.toUpperCase()); + context.record = { ...testMomentRecord(), feeds: [feedRecord(past.toUpperCase())] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", source: { commit: past } }); + expect(await readFile(path.join(context.destination, "current/prepared.txt"), "utf8")).toBe("retained bytes\n"); + }); + + it.each(["blob", "tree", "tag"] as const)("refuses a retained tree whose recorded commit is a %s", async (type) => { + const { context, objects, log } = await retainedFeed(); + await log(objects[type]); + await expect(materializeFeed(context)).rejects.toThrow(`Recorded object ${objects[type]} must be a commit object; found ${type}`); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it("refuses retained bytes when a recorded commit is absent from its history repository", async () => { + const { context, log } = await retainedFeed(); + await log("0".repeat(40)); + await expect(materializeFeed(context)).rejects.toThrow("Rewind git cat-file failed"); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it("requires the history repository even when retained tree bytes exist", async () => { + const { context, past, repository, log } = await retainedFeed(); + await log(past); await rm(repository, { recursive: true }); + await expect(materializeFeed(context)).rejects.toThrow("Missing feed source repository for news"); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it.each(["blob", "tree", "tag"] as const)("validates an unselected landing's %s commit object", async (type) => { + const { context, past, objects, directory } = await retainedFeed(); + const entries = [landing(past), { ...landing(objects[type]), landedAt: "2026-10-09T09:00:00Z" }]; + await writeFile(path.join(directory, "landings.jsonl"), `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`); + await expect(materializeFeed(context)).rejects.toThrow(`Recorded object ${objects[type]} must be a commit object; found ${type}`); + await expect(readdir(context.destination)).rejects.toThrow(); + }); +}); + +describe("feed moment records without landing logs", () => { + const fallbackFeed = async () => { + const value = await retainedFeed(), input = value.context.input; + if (input.section !== "resources" || input.declaration.kind !== "volume") throw new Error("Expected volume feed"); + input.declaration.feed = { git: { repo: "never-fetch", ref: "main" } }; + return value; + }; + + it.each(["blob", "tree", "tag"] as const)("refuses matching %s moment commits before feed-rule fallback", async (type) => { + const { context, objects } = await fallbackFeed(); + context.record = { ...testMomentRecord(), feeds: [feedRecord(objects[type])] }; + await expect(materializeFeed(context)).rejects.toThrow(`Recorded object ${objects[type]} must be a commit object; found ${type}`); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + + it("validates a moment commit but keeps feed-rule selection without a landing log", async () => { + const { context, past } = await fallbackFeed(); + context.record = { ...testMomentRecord(), feeds: [feedRecord(past.toUpperCase())] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", basis: "feed-rule-at", source: { commit: past } }); + }); + + it("does not verify unrelated feed moment records", async () => { + const { context, objects, past } = await fallbackFeed(); + context.record = { ...testMomentRecord(), feeds: [{ ...feedRecord(objects.blob), id: "other" }] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", source: { commit: past } }); + }); + + it("does not verify the replaced feed's moment record for an override", async () => { + const { context, objects } = await fallbackFeed(); + context.input.overridden = true; context.history = undefined; + context.record = { ...testMomentRecord(), feeds: [feedRecord(objects.blob)] }; + context.candidateRecords = [context.record]; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "overridden", basis: "override", source: {} }); + }); +}); + +describe("recorded commit object format", () => { + it("verifies full SHA-256 commit IDs without changing object format", async () => { + const root = await gitTestRoot(); roots.push(root); + testGit(root, ["init", "--quiet", "--initial-branch=main", "--template=", "--object-format=sha256"]); + const commit = await commitTestGit(root, { "readme.txt": "SHA-256\n" }, EARLY); + expect(commit).toHaveLength(64); + expect(await resolveRecordedCommit(root, commit.toUpperCase())).toBe(commit); + }); +}); diff --git a/src/compiler/training/rewind/recordedCommit.ts b/src/compiler/training/rewind/recordedCommit.ts new file mode 100644 index 00000000..80746b81 --- /dev/null +++ b/src/compiler/training/rewind/recordedCommit.ts @@ -0,0 +1,10 @@ +import { recordedCommitSchema } from "./contract.js"; +import { rewindGit } from "./gitProcess.js"; + +/** Recorded identity names the commit object itself, never a selector or a peeled tag. */ +export const resolveRecordedCommit = async (repository: string, value: string): Promise => { + const commit = recordedCommitSchema.parse(value); + const type = (await rewindGit(repository, ["cat-file", "-t", commit])).toString().trim(); + if (type !== "commit") throw new Error(`Recorded object ${commit} must be a commit object; found ${type}`); + return commit; +}; diff --git a/src/compiler/training/rewind/select.ts b/src/compiler/training/rewind/select.ts index 61dbdced..ecccc6ce 100644 --- a/src/compiler/training/rewind/select.ts +++ b/src/compiler/training/rewind/select.ts @@ -32,11 +32,14 @@ const canonical = (value: unknown): string => JSON.stringify(value, (_key, item: item && typeof item === "object" && !Array.isArray(item) ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a.localeCompare(b))) : item); +export const matchingMomentRecords = (records: readonly MomentRecord[], agent: string, at: string): MomentRecord[] => + records.filter((record) => record.agent === agent && compareInstants(record.wake.at, at) === 0); + /** Capture time and wake payload are not used to materialize data. Normalize actual used instants. */ export const selectMomentRecord = ( records: MomentRecord[], agent: string, at: string, inputs?: Array ): MomentRecord | null => { - const matches = records.filter((record) => record.agent === agent && compareInstants(record.wake.at, at) === 0); + const matches = matchingMomentRecords(records, agent, at); if (!matches.length) return null; const includes = (section: InputIdentity["section"], id: string, declaredBy?: string, field?: "resources" | "feeds" | "volumes"): boolean => !inputs || inputs.some((input) => { diff --git a/src/compiler/training/rewind/types.ts b/src/compiler/training/rewind/types.ts index 7dd09741..76d5f433 100644 --- a/src/compiler/training/rewind/types.ts +++ b/src/compiler/training/rewind/types.ts @@ -14,6 +14,7 @@ export interface InputContext { input: RewindInput; at: string; record: MomentRecord | null; + candidateRecords?: readonly MomentRecord[]; history: History | undefined; destination: string; } From e7b4c25a47c7588027dd9df456e97c481cfc6928 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sun, 11 Oct 2026 03:28:37 +0200 Subject: [PATCH 4/9] fix: check feed commit dates before copying retained trees and give each process one clock owner --- specs/TRAINING.md | 93 ++++++++++--- src/compiler/training/rewind/AGENTS.md | 5 +- src/compiler/training/rewind/contract.ts | 2 +- src/compiler/training/rewind/feed.test.ts | 18 +++ src/compiler/training/rewind/feed.ts | 5 +- src/compiler/training/rewind/git.test.ts | 15 +++ src/compiler/training/rewind/git.ts | 5 +- src/compiler/training/rewind/gitProcess.ts | 2 +- src/compiler/training/rewind/leakCheck.ts | 2 + .../training/rewind/leakProvenance.test.ts | 122 ++++++++++++++++++ .../training/rewind/leakProvenance.ts | 54 ++++++++ .../training/rewind/taskClock.test.ts | 57 ++++++-- src/compiler/training/rewind/taskClock.ts | 27 +++- 13 files changed, 363 insertions(+), 44 deletions(-) create mode 100644 src/compiler/training/rewind/leakProvenance.test.ts create mode 100644 src/compiler/training/rewind/leakProvenance.ts diff --git a/specs/TRAINING.md b/specs/TRAINING.md index 76d1ac70..9627e253 100644 --- a/specs/TRAINING.md +++ b/specs/TRAINING.md @@ -293,6 +293,9 @@ first-parent history and choose its newest commit with committer time `<= at` (`approximate`, `selector-at`); an absent selector means `HEAD`. Commit dates do not establish when a commit reached a branch or tag, so this fallback cannot prove historical ref membership. A missing eligible commit refuses. +The selected commit's author AND committer dates MUST be `<= at` before any +git checkout or bundle extraction, including pinned, recorded, and overridden +git sources. All paths MUST use the same selected-commit guard. Git output MUST be a clean new repository containing only objects reachable from the selected commit, with one checked-out branch, no remotes, tags, source @@ -324,6 +327,11 @@ feeds, and freeze/waiting behavior cannot reconstruct historical landings without a log and MUST refuse. Ref dates alone do not capture earlier freeze holds or when a producer made a ref available. +Before choosing retained bytes versus extraction, the selected feed source +commit MUST be verified as a commit object with author AND committer dates +`<= at`. This single selection guard applies equally to plain, retained, +prepared, and included git feeds; an eligible landing time does not make a +future source commit eligible. Directory feeds have no git source commit. Feed bytes come from OPTIONAL retained `trees//` when available. Otherwise a plain git feed re-extracts the landing's expanded paths at its commit and MUST verify the revision with the existing feed revision function. A mismatch @@ -397,25 +405,53 @@ MUST be set with `lutimes`; following a symlink to set times is insufficient. ### Task clock `task-clock.json` is strict `spawnfile.task-clock.v1` with `{version, origin}`; -`origin` is the example instant normalized to ISO-8601 with `Z`. The pure helper -`taskClockEnvironment(origin, anchorEpochMs)` accepts a caller-supplied safe -integer real launch anchor in milliseconds and returns these string variables: - -| Variable | Value | -| --- | --- | -| `NOOPOLIS_TASK_CLOCK` | Compact JSON `{"version":"noopolis.task-clock.v1","origin":"","anchorEpochMs":}` | -| `MNEME_CLOCK_ORIGIN` | The same normalized origin | -| `MNEME_CLOCK_ANCHOR_MS` | Decimal anchor milliseconds | -| `FAKETIME` | `@YYYY-MM-DD HH:MM:SS`, expressed in UTC | -| `FAKETIME_DONT_FAKE_MONOTONIC` | `1` | - -Daimon consumes the cross-repository `noopolis.task-clock.v1` contract: reported -agent wall time is `origin + (realNow - anchorEpochMs)`. Mneme consumes its generic -clock variables. The libfaketime values take effect only in capable images with -`LD_PRELOAD` configured by trial preparation. The clock offsets reported wall -time; real timeouts and monotonic elapsed time MUST remain real. Rewind neither -sleeps on nor advances a synthetic scheduler. Trial preparation owns the anchor -and runtime environment; the immutable bundle carries only the origin. +`origin` is the example instant normalized to ISO-8601 with `Z`. Trial preparation +supplies one safe-integer real launch anchor in milliseconds. All processes MUST +share `originEpochMs + (realNowEpochMs - anchorEpochMs)`; independently started +children MUST NOT reset their clock to the origin. `originEpochMs` uses JavaScript +`Date.parse(origin)` millisecond precision. + +Each process MUST have exactly one offset owner. These pure helpers replace +`taskClockEnvironment`; their outputs MUST NOT be merged for a single process: + +| Helper `(origin, anchorEpochMs)` | Recipient | Returned variables | +| --- | --- | --- | +| `harnessTaskClockEnvironment` | Top-level agent harness (Daimon), with an unshifted process clock | Only `NOOPOLIS_TASK_CLOCK`: compact JSON `{"version":"noopolis.task-clock.v1","origin":"","anchorEpochMs":}` | +| `processClockEnvironment` | Child/tool process under libfaketime | Only `FAKETIME`: signed relative seconds `(originEpochMs - anchorEpochMs) / 1000`, and `FAKETIME_DONT_FAKE_MONOTONIC=1` | +| `clockAwareEnvironment` | Clock-aware child (such as Mneme) without libfaketime | Only `MNEME_CLOCK_ORIGIN`: normalized origin, and `MNEME_CLOCK_ANCHOR_MS`: decimal anchor milliseconds | + +Daimon consumes the top-level contract and derives each child's environment itself. +Trial preparation supplies the installed libfaketime path through `LD_PRELOAD` for +the process-clock case. The helper cannot choose an image-specific library path. +That child's environment MUST omit `NOOPOLIS_TASK_CLOCK`, `MNEME_CLOCK_ORIGIN`, +and `MNEME_CLOCK_ANCHOR_MS`, including inherited values. A clock-aware child MUST +instead omit libfaketime from `LD_PRELOAD` and omit `FAKETIME` and +`NOOPOLIS_TASK_CLOCK`. Applying both offsets would shift time twice. The top-level +harness MUST NOT itself run under libfaketime or receive the Mneme offset variables. + +Relative `FAKETIME` MUST include a sign, use seconds without a suffix, and preserve +milliseconds (three fractional digits when nonzero; no fraction for integral +seconds). Integer-second rounding and the `@` start-at form are forbidden. +[Upstream libfaketime documentation](https://github.com/wolfcw/libfaketime#readme), +sections 4d/4e, supports fractional signed offsets since 0.8 and notes that the +decimal delimiter depends on locale. Trial preparation MUST ensure a dot-decimal +numeric locale (for example `LC_ALL=C`) for these values. + +Shared Spawnfile/Daimon conformance vectors: + +| Origin | Anchor epoch ms | `FAKETIME` | +| --- | --- | --- | +| `2001-01-01T00:00:00.000Z` | `1821692800250` | `-843385600.250` | +| `2001-01-01T00:00:00.000Z` | `1821692860250` | `-843385660.250` | +| `2001-01-01T00:00:00.250Z` | `978307200000` | `+0.250` | +| `2001-01-01T00:00:00.000Z` | `978307200001` | `-0.001` | +| `2001-01-01T00:00:00.000Z` | `978307200000` | `+0` | +| `2001-01-01T00:00:00.000Z` | `978307199000` | `+1` | + +The clock offsets reported wall time; real timeouts and monotonic elapsed time +MUST remain real. Rewind neither sleeps on nor advances a synthetic scheduler. +Trial preparation owns the anchor and runtime environment; the immutable bundle +carries only the origin. ### Independent leak check and output @@ -426,13 +462,26 @@ messages/events, and snapshots. Each finding carries `{input, class, path, detail}`; `input` is an input identity or `null` for bundle-level problems. Metadata is captured before payload reads; the checker restores the captured access times after inspection, preserving pre-existing future atimes for refusal. +Every git-backed input MUST record its selected full commit ID in +`moment.json`'s `inputs[].source.commit`; git feed identity MUST record the same +ID. Acceptance MUST independently open the source history repository read-only, +verify that ID names a commit object, and inspect its raw author and committer +dates against `at`, regardless of retained bytes or normalized filesystem times. +Missing/unreadable provenance and inconsistent feed commit IDs also refuse. +History repositories are resolved by input identity from `--history`. A local +pin or git-backed override resolved without that history entry MUST additionally +record its absolute local repository path in `source.repository`; acceptance +reopens it read-only. That source repository must remain available for subsequent +verification. Directory feeds and explicitly overridden prebuilt tars have no git +provenance. This source check is independent of the materialized git object's +reachable-history check and MUST NOT reuse the producer's date assertion. Every finding refuses the moment: | Class | Refusal condition | | --- | --- | | `file-time` | A file, directory, or symlink atime or mtime is after `at`. | | `symlink-escape` | An absolute link or a lexically/physically escaping link resolves outside the bundle. | -| `git-object` | Any commit author/committer time is after `at`, any object is unreachable from the selected commit, or refs/remotes/reflogs/fetch metadata exceed the single branch. Any `.git` directory/gitfile or bare repository (`HEAD` + `objects/` + `refs/`) anywhere outside the materialized git resource repository also refuses, including inside feed trees and bundles. | +| `git-object` | Source provenance is missing, inconsistent, not a commit, or has author/committer time after `at`; any materialized commit has a future date, any object is unreachable from the selected commit, or refs/remotes/reflogs/fetch metadata exceed the single branch. Any `.git` directory/gitfile or bare repository (`HEAD` + `objects/` + `refs/`) anywhere outside the materialized git resource repository also refuses, including inside feed trees and bundles. | | `memory-event` | Future ledger events, sequence gaps, invalid prefix ordering, or derived/audit files are present. | | `moltnet-message` / `moltnet-event` / `moltnet-room` | Retained messages, delivery events, or rooms are after `at`. | | `feed-identity` | `landed_at` is after `at` or `current` does not target a tree inside the volume. | @@ -1532,6 +1581,10 @@ The child deadline is Paideia's declared deadline plus five seconds for cleanup. "type": "string", "minLength": 1 }, + "repository": { + "type": "string", + "minLength": 1 + }, "revision": { "type": "string", "minLength": 1 diff --git a/src/compiler/training/rewind/AGENTS.md b/src/compiler/training/rewind/AGENTS.md index 3cf95171..c2e4447b 100644 --- a/src/compiler/training/rewind/AGENTS.md +++ b/src/compiler/training/rewind/AGENTS.md @@ -8,14 +8,15 @@ - `feed.ts`, `memory.ts`, and `volumes.ts` own their cuts and history-log schemas; `memorySchemas.ts` validates native ledger events. - `moltnet.ts` dispatches stores; `moltnetJson.ts`, `moltnetSqlite.ts`, `moltnetSqliteData.ts`, and `moltnetSchemas.ts` own snapshots, copied databases, SQL decoding, and strict native shapes. - `materialize.ts` stages all inputs and publishes only after the independent disk reader in `leakCheck.ts` accepts them. -- `leakFiles.ts`, `leakGit.ts`, and `leakInputs.ts` inspect disk metadata, every git object, and native payloads independently of materialization decisions. Discover unexpected `.git` directories/gitfiles and bare repositories throughout every input; only the materialized git resource repository is allowed. +- `leakFiles.ts`, `leakGit.ts`, and `leakInputs.ts` inspect disk metadata, every git object, and native payloads independently of materialization decisions. `leakProvenance.ts` reopens source repositories read-only using disk commit receipts, checking type and both raw dates without the producer's guard. Discover unexpected `.git` directories/gitfiles and bare repositories throughout every input; only the materialized git resource repository is allowed. - `instant.ts` compares parsed nanosecond instants; `timestamps.ts` stamps metadata without following symlinks. -- `taskClock.ts` defines offset wall-clock environment only; timeouts remain real. libfaketime requires trial-owned LD_PRELOAD. +- `taskClock.ts` separates top-level harness JSON, libfaketime child offsets, and clock-aware child variables: one offset owner per process. Relative signed seconds preserve milliseconds and the shared anchor; trial preparation supplies LD_PRELOAD and a dot-decimal numeric locale. Timeouts remain real. - Adjacent tests use local temporary repositories and fixtures. Never use Docker, production hosts or network fetches. - `fixtures.test-helper.ts` and `git.test-helper.ts` build disposable real projects and repositories; package emission excludes them. - Selector-at git history is approximate: committer dates cannot prove when a commit reached a ref. Feed-rule approximation cannot reconstruct freeze holds. - Reuse P2 effective input identity resolution. No input may silently use present-day data. - Feed/bundle extraction reads raw blobs, never `git archive` or attribute-driven transformations. Preserve symlinks for the escape check; refuse gitlinks. Recorded commits must be full lowercase 40/64-hex IDs verified as commit objects, including retained feeds. +- Guard every selected git/bundle/feed commit's author and committer dates before choosing output bytes, including retained/prepared feeds. Record the selected commit in the moment/identity; local pins or git-backed overrides without a history entry also record their repository path for independent acceptance. - Validate each non-overridden input's recorded commit IDs in every same-agent/same-wake candidate before accepting that input, even when pin precedence omits those fields from selection comparisons. - Validate memory JSONL structure locally; rewind never imports Mneme or ships an index/audit state. Checksum verification is skipped because persisted normalized events cannot recover the original JSON checksum input (`node_modules/@noopolis/mneme/dist/identity/ids.js:15-17`, `dist/store/store.js:249-274`). - Mneme 0.1.1's index is rebuilt from the ledger before every index read, not on open: `node_modules/@noopolis/mneme/dist/runtime/runtime.js:86-87` (embedding recall), `node_modules/@noopolis/mneme/dist/kernel/kernel.js:152-153` (search), and `:207-208` (locate). A ledger-only materialized bank is complete. diff --git a/src/compiler/training/rewind/contract.ts b/src/compiler/training/rewind/contract.ts index 02f497c9..3cb56800 100644 --- a/src/compiler/training/rewind/contract.ts +++ b/src/compiler/training/rewind/contract.ts @@ -36,7 +36,7 @@ export type MomentRecord = z.infer; export const materializedInputSchema = inputIdentitySchema.extend({ kind: z.enum(["git", "bundle", "feed", "volume", "memory", "moltnet"]), fidelity: z.enum(["exact", "approximate", "overridden"]).nullable(), basis: text, - source: z.object({ commit: text.optional(), revision: text.optional(), seq: count.optional(), + source: z.object({ commit: text.optional(), repository: text.optional(), revision: text.optional(), seq: count.optional(), generation: count.optional(), snapshot: text.optional(), cursor: text.optional() }).strict(), path: text.nullable(), notes: z.array(z.string()), refusal: text.nullable(), build: z.literal("deferred").optional() }).strict(); diff --git a/src/compiler/training/rewind/feed.test.ts b/src/compiler/training/rewind/feed.test.ts index 4d6dbcf2..0f88c888 100644 --- a/src/compiler/training/rewind/feed.test.ts +++ b/src/compiler/training/rewind/feed.test.ts @@ -7,6 +7,7 @@ import { feedLandingSchema, materializeFeed, selectFeedLanding, type FeedLanding import { History } from "./history.js"; import { commitTestGit, gitTestRoot, initTestGit, testGit, testMomentRecord } from "./git.test-helper.js"; import type { InputContext } from "./types.js"; +import { stampTree } from "./timestamps.js"; const AT = "2026-10-09T08:20:31Z", EARLY = "2026-10-09T08:00:00Z", LATE = "2026-10-09T09:00:00Z"; const roots: string[] = []; @@ -28,6 +29,23 @@ const fixture = async () => { }; describe("feed rewind", () => { + it.each(["retained", "reconstructed"].flatMap((bytes) => ["author", "committer"].map((role) => [bytes, role]))) + ("refuses a future selected commit before using %s bytes (%s date)", async (bytes, role) => { + const { context, repository, past, directory, landing, log } = await fixture(); + testGit(repository, ["checkout", "--detach", past]); + testGit(repository, ["commit", "--allow-empty", "-m", "future metadata", `--date=${role === "author" ? LATE : EARLY}`], role === "committer" ? LATE : EARLY); + const commit = testGit(repository, ["rev-parse", "HEAD"]); + const revision = resolveFeedSource({ kind: "git", fetch: false, repo: repository, ref: commit, paths: ["published"] }).revision; + await log([{ ...landing, commit, revision, tree: revision, landedAt: EARLY }]); + if (bytes === "retained") { + const retained = path.join(directory, "trees", revision); + await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "future.txt"), "future retained bytes"); + await stampTree(retained, EARLY, AT); + } + await expect(materializeFeed(context)).rejects.toThrow(`Selected commit ${commit} has an author or committer date after ${AT}`); + await expect(readdir(context.destination)).rejects.toThrow(); + }); + it("selects exact boundary landings and matches offset-equivalent records", async () => { const { context, landing, log, revision } = await fixture(); const selected = { ...landing, landedAt: AT }; diff --git a/src/compiler/training/rewind/feed.ts b/src/compiler/training/rewind/feed.ts index 4850b3d9..e78dea76 100644 --- a/src/compiler/training/rewind/feed.ts +++ b/src/compiler/training/rewind/feed.ts @@ -109,6 +109,9 @@ export const materializeFeed = async (context: InputContext): Promise { }; describe("git rewind", () => { + it.each(["git", "bundle"] as const)("refuses future recorded %s commits before materializing", async (kind) => { + const { repository, past, context } = await fixture(); + context.input.declaration = kind === "git" + ? { id: "archive", kind, mode: "readonly", mount: "./archive", url: repository } + : { id: "archive", kind, mode: "readonly", mount: "./archive", build: { files: { root: "package" } } }; + for (const role of ["author", "committer"]) { + testGit(repository, ["checkout", "--detach", past]); + testGit(repository, ["commit", "--allow-empty", "-m", "future metadata", `--date=${role === "author" ? LATE : EARLY}`], role === "committer" ? LATE : EARLY); + const commit = testGit(repository, ["rev-parse", "HEAD"]); + context.record = { ...testMomentRecord(), resources: [{ id: "archive", kind, commit }] }; + await expect(materializeGit(context)).rejects.toThrow(`Selected commit ${commit} has an author or committer date after ${AT}`); + await expect(readdir(context.destination)).rejects.toThrow(); + } + }); + it("records exactly one commit's reachable closure and removes future objects, refs and configuration", async () => { const { repository, past, future, context } = await fixture(); context.record = { ...testMomentRecord(), resources: [{ id: "archive", kind: "git", commit: past }] }; diff --git a/src/compiler/training/rewind/git.ts b/src/compiler/training/rewind/git.ts index 8f291904..7d6829c6 100644 --- a/src/compiler/training/rewind/git.ts +++ b/src/compiler/training/rewind/git.ts @@ -101,6 +101,7 @@ export const materializeGit = async (context: InputContext): Promise ({ commit, instant: await commitInstant(repository!, commit) })) : await commitAt(repository, selector, context.at); if (!selected) throw new Error(`No first-parent commit for ${selector} exists at or before ${context.at}`); - if (resource.kind === "bundle") await assertCommitNotFuture(repository, selected.commit, context.at); + await assertCommitNotFuture(repository, selected.commit, context.at); const deferred = resource.kind === "bundle" ? await materializeBundle(resource, context, repository, selected.commit) : false; if (resource.kind === "git") await materializeCleanRepository(repository, selected.commit, context.destination); return { fidelity: input.overridden ? "overridden" : pinned || record ? "exact" : "approximate", basis: input.overridden ? "override" : pinned ? "pinned" : record ? "record" : "selector-at", - source: { commit: selected.commit }, instant: selected.instant, + source: { commit: selected.commit, ...(!fromHistory ? { repository } : {}) }, instant: selected.instant, notes: pinned || record || input.overridden ? [] : ["Commit dates do not establish when a commit reached the selected ref."], ...(deferred ? { build: "deferred" } : {}) }; diff --git a/src/compiler/training/rewind/gitProcess.ts b/src/compiler/training/rewind/gitProcess.ts index 8e55ea34..0a4e7a3d 100644 --- a/src/compiler/training/rewind/gitProcess.ts +++ b/src/compiler/training/rewind/gitProcess.ts @@ -45,7 +45,7 @@ export const commitInstant = async (repo: string, commit: string): Promise => { const dates = (await rewindGit(repo, ["show", "-s", "--format=%at %ct", commit, "--"])).toString().trim().split(" "); if (dates.length !== 2 || dates.some((value) => !Number.isSafeInteger(Number(value)) || compareInstants(new Date(Number(value) * 1000).toISOString(), at) > 0)) { diff --git a/src/compiler/training/rewind/leakCheck.ts b/src/compiler/training/rewind/leakCheck.ts index 5aea63eb..269035fd 100644 --- a/src/compiler/training/rewind/leakCheck.ts +++ b/src/compiler/training/rewind/leakCheck.ts @@ -4,6 +4,7 @@ import { materializedMomentSchema, leakCheckSchema, type LeakCheck, type LeakFin import { openHistory } from "./history.js"; import { diskPath, inspectBundleFiles, restoreInspectedAccessTimes, snapshotBundleFiles } from "./leakFiles.js"; import { inspectGitBundle } from "./leakGit.js"; +import { inspectInputProvenance } from "./leakProvenance.js"; import { addInputFinding, inspectFeedBundle, inspectMemoryBundle, inspectMoltnetBundle, inspectVolumeBundle } from "./leakInputs.js"; export interface LeakCheckOptions { historyDir?: string } @@ -30,6 +31,7 @@ export const inspectTrainingMoment = async ( // Do not hand git/SQLite a payload with escaping paths, even for a failed check. if (report.findings.some((finding) => finding.class === "symlink-escape" && (finding.path === input.path || finding.path.startsWith(`${input.path}/`)))) continue; + if (["git", "bundle", "feed"].includes(input.kind)) await inspectInputProvenance(root, input, report, options.historyDir); if (input.kind === "git") await inspectGitBundle(directory, input, report); if (input.kind === "memory") await inspectMemoryBundle(root, input, report); if (input.kind === "feed") await inspectFeedBundle(root, input, report); diff --git a/src/compiler/training/rewind/leakProvenance.test.ts b/src/compiler/training/rewind/leakProvenance.test.ts new file mode 100644 index 00000000..3f7b0a2a --- /dev/null +++ b/src/compiler/training/rewind/leakProvenance.test.ts @@ -0,0 +1,122 @@ +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { resolveFeedSource } from "../../../volume/index.js"; +import type { MaterializedMoment } from "./contract.js"; +import { materializeFeed } from "./feed.js"; +import { materializeGit } from "./git.js"; +import * as gitProcess from "./gitProcess.js"; +import { commitTestGit, gitTestRoot, initTestGit, testGit } from "./git.test-helper.js"; +import { History } from "./history.js"; +import { leakCheckTrainingMoment } from "./leakCheck.js"; +import { finalizeTrainingMoment } from "./materialize.js"; +import { stampTree } from "./timestamps.js"; +import type { InputContext, InputResult } from "./types.js"; + +const AT = "2026-10-09T08:20:31Z", BEFORE = "2026-10-09T08:00:00Z", AFTER = "2026-10-11T00:00:00Z"; +const roots: string[] = []; +afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); +type Kind = "git" | "bundle" | "feed"; +const fixture = async (kind: Kind) => { + const root = await gitTestRoot(); roots.push(root); + const historyDir = path.join(root, "history"), key = "agent%3Awriter/source"; + const repository = path.join(historyDir, kind === "feed" ? `feeds/${key}/repo` : `git/${key}`); + await initTestGit(repository); + const commit = await commitTestGit(repository, { "data.txt": "past bytes\n" }, BEFORE); + const bundle = path.join(root, "stage"), relative = `resources/${key}`, destination = path.join(bundle, relative); + const declaration = kind === "feed" + ? { id: "source", kind: "volume" as const, mode: "mutable" as const, mount: "./data", name: "source", feed: { git: { repo: repository } } } + : kind === "git" ? { id: "source", kind, mode: "readonly" as const, mount: "./data", url: repository } + : { id: "source", kind, mode: "readonly" as const, mount: "./data", build: { files: { root: repository } } }; + const context: InputContext = { at: AT, destination, history: new History(historyDir), record: null, + input: { section: "resources", id: "source", declaredBy: "agent:writer", declarationBase: root, declaration, overridden: false } }; + const landing = async (selected: string) => { + const revision = resolveFeedSource({ kind: "git", fetch: false, repo: repository, ref: selected }).revision; + const directory = path.dirname(repository), retained = path.join(directory, "trees", revision); + await mkdir(retained, { recursive: true }); + await writeFile(path.join(retained, "data.txt"), selected === commit ? "past bytes\n" : "future retained bytes\n"); + await writeFile(path.join(directory, "landings.jsonl"), JSON.stringify({ revision, tree: revision, commit: selected, + ref: "main", paths: null, landedAt: BEFORE }) + "\n"); + await stampTree(retained, BEFORE, AT); + }; + const stage = async () => { + const result: InputResult = await (kind === "feed" ? materializeFeed(context) : materializeGit(context)); + const moment: MaterializedMoment = { version: "spawnfile.materialized-moment.v1", status: "materialized", agent: "agent:writer", + example: { id: "example", role: "test", dataset: "dataset" }, at: AT, record: { matched: false }, leakCheck: { ok: false, findings: 0 }, + inputs: [{ section: "resources", id: "source", declaredBy: "agent:writer", kind, fidelity: result.fidelity, + basis: result.basis, source: result.source, path: relative, notes: [], refusal: null }] }; + await writeFile(path.join(bundle, "moment.json"), JSON.stringify(moment)); + await stampTree(bundle, BEFORE, AT); + return moment; + }; + const future = (role: string) => { + testGit(repository, ["commit", "--allow-empty", "-m", "future", `--date=${role === "author" ? AFTER : BEFORE}`], role === "committer" ? AFTER : BEFORE); + return testGit(repository, ["rev-parse", "HEAD"]); + }; + if (kind === "feed") await landing(commit); + return { root, historyDir, repository, bundle, relative, destination, context, commit, landing, stage, future }; +}; + +describe("independent source commit provenance", () => { + it.each(["author", "committer"])("refuses retained future feed bytes when the producer's %s guard is deleted", async (role) => { + const f = await fixture("feed"), commit = f.future(role); + await f.landing(commit); // Eligible landing, future source, retained bytes with normalized timestamps. + const guard = vi.spyOn(gitProcess, "assertCommitNotFuture").mockResolvedValue(undefined); + const moment = await f.stage(); guard.mockRestore(); + expect(moment.inputs[0]).toMatchObject({ fidelity: "exact", source: { commit } }); + expect(await readFile(path.join(f.destination, "current/data.txt"), "utf8")).toBe("future retained bytes\n"); + await stampTree(f.bundle, BEFORE, AT); + const out = path.join(f.root, "refused"); + expect((await finalizeTrainingMoment(f.bundle, out, { historyDir: f.historyDir })).status).toBe("refused"); + const check = JSON.parse(await readFile(path.join(out, "leak-check.json"), "utf8")); + expect(check.findings).toEqual([{ input: { section: "resources", id: "source", declaredBy: "agent:writer" }, + class: "git-object", path: f.relative, detail: `Provenance commit ${commit} has a future ${role} date` }]); + expect(await readdir(out)).toEqual(["leak-check.json", "moment.json"]); + }); + + it.each((["git", "bundle", "feed"] as const).flatMap((kind) => ["author", "committer", "blob"].map((role) => ({ kind, role })))) + ("re-reads $kind source provenance and refuses $role on disk", async ({ kind, role }) => { + const f = await fixture(kind), moment = await f.stage(); + expect((await leakCheckTrainingMoment(f.bundle, { historyDir: f.historyDir })).findings).toEqual([]); + const commit = role === "blob" ? testGit(f.repository, ["rev-parse", `${f.commit}:data.txt`]) : f.future(role); + moment.inputs[0]!.source.commit = commit; + await writeFile(path.join(f.bundle, "moment.json"), JSON.stringify(moment)); + if (kind === "feed") { + const file = path.join(f.destination, ".spawnfile-feed.json"), identity = JSON.parse(await readFile(file, "utf8")); + identity.source.commit = commit; await writeFile(file, JSON.stringify(identity)); + } + await stampTree(f.bundle, BEFORE, AT); + const check = await leakCheckTrainingMoment(f.bundle, { historyDir: f.historyDir }); + expect(check.findings).toContainEqual(expect.objectContaining({ class: "git-object", path: f.relative, + detail: role === "blob" ? `Provenance object ${commit} must be a commit object; found blob` + : `Provenance commit ${commit} has a future ${role} date` })); + expect(check.findings.some((finding) => finding.class === "file-time")).toBe(false); + }); + + it.each(["git", "bundle"] as const)("keeps local pinned %s provenance verifiable without a history directory", async (kind) => { + const f = await fixture(kind); + if (f.context.input.section !== "resources") throw new Error("Expected resource"); + const resource = f.context.input.declaration; + if (resource.kind === "git") resource.ref = f.commit; + if (resource.kind === "bundle") resource.build!.files!.ref = f.commit; + f.context.history = undefined; + const moment = await f.stage(); + expect(moment.inputs[0]!.source).toEqual({ commit: f.commit, repository: f.repository }); + expect((await leakCheckTrainingMoment(f.bundle)).findings).toEqual([]); + await rm(f.repository, { recursive: true }); + expect((await leakCheckTrainingMoment(f.bundle)).findings).toContainEqual(expect.objectContaining({ class: "git-object" })); + }); + + it("refuses inconsistent feed commit identities even when both commits are historical", async () => { + const f = await fixture("feed"); await f.stage(); + const other = await commitTestGit(f.repository, {}, BEFORE); + const file = path.join(f.destination, ".spawnfile-feed.json"), identity = JSON.parse(await readFile(file, "utf8")); + identity.source.commit = other; await writeFile(file, JSON.stringify(identity)); + await stampTree(f.bundle, BEFORE, AT); + expect((await leakCheckTrainingMoment(f.bundle, { historyDir: f.historyDir })).findings) + .toContainEqual(expect.objectContaining({ class: "git-object", detail: "Feed source commit disagrees with moment provenance" })); + }); +}); diff --git a/src/compiler/training/rewind/leakProvenance.ts b/src/compiler/training/rewind/leakProvenance.ts new file mode 100644 index 00000000..8b7970b0 --- /dev/null +++ b/src/compiler/training/rewind/leakProvenance.ts @@ -0,0 +1,54 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import type { FeedIdentity } from "../../../volume/feedRecord.js"; +import type { LeakCheck, MaterializedInput } from "./contract.js"; +import { assertLocalGitRepository } from "./gitHistory.js"; +import { rewindGit } from "./gitProcess.js"; +import { historyKey, openHistory } from "./history.js"; +import { instantNanos } from "./instant.js"; +import { diskPath } from "./leakFiles.js"; +import { addInputFinding } from "./leakInputs.js"; + +/** Read receipts and source objects afresh; never trust the producer's date guard or stamped files. */ +export const inspectInputProvenance = async ( + root: string, input: MaterializedInput, report: LeakCheck, historyDir?: string +): Promise => { + const add = (detail: string) => addInputFinding(report, input, "git-object", detail); + try { + const commit = input.source.commit; + if (input.kind === "feed") { + if (input.fidelity === "overridden") return; + const identity: FeedIdentity = JSON.parse(await readFile(await diskPath(root, `${input.path}/.spawnfile-feed.json`), "utf8")); + if (identity.source?.kind === "git") { + if (!commit || identity.source.commit !== commit) add("Feed source commit disagrees with moment provenance"); + } else if (!commit) return; // Directory feeds have no git provenance. + } else if (input.kind === "bundle" && input.fidelity === "overridden" && !commit && !input.source.repository) { + return; // Explicit prebuilt tar overrides have no source commit. + } + if (!commit || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/u.test(commit)) throw new Error("Missing or invalid source commit provenance"); + let repository: string; + if (input.source.repository) { + // Only locally resolved pins/overrides may lack an entry in the history layout. + if (input.kind === "feed" || input.basis !== "pinned" && input.fidelity !== "overridden" + || !path.isAbsolute(input.source.repository)) throw new Error("Invalid local provenance repository"); + repository = input.source.repository; + await assertLocalGitRepository(repository); + } else { + const history = await openHistory(historyDir); + const key = input.kind === "feed" ? `feeds/${historyKey(input)}/repo` : `git/${historyKey(input)}`; + const retained = await history?.repository(key); + if (!retained) throw new Error(`Missing source history repository for provenance: ${key}`); + repository = retained; + await assertLocalGitRepository(repository, history); + } + const type = (await rewindGit(repository, ["cat-file", "-t", commit])).toString().trim(); + if (type !== "commit") throw new Error(`Provenance object ${commit} must be a commit object; found ${type}`); + const headers = (await rewindGit(repository, ["cat-file", "commit", commit])).toString().split("\n\n", 1)[0]!; + report.checked.gitObjects++; + for (const role of ["author", "committer"]) { + const match = new RegExp(`^${role} .* (-?\\d+) [+-]\\d{4}$`, "mu").exec(headers); + if (!match) add(`Provenance commit ${commit} has no valid ${role} date`); + else if (BigInt(match[1]!) * 1_000_000_000n > instantNanos(report.at)) add(`Provenance commit ${commit} has a future ${role} date`); + } + } catch (error) { add(error instanceof Error ? error.message : String(error)); } +}; diff --git a/src/compiler/training/rewind/taskClock.test.ts b/src/compiler/training/rewind/taskClock.test.ts index 9ec1c80d..3dc04c66 100644 --- a/src/compiler/training/rewind/taskClock.test.ts +++ b/src/compiler/training/rewind/taskClock.test.ts @@ -1,17 +1,54 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { createTaskClock, taskClockEnvironment } from "./taskClock.js"; +import { clockAwareEnvironment, createTaskClock, harnessTaskClockEnvironment, processClockEnvironment } from "./taskClock.js"; afterEach(() => vi.restoreAllMocks()); +const helpers = [harnessTaskClockEnvironment, processClockEnvironment, clockAwareEnvironment]; +// Shared Spawnfile/Daimon contract vectors, also documented in specs/TRAINING.md. +const vectors: Array<[string, number, string]> = [ + ["2001-01-01T00:00:00.000Z", 1_821_692_800_250, "-843385600.250"], + ["2001-01-01T00:00:00.000Z", 1_821_692_860_250, "-843385660.250"], + ["2001-01-01T00:00:00.250Z", 978_307_200_000, "+0.250"], + ["2001-01-01T00:00:00.000Z", 978_307_200_001, "-0.001"], + ["2001-01-01T00:00:00.000Z", 978_307_200_000, "+0"], + ["2001-01-01T00:00:00.000Z", 978_307_199_000, "+1"] +]; describe("training task clock contract", () => { - it("normalizes the bundle origin and returns exact cross-repository environment values", () => { + it("normalizes the origin and gives each process exactly one clock owner", () => { const origin = "2026-10-09T10:20:31.120000000+02:00", anchor = 1_790_000_000_123; expect(createTaskClock(origin)).toEqual({ version: "spawnfile.task-clock.v1", origin: "2026-10-09T08:20:31.12Z" }); - expect(taskClockEnvironment(origin, anchor)).toEqual({ - NOOPOLIS_TASK_CLOCK: '{"version":"noopolis.task-clock.v1","origin":"2026-10-09T08:20:31.12Z","anchorEpochMs":1790000000123}', - MNEME_CLOCK_ORIGIN: "2026-10-09T08:20:31.12Z", MNEME_CLOCK_ANCHOR_MS: "1790000000123", - FAKETIME: "@2026-10-09 08:20:31", FAKETIME_DONT_FAKE_MONOTONIC: "1" + expect(harnessTaskClockEnvironment(origin, anchor)).toEqual({ + NOOPOLIS_TASK_CLOCK: '{"version":"noopolis.task-clock.v1","origin":"2026-10-09T08:20:31.12Z","anchorEpochMs":1790000000123}' }); + expect(clockAwareEnvironment(origin, anchor)).toEqual({ + MNEME_CLOCK_ORIGIN: "2026-10-09T08:20:31.12Z", MNEME_CLOCK_ANCHOR_MS: "1790000000123" + }); + expect(Object.keys(processClockEnvironment(origin, anchor)).sort()).toEqual(["FAKETIME", "FAKETIME_DONT_FAKE_MONOTONIC"]); + for (const helper of helpers) { + const env = helper(origin, anchor); + expect(["NOOPOLIS_TASK_CLOCK", "MNEME_CLOCK_ORIGIN", "FAKETIME"].filter((key) => key in env)).toHaveLength(1); + } + }); + + it.each(vectors)("matches the shared offset for %s anchored at %s", (origin, anchor, expected) => { + const processEnv = processClockEnvironment(origin, anchor); + expect(processEnv).toEqual({ FAKETIME: expected, FAKETIME_DONT_FAKE_MONOTONIC: "1" }); + const contract = JSON.parse(harnessTaskClockEnvironment(origin, anchor).NOOPOLIS_TASK_CLOCK!); + const aware = clockAwareEnvironment(origin, anchor); + // Daimon's task-clock formula, including a later independently started process. + for (const delay of [0, 1, 60_123]) { + const realNow = anchor + delay; + const daimonNow = Date.parse(contract.origin) + (realNow - contract.anchorEpochMs); + expect(realNow + Number(processEnv.FAKETIME) * 1000).toBe(daimonNow); + expect(Date.parse(aware.MNEME_CLOCK_ORIGIN!) + realNow - Number(aware.MNEME_CLOCK_ANCHOR_MS)).toBe(daimonNow); + expect(daimonNow).toBe(Date.parse(origin) + delay); + } + }); + + it("changes the relative offset when the shared anchor changes", () => { + const origin = "2001-01-01T00:00:00.000Z"; + expect(processClockEnvironment(origin, 1_821_692_800_250).FAKETIME).toBe("-843385600.250"); + expect(processClockEnvironment(origin, 1_821_692_860_250).FAKETIME).toBe("-843385660.250"); }); it("uses only its supplied anchor without wall-clock reads, sleeping, or process environment mutation", () => { @@ -19,14 +56,14 @@ describe("training task clock contract", () => { const now = vi.spyOn(Date, "now").mockImplementation(() => { throw Error("read live clock"); }); const timeout = vi.spyOn(globalThis, "setTimeout").mockImplementation(() => { throw Error("started timeout"); }); const interval = vi.spyOn(globalThis, "setInterval").mockImplementation(() => { throw Error("started interval"); }); - const first = taskClockEnvironment("2026-10-09T08:20:31Z", 0); - expect(taskClockEnvironment("2026-10-09T10:20:31+02:00", 0)).toEqual(first); - expect(JSON.parse(first.NOOPOLIS_TASK_CLOCK!).anchorEpochMs).toBe(0); + for (const helper of helpers) { + expect(helper("2026-10-09T08:20:31Z", 0)).toEqual(helper("2026-10-09T10:20:31+02:00", 0)); + } expect(now).not.toHaveBeenCalled(); expect(timeout).not.toHaveBeenCalled(); expect(interval).not.toHaveBeenCalled(); expect(process.env).toEqual(environment); }); it.each([NaN, Infinity, 1.5, Number.MAX_SAFE_INTEGER + 1])("refuses non-integer or unsafe anchor %s", (anchor) => { - expect(() => taskClockEnvironment("2026-10-09T08:20:31Z", anchor)).toThrow(/anchorEpochMs must be a safe integer/u); + for (const helper of helpers) expect(() => helper("2026-10-09T08:20:31Z", anchor)).toThrow(/anchorEpochMs must be a safe integer/u); }); }); diff --git a/src/compiler/training/rewind/taskClock.ts b/src/compiler/training/rewind/taskClock.ts index 9bef899f..a98e1efe 100644 --- a/src/compiler/training/rewind/taskClock.ts +++ b/src/compiler/training/rewind/taskClock.ts @@ -4,14 +4,29 @@ import { taskClockSchema } from "./contract.js"; export const createTaskClock = (origin: string) => taskClockSchema.parse({ version: "spawnfile.task-clock.v1", origin: normalizeInstant(origin) }); -/** Offsets reported wall time only. Callers supply the real anchor at trial launch. */ -export const taskClockEnvironment = (origin: string, anchorEpochMs: number): Record => { +const clockOrigin = (origin: string, anchorEpochMs: number): string => { if (!Number.isSafeInteger(anchorEpochMs)) throw new Error("Task clock anchorEpochMs must be a safe integer"); - const normalized = normalizeInstant(origin); + return normalizeInstant(origin); +}; + +/** Top-level Daimon only: its clock stays real and it derives each child's clock environment. */ +export const harnessTaskClockEnvironment = (origin: string, anchorEpochMs: number): Record => ({ + NOOPOLIS_TASK_CLOCK: JSON.stringify({ version: "noopolis.task-clock.v1", origin: clockOrigin(origin, anchorEpochMs), anchorEpochMs }) +}); + +/** Child/tool wall clock. Trial preparation supplies libfaketime LD_PRELOAD and a dot-decimal locale. */ +export const processClockEnvironment = (origin: string, anchorEpochMs: number): Record => { + const normalized = clockOrigin(origin, anchorEpochMs); + const offsetMs = BigInt(Date.parse(normalized)) - BigInt(anchorEpochMs); + const magnitude = offsetMs < 0n ? -offsetMs : offsetMs; + const fraction = magnitude % 1000n; return { - NOOPOLIS_TASK_CLOCK: JSON.stringify({ version: "noopolis.task-clock.v1", origin: normalized, anchorEpochMs }), - MNEME_CLOCK_ORIGIN: normalized, MNEME_CLOCK_ANCHOR_MS: String(anchorEpochMs), - FAKETIME: `@${new Date(normalized).toISOString().slice(0, 19).replace("T", " ")}`, + FAKETIME: `${offsetMs < 0n ? "-" : "+"}${magnitude / 1000n}${fraction ? `.${String(fraction).padStart(3, "0")}` : ""}`, FAKETIME_DONT_FAKE_MONOTONIC: "1" }; }; + +/** Clock-aware child (e.g. Mneme) without libfaketime: apply the offset inside the process exactly once. */ +export const clockAwareEnvironment = (origin: string, anchorEpochMs: number): Record => ({ + MNEME_CLOCK_ORIGIN: clockOrigin(origin, anchorEpochMs), MNEME_CLOCK_ANCHOR_MS: String(anchorEpochMs) +}); From 1533c7254bcb779eb99c649f1d342eb12760b8e3 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sun, 11 Oct 2026 03:55:20 +0200 Subject: [PATCH 5/9] refactor: read moment records, landing logs and recorder snapshots in the recording formats --- specs/TRAINING.md | 367 +++++++++++------- src/cli/trainCommand.test.ts | 2 +- src/compiler/training/rewind/AGENTS.md | 9 +- src/compiler/training/rewind/contract.ts | 29 +- src/compiler/training/rewind/feed.test.ts | 121 ++++-- src/compiler/training/rewind/feed.ts | 114 +++--- .../training/rewind/feedLandings.test.ts | 50 +++ src/compiler/training/rewind/feedLandings.ts | 51 +++ .../training/rewind/fixtures.test-helper.ts | 20 +- .../training/rewind/git.test-helper.ts | 5 +- src/compiler/training/rewind/git.test.ts | 20 +- src/compiler/training/rewind/git.ts | 24 +- .../training/rewind/gitArchive.test.ts | 6 +- src/compiler/training/rewind/gitTree.ts | 6 + src/compiler/training/rewind/history.test.ts | 5 +- src/compiler/training/rewind/history.ts | 19 +- src/compiler/training/rewind/index.ts | 6 +- src/compiler/training/rewind/inputs.ts | 2 +- src/compiler/training/rewind/leakCheck.ts | 4 +- src/compiler/training/rewind/leakFeed.ts | 29 ++ src/compiler/training/rewind/leakInputs.ts | 72 ++-- .../training/rewind/leakProvenance.test.ts | 6 +- .../training/rewind/leakProvenance.ts | 5 + .../training/rewind/materialize.test.ts | 8 +- src/compiler/training/rewind/materialize.ts | 40 +- src/compiler/training/rewind/memory.test.ts | 34 +- src/compiler/training/rewind/memory.ts | 56 +-- src/compiler/training/rewind/memorySchemas.ts | 8 - src/compiler/training/rewind/moltnet.ts | 19 +- .../training/rewind/moltnetJson.test.ts | 32 +- src/compiler/training/rewind/moltnetJson.ts | 6 - .../training/rewind/moltnetSqlite.test.ts | 16 +- .../training/rewind/recordInput.test.ts | 30 ++ src/compiler/training/rewind/recordInput.ts | 43 ++ .../training/rewind/recordedBundle.test.ts | 69 ++++ .../training/rewind/recordedBundle.ts | 54 +++ .../training/rewind/recordedCommit.test.ts | 52 +-- .../training/rewind/recordingFormat.test.ts | 108 ++++++ src/compiler/training/rewind/select.test.ts | 24 +- src/compiler/training/rewind/select.ts | 15 +- .../rewind/snapshotFixtures.test-helper.ts | 50 +++ .../training/rewind/snapshotManifest.ts | 118 ++++++ src/compiler/training/rewind/types.ts | 2 + src/compiler/training/rewind/volumes.test.ts | 164 +++++--- src/compiler/training/rewind/volumes.ts | 138 ++----- 45 files changed, 1439 insertions(+), 619 deletions(-) create mode 100644 src/compiler/training/rewind/feedLandings.test.ts create mode 100644 src/compiler/training/rewind/feedLandings.ts create mode 100644 src/compiler/training/rewind/leakFeed.ts create mode 100644 src/compiler/training/rewind/recordInput.test.ts create mode 100644 src/compiler/training/rewind/recordInput.ts create mode 100644 src/compiler/training/rewind/recordedBundle.test.ts create mode 100644 src/compiler/training/rewind/recordedBundle.ts create mode 100644 src/compiler/training/rewind/recordingFormat.test.ts create mode 100644 src/compiler/training/rewind/snapshotFixtures.test-helper.ts create mode 100644 src/compiler/training/rewind/snapshotManifest.ts diff --git a/specs/TRAINING.md b/specs/TRAINING.md index 9627e253..b6ff3866 100644 --- a/specs/TRAINING.md +++ b/specs/TRAINING.md @@ -155,40 +155,49 @@ appears between the `training-plan-schema` markers below. ## Moment record -`spawnfile.moment-record.v1` specifies one record per wake; rewind consumes this format, -while the recording phase produces it. Recording MUST be off unless an effective `record` exists, -with zero recording cost when off. `record.keep` is snapshot retention. - -```json -{ - "version": "spawnfile.moment-record.v1", - "agent": "agent:writer", - "wake": { "at": "2026-10-09T08:20:31Z", "surface": "moltnet", "room": "assignment", "from": "brass", "messageId": "42" }, - "recordedAt": "2026-10-09T08:20:32Z", - "resources": [{ "id": "archive", "kind": "git", "commit": "0123456789abcdef0123456789abcdef01234567" }], - "feeds": [{ "id": "news", "revision": "revision-id", "tree": "tree-id", "commit": "0123456789abcdef0123456789abcdef01234567", "paths": ["published"], "landedAt": "2026-10-09T08:00:00Z" }], - "memory": [{ "id": "self", "declaredBy": "agent:writer", "seq": 42, "generation": 3 }], - "moltnet": [{ "network": "news", "cursor": "wake-cursor" }], - "volumes": [{ "id": "edition-state", "snapshot": "incremental-snapshot-id" }] -} -``` - -`agent` is the exact node ID. `wake.at` MUST be the real wake time (not the dataset's -authored trial time); `recordedAt` is capture time. Both are instants with an offset or -`Z`; `messageId` is OPTIONAL. Resources identify the git/bundle resolved commit per -build/deploy. Every recorded `resources[].commit`, `feeds[].commit`, and non-null -landing-log `commit` MUST be a full 40- or 64-hex object ID, normalized to lowercase. -Moving selectors and abbreviated IDs refuse. For every non-overridden input, -rewind MUST verify matching moment-record IDs and every non-null landing-log ID -as commit objects in that input's history repository; tree/blob IDs and annotated -tag objects refuse without peeling. This validation also applies to pinned -resources, feed-rule fallback, and retained feed trees; it does not change which -valid commit takes selection precedence. -Feeds identify the fed-volume landing-log entry in effect at the wake. -Memory pins the Mneme ledger sequence and active curation generation for each declared -bank; Moltnet pins each network's per-wake message cursor. Volumes identify an incremental -snapshot for every effective `record.volumes` entry. Rewind MUST use records when present; -otherwise it is best effort and the trial MUST be labelled as such. +The recording PR (`feat/training-recording`, not yet merged) owns the normative +`spawnfile.moment-record.v1` format in `specs/TRAINING.md`, “Moment record” and +“Host histories and retention”. Rewind depends on that PR, not the superseded +format sketch. The consumer-dependent recording text below is carried unchanged +until the recording PR merges; it is the same contract, not a second format. + +All instants MUST use one host clock, shared by its containers, and include an offset +or `Z`. Every pin MUST describe state in effect at `wake.startedAt`, not capture time. +A later `recordedAt` MUST NOT advance a pin. + +- `agent` MUST be the exact compiled node ID; `recordedAt` is the capture instant. +- `wake` MUST contain `at`, `startedAt`, `executionId`, and nonempty `deliveries` of + `{deliveryId, kind, occurredAt}`. `at` is the first delivery's `occurred_at` (real wake + time); `startedAt` is when the turn began. Queued deliveries can precede that instant. + `surface`, `room`, `from`, and `messageId` are OPTIONAL, derived from a + `moltnet:` delivery and its Moltnet message when available; other deliveries + MUST NOT invent Moltnet metadata. +- `resources` MUST use `{id, kind:"git", commit, sha256?}` or + `{id, kind:"bundle", sha256}` from the deploy resource entry in effect at + `startedAt`, selected by deployment record identity and `agent_node_ids` containing + that exact agent node ID. Bundles have no commit. An inexact activation boundary + MUST set the moment's `exact:false`; an overlapping predecessor can still run old pins. +- `feeds` MUST use `{id, revision, tree, commit?, ref?, paths?, landedAt, period?}` + from the landing entry in effect at `startedAt` for each consumed fed volume. + A wake inside the inclusive `[switching_at, landed_at]` transition window MUST + set `exact:false`; the helper returns that landing with `ambiguous:true`. +- `memory` MUST use `{id, declaredBy, seq}` for each accessible bank. `seq` is the + highest Mneme ledger sequence whose `createdAt <= startedAt`; 0 means empty. + `generation` is OPTIONAL and MUST be omitted while Mneme has no generations. + Mneme's append-only `events.jsonl` is the source of truth; there are no curation + archives or generations to pin today. +- `moltnet` MUST use `{network, cursor}` with `cursor = startedAt`. Visible history + contains messages with `created_at <= cursor`: server timestamps order history, + while message IDs do not order messages across rooms. OPTIONAL `messageIds` lists + wake message IDs, not a history cursor. +- `volumes` MUST use `{id, snapshot}` for each successful pre-turn snapshot of an + effective `record.volumes` root. A failed root MUST be omitted and named in OPTIONAL + `errors: [string]`. REQUIRED `exact` MUST be `false` on any snapshot or pin failure; + otherwise it MUST be `true`. Missing or invalid history MUST NOT be guessed. + +Recorded commit fields MUST additionally be full 40- or 64-hex object IDs, +normalized to lowercase and verified as commit objects without peeling tags. +Bundles carry only a `sha256:…` digest, never a commit. Unknown fields refuse. ## Rewind @@ -226,53 +235,71 @@ paths in replacement declarations resolve from the dataset file's directory. Every input MUST materialize successfully; present-day data MUST NOT substitute for unavailable history. -The history consumer layout is normative: +The local history import layout is normative: ```text DIR/ - moments/*.jsonl - git/// + moments/**/*.json # one record per file; .jsonl also accepted + bundles/.tar + git/// # local repository mirror feeds///landings.jsonl - feeds///repo/ - feeds///trees// + feeds///repo/ # local git source mirror + feeds///trees// # preserve .N re-land suffixes memory///events.jsonl - memory///generations.jsonl moltnet///state.json | moltnet.sqlite - volumes///snapshots.jsonl - volumes///objects/ + recording//wake-moments.jsonl + recording//snapshots/// + recording//snapshots//.manifest.json ``` `` MUST be `encodeURIComponent` of the node ID: `agent:writer` -becomes `agent%3Awriter`. IDs and content-addressed names MUST be safe path -segments. Every read MUST remain inside `DIR` both lexically and after realpath -resolution; symlink escapes MUST refuse. History repositories MAY be bare or -ordinary repositories and MUST remain unmodified. Every JSONL line MUST be -validated strictly; an invalid line MUST refuse affected inputs and identify -the history file, line, and reason. History writers use these line shapes: - -- `moments/*.jsonl`: one `spawnfile.moment-record.v1` object per line, any filename. -- `landings.jsonl`: `{revision, tree, commit, ref, paths, landedAt}`. - `revision` is the lowercase 64-hex feed digest; `tree` names that revision, - optionally with its re-land generation or `trees/` prefix. Directory sources - use `null` commit/ref/paths; git sources require commit/ref, with `null` or `[]` - paths denoting the whole tree. -- OPTIONAL `generations.jsonl`: `{generation, switchedAt, ledger}`, with `ledger` - relative to that bank's directory and naming its current or archived ledger. -- `snapshots.jsonl`: `{snapshot, takenAt, parent, files}`, where `parent` is a - snapshot ID or `null`; `files` maps changed relative paths to SHA-256 digests or - `null` for deletion. `objects/` contains each referenced file's bytes. - -Moment records match when `agent` equals the plan agent ID and `wake.at` is the -same parsed instant as example `at`; different offset spellings are equivalent. -Several matching records MUST agree in every field used for the selected -inputs, otherwise the moment refuses. Every matching record's commit IDs for each -non-overridden input MUST pass history-object validation, including fields omitted -from selection comparisons for pinned resources. An invalid candidate refuses -its affected input. No matching record triggers each input's -fallback rule. Fallbacks are labelled `approximate`; independent immutable pins -and landing logs retain the exact fidelity specified below. All timestamp -comparisons MUST compare instants, preserving RFC3339 nanoseconds where present, -never timestamp strings or UUID order. Boundary instants equal to `at` are included. +becomes `agent%3Awriter`. `` is the selected compiled node's existing +slug, including the compiler's collision handling. `` is the +`record.volumes` resource ID. IDs and archive names MUST be safe path segments. +Every read MUST remain inside `DIR` lexically and after realpath resolution; +symlink escapes refuse. JSON and every JSONL line are validated strictly; +invalid lines refuse the affected input with file, line number, and reason. + +Import copies deployment evidence verbatim: composed moment JSON files; the +host feed state's `landings.jsonl` beside `landed.json`; retained feed volume +`trees/` directories; deployed bundle tars from `container/workspace-bundles/`; +Daimon's persisted `recording//` rows, snapshot trees and manifests; +Mneme's `/memory/events.jsonl`; and the network's consistent Moltnet store +(including SQLite recovery sidecars when needed). The `git/` repositories and +feed `repo/` directories are local mirrors retaining historical commits, not +recorder output. Rewind neither fetches nor changes these mirrors. + +Feed logs follow the recording PR's `spawnfile.volume-feed-landing.v1` shape: +`{version, resource, volume, revision, tree, source, commit?, ref?, paths?, +switching_at?, landed_at, period?}`. Source identifies `{kind:"git", repo}` or +`{kind:"directory", directory}`, with optional prepared provenance. This reader +accepts older entries without `switching_at`; new recording writes transition +boundaries. The isolated compatibility reader must be replaced by P3's +`readFeedLandings` / `feedLandingAt` after that PR merges. + +Moment records match exact compiled `agent` and parsed `wake.at == example.at`. +For several attempts of that wake, select the EARLIEST `wake.startedAt` (original +turn), even when replay pins differ or append/capture order differs. Conflicting +copies at that same earliest instant still refuse when their used pins differ. +Validate recorded commit IDs in every earliest-attempt copy for each used input, +including IDs hidden by declared pin precedence; later replays cannot replace +or invalidate the original turn's pins. + +The report retains dataset `at` and adds `effectiveAt = selected wake.startedAt`, +or `effectiveAt = at` if unmatched. Task-clock origin, historical selection, and +leak bounds MUST use `effectiveAt`. Report the matching count, selected execution +ID, selected start instant, and `earliest-startedAt` choice. All comparisons parse +instants, preserving RFC3339 nanoseconds rather than ordering strings or UUIDs. + +An `exact:false` record does not invalidate healthy pins: a present pin is used +with `exact` fidelity and `record` basis (`record-cursor` for Moltnet), unless its +`errors` names that input. Errored pins are ignored, absent pins use fallback +rules, and those inputs are labelled `approximate`, with the record's error text +in notes. Explicit section or declaring-node qualifications apply only to that +input; unqualified ID errors conservatively apply to every input with that ID. +A missing fallback still refuses. No matching record uses the normal +rules below; independent immutable git pins and unambiguous landing logs may +still be exact. ### Per-input reconstruction and fidelity @@ -286,14 +313,14 @@ regardless of its declared storage backend; trial preparation consumes or import that snapshot. With no `--history`, only locally materializable pins and explicit overrides can succeed; everything else refuses. -Git resources and source-backed bundles select a full 40- or 64-hex declared +Git resources select a full 40- or 64-hex declared commit first (`exact`, `pinned`), otherwise the matching record's resource commit (`exact`, `record`). Otherwise they walk the declared selector's -first-parent history and choose its newest commit with committer time `<= at` +first-parent history and choose its newest commit with committer time `<= effectiveAt` (`approximate`, `selector-at`); an absent selector means `HEAD`. Commit dates do not establish when a commit reached a branch or tag, so this fallback cannot prove historical ref membership. A missing eligible commit refuses. -The selected commit's author AND committer dates MUST be `<= at` before any +The selected commit's author AND committer dates MUST be `<= effectiveAt` before any git checkout or bundle extraction, including pinned, recorded, and overridden git sources. All paths MUST use the same selected-commit guard. @@ -302,11 +329,18 @@ from the selected commit, with one checked-out branch, no remotes, tags, source packed refs, reflogs, `FETCH_HEAD`, or `ORIG_HEAD`. Git commands use argv without a shell and sanitized configuration. History MUST NOT be mutated. The leak check also rejects any reachable ancestor with author or committer time after -`at`. Bundle `build.files` extracts `files.root` at the selected commit and honors +`effectiveAt`. Bundle `build.files` extracts `files.root` at the selected commit and honors `exclude`. `build.dependencies` and `build.generated` retain their source directory or declared inputs at that commit and report `build: deferred`; -trial preparation owns builds. Bundle output MAY be a plain tree. A prebuilt -`source` tar without a record or explicit override is un-rewindable and refuses. +trial preparation owns builds. Bundle output MAY be a plain tree. A record pins only its bundle SHA-256. +Exact bundle output requires `bundles/.tar` whose actual digest +matches that pin. Validate the archive before extraction: escaping paths, +symlinks, hard links, devices, and other special entries refuse. Acceptance +reopens the archive and compares output paths, bytes, and executable modes. +Absent or digest-mismatched archives fall back to source reconstruction where +available, always `approximate` even with a pinned source commit; notes explain +that the recorded SHA-256 could not be reproduced. A prebuilt `source` tar has +no historical source fallback and refuses unless overridden. Generated inputs spanning multiple repositories or independently pinned input refs cannot be represented by this one-repository/one-commit history contract and MUST refuse. Bundle roots whose historical repository prefix cannot be @@ -316,43 +350,51 @@ attribute-driven transforms, `export-subst`, or `export-ignore` semantics. Symlinks MUST be recreated as symlinks and pass the independent escape check; gitlink/submodule entries MUST refuse with an explicit unsupported-history error. -Fed volumes select the newest landing with `landedAt <= at` when a landing log -exists (`exact`, `landing-log`); no eligible landing refuses. A matching record -feed entry MUST agree with that landing. Without a log, a plain git feed expands -its ref template and paths using `at`, including `${date}` in UTC and -`${date:}` in that zone. The primary ref is used only if it has an -eligible first-parent commit; otherwise the declared fallback is tried. Its -newest eligible commit is `approximate`, `feed-rule-at`. Commands, directory -feeds, and freeze/waiting behavior cannot reconstruct historical landings -without a log and MUST refuse. Ref dates alone do not capture earlier freeze -holds or when a producer made a ref available. +Fed volumes use a matching record pin with `exact`, `record`, including when no +landing log is retained. Otherwise, lookup inside the inclusive +`[switching_at, landed_at]` window selects the incoming entry with `approximate`, +`landing-log` and an ambiguous-transition note. Outside windows, choose the +newest `landed_at <= effectiveAt`, breaking equal timestamps by last append +(`exact`, `landing-log`). Before the first entry, there is no landed state. + +Without selected recorded state, a plain git feed evaluates its ref template +and paths at `effectiveAt`, including `${date}` in UTC and `${date:}`. +Use the primary ref only if it has an eligible first-parent commit; otherwise +try the fallback. Its newest eligible commit is `approximate`, `feed-rule-at`. +Command refs, directory feeds, prepared/include recipes and freeze/waiting state +cannot be reconstructed this way and refuse. Commit dates do not establish ref +availability or historical freeze holds. Before choosing retained bytes versus extraction, the selected feed source commit MUST be verified as a commit object with author AND committer dates -`<= at`. This single selection guard applies equally to plain, retained, +`<= effectiveAt`. This single selection guard applies equally to plain, retained, prepared, and included git feeds; an eligible landing time does not make a future source commit eligible. Directory feeds have no git source commit. -Feed bytes come from OPTIONAL retained `trees//` when available. +Feed bytes come from OPTIONAL retained `trees//` when available. Otherwise a plain git feed re-extracts the landing's expanded paths at its commit and MUST verify the revision with the existing feed revision function. A mismatch refuses. `include` or `prepare` requires retained bytes; rewind MUST NOT run them. Raw blob extraction ignores repository-local `info/attributes` as well as committed archive attributes; later tags cannot change historical file bytes. -Output preserves the agent-visible `trees//`, relative `current` +Output preserves the agent-visible `trees//`, including re-land suffixes, relative `current` symlink, and `.spawnfile-feed.json` (`spawnfile.volume-feed.v1`). Its `landed_at` is the selected landing time, or the selected commit's committer time for an -approximation, always `<= at`. - -Memory selects the generation with greatest `switchedAt <= at`, refusing if -the generation log contains none or its archived ledger is unavailable. Without -a generation log, the sole current ledger is treated as original generation 0; -a record naming another generation refuses. A matching record MUST name the -selected generation. Its `seq` selects the first `seq` ledger lines (`exact`, -`record`). Otherwise selection stops before the FIRST event after `at` -(`approximate`, `created-at-prefix`): subsequent clock regression never reopens -the cut. The prefix MUST end in LF, contain contiguous sequences from 1, and -contain only nondecreasing event times `<= at`. Legacy seq-less events are -allowed only as a leading run. Invalid sequence or time ordering refuses. +feed-rule approximation. A recorded pin or ambiguous incoming transition may +have publication metadata after `effectiveAt`; this does not advance the trial +clock or filesystem timestamps. The independent gate reopens its record/window +before allowing such publication metadata. Directory feeds are not refetchable: +retained tree bytes MUST recompute to the recorded revision or refuse (including +prepared directory revisions whose composite identity cannot be reproduced). + +Memory uses Mneme's single append-only ledger. Any recorded `generation`, +including 0, refuses as unsupported: Mneme has no generations or curation +archives. There is no `generations.jsonl` reader. A record's `seq` selects the +first `seq` ledger lines (`exact`, `record`); `seq:0` is an exact empty bank and +requires no ledger. Otherwise selection stops before the FIRST event after +`effectiveAt` (`approximate`, `created-at-prefix`): a subsequent clock regression +never reopens the cut. The prefix MUST end in LF, contain contiguous sequences +from 1, and only nondecreasing event times `<= effectiveAt`. Legacy seq-less +events are allowed only as a leading run. Invalid sequence or time ordering refuses. Memory output is `memory///memory/events.jsonl`. It MUST exclude `index.sqlite`, `causal*`, `authority-receipts.jsonl`, and `authority-receipts/`. @@ -374,38 +416,57 @@ Opening an index alone does NOT rebuild it. A materialized bank without `index.sqlite` is complete because this index is derived state; trial preparation MUST preserve those rebuild-before-read consumers. Rewind requires no Mneme runtime dependency and does not create even a temporary index. -Missing bank history also refuses unless the bank is overridden. +Missing bank history refuses unless the bank is overridden or recorded at `seq:0`. Moltnet JSON and SQLite stores remove messages, rooms, and attachment-delivery -events after `at`. Empty threads/DMs and threads/DMs whose root is after the cut +events after `effectiveAt`. Empty threads/DMs and threads/DMs whose root is after the cut are dropped. Thread/DM message counts and last-message times are recomputed; attachment cursors are clamped/remapped to retained events. Room removals after -`at` are undone when removal time and restorable room metadata exist; a missing +`effectiveAt` are undone when removal time and restorable room metadata exist; a missing historical removal time retains current removal state. Agents and memberships have no historical log and are retained as now, explicitly disclosed in input `notes`: even a cursor-exact content cut cannot prove historical membership. SQLite changes apply only to a copy through `node:sqlite` on the supported Node >=22.19 line; if that API is unavailable, SQLite rewind refuses. RFC3339Nano fractions with omitted trailing zeros MUST be compared as parsed instants. -A record cursor yields `exact`, `record-cursor` only if it identifies a retained -message and no retained message is later than that cursor's message. Otherwise -fidelity is `approximate`, `created-at-cut`. Missing stores refuse unless overridden. - -Unfed volumes choose the newest snapshot with `takenAt <= at`. A record MUST -name that snapshot or an ancestor-consistent exact snapshot; a valid record is -`exact`, otherwise selection is `approximate`. The full parent chain is replayed -from its root, including deletions, and every copied object is SHA-256 verified. -Missing history, missing parents/objects, or no eligible snapshot refuses. -Materialized content must equal the replayed snapshot, checked independently. - -All materialized files and directories MUST have atime/mtime `<= at`, using the -source commit, landing, event, or snapshot instant capped at `at`. Symlink times +A record cursor is an instant, MUST equal `wake.startedAt`, and cuts messages at +`created_at <= cursor` (`exact`, `record-cursor`). When `messageIds` is present, +every listed wake message MUST survive the cut; otherwise refuse. With no pin, +cut at `effectiveAt` (`approximate`, `created-at-cut`). Missing stores refuse +unless overridden; UUIDs never order messages. + +Unfed volumes consume full point-in-time Daimon trees, not deltas or object +chains. Rows in `recording//wake-moments.jsonl` use +`noopolis.daimon.wake-moment.v1` with `agent_id`, `execution_id`, `started_at`, +`deliveries`, `snapshots:[{id,snapshot,files,copied,linked,skipped}]`, and optional +`error`. Snapshot names are compact `started_at` plus execution ID (for example +`20261009T082031123Z-execution-17`), but time selection MUST use the row's +`started_at`, not that prefix. A record names its exact snapshot (`exact`, +`record`); otherwise use the newest complete snapshot row `<= effectiveAt` +(`approximate`, `snapshot-at`). `.partial` directories never count. Missing +named history or no complete eligible snapshot refuses. + +The sibling manifest contains `{version,id,snapshot,started_at,sourcePath,entries}`. +Entries map relative paths to `{type,mode,size,source,target?}`: type is file, +directory or symlink; size is a decimal string; source preserves recorder +`dev`, `ino`, `size`, `mtimeNs`, `ctimeNs`, and `mode`; target is symlink text. +Copy file bytes into new inodes; do not hard-link into the bundle. Verify paths, +entry types, modes, file/symlink sizes and targets against the manifest; missing +or extra entries refuse. Restore source directory modes (the recorder protects +retained directories as 0700). Directory inode size and copied inode identity +are filesystem metadata, not portable tree invariants. Verify their manifest +structure but not equality to new inodes. Acceptance independently checks the +manifest and compares copied bytes with the retained tree, catching same-size +byte mutations even though recorder manifests contain no content digest. + +All materialized files and directories MUST have atime/mtime `<= effectiveAt`, using the +source commit, landing, event, or snapshot instant capped at `effectiveAt`. Symlink times MUST be set with `lutimes`; following a symlink to set times is insufficient. ### Task clock `task-clock.json` is strict `spawnfile.task-clock.v1` with `{version, origin}`; -`origin` is the example instant normalized to ISO-8601 with `Z`. Trial preparation +`origin` is `effectiveAt` normalized to ISO-8601 with `Z`. Trial preparation supplies one safe-integer real launch anchor in milliseconds. All processes MUST share `originEpochMs + (realNowEpochMs - anchorEpochMs)`; independently started children MUST NOT reset their clock to the origin. `originEpochMs` uses JavaScript @@ -466,27 +527,27 @@ Every git-backed input MUST record its selected full commit ID in `moment.json`'s `inputs[].source.commit`; git feed identity MUST record the same ID. Acceptance MUST independently open the source history repository read-only, verify that ID names a commit object, and inspect its raw author and committer -dates against `at`, regardless of retained bytes or normalized filesystem times. +dates against `effectiveAt`, regardless of retained bytes or normalized filesystem times. Missing/unreadable provenance and inconsistent feed commit IDs also refuse. History repositories are resolved by input identity from `--history`. A local pin or git-backed override resolved without that history entry MUST additionally record its absolute local repository path in `source.repository`; acceptance reopens it read-only. That source repository must remain available for subsequent -verification. Directory feeds and explicitly overridden prebuilt tars have no git -provenance. This source check is independent of the materialized git object's +verification. Directory feeds and digest-pinned or explicitly overridden prebuilt tars have no git +provenance. Recorded bundles are independently checked against their digest-pinned archive. This source check is independent of the materialized git object's reachable-history check and MUST NOT reuse the producer's date assertion. Every finding refuses the moment: | Class | Refusal condition | | --- | --- | -| `file-time` | A file, directory, or symlink atime or mtime is after `at`. | +| `file-time` | A file, directory, or symlink atime or mtime is after `effectiveAt`. | | `symlink-escape` | An absolute link or a lexically/physically escaping link resolves outside the bundle. | -| `git-object` | Source provenance is missing, inconsistent, not a commit, or has author/committer time after `at`; any materialized commit has a future date, any object is unreachable from the selected commit, or refs/remotes/reflogs/fetch metadata exceed the single branch. Any `.git` directory/gitfile or bare repository (`HEAD` + `objects/` + `refs/`) anywhere outside the materialized git resource repository also refuses, including inside feed trees and bundles. | +| `git-object` | Source provenance is missing, inconsistent, not a commit, or has author/committer time after `effectiveAt`; any materialized commit has a future date, any object is unreachable from the selected commit, or refs/remotes/reflogs/fetch metadata exceed the single branch. Any `.git` directory/gitfile or bare repository (`HEAD` + `objects/` + `refs/`) anywhere outside the materialized git resource repository also refuses, including inside feed trees and bundles. | | `memory-event` | Future ledger events, sequence gaps, invalid prefix ordering, or derived/audit files are present. | -| `moltnet-message` / `moltnet-event` / `moltnet-room` | Retained messages, delivery events, or rooms are after `at`. | -| `feed-identity` | `landed_at` is after `at` or `current` does not target a tree inside the volume. | -| `dated-path` | A path segment contains a calendar date `YYYY-MM-DD` later than the UTC date of `at + 14h`, the latest possible local date. | -| `snapshot` | An unfed volume differs from its selected replayed snapshot. | +| `moltnet-message` / `moltnet-event` / `moltnet-room` | Retained messages, delivery events, or rooms are after `effectiveAt`. | +| `feed-identity` | Unproved publication time after `effectiveAt`, a directory revision mismatch, or `current` outside its selected tree. | +| `dated-path` | A path segment contains a calendar date `YYYY-MM-DD` later than the UTC date of `effectiveAt + 14h`, the latest possible local date. | +| `snapshot` | An unfed volume differs from its selected full snapshot manifest or retained bytes. | `OUT` MUST be absent or an empty directory. The materializer builds in a sibling staging directory and renames into place only after the disk leak check passes: @@ -502,7 +563,8 @@ OUT/ ``` `moment.json` is strict `spawnfile.materialized-moment.v1`: status, agent, -`example: {id, role, dataset}`, normalized `at`, sorted inputs, `record: {matched}`, +`example: {id, role, dataset}`, normalized `at`, `effectiveAt`, sorted inputs, and `record: {matched, attempts?, +selectedStartedAt?, executionId?, selection?}`, and `leakCheck: {ok, findings}`. Inputs sort by section, declaring node, then ID; each reports kind, fidelity (or `null` on refusal), basis, source identifiers, relative output path (or `null`), notes, and refusal reason (or `null`). Deferred @@ -1523,6 +1585,11 @@ The child deadline is Paideia's declared deadline plus five seconds for cleanup. "format": "date-time", "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" }, + "effectiveAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, "inputs": { "type": "array", "items": { @@ -1594,10 +1661,13 @@ The child deadline is Paideia's declared deadline plus five seconds for cleanup. "minimum": 0, "maximum": 9007199254740991 }, - "generation": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "sha256": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + }, + "agentSlug": { + "type": "string", + "minLength": 1 }, "snapshot": { "type": "string", @@ -1606,6 +1676,16 @@ The child deadline is Paideia's declared deadline plus five seconds for cleanup. "cursor": { "type": "string", "minLength": 1 + }, + "switchingAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "landedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" } }, "additionalProperties": false @@ -1663,6 +1743,24 @@ The child deadline is Paideia's declared deadline plus five seconds for cleanup. "properties": { "matched": { "type": "boolean" + }, + "attempts": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "selectedStartedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "executionId": { + "type": "string", + "minLength": 1 + }, + "selection": { + "type": "string", + "const": "earliest-startedAt" } }, "required": [ @@ -1695,6 +1793,7 @@ The child deadline is Paideia's declared deadline plus five seconds for cleanup. "agent", "example", "at", + "effectiveAt", "inputs", "record", "leakCheck" diff --git a/src/cli/trainCommand.test.ts b/src/cli/trainCommand.test.ts index 26942102..387e1e5d 100644 --- a/src/cli/trainCommand.test.ts +++ b/src/cli/trainCommand.test.ts @@ -83,7 +83,7 @@ describe("spawnfile train --materialize", () => { const root = await trainingProject({ ...dataFiles, Spawnfile: agentManifest() }); const materializeTrainingMoment = vi.fn(async ({ plan }) => ({ version: "spawnfile.materialized-moment.v1", status: "materialized", agent: plan.agent.id, - example: { id: example.id, role: "train", dataset: "training/train.yaml" }, at: example.at, + example: { id: example.id, role: "train", dataset: "training/train.yaml" }, at: example.at, effectiveAt: example.at, inputs: [], record: { matched: false }, leakCheck: { ok: true, findings: 0 } })); const stdout: string[] = []; diff --git a/src/compiler/training/rewind/AGENTS.md b/src/compiler/training/rewind/AGENTS.md index c2e4447b..9661e8b6 100644 --- a/src/compiler/training/rewind/AGENTS.md +++ b/src/compiler/training/rewind/AGENTS.md @@ -2,24 +2,25 @@ - `contract.ts` owns strict moment, materialization, leak and task-clock documents and generated schemas. - `history.ts` is the only boundary for reading history files, logs and retained bytes; every path must be lexically and physically contained. -- `select.ts` and `inputs.ts` resolve records, examples and effective inputs without I/O. +- `select.ts` and `inputs.ts` resolve original attempts, examples and compiled input identities; `recordInput.ts` isolates per-input recording failures. Dataset `at` matches wake time; `effectiveAt` is the original startedAt and owns all cuts, clocks and leak bounds. - `types.ts` defines input identities, declarations and per-kind contexts/results; `index.ts` exports the public materializer and pure selection/cut helpers. - `git.ts` owns resources; `gitProcess.ts`, `gitHistory.ts`, `gitTree.ts`, and `gitArchive.ts` isolate argv execution, containment, blob trees, and raw feed trees; `recordedCommit.ts` verifies full recorded IDs name commit objects without peeling. -- `feed.ts`, `memory.ts`, and `volumes.ts` own their cuts and history-log schemas; `memorySchemas.ts` validates native ledger events. +- `feed.ts`, `memory.ts`, and `volumes.ts` own input reconstruction. `feedLandings.ts` is the sole P3 log compatibility reader (replace with readFeedLandings/feedLandingAt after merge); `snapshotManifest.ts` verifies Daimon full trees; `memorySchemas.ts` validates ledger events. Mneme generations are unsupported; seq:0 is exact empty. - `moltnet.ts` dispatches stores; `moltnetJson.ts`, `moltnetSqlite.ts`, `moltnetSqliteData.ts`, and `moltnetSchemas.ts` own snapshots, copied databases, SQL decoding, and strict native shapes. - `materialize.ts` stages all inputs and publishes only after the independent disk reader in `leakCheck.ts` accepts them. - `leakFiles.ts`, `leakGit.ts`, and `leakInputs.ts` inspect disk metadata, every git object, and native payloads independently of materialization decisions. `leakProvenance.ts` reopens source repositories read-only using disk commit receipts, checking type and both raw dates without the producer's guard. Discover unexpected `.git` directories/gitfiles and bare repositories throughout every input; only the materialized git resource repository is allowed. - `instant.ts` compares parsed nanosecond instants; `timestamps.ts` stamps metadata without following symlinks. - `taskClock.ts` separates top-level harness JSON, libfaketime child offsets, and clock-aware child variables: one offset owner per process. Relative signed seconds preserve milliseconds and the shared anchor; trial preparation supplies LD_PRELOAD and a dot-decimal numeric locale. Timeouts remain real. - Adjacent tests use local temporary repositories and fixtures. Never use Docker, production hosts or network fetches. -- `fixtures.test-helper.ts` and `git.test-helper.ts` build disposable real projects and repositories; package emission excludes them. +- `fixtures.test-helper.ts`, `snapshotFixtures.test-helper.ts`, and `git.test-helper.ts` build disposable real projects and repositories; package emission excludes them. - Selector-at git history is approximate: committer dates cannot prove when a commit reached a ref. Feed-rule approximation cannot reconstruct freeze holds. - Reuse P2 effective input identity resolution. No input may silently use present-day data. - Feed/bundle extraction reads raw blobs, never `git archive` or attribute-driven transformations. Preserve symlinks for the escape check; refuse gitlinks. Recorded commits must be full lowercase 40/64-hex IDs verified as commit objects, including retained feeds. - Guard every selected git/bundle/feed commit's author and committer dates before choosing output bytes, including retained/prepared feeds. Record the selected commit in the moment/identity; local pins or git-backed overrides without a history entry also record their repository path for independent acceptance. -- Validate each non-overridden input's recorded commit IDs in every same-agent/same-wake candidate before accepting that input, even when pin precedence omits those fields from selection comparisons. +- Validate each non-overridden input's recorded commit IDs in every same-agent/same-wake earliest-attempt copy before accepting that input, even when pin precedence omits those fields from selection comparisons. - Validate memory JSONL structure locally; rewind never imports Mneme or ships an index/audit state. Checksum verification is skipped because persisted normalized events cannot recover the original JSON checksum input (`node_modules/@noopolis/mneme/dist/identity/ids.js:15-17`, `dist/store/store.js:249-274`). - Mneme 0.1.1's index is rebuilt from the ledger before every index read, not on open: `node_modules/@noopolis/mneme/dist/runtime/runtime.js:86-87` (embedding recall), `node_modules/@noopolis/mneme/dist/kernel/kernel.js:152-153` (search), and `:207-208` (locate). A ledger-only materialized bank is complete. - Inspect atime and mtime from metadata captured before payload reads, then restore captured access times without hiding existing future atimes. - Generated inputs spanning repositories refuse: this history contract carries one commit per resource. Never claim that one repository proves another repository's bytes. +- `recordedBundle.ts` requires digest-matching retained tars for exact bundles and independently checks extracted bytes; source reconstruction is approximate. `leakFeed.ts` proves late publication metadata against record/window history. - Keep modules under 400 lines, use named/barrel exports, and never import Mneme internals. diff --git a/src/compiler/training/rewind/contract.ts b/src/compiler/training/rewind/contract.ts index 3cb56800..76df3609 100644 --- a/src/compiler/training/rewind/contract.ts +++ b/src/compiler/training/rewind/contract.ts @@ -3,6 +3,7 @@ import { trainingInstantSchema } from "../../../manifest/index.js"; const text = z.string().min(1); const count = z.number().int().nonnegative(); +export const recordedSha256Schema = z.string().regex(/^sha256:[a-f0-9]{64}$/u); export const recordedCommitSchema = z.string() .regex(/^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/u, "Expected a full 40- or 64-hex commit object id") .toLowerCase(); @@ -20,15 +21,20 @@ const unique = (schema: T, key: (item: z.infer) => strin }); export const momentRecordSchema = z.object({ version: z.literal("spawnfile.moment-record.v1"), agent: text, - wake: z.object({ at: trainingInstantSchema, surface: z.literal("moltnet"), room: z.string(), - from: z.string(), messageId: text.optional() }).strict(), - recordedAt: trainingInstantSchema, - resources: unique(z.object({ id: text, kind: z.enum(["git", "bundle"]), commit: recordedCommitSchema }).strict(), (r) => r.id), - feeds: unique(z.object({ id: text, revision: text, tree: text, commit: recordedCommitSchema, - paths: z.array(text), landedAt: trainingInstantSchema }).strict(), (r) => r.id), - memory: unique(z.object({ id: text, declaredBy: text, seq: count, generation: count }).strict(), + wake: z.object({ at: trainingInstantSchema, startedAt: trainingInstantSchema, executionId: text, + deliveries: z.array(z.object({ deliveryId: text, kind: text, occurredAt: trainingInstantSchema }).strict()).min(1), + surface: z.literal("moltnet").optional(), room: z.string().optional(), + from: z.string().optional(), messageId: text.optional() }).strict(), + recordedAt: trainingInstantSchema, exact: z.boolean(), errors: z.array(z.string()).optional(), + resources: unique(z.discriminatedUnion("kind", [ + z.object({ id: text, kind: z.literal("git"), commit: recordedCommitSchema, sha256: recordedSha256Schema.optional() }).strict(), + z.object({ id: text, kind: z.literal("bundle"), sha256: recordedSha256Schema }).strict() + ]), (r) => r.id), + feeds: unique(z.object({ id: text, revision: text, tree: text, commit: recordedCommitSchema.optional(), ref: text.optional(), + paths: z.array(text).optional(), landedAt: trainingInstantSchema, period: text.optional() }).strict(), (r) => r.id), + memory: unique(z.object({ id: text, declaredBy: text, seq: count, generation: count.optional() }).strict(), (r) => JSON.stringify([r.declaredBy, r.id])), - moltnet: unique(z.object({ network: text, cursor: text }).strict(), (r) => r.network), + moltnet: unique(z.object({ network: text, cursor: trainingInstantSchema, messageIds: z.array(text).optional() }).strict(), (r) => r.network), volumes: unique(z.object({ id: text, snapshot: text }).strict(), (r) => r.id) }).strict(); export type MomentRecord = z.infer; @@ -37,14 +43,17 @@ export const materializedInputSchema = inputIdentitySchema.extend({ kind: z.enum(["git", "bundle", "feed", "volume", "memory", "moltnet"]), fidelity: z.enum(["exact", "approximate", "overridden"]).nullable(), basis: text, source: z.object({ commit: text.optional(), repository: text.optional(), revision: text.optional(), seq: count.optional(), - generation: count.optional(), snapshot: text.optional(), cursor: text.optional() }).strict(), + sha256: recordedSha256Schema.optional(), agentSlug: text.optional(), snapshot: text.optional(), cursor: text.optional(), + switchingAt: trainingInstantSchema.optional(), landedAt: trainingInstantSchema.optional() }).strict(), path: text.nullable(), notes: z.array(z.string()), refusal: text.nullable(), build: z.literal("deferred").optional() }).strict(); export type MaterializedInput = z.infer; export const materializedMomentSchema = z.object({ version: z.literal("spawnfile.materialized-moment.v1"), status: z.enum(["materialized", "refused"]), agent: text, example: z.object({ id: text, role: z.enum(["train", "validation", "test"]), dataset: text }).strict(), - at: trainingInstantSchema, inputs: z.array(materializedInputSchema), record: z.object({ matched: z.boolean() }).strict(), + at: trainingInstantSchema, effectiveAt: trainingInstantSchema, inputs: z.array(materializedInputSchema), + record: z.object({ matched: z.boolean(), attempts: count.optional(), selectedStartedAt: trainingInstantSchema.optional(), + executionId: text.optional(), selection: z.literal("earliest-startedAt").optional() }).strict(), leakCheck: z.object({ ok: z.boolean(), findings: count }).strict() }).strict(); export type MaterializedMoment = z.infer; diff --git a/src/compiler/training/rewind/feed.test.ts b/src/compiler/training/rewind/feed.test.ts index 0f88c888..6c31c23e 100644 --- a/src/compiler/training/rewind/feed.test.ts +++ b/src/compiler/training/rewind/feed.test.ts @@ -3,7 +3,7 @@ import { mkdir, readFile, readdir, readlink, rm, symlink, writeFile } from "node import path from "node:path"; import type { TeamWorkspaceResource } from "../../../manifest/index.js"; import { digestDirectory, feedIdentityFindings, resolveFeedSource } from "../../../volume/index.js"; -import { feedLandingSchema, materializeFeed, selectFeedLanding, type FeedLanding } from "./feed.js"; +import { materializeFeed, type FeedLanding } from "./feed.js"; import { History } from "./history.js"; import { commitTestGit, gitTestRoot, initTestGit, testGit, testMomentRecord } from "./git.test-helper.js"; import type { InputContext } from "./types.js"; @@ -19,7 +19,8 @@ const fixture = async () => { const past = await commitTestGit(repository, { "published/news.txt": "past\n", "ignored/private.txt": "not-fed\n" }, EARLY); const future = await commitTestGit(repository, { "published/news.txt": "future\n" }, LATE); const revision = resolveFeedSource({ kind: "git", fetch: false, repo: repository, ref: past, paths: ["published"] }).revision; - const landing: FeedLanding = { revision, tree: revision, commit: past, ref: "main", paths: ["published"], landedAt: EARLY }; + const landing: FeedLanding = { version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "org-news", + source: { kind: "git", repo: "https://never-fetch.invalid/source" }, revision, tree: `trees/${revision}`, commit: past, ref: "main", paths: ["published"], landed_at: EARLY }; const resource: TeamWorkspaceResource = { id: "news", kind: "volume", mode: "mutable", mount: "./news", name: "org-news", feed: { git: { repo: "https://never-fetch.invalid/source", ref: "main", paths: ["published"], fetch: true } } }; const context: InputContext = { at: AT, destination: path.join(root, "output"), history: new History(path.join(root, "history")), record: null, @@ -36,7 +37,7 @@ describe("feed rewind", () => { testGit(repository, ["commit", "--allow-empty", "-m", "future metadata", `--date=${role === "author" ? LATE : EARLY}`], role === "committer" ? LATE : EARLY); const commit = testGit(repository, ["rev-parse", "HEAD"]); const revision = resolveFeedSource({ kind: "git", fetch: false, repo: repository, ref: commit, paths: ["published"] }).revision; - await log([{ ...landing, commit, revision, tree: revision, landedAt: EARLY }]); + await log([{ ...landing, commit, revision, tree: revision, landed_at: EARLY }]); if (bytes === "retained") { const retained = path.join(directory, "trees", revision); await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "future.txt"), "future retained bytes"); @@ -48,17 +49,51 @@ describe("feed rewind", () => { it("selects exact boundary landings and matches offset-equivalent records", async () => { const { context, landing, log, revision } = await fixture(); - const selected = { ...landing, landedAt: AT }; - await log([selected, { ...landing, revision: "f".repeat(64), tree: "f".repeat(64), landedAt: "2026-10-09T08:20:31.001Z" }]); - context.record = { ...testMomentRecord(), feeds: [{ id: "news", revision, tree: revision, commit: landing.commit!, paths: ["published"], landedAt: "2026-10-09T10:20:31+02:00" }] }; - expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "landing-log", source: { commit: landing.commit, revision }, instant: AT }); + const selected = { ...landing, landed_at: AT }; + await log([selected, { ...landing, revision: "f".repeat(64), tree: "f".repeat(64), landed_at: "2026-10-09T08:20:31.001Z" }]); + context.record = { ...testMomentRecord(), feeds: [{ id: "news", revision, tree: revision, commit: landing.commit!, ref: "main", paths: ["published"], landedAt: "2026-10-09T10:20:31+02:00" }] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: landing.commit, revision }, instant: "2026-10-09T10:20:31+02:00" }); expect(await readlink(path.join(context.destination, "current"))).toBe(`trees/${revision}`); expect(await readFile(path.join(context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); const identity = JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")); expect(feedIdentityFindings(identity)).toEqual([]); - expect(identity).toMatchObject({ landed_at: AT, files: 1, source: { commit: landing.commit, paths: ["published"], ref: "main" } }); + expect(identity).toMatchObject({ landed_at: "2026-10-09T10:20:31+02:00", files: 1, source: { commit: landing.commit, paths: ["published"], ref: "main" } }); }); + it("uses a present pin from an inexact record without a landing log or optional ref", async () => { + const { context, landing } = await fixture(); + context.record = { ...testMomentRecord(), exact: false, errors: ["memory self unavailable"], + feeds: [{ id: "news", revision: landing.revision, tree: landing.tree, commit: landing.commit, + paths: landing.paths, landedAt: EARLY }] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: landing.commit } }); + expect(await readFile(path.join(context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); + expect(JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")).source.ref).toBe(landing.commit); + }); + + it("keeps the original wake pin when another landing precedes the replay", async () => { + const { context, landing, log, repository } = await fixture(); + const commit = await commitTestGit(repository, { "published/news.txt": "later landing\n" }, "2026-10-09T08:10:00Z"); + const revision = resolveFeedSource({ kind: "git", fetch: false, repo: repository, ref: commit, paths: ["published"] }).revision; + await log([landing, { ...landing, commit, revision, tree: `trees/${revision}`, landed_at: "2026-10-09T08:11:00Z" }]); + context.record = { ...testMomentRecord(), feeds: [{ id: "news", revision: landing.revision, tree: landing.tree, + commit: landing.commit, ref: landing.ref, paths: landing.paths, landedAt: EARLY }] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: landing.commit } }); + expect(await readFile(path.join(context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); + }); + + it.each(["2026-10-09T08:20:30Z", AT, "2026-10-09T08:20:32Z"]) + ("uses the incoming transition landing approximately at %s", async (at) => { + const { context, landing, log } = await fixture(); + const incoming = { ...landing, switching_at: "2026-10-09T08:20:30Z", landed_at: "2026-10-09T08:20:32Z" }; + await log([landing, incoming]); context.at = at; + const result = await materializeFeed(context); + expect(result).toMatchObject({ fidelity: "approximate", basis: "landing-log", instant: at, + source: { switchingAt: incoming.switching_at, landedAt: incoming.landed_at } }); + expect(result.notes).toEqual([expect.stringContaining("ambiguous feed transition")]); + expect(JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")).landed_at).toBe(incoming.landed_at); + expect(await readFile(path.join(context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); + }); + it("falls back to the feed rule at the trial instant with historical committer landing time", async () => { const { context, past, revision } = await fixture(); expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", basis: "feed-rule-at", source: { commit: past, revision }, instant: "2026-10-09T08:00:00.000Z" }); @@ -91,11 +126,15 @@ describe("feed rewind", () => { const { context, resource, directory, landing, log } = await fixture(); resource.feed = { git: { repo: "never-used", ref: { command: ["never-run-ref"] } }, freeze: { timezone: "UTC", after: "08:00" }, include: [{ from: "not-on-disk", to: "assets" }], prepare: { host: true, command: ["never-run-prepare"] } }; - const retained = path.join(directory, "trees", landing.revision); + const retained = path.join(directory, "trees", `${landing.revision}.2`); await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "prepared.txt"), "retained\n"); - await log([{ ...landing, tree: `${landing.revision}.2` }]); + await log([{ ...landing, tree: `trees/${landing.revision}.2`, source: { kind: "git", repo: "never-used", prepared: { + source_revision: landing.revision, include: [], prepare: { command: ["never-run-prepare"], image: null, network: null, platform: null } + } } }]); expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "landing-log" }); + expect(await readlink(path.join(context.destination, "current"))).toBe(`trees/${landing.revision}.2`); expect(await readFile(path.join(context.destination, "current/prepared.txt"), "utf8")).toBe("retained\n"); + expect(JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")).source.prepared.prepare.command).toEqual(["never-run-prepare"]); }); it("uses retained directory feed landings and starts overridden feeds empty", async () => { @@ -104,7 +143,8 @@ describe("feed rewind", () => { const retainedTemp = path.join(directory, "retained"); await mkdir(retainedTemp); await writeFile(path.join(retainedTemp, "data.txt"), "directory\n"); const revision = digestDirectory(retainedTemp), retained = path.join(directory, "trees", revision); await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "data.txt"), "directory\n"); - await log([{ revision, tree: revision, commit: null, ref: null, paths: null, landedAt: EARLY }]); + await log([{ version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "org-news", + source: { kind: "directory", directory: "never-read" }, revision, tree: `trees/${revision}`, landed_at: EARLY }]); expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", source: { revision } }); expect(await readFile(path.join(context.destination, "current/data.txt"), "utf8")).toBe("directory\n"); await rm(context.destination, { recursive: true }); context.input.overridden = true; context.history = undefined; @@ -112,14 +152,44 @@ describe("feed rewind", () => { expect(await readdir(context.destination)).toEqual([]); }); - it("refuses revision mismatches and disagreements with the moment record", async () => { + it("uses retained directory record pins without optional git metadata or a log", async () => { + const { context, resource, directory } = await fixture(); + resource.feed = { directory: "never-read" }; + const retained = path.join(directory, "retained"); await mkdir(retained); await writeFile(path.join(retained, "data.txt"), "past\n"); + const revision = digestDirectory(retained), selected = path.join(directory, "trees", `${revision}.2`); + await mkdir(selected, { recursive: true }); await writeFile(path.join(selected, "data.txt"), "past\n"); + context.record = { ...testMomentRecord(), feeds: [{ id: "news", revision, tree: `trees/${revision}.2`, landedAt: EARLY }] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { revision } }); + expect(await readFile(path.join(context.destination, "current/data.txt"), "utf8")).toBe("past\n"); + }); + + it.each(["changed bytes", "extra file", "missing file"])("refuses retained directory feeds with %s", async (change) => { + const { context, directory, landing, log } = await fixture(); + const retained = path.join(directory, "retained"); await mkdir(retained); await writeFile(path.join(retained, "data.txt"), "past\n"); + const revision = digestDirectory(retained), selected = path.join(directory, "trees", `${revision}.3`); + await mkdir(selected, { recursive: true }); + if (change !== "missing file") await writeFile(path.join(selected, "data.txt"), change === "changed bytes" ? "evil\n" : "past\n"); + if (change === "extra file") await writeFile(path.join(selected, "extra.txt"), "extra\n"); + // Historical source kind controls verification even after the declaration switched to git. + const { commit: _commit, ref: _ref, paths: _paths, ...base } = landing; + await log([{ ...base, source: { kind: "directory", directory: "never-read" }, revision, tree: `trees/${revision}.3` }]); + await expect(materializeFeed(context)).rejects.toThrow("Directory feed revision mismatch"); + }); + + it("does not substitute another retained generation for a missing directory tree", async () => { + const { context, directory, landing, log } = await fixture(); + const retained = path.join(directory, "retained"); await mkdir(retained); await writeFile(path.join(retained, "data.txt"), "past\n"); + const revision = digestDirectory(retained), previous = path.join(directory, "trees", revision); + await mkdir(previous, { recursive: true }); await writeFile(path.join(previous, "data.txt"), "past\n"); + const { commit: _commit, ref: _ref, paths: _paths, ...base } = landing; + await log([{ ...base, source: { kind: "directory", directory: "never-read" }, revision, tree: `trees/${revision}.3` }]); + await expect(materializeFeed(context)).rejects.toThrow("Directory/prepared/included feed requires retained landed tree bytes"); + }); + + it("refuses revisions which cannot be reproduced from the selected commit", async () => { const { context, landing, log } = await fixture(); await log([{ ...landing, revision: "0".repeat(64), tree: "0".repeat(64) }]); await expect(materializeFeed(context)).rejects.toThrow("Feed revision mismatch"); - await log([landing]); - context.record = { ...testMomentRecord(), feeds: [{ id: "news", revision: "f".repeat(64), tree: landing.tree, - commit: landing.commit!, paths: ["published"], landedAt: EARLY }] }; - await expect(materializeFeed(context)).rejects.toThrow("disagrees with its landing log"); }); it("refuses nonhistorical feed rules and missing retained prepared bytes", async () => { @@ -131,20 +201,23 @@ describe("feed rewind", () => { resource.feed = { git: { repo: "never-read" }, freeze: { after: "08:00", timezone: "UTC" } }; await expect(materializeFeed(context)).rejects.toThrow("freeze/waiting"); resource.feed = { git: { repo: "never-read" }, prepare: { host: true, command: ["never-execute"] } }; - await log([landing]); + await log([{ ...landing, source: { kind: "git", repo: "never-read", prepared: { source_revision: landing.revision, + include: [], prepare: { command: ["never-execute"], image: null, network: null, platform: null } } } }]); await expect(materializeFeed(context)).rejects.toThrow("retained landed tree bytes"); context.history = undefined; await expect(materializeFeed(context)).rejects.toThrow("--history is required"); }); - it("validates every landing line and refuses no eligible landing", async () => { + it("validates every landing line and falls back before the first eligible landing", async () => { const { context, directory, landing, log } = await fixture(); - await log([{ ...landing, landedAt: LATE }]); - await expect(materializeFeed(context)).rejects.toThrow("No feed landing"); + await log([{ ...landing, landed_at: LATE }]); + expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", basis: "feed-rule-at" }); + await rm(context.destination, { recursive: true }); await writeFile(path.join(directory, "landings.jsonl"), `${JSON.stringify(landing)}\n${JSON.stringify({ ...landing, unknown: true })}\n`); await expect(materializeFeed(context)).rejects.toThrow(/landings.jsonl:2/u); await writeFile(path.join(directory, "landings.jsonl"), ""); - await expect(materializeFeed(context)).rejects.toThrow("No feed landing"); + expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", basis: "feed-rule-at" }); + await rm(context.destination, { recursive: true }); }); it("ignores archive attributes and preserves long names when reconstructing feed bytes", async () => { @@ -166,10 +239,4 @@ describe("feed rewind", () => { await expect(materializeFeed(context)).rejects.toThrow("History symlink escape"); }); - it("selects pure instant boundaries and refuses contradictory equal-instant landings", () => { - const revision = "a".repeat(64), landing = feedLandingSchema.parse({ revision, tree: revision, commit: "a".repeat(40), ref: "main", paths: [], landedAt: AT }); - expect(selectFeedLanding([landing], "2026-10-09T10:20:31+02:00")).toBe(landing); - expect(selectFeedLanding([landing], "2026-10-09T08:20:30.999Z")).toBeNull(); - expect(() => selectFeedLanding([landing, { ...landing, commit: "b".repeat(40), landedAt: "2026-10-09T10:20:31+02:00" }], AT)).toThrow("Conflicting feed landings"); - }); }); diff --git a/src/compiler/training/rewind/feed.ts b/src/compiler/training/rewind/feed.ts index e78dea76..7cf658a6 100644 --- a/src/compiler/training/rewind/feed.ts +++ b/src/compiler/training/rewind/feed.ts @@ -1,11 +1,10 @@ import { lstat, mkdir, readdir, symlink, writeFile } from "node:fs/promises"; import path from "node:path"; -import { z } from "zod"; -import { trainingInstantSchema, type TeamWorkspaceResource } from "../../../manifest/index.js"; -import { expandFeedPaths, expandFeedRefTemplate, FEED_IDENTITY_FILE, FEED_IDENTITY_VERSION, +import type { TeamWorkspaceResource } from "../../../manifest/index.js"; +import { digestDirectory, expandFeedPaths, expandFeedRefTemplate, FEED_IDENTITY_FILE, FEED_IDENTITY_VERSION, resolveFeedSource, type FeedExec, type FeedIdentity } from "../../../volume/index.js"; -import { recordedCommitSchema } from "./contract.js"; import { resolveRecordedCommit } from "./recordedCommit.js"; +import { feedLandingSchema, readFeedLandingHistory, selectFeedLanding, type FeedLanding } from "./feedLandings.js"; import { exportFeedGitTree } from "./gitArchive.js"; import { assertLocalGitRepository } from "./gitHistory.js"; import { assertCommitNotFuture, commitAt, rewindGitSync, type SelectedCommit } from "./gitProcess.js"; @@ -13,34 +12,15 @@ import { historyKey } from "./history.js"; import { compareInstants } from "./instant.js"; import type { InputContext, InputResult } from "./types.js"; +export { feedLandingSchema, selectFeedLanding, type FeedLanding } from "./feedLandings.js"; type VolumeFeed = NonNullable["feed"]>; -const revision = z.string().regex(/^[a-f0-9]{64}$/u); -const plainPath = z.string().min(1).refine((value) => !value.startsWith("/") && !/[\\\0]/u.test(value) - && !value.split("/").some((segment) => !segment || segment === "." || segment === ".." || segment.toLowerCase() === ".git"), "Expected a plain tree-relative path"); -export const feedLandingSchema = z.object({ - revision, tree: z.string().regex(/^(?:trees\/)?[a-f0-9]{64}(?:\.[1-9][0-9]*)?$/u), - commit: recordedCommitSchema.nullable(), ref: z.string().min(1).nullable(), paths: z.array(plainPath).nullable(), - landedAt: trainingInstantSchema -}).strict().superRefine((value, context) => { - if (!value.tree.replace(/^trees\//u, "").startsWith(value.revision)) context.addIssue({ code: "custom", message: "Landing tree must belong to its revision" }); -}); -export type FeedLanding = z.infer; - -/** Equal-instant landings must agree; log order cannot break a contradictory timestamp tie. */ -export const selectFeedLanding = (landings: readonly FeedLanding[], at: string): FeedLanding | null => { - const eligible = landings.filter((landing) => compareInstants(landing.landedAt, at) <= 0); - const selected = eligible.reduce((latest, entry) => !latest || compareInstants(entry.landedAt, latest.landedAt) > 0 ? entry : latest, null); - if (selected && eligible.some((entry) => compareInstants(entry.landedAt, selected.landedAt) === 0 - && JSON.stringify({ ...entry, landedAt: "" }) !== JSON.stringify({ ...selected, landedAt: "" }))) throw new Error("Conflicting feed landings at the same instant"); - return selected; -}; - const revisionExec: FeedExec = (command, args) => { if (command !== "git" || args[0] !== "-C" || !args[1]) throw new Error("Feed revision may only read a local git repository"); return rewindGitSync(args[1], args.slice(2)); }; -const feedPaths = (paths: string[] | null): string[] | undefined => paths?.length ? paths : undefined; +const feedPaths = (paths: string[] | undefined): string[] | undefined => paths?.length ? paths : undefined; +const treeName = (tree: string): string => tree.replace(/^trees\//u, ""); /** Same template/fallback ordering as chooseFeedRef, with historical eligibility replacing existence. */ export const selectFeedCommit = async (feed: VolumeFeed, repository: string, at: string): Promise => { @@ -77,45 +57,64 @@ export const materializeFeed = async (context: InputContext): Promise => { repository ??= await history.repository(`${key}/repo`); if (!repository) throw new Error(`Missing feed source repository for ${input.id}`); await assertLocalGitRepository(repository, history); return repository; }; + for (const [index, entry] of (log ?? []).entries()) { + if (!entry.commit) continue; + try { await resolveRecordedCommit(await openRepository(), entry.commit); } + catch (error) { throw new Error(`Invalid history ${key}/landings.jsonl:${index + 1}: ${error instanceof Error ? error.message : String(error)}`); } + } for (const candidate of context.candidateRecords ?? []) { const entry = candidate.feeds.find((record) => record.id === input.id); - if (entry) await resolveRecordedCommit(await openRepository(), entry.commit); + if (entry?.commit !== undefined) await resolveRecordedCommit(await openRepository(), entry.commit); } const record = context.record?.feeds.find((entry) => entry.id === input.id); - const recordedCommit = record ? await resolveRecordedCommit(await openRepository(), record.commit) : null; - if (log) { - for (const entry of log) if (entry.commit) await resolveRecordedCommit(await openRepository(), entry.commit); - const selected = selectFeedLanding(log, context.at); - if (!selected) throw new Error(`No feed landing exists at or before ${context.at}`); - landing = selected; - if (record && (record.revision !== landing.revision || record.tree !== landing.tree || recordedCommit !== landing.commit - || JSON.stringify(feedPaths(record.paths)) !== JSON.stringify(feedPaths(landing.paths)) || compareInstants(record.landedAt, landing.landedAt) !== 0)) { - throw new Error(`Moment feed ${input.id} disagrees with its landing log`); - } + const recordedCommit = record?.commit === undefined ? undefined : await resolveRecordedCommit(await openRepository(), record.commit); + const matching = record ? log?.findLast((entry) => entry.revision === record.revision && treeName(entry.tree) === treeName(record.tree) + && compareInstants(entry.landed_at, record.landedAt) === 0 && entry.commit === recordedCommit) : undefined; + const selected = record ? null : selectFeedLanding(log ?? [], context.at); + if (record) { + // A wake pin is authoritative even if a later attempt or landing changed the currently selected log row. + landing = { version: "spawnfile.volume-feed-landing.v1", resource: input.id, volume: matching?.volume ?? resource.name!, + revision: record.revision, tree: record.tree, landed_at: record.landedAt, + source: matching?.source ?? (feed.git ? { kind: "git", repo: feed.git.repo } : { kind: "directory", directory: feed.directory! }), + ...(recordedCommit ? { commit: recordedCommit, ref: record.ref ?? recordedCommit } : {}), + ...(record.paths ? { paths: record.paths } : {}), ...(record.period ? { period: record.period } : {}), + ...(matching?.switching_at ? { switching_at: matching.switching_at } : {}) }; + basis = "record"; + } else if (selected) { + landing = selected.landing; ambiguous = selected.ambiguous; basis = "landing-log"; } else { // Refuse unknown historical behavior before touching any declared source or executing any command. if (!feed.git || feed.freeze || typeof feed.git.ref === "object" && feed.git.ref.command) await selectFeedCommit(feed, "", context.at); if (feed.include || feed.prepare) throw new Error("Prepared/included feed requires a landing log and retained tree bytes"); - const repo = await openRepository(), selected = await selectFeedCommit(feed, repo, context.at); + const repo = await openRepository(), commit = await selectFeedCommit(feed, repo, context.at); const paths = expandFeedPaths(feed.git!.paths, new Date(context.at)); - const source = resolveFeedSource({ kind: "git", fetch: false, repo, ref: selected.commit, label: selected.ref, paths }, { exec: revisionExec }); - landing = { revision: source.revision, tree: source.revision, commit: selected.commit, ref: selected.ref, paths: paths ?? null, landedAt: selected.instant }; + const source = resolveFeedSource({ kind: "git", fetch: false, repo, ref: commit.commit, label: commit.ref, paths }, { exec: revisionExec }); + landing = { version: "spawnfile.volume-feed-landing.v1", resource: input.id, volume: resource.name!, + revision: source.revision, tree: source.revision, source: { kind: "git", repo: feed.git!.repo }, + commit: commit.commit, ref: commit.ref, ...(paths ? { paths } : {}), landed_at: commit.instant }; + basis = "feed-rule-at"; } + landing = feedLandingSchema.parse(landing); // Selection owns this check: retained/prepared bytes must obey the same cutoff as extraction. - if (feed.git && (!landing.commit || !landing.ref)) throw new Error("Git feed landing must record its commit and ref"); - if (landing.commit) await assertCommitNotFuture(await openRepository(), landing.commit, context.at); - const tree = path.join(context.destination, "trees", landing.revision); - if (await history.path(`${key}/trees/${landing.revision}`)) await history.copyTree(`${key}/trees/${landing.revision}`, tree); + if (landing.source.kind === "git" && (!landing.commit || !landing.ref)) throw new Error("Git feed landing must record its commit and ref"); + if (landing.commit) { + landing.commit = await resolveRecordedCommit(await openRepository(), landing.commit); + await assertCommitNotFuture(await openRepository(), landing.commit, context.at); + } + const retainedName = treeName(landing.tree), tree = path.join(context.destination, "trees", retainedName); + const needsRetained = landing.source.kind === "directory" || landing.source.prepared !== undefined + || basis === "record" && !matching && (feed.include !== undefined || feed.prepare !== undefined); + if (await history.path(`${key}/trees/${retainedName}`)) await history.copyTree(`${key}/trees/${retainedName}`, tree); else { - if (!feed.git || feed.include || feed.prepare) throw new Error("Directory/prepared/included feed requires retained landed tree bytes"); + if (needsRetained) throw new Error("Directory/prepared/included feed requires retained landed tree bytes"); if (!landing.commit || !landing.ref) throw new Error("Git feed landing must record its commit and ref"); const repo = await openRepository(); const resolved = resolveFeedSource({ kind: "git", fetch: false, repo, ref: landing.commit, label: landing.ref, paths: feedPaths(landing.paths) }, { exec: revisionExec }); @@ -123,14 +122,23 @@ export const materializeFeed = async (context: InputContext): Promise 0 ? context.at : landing.landed_at, + notes: ambiguous ? ["The trial instant falls inside an ambiguous feed transition; the incoming landing is used."] + : basis === "feed-rule-at" ? ["Commit dates do not establish when a ref advanced; historical freeze holds cannot be reconstructed."] : [] }; }; diff --git a/src/compiler/training/rewind/feedLandings.test.ts b/src/compiler/training/rewind/feedLandings.test.ts new file mode 100644 index 00000000..e9b2218c --- /dev/null +++ b/src/compiler/training/rewind/feedLandings.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { feedLandingSchema, selectFeedLanding, type FeedLanding } from "./feedLandings.js"; + +const AT = "2026-10-09T08:20:31Z", REVISION = "a".repeat(64); +const landing = (fields: Partial = {}): FeedLanding => ({ + version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "org-news", revision: REVISION, + tree: `trees/${REVISION}.2`, source: { kind: "git", repo: "https://never-fetch.invalid/source" }, + commit: "a".repeat(40), ref: "daily/2026-10-09", paths: ["published"], landed_at: AT, ...fields +}); + +describe("recording feed landing compatibility", () => { + it("accepts snake_case recording rows and directory sources without git fields", () => { + const row = landing({ switching_at: AT, period: "2026-10-09" }); + expect(feedLandingSchema.parse(row)).toEqual(row); + const { commit: _commit, ref: _ref, paths: _paths, ...directory } = row; + expect(feedLandingSchema.parse({ ...directory, source: { kind: "directory", directory: "/deployed/news" } })).toMatchObject({ + source: { kind: "directory", directory: "/deployed/news" } + }); + }); + + it("refuses malformed provenance, paths, tree identities and reversed transitions", () => { + const row = landing(); + for (const invalid of [ + { ...row, landedAt: AT }, { ...row, revision: "b".repeat(64) }, { ...row, tree: `trees/${REVISION}/../escape` }, + { ...row, paths: ["../escape"] }, { ...row, commit: "HEAD" }, { ...row, commit: null }, + { ...row, source: { kind: "git" } }, { ...row, ref: undefined }, + { ...row, switching_at: "2026-10-09T08:20:31.000000001Z" } + ]) expect(() => feedLandingSchema.parse(invalid)).toThrow(); + }); + + it("uses last-appended state for equal instants, including equivalent offsets", () => { + const first = landing(), last = landing({ tree: `trees/${REVISION}.3`, landed_at: "2026-10-09T10:20:31+02:00" }); + expect(selectFeedLanding([first, last], AT)).toEqual({ landing: last, ambiguous: false }); + expect(selectFeedLanding([last, first], AT)).toEqual({ landing: first, ambiguous: false }); + expect(selectFeedLanding([last, first], "2026-10-09T08:20:30.999999999Z")).toBeNull(); + }); + + it.each(["2026-10-09T08:20:30Z", AT, "2026-10-09T08:20:32Z"]) + ("selects the incoming row inside the inclusive transition at %s", (at) => { + const previous = landing({ landed_at: "2026-10-09T08:00:00Z" }); + const incoming = landing({ switching_at: "2026-10-09T08:20:30Z", landed_at: "2026-10-09T08:20:32Z" }); + expect(selectFeedLanding([previous, incoming], at)).toEqual({ landing: incoming, ambiguous: true }); + expect(selectFeedLanding([previous, incoming], "2026-10-09T08:20:32.000000001Z")).toEqual({ landing: incoming, ambiguous: false }); + }); + + it("marks a zero-duration transition ambiguous at the shared boundary", () => { + const row = landing({ switching_at: AT }); + expect(selectFeedLanding([row], "2026-10-09T10:20:31+02:00")).toEqual({ landing: row, ambiguous: true }); + }); +}); diff --git a/src/compiler/training/rewind/feedLandings.ts b/src/compiler/training/rewind/feedLandings.ts new file mode 100644 index 00000000..975d7c55 --- /dev/null +++ b/src/compiler/training/rewind/feedLandings.ts @@ -0,0 +1,51 @@ +import { z } from "zod"; +import { trainingInstantSchema } from "../../../manifest/index.js"; +import { recordedCommitSchema } from "./contract.js"; +import type { History } from "./history.js"; +import { compareInstants } from "./instant.js"; + +const text = z.string().min(1), revision = z.string().regex(/^[a-f0-9]{64}$/u); +const plainPath = text.refine((value) => !value.startsWith("/") && !/[\\\0]/u.test(value) + && !value.split("/").some((segment) => !segment || segment === "." || segment === ".." || segment.toLowerCase() === ".git"), "Expected a plain tree-relative path"); +const prepared = z.object({ + source_revision: revision, + include: z.array(z.object({ digest: revision, to: plainPath }).strict()), + prepare: z.object({ command: z.array(text).min(1), image: text.nullable(), network: z.boolean().nullable(), platform: text.nullable() }).strict().nullable() +}).strict(); + +// Recording-format compatibility boundary: replace with readFeedLandings/feedLandingAt after P3 merges. +export const feedLandingSchema = z.object({ + version: z.literal("spawnfile.volume-feed-landing.v1"), resource: text, volume: text, + revision, tree: z.string().regex(/^(?:trees\/)?[a-f0-9]{64}(?:\.[1-9][0-9]*)?$/u), + source: z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("git"), repo: text, prepared: prepared.optional() }).strict(), + z.object({ kind: z.literal("directory"), directory: text, prepared: prepared.optional() }).strict() + ]), + commit: recordedCommitSchema.optional(), ref: text.optional(), paths: z.array(plainPath).optional(), + switching_at: trainingInstantSchema.optional(), landed_at: trainingInstantSchema, period: text.optional() +}).strict().superRefine((value, context) => { + if (!value.tree.replace(/^trees\//u, "").startsWith(value.revision)) context.addIssue({ code: "custom", message: "Landing tree must belong to its revision" }); + if (value.source.kind === "git" && (!value.commit || !value.ref)) context.addIssue({ code: "custom", message: "Git feed landing must record its commit and ref" }); + if (value.source.kind === "directory" && (value.commit !== undefined || value.ref !== undefined || value.paths !== undefined)) { + context.addIssue({ code: "custom", message: "Directory feed landing cannot carry git provenance" }); + } + if (value.switching_at && compareInstants(value.switching_at, value.landed_at) > 0) { + context.addIssue({ code: "custom", message: "Landing transition begins after landed_at" }); + } +}); +export type FeedLanding = z.infer; +export interface FeedLandingSelection { landing: FeedLanding; ambiguous: boolean } + +export const readFeedLandingHistory = async (history: History, key: string, resource: string): Promise => + history.jsonl(`${key}/landings.jsonl`, feedLandingSchema.refine((entry) => entry.resource === resource, + "Landing resource disagrees with the history input")); + +/** Inclusive transition windows take priority; equal-instant state uses the last appended row. */ +export const selectFeedLanding = (landings: readonly FeedLanding[], at: string): FeedLandingSelection | null => { + let transition: FeedLanding | null = null, selected: FeedLanding | null = null; + for (const landing of landings) { + if (landing.switching_at && compareInstants(landing.switching_at, at) <= 0 && compareInstants(at, landing.landed_at) <= 0) transition = landing; + if (compareInstants(landing.landed_at, at) <= 0 && (!selected || compareInstants(landing.landed_at, selected.landed_at) >= 0)) selected = landing; + } + return transition ? { landing: transition, ambiguous: true } : selected ? { landing: selected, ambiguous: false } : null; +}; diff --git a/src/compiler/training/rewind/fixtures.test-helper.ts b/src/compiler/training/rewind/fixtures.test-helper.ts index ac6d3a1f..9e3662c9 100644 --- a/src/compiler/training/rewind/fixtures.test-helper.ts +++ b/src/compiler/training/rewind/fixtures.test-helper.ts @@ -1,4 +1,3 @@ -import { createHash } from "node:crypto"; import { execFileSync } from "node:child_process"; import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; @@ -6,6 +5,7 @@ import { trainingProject, writeTrainingFile, teamFiles, teamManifest, volume, ex import { loadTrainingProject, resolveTrainingPlan } from "../plan/index.js"; import { rewindGitEnvironment } from "./gitProcess.js"; import { resolveFeedSource } from "../../../volume/feedSource.js"; +import { writeRecordedVolumeSnapshot } from "./snapshotFixtures.test-helper.js"; import type { MomentRecord } from "./contract.js"; import type { MemoryLedgerEvent } from "./memorySchemas.js"; @@ -18,8 +18,9 @@ export const memoryEvent = (seq: number, createdAt = BEFORE): MemoryLedgerEvent tags: [], entities: [], sensitivity: "normal", parentEventIds: [], checksum: `checksum-${seq}` }); export const emptyRecord = (agent = "agent:a", at = AT): MomentRecord => ({ - version: "spawnfile.moment-record.v1", agent, wake: { at, surface: "moltnet", room: "assignment", from: "brass" }, - recordedAt: FUTURE, resources: [], feeds: [], memory: [], moltnet: [], volumes: [] + version: "spawnfile.moment-record.v1", agent, wake: { at, startedAt: at, executionId: "execution-17", + deliveries: [{ deliveryId: "delivery-17", kind: "message", occurredAt: at }], surface: "moltnet", room: "assignment", from: "brass" }, + recordedAt: FUTURE, exact: true, resources: [], feeds: [], memory: [], moltnet: [], volumes: [] }); export const jsonl = async (root: string, relative: string, values: unknown[]): Promise => { const file = path.join(root, relative); @@ -44,8 +45,8 @@ export const rewindFixture = async () => { await mkdir(path.dirname(feedRepo), { recursive: true }); testGit(root, ["clone", "--quiet", "--no-local", gitRepo, feedRepo]); const feedSource = resolveFeedSource({ kind: "git", repo: feedRepo, ref: commit, fetch: false }); - await jsonl(historyDir, "feeds/team%3Apublication/news/landings.jsonl", [{ revision: feedSource.revision, - tree: feedSource.revision, commit, ref: "main", paths: null, landedAt: BEFORE }]); + await jsonl(historyDir, "feeds/team%3Apublication/news/landings.jsonl", [{ version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "news", source: { kind: "git", repo: feedRepo }, revision: feedSource.revision, + tree: `trees/${feedSource.revision}`, commit, ref: "main", landed_at: BEFORE }]); await writeFile(path.join(gitRepo, "future.txt"), "secret after wake\n"); testGit(gitRepo, ["add", "."]); testGit(gitRepo, ["commit", "--quiet", "-m", "future"], "2026-10-10T00:00:00Z"); const futureCommit = testGit(gitRepo, ["rev-parse", "HEAD"]); @@ -56,13 +57,10 @@ export const rewindFixture = async () => { await writeTrainingFile(root, "Spawnfile", manifest); await writeTrainingFile(root, "training/train.yaml", { examples: [{ ...example, networks: [network] }] }); await jsonl(historyDir, "memory/agent%3Aa/self/events.jsonl", [memoryEvent(1), memoryEvent(2, FUTURE)]); - const data = Buffer.from("snapshot bytes\n"), digest = createHash("sha256").update(data).digest("hex"); - await jsonl(historyDir, "volumes/team%3Apublication/edition-state/snapshots.jsonl", [ - { snapshot: "snapshot-1", takenAt: BEFORE, parent: null, files: { "saved.txt": digest } } - ]); - const objectDir = path.join(historyDir, "volumes/team%3Apublication/edition-state/objects"); - await mkdir(objectDir, { recursive: true }); await writeFile(path.join(objectDir, digest), data); const project = await loadTrainingProject(root), plan = resolveTrainingPlan(project, "agent:a", "test"); + const agentSlug = project.compilePlan.nodes.find((node) => node.id === plan.agent.id)!.slug; + await writeRecordedVolumeSnapshot(historyDir, { agentSlug, agentId: plan.agent.id, rootId: "edition-state", + snapshot: "20261009T070000000Z-execution-17", startedAt: BEFORE, files: { "saved.txt": "snapshot bytes\n" } }); return { root, historyDir, project, plan, gitRepo, futureCommit, commit, feedRepo, options: { project, plan, example: example.id, historyDir, out: path.join(root, "out") } }; }; diff --git a/src/compiler/training/rewind/git.test-helper.ts b/src/compiler/training/rewind/git.test-helper.ts index 7857ff66..de85e592 100644 --- a/src/compiler/training/rewind/git.test-helper.ts +++ b/src/compiler/training/rewind/git.test-helper.ts @@ -25,6 +25,7 @@ export const commitTestGit = async (repository: string, files: Record ({ - version: "spawnfile.moment-record.v1", agent: "agent:writer", wake: { at, surface: "moltnet", room: "assignment", from: "brass" }, - recordedAt: at, resources: [], feeds: [], memory: [], moltnet: [], volumes: [] + version: "spawnfile.moment-record.v1", agent: "agent:writer", wake: { at, startedAt: at, executionId: "execution-17", + deliveries: [{ deliveryId: "delivery-17", kind: "message", occurredAt: at }], surface: "moltnet", room: "assignment", from: "brass" }, + recordedAt: at, exact: true, resources: [], feeds: [], memory: [], moltnet: [], volumes: [] }); diff --git a/src/compiler/training/rewind/git.test.ts b/src/compiler/training/rewind/git.test.ts index f2cb91bb..57537f2e 100644 --- a/src/compiler/training/rewind/git.test.ts +++ b/src/compiler/training/rewind/git.test.ts @@ -35,7 +35,8 @@ describe("git rewind", () => { testGit(repository, ["checkout", "--detach", past]); testGit(repository, ["commit", "--allow-empty", "-m", "future metadata", `--date=${role === "author" ? LATE : EARLY}`], role === "committer" ? LATE : EARLY); const commit = testGit(repository, ["rev-parse", "HEAD"]); - context.record = { ...testMomentRecord(), resources: [{ id: "archive", kind, commit }] }; + context.record = { ...testMomentRecord(), resources: kind === "git" ? [{ id: "archive", kind, commit }] : [] }; + if (context.input.declaration.kind === "bundle") context.input.declaration.build!.files!.ref = commit; await expect(materializeGit(context)).rejects.toThrow(`Selected commit ${commit} has an author or committer date after ${AT}`); await expect(readdir(context.destination)).rejects.toThrow(); } @@ -93,7 +94,7 @@ describe("git rewind", () => { const { past, context } = await fixture(); context.history = undefined; await expect(materializeGit(context)).rejects.toThrow("unpinned resources require --history"); context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", source: "today.tar" }; - await expect(materializeGit(context)).rejects.toThrow("Prebuilt bundle has no recorded commit"); + await expect(materializeGit(context)).rejects.toThrow("Prebuilt bundle has no matching recorded archive"); context.input.declaration = { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: "https://example.invalid/repo", ref: past }; await expect(materializeGit(context)).rejects.toThrow("network access is forbidden"); }); @@ -101,7 +102,7 @@ describe("git rewind", () => { it("exports files bundle roots and exclusions at their pinned commit", async () => { const { past, context } = await fixture(); context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { files: { root: "package", ref: past, exclude: ["secret.txt"] } } }; - expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "pinned", source: { commit: past } }); + expect(await materializeGit(context)).toMatchObject({ fidelity: "approximate", basis: "pinned", source: { commit: past } }); expect(await readdir(context.destination)).toEqual(["package.json"]); }); @@ -117,15 +118,6 @@ describe("git rewind", () => { await expect(readdir(context.destination)).rejects.toThrow(); }); - it("exports a prebuilt bundle's recorded git tree instead of today's tar", async () => { - const { past, context } = await fixture(); - context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", source: "does-not-exist.tar" }; - context.record = { ...testMomentRecord(), resources: [{ id: "archive", kind: "bundle", commit: past }] }; - expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "record" }); - expect(await readFile(path.join(context.destination, "readme.txt"), "utf8")).toBe("past\n"); - await expect(readFile(path.join(context.destination, ".git"))).rejects.toThrow(); - }); - it("defers dependency and generated recipes while keeping all declared historical inputs", async () => { const { context } = await fixture(); context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { dependencies: { directory: "package", image: `node@sha256:${"0".repeat(64)}` } } }; @@ -154,12 +146,12 @@ describe("git rewind", () => { await symlink("package.json", path.join(repository, "package/link")); const ref = await commitTestGit(repository, {}, "2026-10-09T08:10:00Z"); context.input.declaration = { id: "archive", kind: "bundle", mode: "readonly", mount: "./archive", build: { files: { root: "package", ref } } }; - expect(await materializeGit(context)).toMatchObject({ fidelity: "exact" }); + expect(await materializeGit(context)).toMatchObject({ fidelity: "approximate" }); expect(await readlink(path.join(context.destination, "link"))).toBe("package.json"); expect(await readFile(path.join(context.destination, "link"), "utf8")).toBe("{}\n"); await rm(context.destination, { recursive: true, force: true }); context.input.declaration.build!.files!.exclude = ["link"]; - expect(await materializeGit(context)).toMatchObject({ fidelity: "exact" }); + expect(await materializeGit(context)).toMatchObject({ fidelity: "approximate" }); expect(await readdir(context.destination)).not.toContain("link"); }); diff --git a/src/compiler/training/rewind/git.ts b/src/compiler/training/rewind/git.ts index 7d6829c6..d038e0b3 100644 --- a/src/compiler/training/rewind/git.ts +++ b/src/compiler/training/rewind/git.ts @@ -8,6 +8,7 @@ import { assertLocalGitRepository } from "./gitHistory.js"; import { assertCommitNotFuture, commitAt, commitInstant, resolveRewindCommit, rewindGit } from "./gitProcess.js"; import { exportGitTree, extractOverrideBundle } from "./gitTree.js"; import { resolveRecordedCommit } from "./recordedCommit.js"; +import { materializeRecordedBundle } from "./recordedBundle.js"; import type { InputContext, InputResult } from "./types.js"; export { commitAt, commitInstant, resolveRewindCommit, rewindGit, rewindGitEnvironment, rewindGitSync, selectCommitAt } from "./gitProcess.js"; @@ -92,17 +93,23 @@ export const materializeGit = async (context: InputContext): Promise entry.id === input.id); const candidateCommits = input.overridden ? [] : context.candidateRecords?.flatMap((candidate) => - candidate.resources.filter((entry) => entry.id === input.id).map((entry) => entry.commit)) ?? []; - const record = pinned ? undefined : matchingRecord; - if (record && record.kind !== resource.kind) throw new Error(`Moment resource ${input.id} kind disagrees with its declaration`); + candidate.resources.flatMap((entry) => entry.id === input.id && entry.kind === "git" ? [entry.commit] : [])) ?? []; + if (matchingRecord && matchingRecord.kind !== resource.kind) throw new Error(`Moment resource ${input.id} kind disagrees with its declaration`); + const bundlePin = matchingRecord?.kind === "bundle" ? matchingRecord : undefined; + if (bundlePin && await materializeRecordedBundle(history, bundlePin.sha256, context.destination)) { + return { fidelity: "exact", basis: "record", source: { sha256: bundlePin.sha256 }, notes: [], instant: context.at }; + } + const bundleNote = bundlePin ? `Recorded bundle ${bundlePin.sha256} could not be reproduced from retained archive bytes.` : undefined; + const gitPin = matchingRecord?.kind === "git" ? matchingRecord : undefined; + const record = pinned ? undefined : gitPin; if (input.overridden && resource.kind === "bundle" && resource.source) { await extractOverrideBundle(localRepositoryPath(resource.source, input.declarationBase), context.destination, resource.sha256); return { fidelity: "overridden", basis: "override", source: {}, notes: [], instant: context.at }; } - if (resource.kind === "bundle" && resource.source && !record) throw new Error("Prebuilt bundle has no recorded commit and cannot be rewound; provide an override"); + if (resource.kind === "bundle" && resource.source) throw new Error(`${bundleNote ?? "Prebuilt bundle has no matching recorded archive"} and cannot be rewound; provide an override`); let repository = input.overridden ? null : await history?.repository(`git/${historyKey(input)}`); const fromHistory = Boolean(repository); - if ((matchingRecord || candidateCommits.length) && !repository) throw new Error(`Missing git history for recorded resource ${input.id}`); + if ((gitPin || candidateCommits.length) && !repository) throw new Error(`Missing git history for recorded resource ${input.id}`); if (!repository && (pinned || input.overridden)) { const declared = localRoot(resource); if (declared) repository = localRepositoryPath(declared, input.declarationBase); @@ -110,7 +117,7 @@ export const materializeGit = async (context: InputContext): Promise ({ commit, instant: await commitInstant(repository!, commit) })) @@ -120,10 +127,11 @@ export const materializeGit = async (context: InputContext): Promise { await mkdir(path.join(repository, ".git/info"), { recursive: true }); await writeFile(path.join(repository, ".git/info/attributes"), "hidden.txt export-ignore\n"); const context: InputContext = { at: AT, destination: path.join(root, "out"), history: new History(path.join(root, "history")), - record: { ...testMomentRecord(), resources: kind === "bundle" ? [{ id: "input", kind: "bundle", commit }] : [] }, + record: { ...testMomentRecord(), resources: [] }, input: { section: "resources", id: "input", declaredBy: "agent:writer", overridden: false, declarationBase: repository, declaration: kind === "feed" ? { id: "input", kind: "volume", mode: "mutable", mount: "./input", name: "input", feed: { git: { repo: repository } } } @@ -36,10 +36,10 @@ describe("raw historical blob extraction", () => { let output = context.destination; if (kind === "feed") { const revision = resolveFeedSource({ kind: "git", repo: repository, ref: commit, fetch: false }).revision; - await writeFile(path.join(repository, "../landings.jsonl"), `${JSON.stringify({ revision, tree: revision, commit, ref: "main", paths: null, landedAt: BEFORE })}\n`); + await writeFile(path.join(repository, "../landings.jsonl"), `${JSON.stringify({ version: "spawnfile.volume-feed-landing.v1", resource: "input", volume: "input", source: { kind: "git", repo: repository }, revision, tree: revision, commit, ref: "main", landed_at: BEFORE })}\n`); expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", source: { revision, commit } }); output = path.join(output, "current"); - } else expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", source: { commit } }); + } else expect(await materializeGit(context)).toMatchObject({ fidelity: "approximate", source: { commit } }); expect(await readFile(path.join(output, "refs.txt"), "utf8")).toBe("$Format:%D$\n"); expect(await readFile(path.join(output, "hidden.txt"), "utf8")).toBe("raw retained bytes\n"); }); diff --git a/src/compiler/training/rewind/gitTree.ts b/src/compiler/training/rewind/gitTree.ts index 99768890..e05d48b7 100644 --- a/src/compiler/training/rewind/gitTree.ts +++ b/src/compiler/training/rewind/gitTree.ts @@ -62,7 +62,13 @@ export const extractOverrideBundle = async (file: string, destination: string, e if (!info.isFile() || info.size > 536_870_912) throw new Error("Override bundle must be a bounded local tar file"); const bytes = await readFile(file); if (expected && `sha256:${createHash("sha256").update(bytes).digest("hex")}` !== expected) throw new Error("Override bundle checksum mismatch"); + await extractBundleBytes(bytes, destination); +}; + +/** Validated archive bytes are extracted only after digest validation by the caller. */ +export const extractBundleBytes = async (bytes: Buffer, destination: string): Promise => { validateWorkspaceBundleTar(bytes); + await mkdir(destination, { recursive: true }); const text = (buffer: Buffer) => buffer.toString("utf8").replace(/\0.*$/su, ""); for (let offset = 0; offset < bytes.length && !bytes.subarray(offset, offset + 512).every((byte) => byte === 0);) { const header = bytes.subarray(offset, offset + 512), prefix = text(header.subarray(345, 500)), name = text(header.subarray(0, 100)); diff --git a/src/compiler/training/rewind/history.test.ts b/src/compiler/training/rewind/history.test.ts index fc853944..26e1fe4d 100644 --- a/src/compiler/training/rewind/history.test.ts +++ b/src/compiler/training/rewind/history.test.ts @@ -3,6 +3,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { z } from "zod"; +import { emptyRecord } from "./fixtures.test-helper.js"; import { momentRecordSchema } from "./contract.js"; import { History, historyKey, openHistory, safeSegment } from "./history.js"; @@ -16,9 +17,7 @@ const fixture = async () => { await writeFile(path.join(outside, "secret"), "outside bytes"); return { root, directory, outside, history: (await openHistory(directory))! }; }; -const record = { version: "spawnfile.moment-record.v1", agent: "agent:writer", - wake: { at: "2026-10-09T08:20:31Z", surface: "moltnet", room: "assignment", from: "brass" }, - recordedAt: "2026-10-09T08:20:31Z", resources: [], feeds: [], memory: [], moltnet: [], volumes: [] }; +const record = emptyRecord("agent:writer"); describe("history containment and strict reads", () => { it("allows missing history and entries but rejects non-directory roots and non-file reads", async () => { diff --git a/src/compiler/training/rewind/history.ts b/src/compiler/training/rewind/history.ts index 3f558ba7..5c633d97 100644 --- a/src/compiler/training/rewind/history.ts +++ b/src/compiler/training/rewind/history.ts @@ -69,9 +69,22 @@ export class History { const directory = await this.path("moments"); if (!directory) return []; const records: MomentRecord[] = []; - for (const name of (await readdir(directory)).sort()) { - if (name.endsWith(".jsonl")) records.push(...(await this.jsonl(`moments/${name}`, momentRecordSchema) ?? [])); - } + const seen = new Set(); + const walk = async (relative: string): Promise => { + const file = await this.path(relative); + if (!file) throw new Error(`History entry vanished: ${relative}`); + const physical = await realpath(file); + if ((await lstat(physical)).isDirectory()) { + if (seen.has(physical)) throw new Error(`Repeated moment history directory: ${relative}`); + seen.add(physical); + for (const name of (await readdir(file)).sort()) await walk(`${relative}/${name}`); + } else if (relative.endsWith(".jsonl")) records.push(...(await this.jsonl(relative, momentRecordSchema) ?? [])); + else if (relative.endsWith(".json")) { + try { records.push(momentRecordSchema.parse(JSON.parse((await this.text(relative))!))); } + catch (error) { throw new Error(`Invalid history ${relative}: ${reason(error)}`); } + } + }; + await walk("moments"); return records; } /** Validate all tree entries before git or another local parser receives the directory. */ diff --git a/src/compiler/training/rewind/index.ts b/src/compiler/training/rewind/index.ts index b2b233db..d07dd1d7 100644 --- a/src/compiler/training/rewind/index.ts +++ b/src/compiler/training/rewind/index.ts @@ -8,7 +8,7 @@ export * from "./select.js"; export * from "./taskClock.js"; export { selectFeedLanding, selectFeedCommit } from "./feed.js"; export { selectCommitAt } from "./gitProcess.js"; -export { selectMemoryPrefix, selectMemoryGeneration, parseMemoryLedger } from "./memory.js"; -export { rewindMoltnetJson, moltnetCursorExact } from "./moltnetJson.js"; -export { selectVolumeSnapshot, replayVolumeSnapshot } from "./volumes.js"; +export { selectMemoryPrefix, parseMemoryLedger } from "./memory.js"; +export { rewindMoltnetJson } from "./moltnetJson.js"; +export { selectVolumeSnapshot } from "./volumes.js"; export type * from "./types.js"; diff --git a/src/compiler/training/rewind/inputs.ts b/src/compiler/training/rewind/inputs.ts index e66d3653..e9d48835 100644 --- a/src/compiler/training/rewind/inputs.ts +++ b/src/compiler/training/rewind/inputs.ts @@ -60,5 +60,5 @@ export const resolveRewindInputs = ( } } inputs.sort((a, b) => a.section.localeCompare(b.section) || a.declaredBy.localeCompare(b.declaredBy) || a.id.localeCompare(b.id)); - return { inputs, selected, at: normalizeInstant(raw.at) }; + return { inputs, selected, at: normalizeInstant(raw.at), agentSlug: agent.slug }; }; diff --git a/src/compiler/training/rewind/leakCheck.ts b/src/compiler/training/rewind/leakCheck.ts index 269035fd..0eb64e0e 100644 --- a/src/compiler/training/rewind/leakCheck.ts +++ b/src/compiler/training/rewind/leakCheck.ts @@ -18,7 +18,7 @@ export const inspectTrainingMoment = async ( try { const moment = materializedMomentSchema.parse(JSON.parse(await readFile(await diskPath(root, "moment.json"), "utf8"))); if (skipRefused && moment.status === "refused") return { moment, check: null }; - const report: LeakCheck = { version: "spawnfile.leak-check.v1", at: moment.at, ok: false, + const report: LeakCheck = { version: "spawnfile.leak-check.v1", at: moment.effectiveAt, ok: false, checked: { files: 0, gitObjects: 0, memoryEvents: 0, moltnetMessages: 0, moltnetEvents: 0, snapshots: 0 }, findings: [] }; await inspectBundleFiles(root, moment.inputs, report, files); for (const input of moment.inputs) { @@ -34,7 +34,7 @@ export const inspectTrainingMoment = async ( if (["git", "bundle", "feed"].includes(input.kind)) await inspectInputProvenance(root, input, report, options.historyDir); if (input.kind === "git") await inspectGitBundle(directory, input, report); if (input.kind === "memory") await inspectMemoryBundle(root, input, report); - if (input.kind === "feed") await inspectFeedBundle(root, input, report); + if (input.kind === "feed") await inspectFeedBundle(root, input, report, moment, await openHistory(options.historyDir)); if (input.kind === "moltnet") await inspectMoltnetBundle(root, input, report); if (input.kind === "volume") await inspectVolumeBundle(root, input, report, await openHistory(options.historyDir)); } catch (error) { diff --git a/src/compiler/training/rewind/leakFeed.ts b/src/compiler/training/rewind/leakFeed.ts new file mode 100644 index 00000000..fd70b5e9 --- /dev/null +++ b/src/compiler/training/rewind/leakFeed.ts @@ -0,0 +1,29 @@ +import type { FeedIdentity } from "../../../volume/index.js"; +import type { MaterializedInput, MaterializedMoment } from "./contract.js"; +import { readFeedLandingHistory, selectFeedLanding } from "./feedLandings.js"; +import { historyKey, type History } from "./history.js"; +import { compareInstants } from "./instant.js"; +import { recordInputErrors } from "./recordInput.js"; +import { selectMomentRecord } from "./select.js"; + +/** Publication metadata can follow the bound only when independent history proves the incoming pin/window. */ +export const permitsFeedPublicationTime = async ( + identity: FeedIdentity, input: MaterializedInput, moment: MaterializedMoment, history?: History +): Promise => { + if (!history || identity.landed_at !== input.source.landedAt) return false; + if (input.basis === "record" && input.fidelity === "exact" && moment.record.matched) { + const record = selectMomentRecord(await history.records(), moment.agent, moment.at, [input]); + const pin = record?.feeds.find((entry) => entry.id === input.id); + return Boolean(record && pin && compareInstants(record.wake.startedAt, moment.effectiveAt) === 0 + && !recordInputErrors(record, input).length && pin.revision === identity.revision + && pin.tree.replace(/^trees\//u, "") === identity.tree.replace(/^trees\//u, "") + && pin.commit === input.source.commit && compareInstants(pin.landedAt, identity.landed_at) === 0); + } + if (input.basis !== "landing-log" || input.fidelity !== "approximate") return false; + const selection = selectFeedLanding(await readFeedLandingHistory(history, `feeds/${historyKey(input)}`, input.id) ?? [], moment.effectiveAt); + const landing = selection?.landing; + return Boolean(selection?.ambiguous && landing && landing.revision === identity.revision + && landing.tree.replace(/^trees\//u, "") === identity.tree.replace(/^trees\//u, "") + && landing.commit === input.source.commit && landing.switching_at === input.source.switchingAt + && compareInstants(landing.landed_at, identity.landed_at) === 0); +}; diff --git a/src/compiler/training/rewind/leakInputs.ts b/src/compiler/training/rewind/leakInputs.ts index edd239ae..08c1922b 100644 --- a/src/compiler/training/rewind/leakInputs.ts +++ b/src/compiler/training/rewind/leakInputs.ts @@ -1,15 +1,17 @@ -import { createHash } from "node:crypto"; import { lstat, readFile, readdir, readlink, realpath } from "node:fs/promises"; import path from "node:path"; import { feedIdentityFindings, type FeedIdentity } from "../../../volume/feedRecord.js"; -import type { LeakCheck, LeakFinding, MaterializedInput } from "./contract.js"; -import { historyKey, inside, type History } from "./history.js"; +import type { LeakCheck, LeakFinding, MaterializedInput, MaterializedMoment } from "./contract.js"; +import { digestDirectory } from "../../../volume/index.js"; +import { permitsFeedPublicationTime } from "./leakFeed.js"; +import { inside, safeSegment, type History } from "./history.js"; import { compareInstants } from "./instant.js"; import { diskPath } from "./leakFiles.js"; import { parseMemoryLedger } from "./memory.js"; import { inspectMoltnetJson } from "./moltnetJson.js"; import { inspectMoltnetSqlite } from "./moltnetSqlite.js"; -import { replayVolumeSnapshot, volumeSnapshotSchema } from "./volumes.js"; +import { readSnapshotManifest, snapshotBytes, snapshotHistoryPath, snapshotTreeFindings } from "./snapshotManifest.js"; +import { wakeSnapshotRowSchema } from "./volumes.js"; export const addInputFinding = (report: LeakCheck, input: MaterializedInput, kind: LeakFinding["class"], detail: string, file = input.path!): void => { report.findings.push({ input: { section: input.section, id: input.id, declaredBy: input.declaredBy }, class: kind, path: file, detail }); @@ -42,7 +44,7 @@ export const inspectMemoryBundle = async (root: string, input: MaterializedInput if (input.source.seq !== ledger.events.length) addInputFinding(report, input, "memory-event", "Ledger length disagrees with the materialized sequence"); }; -export const inspectFeedBundle = async (root: string, input: MaterializedInput, report: LeakCheck): Promise => { +export const inspectFeedBundle = async (root: string, input: MaterializedInput, report: LeakCheck, moment?: MaterializedMoment, history?: History): Promise => { const directory = await diskPath(root, input.path!); if (input.fidelity === "overridden") { if ((await readdir(directory)).length) addInputFinding(report, input, "feed-identity", "Overridden feed volume must be empty"); @@ -50,7 +52,9 @@ export const inspectFeedBundle = async (root: string, input: MaterializedInput, } const identity: FeedIdentity = JSON.parse(await readFile(await diskPath(root, `${input.path}/.spawnfile-feed.json`), "utf8")); for (const issue of feedIdentityFindings(identity)) addInputFinding(report, input, "feed-identity", issue); - if (compareInstants(identity.landed_at, report.at) > 0) addInputFinding(report, input, "feed-identity", "Feed landed_at is later than the moment"); + if (compareInstants(identity.landed_at, report.at) > 0 && !(moment && await permitsFeedPublicationTime(identity, input, moment, history))) { + addInputFinding(report, input, "feed-identity", "Feed landed_at is later than the moment"); + } if (identity.resource !== input.id || identity.revision !== input.source.revision) { addInputFinding(report, input, "feed-identity", "Feed identity disagrees with the moment input"); } @@ -61,6 +65,9 @@ export const inspectFeedBundle = async (root: string, input: MaterializedInput, || !inside(path.join(directory, "trees"), await realpath(current))) { addInputFinding(report, input, "feed-identity", "Feed current does not point to its tree inside the volume"); } else if (!(await lstat(await realpath(current))).isDirectory()) addInputFinding(report, input, "feed-identity", "Feed tree must be a directory"); + else if (identity.source.kind === "directory" && digestDirectory(await realpath(current)) !== identity.revision) { + addInputFinding(report, input, "feed-identity", "Directory feed revision disagrees with retained bytes"); + } }; export const inspectMoltnetBundle = async (root: string, input: MaterializedInput, report: LeakCheck): Promise => { @@ -75,38 +82,27 @@ export const inspectMoltnetBundle = async (root: string, input: MaterializedInpu }; export const inspectVolumeBundle = async (root: string, input: MaterializedInput, report: LeakCheck, history?: History): Promise => { - let expected: Record = {}; - if (input.fidelity !== "overridden") { - if (!history || !input.source.snapshot) throw new Error("Snapshot verification requires its history and selected snapshot"); - const snapshots = await history.jsonl(`volumes/${historyKey(input)}/snapshots.jsonl`, volumeSnapshotSchema); - if (!snapshots) throw new Error("Snapshot history is missing"); - const replay = replayVolumeSnapshot(snapshots, input.source.snapshot); - if (compareInstants(replay.takenAt, report.at) > 0) throw new Error("Selected snapshot is later than the moment"); - expected = replay.files; - report.checked.snapshots += replay.chain.length; + const directory = await diskPath(root, input.path!); + if (input.fidelity === "overridden") { + for (const name of await readdir(directory)) addInputFinding(report, input, "snapshot", `Unexpected snapshot entry ${name}`); + return; } - const found = new Set(); - const directories = new Set(); - for (const name of Object.keys(expected)) { - for (let parent = path.posix.dirname(name); parent !== "."; parent = path.posix.dirname(parent)) directories.add(parent); + if (!history || !input.source.snapshot || !input.source.agentSlug) throw new Error("Snapshot verification requires its history, agent slug and selected snapshot"); + if (input.source.snapshot.endsWith(".partial")) throw new Error("Selected snapshot is partial"); + const relative = snapshotHistoryPath(input.source.agentSlug, input.id, input.source.snapshot); + const manifest = await readSnapshotManifest(history, relative, input.id, input.source.snapshot); + const retained = await history.path(relative); + if (!manifest || !retained) throw new Error("Snapshot tree or manifest is missing"); + if (compareInstants(manifest.started_at, report.at) > 0) throw new Error("Selected snapshot is later than the moment"); + const rows = await history.jsonl(`recording/${safeSegment(input.source.agentSlug)}/wake-moments.jsonl`, wakeSnapshotRowSchema); + if (!rows?.some((row) => compareInstants(row.started_at, manifest.started_at) === 0 + && row.snapshots.some((entry) => entry.id === input.id && entry.snapshot === input.source.snapshot))) { + throw new Error("Selected snapshot has no matching wake row"); } - const visit = async (relative: string): Promise => { - const folder = await diskPath(root, `${input.path}${relative ? `/${relative}` : ""}`); - for (const entry of await readdir(folder, { withFileTypes: true })) { - const name = relative ? `${relative}/${entry.name}` : entry.name; - if (entry.isDirectory()) { - if (!directories.has(name)) addInputFinding(report, input, "snapshot", `Unexpected snapshot directory ${name}`); - await visit(name); - } else { - found.add(name); - if (!entry.isFile() || !Object.hasOwn(expected, name)) addInputFinding(report, input, "snapshot", `Unexpected snapshot entry ${name}`); - else { - const digest = createHash("sha256").update(await readFile(await diskPath(root, `${input.path}/${name}`))).digest("hex"); - if (digest !== expected[name]) addInputFinding(report, input, "snapshot", `Snapshot digest mismatch for ${name}`); - } - } - } - }; - await visit(""); - for (const name of Object.keys(expected)) if (!found.has(name)) addInputFinding(report, input, "snapshot", `Missing snapshot file ${name}`); + report.checked.snapshots++; + for (const detail of await snapshotTreeFindings(retained, manifest, { retained: true })) addInputFinding(report, input, "snapshot", `Retained ${detail}`); + for (const detail of await snapshotTreeFindings(directory, manifest, { + readBytes: async (name) => readFile(await diskPath(root, `${input.path}/${name}`)), + expectedBytes: (name) => snapshotBytes(history, relative, name) + })) addInputFinding(report, input, "snapshot", detail); }; diff --git a/src/compiler/training/rewind/leakProvenance.test.ts b/src/compiler/training/rewind/leakProvenance.test.ts index 3f7b0a2a..c01e65ee 100644 --- a/src/compiler/training/rewind/leakProvenance.test.ts +++ b/src/compiler/training/rewind/leakProvenance.test.ts @@ -38,14 +38,14 @@ const fixture = async (kind: Kind) => { const directory = path.dirname(repository), retained = path.join(directory, "trees", revision); await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "data.txt"), selected === commit ? "past bytes\n" : "future retained bytes\n"); - await writeFile(path.join(directory, "landings.jsonl"), JSON.stringify({ revision, tree: revision, commit: selected, - ref: "main", paths: null, landedAt: BEFORE }) + "\n"); + await writeFile(path.join(directory, "landings.jsonl"), JSON.stringify({ version: "spawnfile.volume-feed-landing.v1", resource: "source", volume: "source", source: { kind: "git", repo: repository }, revision, tree: revision, commit: selected, + ref: "main", landed_at: BEFORE }) + "\n"); await stampTree(retained, BEFORE, AT); }; const stage = async () => { const result: InputResult = await (kind === "feed" ? materializeFeed(context) : materializeGit(context)); const moment: MaterializedMoment = { version: "spawnfile.materialized-moment.v1", status: "materialized", agent: "agent:writer", - example: { id: "example", role: "test", dataset: "dataset" }, at: AT, record: { matched: false }, leakCheck: { ok: false, findings: 0 }, + example: { id: "example", role: "test", dataset: "dataset" }, at: AT, effectiveAt: AT, record: { matched: false }, leakCheck: { ok: false, findings: 0 }, inputs: [{ section: "resources", id: "source", declaredBy: "agent:writer", kind, fidelity: result.fidelity, basis: result.basis, source: result.source, path: relative, notes: [], refusal: null }] }; await writeFile(path.join(bundle, "moment.json"), JSON.stringify(moment)); diff --git a/src/compiler/training/rewind/leakProvenance.ts b/src/compiler/training/rewind/leakProvenance.ts index 8b7970b0..c38c8e3b 100644 --- a/src/compiler/training/rewind/leakProvenance.ts +++ b/src/compiler/training/rewind/leakProvenance.ts @@ -8,6 +8,7 @@ import { historyKey, openHistory } from "./history.js"; import { instantNanos } from "./instant.js"; import { diskPath } from "./leakFiles.js"; import { addInputFinding } from "./leakInputs.js"; +import { verifyRecordedBundle } from "./recordedBundle.js"; /** Read receipts and source objects afresh; never trust the producer's date guard or stamped files. */ export const inspectInputProvenance = async ( @@ -15,6 +16,10 @@ export const inspectInputProvenance = async ( ): Promise => { const add = (detail: string) => addInputFinding(report, input, "git-object", detail); try { + if (input.kind === "bundle" && input.basis === "record" && input.source.sha256) { + await verifyRecordedBundle(await openHistory(historyDir), input.source.sha256, await diskPath(root, input.path!)); + return; + } const commit = input.source.commit; if (input.kind === "feed") { if (input.fidelity === "overridden") return; diff --git a/src/compiler/training/rewind/materialize.test.ts b/src/compiler/training/rewind/materialize.test.ts index 5be116bb..1e5881e0 100644 --- a/src/compiler/training/rewind/materialize.test.ts +++ b/src/compiler/training/rewind/materialize.test.ts @@ -42,8 +42,8 @@ describe("materializeTrainingMoment", () => { await mkdir(fixture.options.out); const record = { ...emptyRecord("agent:a", "2026-10-09T10:20:31+02:00"), resources: [{ id: "code", kind: "git", commit: fixture.commit }], - memory: [{ id: "self", declaredBy: "agent:a", seq: 1, generation: 0 }], - volumes: [{ id: "edition-state", snapshot: "snapshot-1" }] }; + memory: [{ id: "self", declaredBy: "agent:a", seq: 1 }], + volumes: [{ id: "edition-state", snapshot: "20261009T070000000Z-execution-17" }] }; await jsonl(fixture.historyDir, "moments/anything.jsonl", [record]); const result = await materializeTrainingMoment(fixture.options); expect(result.status).toBe("materialized"); @@ -72,7 +72,7 @@ describe("materializeTrainingMoment", () => { .toBe(`Recorded object ${blob} must be a commit object; found blob`); expect(result.inputs.filter((input) => input.refusal !== null).map((input) => input.id)).toEqual(["code"]); expect(result.inputs.find((input) => input.id === "edition-state")) - .toMatchObject({ fidelity: "approximate", source: { snapshot: "snapshot-1" }, refusal: null }); + .toMatchObject({ fidelity: "approximate", source: { snapshot: "20261009T070000000Z-execution-17" }, refusal: null }); expect(snapshotBeforeDiscard).toBe("snapshot bytes\n"); expect(await readdir(fixture.options.out)).toEqual(["moment.json"]); }); @@ -114,7 +114,7 @@ describe("materializeTrainingMoment", () => { it("refuses contradictory matching records and malformed history lines without a usable payload", async () => { const fixture = await rewindFixture(); await jsonl(fixture.historyDir, "moments/conflict.jsonl", [emptyRecord(), { - ...emptyRecord(), memory: [{ id: "self", declaredBy: "agent:a", seq: 1, generation: 0 }] + ...emptyRecord(), memory: [{ id: "self", declaredBy: "agent:a", seq: 1 }] }]); const refused = await materializeTrainingMoment(fixture.options); expect(refused.inputs.filter((input) => input.fidelity !== "overridden") diff --git a/src/compiler/training/rewind/materialize.ts b/src/compiler/training/rewind/materialize.ts index 1df637ad..43a9ab04 100644 --- a/src/compiler/training/rewind/materialize.ts +++ b/src/compiler/training/rewind/materialize.ts @@ -9,7 +9,8 @@ import { resolveRewindInputs } from "./inputs.js"; import { inspectTrainingMoment, type LeakCheckOptions } from "./leakCheck.js"; import { materializeMemory } from "./memory.js"; import { materializeMoltnet } from "./moltnet.js"; -import { matchingMomentRecords, selectMomentRecord } from "./select.js"; +import { matchingMomentRecords, originalMomentRecords, selectMomentRecord } from "./select.js"; +import { hasRecordPin, recordForInput } from "./recordInput.js"; import { createTaskClock } from "./taskClock.js"; import { restoreAccessTimes, stampTree } from "./timestamps.js"; import type { InputContext, InputResult, RewindInput } from "./types.js"; @@ -28,9 +29,9 @@ export const resolveTrainingMoment = ( plan: TrainingPlan, project: LoadedTrainingProject, example: string, records: MomentRecord[] = [] ) => { const resolved = resolveRewindInputs(plan, project, example); - const candidateRecords = matchingMomentRecords(records, plan.agent.id, resolved.at); - return { ...resolved, candidateRecords, record: selectMomentRecord(candidateRecords, plan.agent.id, resolved.at, - resolved.inputs.filter((input) => !input.overridden)) }; + const candidateRecords = originalMomentRecords(matchingMomentRecords(records, plan.agent.id, resolved.at)); + const record = selectMomentRecord(candidateRecords, plan.agent.id, resolved.at, resolved.inputs.filter((input) => !input.overridden)); + return { ...resolved, candidateRecords, record, effectiveAt: record?.wake.startedAt ?? resolved.at }; }; const reason = (error: unknown): string => error instanceof Error ? error.message : String(error); @@ -80,12 +81,12 @@ export const finalizeTrainingMoment = async (staging: string, out: string, optio await discardPayload(staging, moment); } await writeFile(path.join(staging, "leak-check.json"), serialize(check)); - await stampTree(path.join(staging, "leak-check.json"), moment.at, moment.at); + await stampTree(path.join(staging, "leak-check.json"), moment.effectiveAt, moment.effectiveAt); } else await discardPayload(staging, moment); await writeFile(path.join(staging, "moment.json"), serialize(materializedMomentSchema.parse(moment))); - await stampTree(path.join(staging, "moment.json"), moment.at, moment.at); - await utimes(staging, new Date(moment.at), new Date(moment.at)); - await restoreAccessTimes(staging, moment.at); + await stampTree(path.join(staging, "moment.json"), moment.effectiveAt, moment.effectiveAt); + await utimes(staging, new Date(moment.effectiveAt), new Date(moment.effectiveAt)); + await restoreAccessTimes(staging, moment.effectiveAt); await assertEmptyOutput(out); await rename(staging, out); return moment; @@ -112,10 +113,15 @@ export const materializeTrainingMoment = async (options: MaterializeTrainingMome record = selectMomentRecord(candidateRecords, plan.agent.id, resolved.at, resolved.inputs.filter((input) => !input.overridden)); } catch (error) { recordError = reason(error); } + const attempts = candidateRecords.length; + candidateRecords = originalMomentRecords(candidateRecords); + const effectiveAt = record?.wake.startedAt ?? resolved.at; const moment: MaterializedMoment = { version: "spawnfile.materialized-moment.v1", status: "materialized", agent: plan.agent.id, example: { id: resolved.selected.id, role: resolved.selected.role, dataset: resolved.selected.dataset }, - at: resolved.at, inputs: [], record: { matched: record !== null }, leakCheck: { ok: false, findings: 0 } + at: resolved.at, effectiveAt, inputs: [], record: { matched: record !== null, ...(record ? { + attempts, selectedStartedAt: record.wake.startedAt, executionId: record.wake.executionId, selection: "earliest-startedAt" as const + } : {}) }, leakCheck: { ok: false, findings: 0 } }; for (const input of resolved.inputs) { const entry = reportInput(input); @@ -124,8 +130,14 @@ export const materializeTrainingMoment = async (options: MaterializeTrainingMome if (!input.overridden && (historyError || recordError)) throw new Error(historyError ?? recordError); const relative = `${input.section === "networks" ? "moltnet" : input.section}/${historyKey(input)}`; const destination = path.join(staging, relative); - const result = await dispatch({ input, at: resolved.at, record, candidateRecords, history, destination }); - await stampTree(destination, result.instant, resolved.at); + const usableRecord = recordForInput(record, input); + const result = await dispatch({ input, at: effectiveAt, agentSlug: resolved.agentSlug, agentId: plan.agent.id, + record: usableRecord, candidateRecords: candidateRecords.filter((candidate) => recordForInput(candidate, input)), history, destination }); + if (record && !record.exact && !input.overridden && !hasRecordPin(usableRecord, input)) { + result.fidelity = "approximate"; + result.notes.push(...(record.errors ?? [])); + } + await stampTree(destination, result.instant, effectiveAt); Object.assign(entry, { fidelity: result.fidelity, basis: result.basis, source: result.source, notes: result.notes, path: relative, ...(result.build ? { build: result.build } : {}) }); } catch (error) { @@ -134,7 +146,7 @@ export const materializeTrainingMoment = async (options: MaterializeTrainingMome } } if (recordError && resolved.inputs.length === 0) throw new Error(recordError); - await writeFile(path.join(staging, "task-clock.json"), serialize(createTaskClock(resolved.at))); + await writeFile(path.join(staging, "task-clock.json"), serialize(createTaskClock(effectiveAt))); await writeFile(path.join(staging, "moment.json"), serialize(moment)); // Set only unstamped wrapper directories; per-input source times remain intact. const wrappers = async (directory: string): Promise => { @@ -142,9 +154,9 @@ export const materializeTrainingMoment = async (options: MaterializeTrainingMome if (moment.inputs.some((input) => input.path === relative)) return; for (const entry of await readdir(directory, { withFileTypes: true })) { if (entry.isDirectory()) await wrappers(path.join(directory, entry.name)); - else if (directory === staging) await stampTree(path.join(directory, entry.name), resolved.at, resolved.at); + else if (directory === staging) await stampTree(path.join(directory, entry.name), effectiveAt, effectiveAt); } - await utimes(directory, new Date(resolved.at), new Date(resolved.at)); + await utimes(directory, new Date(effectiveAt), new Date(effectiveAt)); }; await wrappers(staging); return await finalizeTrainingMoment(staging, out, { historyDir: options.historyDir }); diff --git a/src/compiler/training/rewind/memory.test.ts b/src/compiler/training/rewind/memory.test.ts index 238f54fa..dffcf125 100644 --- a/src/compiler/training/rewind/memory.test.ts +++ b/src/compiler/training/rewind/memory.test.ts @@ -3,8 +3,8 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { openHistory } from "./history.js"; -import { materializeMemory, parseMemoryLedger, selectMemoryGeneration, selectMemoryPrefix } from "./memory.js"; -import { AT, BEFORE, FUTURE, emptyRecord, jsonl, memoryEvent } from "./fixtures.test-helper.js"; +import { materializeMemory, parseMemoryLedger, selectMemoryPrefix } from "./memory.js"; +import { AT, FUTURE, emptyRecord, jsonl, memoryEvent } from "./fixtures.test-helper.js"; import type { InputContext, RewindInput } from "./types.js"; const roots: string[] = []; @@ -45,9 +45,10 @@ describe("memory rewind", () => { it("materializes an exact record prefix without an index or a Mneme runtime dependency", async () => { const context = await setup(); await jsonl(context.history!.root, `${base}/events.jsonl`, [memoryEvent(1), memoryEvent(2, AT)]); - context.record = { ...emptyRecord(), memory: [{ id: "self", declaredBy: "agent:a", seq: 1, generation: 0 }] }; + context.record = { ...emptyRecord(), memory: [{ id: "self", declaredBy: "agent:a", seq: 1 }] }; const result = await materializeMemory(context); - expect(result).toMatchObject({ fidelity: "exact", basis: "record", source: { seq: 1, generation: 0 } }); + expect(result).toMatchObject({ fidelity: "exact", basis: "record", source: { seq: 1 } }); + expect(result.source).not.toHaveProperty("generation"); expect(await readdir(path.join(context.destination, "memory"))).toEqual(["events.jsonl"]); expect(await readFile(path.join(context.destination, "memory/events.jsonl"), "utf8")).toBe(`${JSON.stringify(memoryEvent(1))}\n`); expect(result.notes.join(" ")).toContain("without a Mneme runtime dependency"); @@ -58,18 +59,21 @@ describe("memory rewind", () => { await jsonl(context.history!.root, `${base}/events.jsonl`, [memoryEvent(1, AT), memoryEvent(2, FUTURE)]); expect(await materializeMemory(context)).toMatchObject({ fidelity: "approximate", basis: "created-at-prefix", source: { seq: 1 } }); }); - it("selects an archived generation before a switch and refuses a missing archive", async () => { + it.each([{ generation: 0, seq: 0 }, { generation: 0, seq: 1 }, { generation: 1, seq: 0 }, { generation: 1, seq: 1 }])( + "refuses unsupported recorded memory generation $generation at seq $seq", async ({ generation, seq }) => { + const context = await setup(); + await jsonl(context.history!.root, `${base}/events.jsonl`, [memoryEvent(1)]); + context.record = { ...emptyRecord(), memory: [{ id: "self", declaredBy: "agent:a", seq, generation }] }; + await expect(materializeMemory(context)).rejects.toThrow(/Mneme generations are unsupported/); + }); + it.each(["missing", "malformed", "no history"])("uses seq zero as an exact empty bank with %s ledger", async (state) => { const context = await setup(); - await jsonl(context.history!.root, `${base}/generations.jsonl`, [ - { generation: 3, switchedAt: BEFORE, ledger: "archive/g3.jsonl" }, - { generation: 4, switchedAt: FUTURE, ledger: "events.jsonl" } - ]); - await jsonl(context.history!.root, `${base}/events.jsonl`, [memoryEvent(1, FUTURE)]); - await expect(materializeMemory(context)).rejects.toThrow(/Missing memory ledger.*g3/); - await jsonl(context.history!.root, `${base}/archive/g3.jsonl`, [memoryEvent(1)]); - expect(await materializeMemory(context)).toMatchObject({ source: { generation: 3, seq: 1 } }); - expect(() => selectMemoryGeneration([{ generation: 4, switchedAt: FUTURE, ledger: "events.jsonl" }], AT)).toThrow(/archived ledger/); - expect(() => selectMemoryGeneration(null, AT, 3)).toThrow(/disagrees/); + if (state === "malformed") await jsonl(context.history!.root, `${base}/events.jsonl`, [{ bad: true }]); + if (state === "no history") context.history = undefined; + context.record = { ...emptyRecord(), memory: [{ id: "self", declaredBy: "agent:a", seq: 0 }] }; + expect(await materializeMemory(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { seq: 0 } }); + expect(await readdir(path.join(context.destination, "memory"))).toEqual(["events.jsonl"]); + expect(await readFile(path.join(context.destination, "memory/events.jsonl"), "utf8")).toBe(""); }); it("creates an empty override without history and refuses a missing non-overridden bank", async () => { const context = await setup(); diff --git a/src/compiler/training/rewind/memory.ts b/src/compiler/training/rewind/memory.ts index 7e67c882..72e33ee0 100644 --- a/src/compiler/training/rewind/memory.ts +++ b/src/compiler/training/rewind/memory.ts @@ -2,7 +2,7 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { historyKey } from "./history.js"; import { compareInstants } from "./instant.js"; -import { parseMemoryLedgerEvent, memoryGenerationSchema, type MemoryGeneration, type MemoryLedgerEvent } from "./memorySchemas.js"; +import { parseMemoryLedgerEvent, type MemoryLedgerEvent } from "./memorySchemas.js"; import type { InputContext, InputResult } from "./types.js"; export interface MemoryLedger { lines: string[]; events: MemoryLedgerEvent[] } @@ -41,36 +41,6 @@ export const selectMemoryPrefix = (ledger: MemoryLedger, at: string, seq?: numbe return { events, lines: ledger.lines.slice(0, length) }; }; -export const selectMemoryGeneration = ( - generations: MemoryGeneration[] | null, at: string, recordGeneration?: number -): { generation: number; ledger: string; switchedAt: string | null } => { - // With no curation log, only the original generation can be established. - let selected: { generation: number; ledger: string; switchedAt: string | null } = { - generation: 0, ledger: "events.jsonl", switchedAt: null - }; - if (generations !== null) { - const known = new Map(); - for (const generation of generations) { - const prior = known.get(generation.generation); - if (prior && JSON.stringify(prior) !== JSON.stringify(generation)) { - throw new Error(`Conflicting memory generation ${generation.generation}`); - } - known.set(generation.generation, generation); - } - const eligible = [...known.values()].filter((entry) => compareInstants(entry.switchedAt, at) <= 0) - .sort((a, b) => compareInstants(b.switchedAt, a.switchedAt)); - if (!eligible.length) throw new Error("No memory generation switched at or before the moment; an archived ledger is required"); - if (eligible[1] && compareInstants(eligible[0]!.switchedAt, eligible[1].switchedAt) === 0) { - throw new Error("Ambiguous memory generations switched at the same instant"); - } - selected = eligible[0]!; - } - if (recordGeneration !== undefined && recordGeneration !== selected.generation) { - throw new Error(`Memory record generation ${recordGeneration} disagrees with selected generation ${selected.generation}`); - } - return selected; -}; - export const materializeMemory = async (context: InputContext): Promise => { const { input, at, record, history, destination } = context; if (input.section !== "memory") throw new Error("Memory materialization requires a memory input"); @@ -78,25 +48,29 @@ export const materializeMemory = async (context: InputContext): Promise entry.id === input.id && entry.declaredBy === input.declaredBy); - const generation = selectMemoryGeneration(await history.jsonl(`${base}/generations.jsonl`, memoryGenerationSchema), at, pin?.generation); - const ledgerPath = `${base}/${generation.ledger}`; + if (pin?.generation !== undefined) throw new Error(`Memory bank ${input.id}: Mneme generations are unsupported`); + if (pin?.seq === 0) { + await mkdir(directory, { recursive: true }); + await writeFile(path.join(directory, "events.jsonl"), ""); + return { fidelity: "exact", basis: "record", source: { seq: 0 }, + notes: ["The record pins an empty bank (seq: 0); no history ledger is required."], instant: at }; + } + if (!history) throw new Error(`Memory bank ${input.id} requires --history or an override`); + const ledgerPath = `memory/${historyKey(input)}/events.jsonl`; const payload = await history.text(ledgerPath); - if (payload === null) throw new Error(`Missing memory ledger ${ledgerPath}; an archived ledger is required for this generation`); + if (payload === null) throw new Error(`Missing memory ledger ${ledgerPath}`); const prefix = selectMemoryPrefix(parseMemoryLedger(payload, ledgerPath), at, pin?.seq); await mkdir(directory, { recursive: true }); await writeFile(path.join(directory, "events.jsonl"), prefix.lines.length ? `${prefix.lines.join("\n")}\n` : ""); return { fidelity: pin ? "exact" : "approximate", basis: pin ? "record" : "created-at-prefix", - source: { seq: prefix.events.length, generation: generation.generation }, + source: { seq: prefix.events.length }, notes: ["Spawnfile structurally validated the selected ledger without a Mneme runtime dependency; no derived or authority audit state is included.", "Mneme 0.1.1 rebuilds its index from the ledger before every index read.", - "Checksum verification is skipped: Mneme 0.1.1 hashes the original input before normalization, which cannot be reconstructed exactly from stored events.", - ...(generation.switchedAt === null ? ["No generation history: the current ledger is treated as original generation 0."] : [])], - instant: prefix.events.at(-1)?.createdAt ?? generation.switchedAt ?? at + "Checksum verification is skipped: Mneme 0.1.1 hashes the original input before normalization, which cannot be reconstructed exactly from stored events."], + instant: prefix.events.at(-1)?.createdAt ?? at }; }; diff --git a/src/compiler/training/rewind/memorySchemas.ts b/src/compiler/training/rewind/memorySchemas.ts index bead69eb..9e437bcc 100644 --- a/src/compiler/training/rewind/memorySchemas.ts +++ b/src/compiler/training/rewind/memorySchemas.ts @@ -2,14 +2,6 @@ import { z } from "zod"; import { trainingInstantSchema } from "../../../manifest/index.js"; const safeSequence = z.number().int().nonnegative().refine(Number.isSafeInteger); -const relativeLedger = z.string().min(1).refine((value) => !value.includes("\\") - && !value.includes("\0") && value.split("/").every((part) => part !== "" && part !== "." && part !== ".."), -"ledger must be a relative path inside its bank directory"); -export const memoryGenerationSchema = z.object({ - generation: safeSequence, switchedAt: trainingInstantSchema, ledger: relativeLedger -}).strict(); -export type MemoryGeneration = z.infer; - /** Mirrors the public MemoryEvent contract and Mneme 0.1.1's strict ledger reader. */ export const memoryEventSchema = z.object({ id: z.string().min(1), diff --git a/src/compiler/training/rewind/moltnet.ts b/src/compiler/training/rewind/moltnet.ts index bc0da50e..5679ac31 100644 --- a/src/compiler/training/rewind/moltnet.ts +++ b/src/compiler/training/rewind/moltnet.ts @@ -1,7 +1,8 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { historyKey } from "./history.js"; -import { emptyMoltnetSnapshot, moltnetCursorExact, moltnetMessages, parseMoltnetSnapshot, rewindMoltnetJson } from "./moltnetJson.js"; +import { compareInstants } from "./instant.js"; +import { emptyMoltnetSnapshot, moltnetMessages, parseMoltnetSnapshot, rewindMoltnetJson } from "./moltnetJson.js"; import { rewindMoltnetSqlite } from "./moltnetSqlite.js"; import type { InputContext, InputResult } from "./types.js"; @@ -13,6 +14,10 @@ export const materializeMoltnet = async (context: InputContext): Promise entry.network === input.id); + if (pin && compareInstants(pin.cursor, at) !== 0) { + throw new Error(`Moltnet network ${input.id}: recorded cursor must equal wake.startedAt (${at})`); + } if (!history) throw new Error(`Moltnet network ${input.id} requires --history or an override`); const base = `moltnet/${historyKey(input)}`; const json = await history.bytes(`${base}/state.json`), sqlite = await history.bytes(`${base}/moltnet.sqlite`); @@ -34,11 +39,11 @@ export const materializeMoltnet = async (context: InputContext): Promise entry.network === input.id)?.cursor; - const exact = moltnetCursorExact(messages, cursor); - return { fidelity: exact ? "exact" : "approximate", basis: exact ? "record-cursor" : "created-at-cut", - source: cursor === undefined ? {} : { cursor }, instant: at, + const retainedIds = new Set(messages.map((message) => message.id)); + const missing = pin?.messageIds?.filter((id) => !retainedIds.has(id)) ?? []; + if (missing.length) throw new Error(`Moltnet network ${input.id}: recorded wake message IDs ${missing.join(", ")} are not retained after the cursor cut`); + return { fidelity: pin ? "exact" : "approximate", basis: pin ? "record-cursor" : "created-at-cut", + source: pin ? { cursor: pin.cursor } : {}, instant: at, notes: ["Agent registrations and memberships have no historical log and are kept as now (best effort).", - "Room removals without a recorded removal instant retain the current removal state; current JSON Moltnet uses a zero-time marker.", - ...(cursor !== undefined && !exact ? ["The recorded cursor is absent after the cut or is earlier than a retained message; exact cursor fidelity cannot be established."] : [])] }; + "Room removals without a recorded removal instant retain the current removal state; current JSON Moltnet uses a zero-time marker."] }; }; diff --git a/src/compiler/training/rewind/moltnetJson.test.ts b/src/compiler/training/rewind/moltnetJson.test.ts index a37004d7..5874df73 100644 --- a/src/compiler/training/rewind/moltnetJson.test.ts +++ b/src/compiler/training/rewind/moltnetJson.test.ts @@ -5,7 +5,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { materializeMoltnet } from "./moltnet.js"; import { momentRecordSchema } from "./contract.js"; import { openHistory } from "./history.js"; -import { emptyMoltnetSnapshot, inspectMoltnetJson, moltnetCursorExact, moltnetMessages, parseMoltnetSnapshot, rewindMoltnetJson } from "./moltnetJson.js"; +import { emptyMoltnetSnapshot, inspectMoltnetJson, moltnetMessages, parseMoltnetSnapshot, rewindMoltnetJson } from "./moltnetJson.js"; import type { MoltnetMessage } from "./moltnetSchemas.js"; import type { InputContext } from "./types.js"; @@ -44,9 +44,6 @@ describe("Moltnet JSON rewind", () => { expect(cut.attachment_delivery_cursors).toEqual({ writer: 2, brass: 1 }); expect(moltnetMessages(cut).map((entry) => entry.id)).toEqual(["old", "boundary", "kept-old"]); expect(source.threads.kept?.message_count).toBe(99); - expect(moltnetCursorExact(moltnetMessages(cut), "boundary")).toBe(true); - expect(moltnetCursorExact(moltnetMessages(cut), "old")).toBe(false); - expect(moltnetCursorExact(moltnetMessages(cut), "missing")).toBe(false); }); it("restores known later removals, keeps zero-time unknown removals and refuses missing restoration metadata", () => { @@ -63,19 +60,34 @@ describe("Moltnet JSON rewind", () => { it("materializes exact cursor and approximate cuts, keeps history unchanged and makes fresh overridden rooms", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "rewind-moltnet-")); temps.push(root); const bank = path.join(root, "moltnet/team%3Aorg/news"); await mkdir(bank, { recursive: true }); - const source = emptyMoltnetSnapshot(network, before); source.room_messages.assignment = [message("cursor", at), message("future", future)]; + const source = emptyMoltnetSnapshot(network, before); + source.room_messages.assignment = [message("wake", before), message("boundary", at), message("future", future)]; const bytes = JSON.stringify(source); await writeFile(path.join(bank, "state.json"), bytes); const context: InputContext = { at, record: null, history: await openHistory(root), destination: path.join(root, "out"), input: { section: "networks", id: "news", declaredBy: "team:org", overridden: false, declarationBase: root, declaration: network } }; expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "approximate", basis: "created-at-cut" }); expect(await readFile(path.join(bank, "state.json"), "utf8")).toBe(bytes); - expect(await inspectMoltnetJson(path.join(context.destination, "state.json"), at)).toMatchObject({ messages: 1, findings: [] }); + expect(await inspectMoltnetJson(path.join(context.destination, "state.json"), at)).toMatchObject({ messages: 2, findings: [] }); context.record = momentRecordSchema.parse({ version: "spawnfile.moment-record.v1", agent: "agent:writer", - wake: { at, surface: "moltnet", room: "assignment", from: "brass" }, recordedAt: at, - resources: [], feeds: [], memory: [], moltnet: [{ network: "news", cursor: "cursor" }], volumes: [] }); + wake: { at, startedAt: at, executionId: "turn-1", deliveries: [{ deliveryId: "moltnet:wake", kind: "message", occurredAt: at }], + surface: "moltnet", room: "assignment", from: "brass" }, recordedAt: at, exact: true, + resources: [], feeds: [], memory: [], moltnet: [{ network: "news", cursor: at, messageIds: ["wake", "boundary"] }], volumes: [] }); + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "exact", basis: "record-cursor" }); + expect(moltnetMessages(parseMoltnetSnapshot(JSON.parse(await readFile(path.join(context.destination, "state.json"), "utf8")))) + .map((entry) => entry.id)).toEqual(["wake", "boundary"]); + context.record.moltnet[0]!.cursor = "2026-10-09T12:00:00.100000000+02:00"; + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "exact", basis: "record-cursor" }); + for (const cursor of [before, future]) { + context.record.moltnet[0]!.cursor = cursor; + await expect(materializeMoltnet(context)).rejects.toThrow(/cursor must equal.*startedAt/); + } + context.record.moltnet[0]!.cursor = at; + for (const messageIds of [["missing"], ["wake", "future"]]) { + context.record.moltnet[0]!.messageIds = messageIds; + await expect(materializeMoltnet(context)).rejects.toThrow(/wake message.*not retained/); + } + delete context.record.moltnet[0]!.messageIds; expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "exact", basis: "record-cursor" }); - context.record.moltnet[0]!.cursor = "missing"; - expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "approximate" }); context.input.overridden = true; context.history = undefined; expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "overridden" }); const output = parseMoltnetSnapshot(JSON.parse(await readFile(path.join(context.destination, "state.json"), "utf8"))); diff --git a/src/compiler/training/rewind/moltnetJson.ts b/src/compiler/training/rewind/moltnetJson.ts index e2309e1d..b7fba7d3 100644 --- a/src/compiler/training/rewind/moltnetJson.ts +++ b/src/compiler/training/rewind/moltnetJson.ts @@ -113,12 +113,6 @@ export const rewindMoltnetJson = (source: MoltnetSnapshot, at: string): MoltnetS return snapshot; }; -/** UUID cursors have no ordering: exactness is established only from the retained message instant. */ -export const moltnetCursorExact = (messages: Array<{ id: string; created_at: string }>, cursor?: string): boolean => { - const message = cursor === undefined ? undefined : messages.find((entry) => entry.id === cursor); - return Boolean(message && messages.every((entry) => compareInstants(entry.created_at, message.created_at) <= 0)); -}; - /** Independent timestamp inspection of event payloads, including embedded messages and rooms. */ export const inspectMoltnetEvent = (event: MoltnetEvent, at: string, path: string): MoltnetFinding[] => { const findings: MoltnetFinding[] = []; diff --git a/src/compiler/training/rewind/moltnetSqlite.test.ts b/src/compiler/training/rewind/moltnetSqlite.test.ts index b57c70dd..2303d9dd 100644 --- a/src/compiler/training/rewind/moltnetSqlite.test.ts +++ b/src/compiler/training/rewind/moltnetSqlite.test.ts @@ -101,10 +101,22 @@ describe("Moltnet SQLite rewind", () => { expect(await readdir(context.destination)).toEqual(["moltnet.sqlite"]); expect(await inspectMoltnetSqlite(output, at)).toMatchObject({ messages: 4, events: 2, findings: [] }); context.record = momentRecordSchema.parse({ version: "spawnfile.moment-record.v1", agent: "agent:writer", - wake: { at, surface: "moltnet", room: "assignment", from: "brass" }, recordedAt: at, - resources: [], feeds: [], memory: [], moltnet: [{ network: "news", cursor: "boundary" }], volumes: [] }); + wake: { at, startedAt: at, executionId: "turn-1", deliveries: [{ deliveryId: "moltnet:old", kind: "message", occurredAt: at }], + surface: "moltnet", room: "assignment", from: "brass" }, recordedAt: at, exact: true, + resources: [], feeds: [], memory: [], moltnet: [{ network: "news", cursor: at, messageIds: ["old", "boundary"] }], volumes: [] }); context.destination += "-exact"; expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "exact", basis: "record-cursor" }); + context.record.moltnet[0]!.messageIds = ["old", "future-root"]; + await expect(materializeMoltnet(context)).rejects.toThrow(/wake message.*future-root.*not retained/); + context.record.moltnet[0]!.messageIds = ["missing"]; + await expect(materializeMoltnet(context)).rejects.toThrow(/wake message.*missing.*not retained/); + delete context.record.moltnet[0]!.messageIds; + context.record.moltnet[0]!.cursor = "2026-10-09T12:00:00.100000000+02:00"; + expect(await materializeMoltnet(context)).toMatchObject({ fidelity: "exact", basis: "record-cursor" }); + for (const cursor of [before, future]) { + context.record.moltnet[0]!.cursor = cursor; + await expect(materializeMoltnet(context)).rejects.toThrow(/cursor must equal.*startedAt/); + } }); it("copies WAL state and independently detects message, room and attachment-event mutations without changing the file", async () => { diff --git a/src/compiler/training/rewind/recordInput.test.ts b/src/compiler/training/rewind/recordInput.test.ts new file mode 100644 index 00000000..b58fbfb9 --- /dev/null +++ b/src/compiler/training/rewind/recordInput.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { emptyRecord } from "./fixtures.test-helper.js"; +import { recordInputErrors } from "./recordInput.js"; +import type { InputIdentity } from "./types.js"; + +const memory: InputIdentity = { section: "memory", id: "self", declaredBy: "agent:a" }; +const otherBank: InputIdentity = { ...memory, declaredBy: "team:publication" }; +const feed: InputIdentity = { section: "resources", id: "news", declaredBy: "team:publication" }; +const network: InputIdentity = { ...feed, section: "networks" }; + +describe("record input error attribution", () => { + it.each([ + ["memory agent:a/self: capture failed", memory, otherBank], + ["memory self (agent:a): capture failed", memory, otherBank], + ["memory self declaredBy=agent:a: capture failed", memory, otherBank], + ["memory/agent%3Aa/self: capture failed", memory, otherBank], + ["feed news: capture failed", feed, network], + ["moltnet news: capture failed", network, feed] + ] as const)("honors qualified input identity in %s", (error, affected, healthy) => { + const record = emptyRecord(); record.exact = false; record.errors = [error]; + expect(recordInputErrors(record, affected)).toEqual([error]); + expect(recordInputErrors(record, healthy)).toEqual([]); + }); + + it("keeps unqualified errors conservative while respecting complete ID tokens", () => { + const record = emptyRecord(); record.errors = ["self: capture failed", "other-self: failed", "myself: failed"]; + expect(recordInputErrors(record, memory)).toEqual([record.errors[0]]); + expect(recordInputErrors(record, otherBank)).toEqual([record.errors[0]]); + }); +}); diff --git a/src/compiler/training/rewind/recordInput.ts b/src/compiler/training/rewind/recordInput.ts new file mode 100644 index 00000000..090cc380 --- /dev/null +++ b/src/compiler/training/rewind/recordInput.ts @@ -0,0 +1,43 @@ +import type { MomentRecord } from "./contract.js"; +import type { InputIdentity, RewindInput } from "./types.js"; + +const sections: Record = { + resource: "resources", resources: "resources", git: "resources", bundle: "resources", bundles: "resources", + feed: "resources", feeds: "resources", volume: "resources", volumes: "resources", + memory: "memory", bank: "memory", banks: "memory", moltnet: "networks", network: "networks", networks: "networks" +}; +const ownerPattern = "(?:agent|team|org|organization):[A-Za-z0-9_.:/-]+?"; +const qualifiedPrefix = new RegExp(`(?:^|[^A-Za-z0-9_.-])(${Object.keys(sections).join("|")})` + + `(?:[\\s/]+(${ownerPattern}))?[\\s/:]+$`, "iu"); +const ownerPrefix = new RegExp(`(?:^|[^A-Za-z0-9_.-])(${ownerPattern})[\\s/]+$`, "u"); +const ownerSuffix = new RegExp(`^\\s*(?:\\(\\s*|declaredBy\\s*[=:]\\s*|@\\s*)(${ownerPattern})(?=[\\s):,;]|$)`, "u"); + +/** Errors are free text in P3: honor explicit qualifications; otherwise conservatively match the whole ID. */ +export const recordInputErrors = (record: MomentRecord, input: InputIdentity): string[] => { + const escaped = input.id.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const namesInput = new RegExp(`(? { + let decoded = error; + try { decoded = decodeURIComponent(error); } catch { /* A literal percent is valid free text. */ } + return [...decoded.matchAll(namesInput)].some((match) => { + const prefix = decoded.slice(0, match.index); + const qualification = qualifiedPrefix.exec(prefix); + if (qualification && sections[qualification[1]!.toLowerCase()] !== input.section) return false; + const owner = qualification?.[2] ?? ownerPrefix.exec(prefix)?.[1] + ?? ownerSuffix.exec(decoded.slice(match.index + match[0].length))?.[1]; + return !owner || owner === input.declaredBy; + }); + }); +}; + +/** Do not discard healthy pins just because another input made the whole record inexact. */ +export const recordForInput = (record: MomentRecord | null, input: RewindInput): MomentRecord | null => + record && !recordInputErrors(record, input).length ? record : null; + +export const hasRecordPin = (record: MomentRecord | null, input: RewindInput): boolean => { + if (!record) return false; + if (input.section === "memory") return record.memory.some((entry) => entry.id === input.id && entry.declaredBy === input.declaredBy); + if (input.section === "networks") return record.moltnet.some((entry) => entry.network === input.id); + const pins = input.declaration.kind !== "volume" ? record.resources : input.declaration.feed ? record.feeds : record.volumes; + return pins.some((entry) => entry.id === input.id); +}; diff --git a/src/compiler/training/rewind/recordedBundle.test.ts b/src/compiler/training/rewind/recordedBundle.test.ts new file mode 100644 index 00000000..059fc4f3 --- /dev/null +++ b/src/compiler/training/rewind/recordedBundle.test.ts @@ -0,0 +1,69 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { BundleTarWriter } from "../../workspaceBundleTar.js"; +import { materializeGit } from "./git.js"; +import { commitTestGit, gitTestRoot, initTestGit, testMomentRecord } from "./git.test-helper.js"; +import { History } from "./history.js"; +import { verifyRecordedBundle } from "./recordedBundle.js"; +import type { InputContext } from "./types.js"; + +const AT = "2026-10-09T08:20:31Z", EARLY = "2026-10-09T07:00:00Z"; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const fixture = async () => { + const root = await gitTestRoot(); roots.push(root); + const historyRoot = path.join(root, "history"), archive = path.join(root, "bundle.tar"); + await mkdir(path.join(historyRoot, "bundles"), { recursive: true }); + const writer = await BundleTarWriter.create(archive); + await writer.begin("tools/run", 6, 0o755); await writer.data(Buffer.from("saved\n")); await writer.end(); await writer.finish(); + const bytes = await readFile(archive), sha256 = `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + const context: InputContext = { at: AT, destination: path.join(root, "out"), history: new History(historyRoot), + record: { ...testMomentRecord(), resources: [{ id: "tools", kind: "bundle", sha256 }] }, + input: { section: "resources", id: "tools", declaredBy: "agent:writer", declarationBase: root, overridden: false, + declaration: { id: "tools", kind: "bundle", mode: "readonly", mount: "./tools", source: "never-read-current.tar" } } }; + return { root, historyRoot, context, bytes, sha256 }; +}; + +describe("recorded bundle archive", () => { + it("is exact only from the matching retained tar without a repository", async () => { + const f = await fixture(); + await writeFile(path.join(f.historyRoot, `bundles/${f.sha256.slice(7)}.tar`), f.bytes); + expect(await materializeGit(f.context)).toMatchObject({ fidelity: "exact", basis: "record", source: { sha256: f.sha256 } }); + expect(await readFile(path.join(f.context.destination, "tools/run"), "utf8")).toBe("saved\n"); + await verifyRecordedBundle(f.context.history, f.sha256, f.context.destination); + await writeFile(path.join(f.context.destination, "tools/run"), "other\n"); + await expect(verifyRecordedBundle(f.context.history, f.sha256, f.context.destination)).rejects.toThrow("Recorded bundle entry mismatch"); + }); + + it.each(["missing", "mismatch"])("falls back approximately when a recorded digest is %s even with a pinned source commit", async (kind) => { + const f = await fixture(), repository = path.join(f.historyRoot, "git/agent%3Awriter/tools"); + await initTestGit(repository); + const commit = await commitTestGit(repository, { "source.txt": "historical source\n" }, EARLY); + f.context.input.declaration = { id: "tools", kind: "bundle", mode: "readonly", mount: "./tools", build: { files: { root: repository, ref: commit } } }; + if (kind === "mismatch") await writeFile(path.join(f.historyRoot, `bundles/${f.sha256.slice(7)}.tar`), Buffer.concat([f.bytes, Buffer.alloc(512)])); + const result = await materializeGit(f.context); + expect(result).toMatchObject({ fidelity: "approximate", source: { commit }, notes: expect.arrayContaining([expect.stringContaining("could not be reproduced")]) }); + expect(await readFile(path.join(f.context.destination, "source.txt"), "utf8")).toBe("historical source\n"); + }); + + it("refuses a prebuilt resource when its recorded archive is unavailable", async () => { + const f = await fixture(); + await expect(materializeGit(f.context)).rejects.toThrow("could not be reproduced"); + }); + + it.each(["escape", "symlink", "hardlink", "device"])("refuses a digest-matching tar with a %s entry before writing output", async (kind) => { + const f = await fixture(), bytes = Buffer.from(f.bytes); + if (kind === "escape") { bytes.fill(0, 0, 100); bytes.write("../../escape", 0); } + else bytes[156] = { symlink: 50, hardlink: 49, device: 51 }[kind]!; + bytes.fill(32, 148, 156); + const sum = bytes.subarray(0, 512).reduce((total, byte) => total + byte, 0); + bytes.write(`${sum.toString(8).padStart(6, "0")}\0 `, 148, "ascii"); + const sha256 = `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + f.context.record!.resources = [{ id: "tools", kind: "bundle", sha256 }]; + await writeFile(path.join(f.historyRoot, `bundles/${sha256.slice(7)}.tar`), bytes); + await expect(materializeGit(f.context)).rejects.toThrow(/unsafe tar entry/); + await expect(readdir(f.context.destination)).rejects.toThrow(); + }); +}); diff --git a/src/compiler/training/rewind/recordedBundle.ts b/src/compiler/training/rewind/recordedBundle.ts new file mode 100644 index 00000000..7d523f86 --- /dev/null +++ b/src/compiler/training/rewind/recordedBundle.ts @@ -0,0 +1,54 @@ +import { createHash } from "node:crypto"; +import { lstat, readFile, readdir } from "node:fs/promises"; +import path from "node:path"; +import { validateWorkspaceBundleTar } from "../../workspaceBundleArtifacts.js"; +import { recordedSha256Schema } from "./contract.js"; +import { extractBundleBytes } from "./gitTree.js"; +import type { History } from "./history.js"; + +const recordedBytes = async (history: History | undefined, sha256: string): Promise => { + recordedSha256Schema.parse(sha256); + const bytes = await history?.bytes(`bundles/${sha256.slice(7)}.tar`); + if (!bytes || bytes.length > 536_870_912 || `sha256:${createHash("sha256").update(bytes).digest("hex")}` !== sha256) return null; + return bytes; +}; + +export const materializeRecordedBundle = async (history: History | undefined, sha256: string, destination: string): Promise => { + const bytes = await recordedBytes(history, sha256); + if (!bytes) return false; + await extractBundleBytes(bytes, destination); + return true; +}; + +/** Acceptance reopens the digest-pinned archive and compares disk bytes, without trusting producer metadata. */ +export const verifyRecordedBundle = async (history: History | undefined, sha256: string, directory: string): Promise => { + const bytes = await recordedBytes(history, sha256); + if (!bytes) throw new Error(`Recorded bundle ${sha256} could not be reproduced`); + validateWorkspaceBundleTar(bytes); + const text = (field: Buffer) => field.toString("utf8").replace(/\0.*$/su, ""); + const files = new Map(), directories = new Set(); + for (let offset = 0; offset < bytes.length && !bytes.subarray(offset, offset + 512).every((byte) => byte === 0);) { + const header = bytes.subarray(offset, offset + 512), prefix = text(header.subarray(345, 500)); + const name = `${prefix ? `${prefix}/` : ""}${text(header.subarray(0, 100))}`.replace(/\/$/u, ""); + const size = Number.parseInt(text(header.subarray(124, 136)).trim(), 8); + for (let parent = path.posix.dirname(name); parent !== "."; parent = path.posix.dirname(parent)) directories.add(parent); + if (header[156] === 53) directories.add(name); + else files.set(name, { bytes: bytes.subarray(offset + 512, offset + 512 + size), executable: Boolean(Number.parseInt(text(header.subarray(100, 108)).trim(), 8) & 0o111) }); + offset += 512 + Math.ceil(size / 512) * 512; + } + const walk = async (folder: string, prefix = ""): Promise => { + for (const name of await readdir(folder)) { + const relative = prefix ? `${prefix}/${name}` : name, full = path.join(folder, name), stats = await lstat(full); + if (stats.isDirectory() && directories.delete(relative)) await walk(full, relative); + else { + const expected = files.get(relative); + if (!stats.isFile() || !expected || Boolean(stats.mode & 0o111) !== expected.executable || !(await readFile(full)).equals(expected.bytes)) { + throw new Error(`Recorded bundle entry mismatch: ${relative}`); + } + files.delete(relative); + } + } + }; + await walk(directory); + if (files.size || directories.size) throw new Error(`Missing recorded bundle entries: ${[...files.keys(), ...directories].join(", ")}`); +}; diff --git a/src/compiler/training/rewind/recordedCommit.test.ts b/src/compiler/training/rewind/recordedCommit.test.ts index 9e42180e..4cbcb17b 100644 --- a/src/compiler/training/rewind/recordedCommit.test.ts +++ b/src/compiler/training/rewind/recordedCommit.test.ts @@ -12,16 +12,17 @@ import type { InputContext } from "./types.js"; const AT = "2026-10-09T08:20:31Z", EARLY = "2026-10-09T08:00:00Z", REVISION = "e".repeat(64); const roots: string[] = []; afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); -const landing = (commit: string | null) => ({ revision: REVISION, tree: REVISION, commit, ref: "main", paths: [], landedAt: EARLY }); +const landing = (commit: string) => ({ version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "org-news", source: { kind: "git", repo: "recorded-repo" }, revision: REVISION, tree: REVISION, commit, ref: "main", paths: [], landed_at: EARLY }); const feedRecord = (commit: string) => ({ id: "news", revision: REVISION, tree: REVISION, commit, paths: [], landedAt: EARLY }); -const recordedValue = (field: "resources" | "feeds" | "landing", commit: string): string | null => { +const recordedValue = (field: "resources" | "feeds" | "landing", commit: string): string | undefined => { if (field === "landing") return feedLandingSchema.parse(landing(commit)).commit; const value = momentRecordSchema.parse({ ...testMomentRecord(), [field]: field === "resources" ? [{ id: "archive", kind: "git", commit }] : [{ id: "news", revision: REVISION, tree: REVISION, commit, paths: [], landedAt: EARLY }] }); - return value[field][0]!.commit; + const entry = value[field][0]!; + return "commit" in entry ? entry.commit : undefined; }; -const fixture = async (kind: "git" | "bundle" = "git") => { +const fixture = async () => { const root = await gitTestRoot(); roots.push(root); const repository = path.join(root, "history/git/agent%3Awriter/archive"); await initTestGit(repository); @@ -30,10 +31,9 @@ const fixture = async (kind: "git" | "bundle" = "git") => { const objects = { blob: testGit(repository, ["rev-parse", `${past}:readme.txt`]), tree: testGit(repository, ["rev-parse", `${past}^{tree}`]), tag: testGit(repository, ["rev-parse", "refs/tags/record-tag"]) }; const context: InputContext = { at: AT, destination: path.join(root, "output"), history: new History(path.join(root, "history")), - record: { ...testMomentRecord(), resources: [{ id: "archive", kind, commit: past }] }, + record: { ...testMomentRecord(), resources: [{ id: "archive", kind: "git", commit: past }] }, input: { section: "resources", declaredBy: "agent:writer", id: "archive", declarationBase: root, overridden: false, - declaration: kind === "git" ? { id: "archive", kind, mode: "readonly", mount: "./archive", url: repository } - : { id: "archive", kind, mode: "readonly", mount: "./archive", source: "never-open-todays.tar" } } }; + declaration: { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: repository } } }; return { root, repository, past, objects, context }; }; const retainedFeed = async () => { @@ -62,30 +62,32 @@ describe("recorded commit contracts", () => { } }); - it("retains null commit support for directory landing logs", () => { - expect(feedLandingSchema.parse(landing(null)).commit).toBeNull(); + it("directory landing logs omit git commit fields", () => { + const { commit: _commit, ref: _ref, paths: _paths, ...entry } = landing("a".repeat(40)); + expect(feedLandingSchema.parse({ ...entry, source: { kind: "directory", directory: "/source" } }).commit).toBeUndefined(); }); }); -describe.each(["git", "bundle"] as const)("recorded %s commit objects", (kind) => { +describe("recorded git commit objects", () => { + const kind = "git" as const; it("materializes the full recorded ID case-insensitively", async () => { - const { past, context } = await fixture(kind); - context.record!.resources[0]!.commit = past.toUpperCase(); + const { past, context } = await fixture(); + context.record!.resources[0] = { id: "archive", kind: "git", commit: past.toUpperCase() }; expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: past } }); expect(await readFile(path.join(context.destination, "readme.txt"), "utf8")).toBe("recorded bytes\n"); }); it.each(["blob", "tree", "tag"] as const)("refuses a recorded %s object without peeling it", async (type) => { - const { objects, context } = await fixture(kind); - context.record!.resources[0]!.commit = objects[type]; + const { objects, context } = await fixture(); + context.record!.resources[0] = { id: "archive", kind: "git", commit: objects[type] }; await expect(materializeGit(context)).rejects.toThrow(`Recorded object ${objects[type]} must be a commit object; found ${type}`); await expect(readdir(context.destination)).rejects.toThrow(); }); it("refuses moving and abbreviated records even when their selectors resolve", async () => { - const { past, context } = await fixture(kind); + const { past, context } = await fixture(); for (const commit of ["HEAD", past.slice(0, 12)]) { - context.record!.resources[0]!.commit = commit; + context.record!.resources[0] = { id: "archive", kind: "git", commit }; await expect(materializeGit(context)).rejects.toThrow("Expected a full 40- or 64-hex commit object id"); } await expect(readdir(context.destination)).rejects.toThrow(); @@ -98,22 +100,22 @@ describe.each(["git", "bundle"] as const)("recorded %s commit objects", (kind) = }; it.each(["blob", "tree", "tag"] as const)("refuses a matching %s record despite a valid declared pin", async (type) => { - const { context, past, objects } = await fixture(kind); - pin(context, past); context.record!.resources[0]!.commit = objects[type]; + const { context, past, objects } = await fixture(); + pin(context, past); context.record!.resources[0] = { id: "archive", kind: "git", commit: objects[type] }; await expect(materializeGit(context)).rejects.toThrow(`Recorded object ${objects[type]} must be a commit object; found ${type}`); await expect(readdir(context.destination)).rejects.toThrow(); }); it("keeps pin precedence after verifying a different valid recorded commit", async () => { - const { context, repository, past } = await fixture(kind); + const { context, repository, past } = await fixture(); const next = await commitTestGit(repository, { "readme.txt": "other recorded commit\n" }, "2026-10-09T08:10:00Z"); - pin(context, past); context.record!.resources[0]!.commit = next.toUpperCase(); + pin(context, past); context.record!.resources[0] = { id: "archive", kind: "git", commit: next.toUpperCase() }; expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "pinned", source: { commit: past } }); expect(await readFile(path.join(context.destination, "readme.txt"), "utf8")).toBe("recorded bytes\n"); }); it("does not verify unrelated recorded inputs", async () => { - const { context, past, objects } = await fixture(kind); + const { context, past, objects } = await fixture(); pin(context, past); context.record!.resources[0] = { id: "other", kind, commit: objects.blob }; expect(await materializeGit(context)).toMatchObject({ fidelity: "exact", basis: "pinned", source: { commit: past } }); }); @@ -130,7 +132,7 @@ describe("recorded commit validation authority", () => { it("does not verify the replaced input's recorded commit for an override", async () => { const { context, repository, past, objects } = await fixture(); - context.input.overridden = true; context.history = undefined; context.record!.resources[0]!.commit = objects.blob; + context.input.overridden = true; context.history = undefined; context.record!.resources[0] = { id: "archive", kind: "git", commit: objects.blob }; context.candidateRecords = [context.record!]; context.input.declaration = { id: "archive", kind: "git", mode: "readonly", mount: "./archive", url: repository, ref: past }; expect(await materializeGit(context)).toMatchObject({ fidelity: "overridden", source: { commit: past } }); @@ -169,7 +171,7 @@ describe("retained feed recorded commit objects", () => { it.each(["blob", "tree", "tag"] as const)("validates an unselected landing's %s commit object", async (type) => { const { context, past, objects, directory } = await retainedFeed(); - const entries = [landing(past), { ...landing(objects[type]), landedAt: "2026-10-09T09:00:00Z" }]; + const entries = [landing(past), { ...landing(objects[type]), landed_at: "2026-10-09T09:00:00Z" }]; await writeFile(path.join(directory, "landings.jsonl"), `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`); await expect(materializeFeed(context)).rejects.toThrow(`Recorded object ${objects[type]} must be a commit object; found ${type}`); await expect(readdir(context.destination)).rejects.toThrow(); @@ -191,10 +193,10 @@ describe("feed moment records without landing logs", () => { await expect(readdir(context.destination)).rejects.toThrow(); }); - it("validates a moment commit but keeps feed-rule selection without a landing log", async () => { + it("uses a verified moment commit without a landing log", async () => { const { context, past } = await fallbackFeed(); context.record = { ...testMomentRecord(), feeds: [feedRecord(past.toUpperCase())] }; - expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", basis: "feed-rule-at", source: { commit: past } }); + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: past } }); }); it("does not verify unrelated feed moment records", async () => { diff --git a/src/compiler/training/rewind/recordingFormat.test.ts b/src/compiler/training/rewind/recordingFormat.test.ts new file mode 100644 index 00000000..c72367ec --- /dev/null +++ b/src/compiler/training/rewind/recordingFormat.test.ts @@ -0,0 +1,108 @@ +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { momentRecordSchema } from "./contract.js"; +import { AT, BEFORE, emptyRecord, jsonl, memoryEvent, rewindFixture } from "./fixtures.test-helper.js"; +import { gitTestRoot } from "./git.test-helper.js"; +import { History } from "./history.js"; +import { materializeTrainingMoment } from "./materialize.js"; +import { selectMomentRecord } from "./select.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const recorded = () => ({ + version: "spawnfile.moment-record.v1" as const, agent: "agent:a", + wake: { at: AT, startedAt: AT, executionId: "execution-17", + deliveries: [{ deliveryId: "delivery-17", kind: "schedule", occurredAt: AT }] }, + recordedAt: AT, exact: true, resources: [], feeds: [], memory: [], moltnet: [], volumes: [] +}); + +describe("P3 recording format", () => { + it("accepts the recording shape and rejects the superseded sketch", () => { + const value = { ...recorded(), exact: false, errors: ["volume state: capture failed"], + resources: [{ id: "git", kind: "git", commit: "a".repeat(40) }, { id: "bundle", kind: "bundle", sha256: `sha256:${"b".repeat(64)}` }], + feeds: [{ id: "directory", revision: "c".repeat(64), tree: `trees/${"c".repeat(64)}.1`, landedAt: BEFORE }], + memory: [{ id: "self", declaredBy: "agent:a", seq: 0 }], + moltnet: [{ network: "news", cursor: AT, messageIds: ["wake-message"] }] }; + expect(momentRecordSchema.parse(value)).toEqual(value); + for (const invalid of [{ ...value, exact: undefined }, { ...value, wake: { at: AT } }, + { ...value, resources: [{ id: "bundle", kind: "bundle", commit: "a".repeat(40) }] }, + { ...value, moltnet: [{ network: "news", cursor: "message-id" }] }]) { + expect(momentRecordSchema.safeParse(invalid).success).toBe(false); + } + }); + + it("selects the original attempt by earliest startedAt, not capture or append order", () => { + const first = { ...recorded(), memory: [{ id: "self", declaredBy: "agent:a", seq: 1 }] }; + const replay = { ...first, wake: { ...first.wake, at: "2026-10-09T10:20:31+02:00", startedAt: "2026-10-09T09:00:00Z" }, + memory: [{ id: "self", declaredBy: "agent:a", seq: 9 }] }; + expect(selectMomentRecord([replay, first], "agent:a", AT)).toBe(first); + }); + + it("reads nested per-file JSON moments as well as JSONL", async () => { + const root = await gitTestRoot(); roots.push(root); + await mkdir(path.join(root, "moments/a/day"), { recursive: true }); + await writeFile(path.join(root, "moments/a/day/turn.json"), JSON.stringify(recorded())); + await jsonl(root, "moments/other/turns.jsonl", [{ ...recorded(), agent: "agent:other" }]); + expect((await new History(root).records()).map((entry) => entry.agent)).toEqual(["agent:a", "agent:other"]); + }); +}); + +describe("recording pins in a materialized trial", () => { + it.each([false, true])("accepts incoming feed publication metadata after the bound with record=%s", async (withRecord) => { + const f = await rewindFixture(); roots.push(f.root); + const log = "feeds/team%3Apublication/news/landings.jsonl"; + const landing = JSON.parse((await readFile(path.join(f.historyDir, log), "utf8")).trim()); + landing.switching_at = AT; landing.landed_at = "2026-10-09T08:20:32Z"; + if (withRecord) { + const record = emptyRecord(); record.exact = false; record.errors = ["volume edition-state: snapshot failed"]; + record.feeds = [{ id: "news", revision: landing.revision, tree: landing.tree, commit: f.commit, ref: "main", landedAt: landing.landed_at }]; + await jsonl(f.historyDir, "moments/incoming.jsonl", [record]); + await rm(path.join(f.historyDir, log)); + } else await jsonl(f.historyDir, log, [landing]); + const result = await materializeTrainingMoment(f.options); + expect(result, JSON.stringify(result.inputs)).toMatchObject({ status: "materialized", leakCheck: { ok: true } }); + expect(result.inputs.find((input) => input.kind === "feed")).toMatchObject({ fidelity: withRecord ? "exact" : "approximate", + basis: withRecord ? "record" : "landing-log", source: { landedAt: landing.landed_at } }); + }); + + it("uses startedAt for task time and leak bounds, reports original selection and keeps wake at", async () => { + const f = await rewindFixture(); roots.push(f.root); + const first = emptyRecord(); first.wake.startedAt = "2026-10-09T08:21:00Z"; + first.memory = [{ id: "self", declaredBy: "agent:a", seq: 2 }]; + const replay = structuredClone(first); replay.wake.startedAt = "2026-10-09T09:00:00Z"; replay.memory[0]!.seq = 3; + await jsonl(f.historyDir, "moments/attempts.jsonl", [replay, first]); + const result = await materializeTrainingMoment(f.options); + expect(result, JSON.stringify(result.inputs)).toMatchObject({ status: "materialized", at: AT, effectiveAt: first.wake.startedAt, + record: { matched: true, attempts: 2, selectedStartedAt: first.wake.startedAt, selection: "earliest-startedAt" } }); + expect(result.inputs.find((input) => input.kind === "memory")).toMatchObject({ fidelity: "exact", source: { seq: 2 } }); + expect(JSON.parse(await readFile(path.join(f.options.out, "task-clock.json"), "utf8")).origin).toBe(first.wake.startedAt); + expect(JSON.parse(await readFile(path.join(f.options.out, "leak-check.json"), "utf8"))).toMatchObject({ at: first.wake.startedAt, ok: true }); + }); + + it("keeps healthy pins exact in an inexact record and annotates absent pins", async () => { + const f = await rewindFixture(); roots.push(f.root); + const record = emptyRecord(); record.exact = false; record.errors = ["volume edition-state: snapshot failed"]; + record.resources = [{ id: "code", kind: "git", commit: f.commit }]; + record.memory = [{ id: "self", declaredBy: "agent:a", seq: 0 }]; + await jsonl(f.historyDir, "moments/inexact.jsonl", [record]); + const result = await materializeTrainingMoment(f.options); + expect(result.status).toBe("materialized"); + expect(result.inputs.find((input) => input.id === "code")).toMatchObject({ fidelity: "exact", basis: "record" }); + expect(result.inputs.find((input) => input.id === "self")).toMatchObject({ fidelity: "exact", basis: "record", source: { seq: 0 } }); + expect(result.inputs.find((input) => input.id === "edition-state")).toMatchObject({ fidelity: "approximate", + notes: expect.arrayContaining(record.errors) }); + }); + + it("does not trust a present pin named by the record errors", async () => { + const f = await rewindFixture(); roots.push(f.root); + const record = emptyRecord(); record.exact = false; record.errors = ["memory self: sequence capture failed"]; + record.memory = [{ id: "self", declaredBy: "agent:a", seq: 0 }]; + await jsonl(f.historyDir, "moments/inexact.jsonl", [record]); + await jsonl(f.historyDir, "memory/agent%3Aa/self/events.jsonl", [memoryEvent(1)]); + const result = await materializeTrainingMoment(f.options); + expect(result.status).toBe("materialized"); + expect(result.inputs.find((input) => input.id === "self")).toMatchObject({ fidelity: "approximate", basis: "created-at-prefix", + source: { seq: 1 }, notes: expect.arrayContaining(record.errors) }); + }); +}); diff --git a/src/compiler/training/rewind/select.test.ts b/src/compiler/training/rewind/select.test.ts index fc70c354..e1d55357 100644 --- a/src/compiler/training/rewind/select.test.ts +++ b/src/compiler/training/rewind/select.test.ts @@ -6,11 +6,11 @@ import { selectMomentRecord, selectTrainingExample } from "./select.js"; const at = "2026-10-09T08:20:31Z"; const record = (): MomentRecord => ({ version: "spawnfile.moment-record.v1", agent: "agent:writer", - wake: { at, surface: "moltnet", room: "assignment", from: "brass" }, recordedAt: at, + wake: { at, startedAt: at, executionId: "execution-17", deliveries: [{ deliveryId: "delivery-17", kind: "message", occurredAt: at }], surface: "moltnet", room: "assignment", from: "brass" }, recordedAt: at, exact: true, resources: [{ id: "repo", kind: "git", commit: "a".repeat(40) }], feeds: [{ id: "news", revision: "rev", tree: "tree", commit: "b".repeat(40), paths: ["published"], landedAt: at }], memory: [{ id: "self", declaredBy: "agent:writer", seq: 2, generation: 0 }], - moltnet: [{ network: "network", cursor: "cursor" }], volumes: [{ id: "state", snapshot: "snapshot" }] + moltnet: [{ network: "network", cursor: at }], volumes: [{ id: "state", snapshot: "snapshot" }] }); const dataset = (name: string, id: string) => ({ sourcePath: `/project/${name}.yaml`, path: `${name}.yaml`, examples: [{ id, at, wake: { surface: "moltnet" as const, room: "assignment", from: "brass", text: "write" }, overrides: [] }] }); @@ -39,12 +39,12 @@ describe("pure rewind record selection", () => { }); const changes: Array<[string, (value: MomentRecord) => void]> = [ - ["resource commit", (value) => { value.resources[0]!.commit = "c".repeat(40); }], + ["resource commit", (value) => { value.resources[0] = { id: "repo", kind: "git", commit: "c".repeat(40) }; }], ["feed revision", (value) => { value.feeds[0]!.revision = "different"; }], ["feed paths", (value) => { value.feeds[0]!.paths = ["private"]; }], ["memory sequence", (value) => { value.memory[0]!.seq = 3; }], ["memory generation", (value) => { value.memory[0]!.generation = 1; }], - ["network cursor", (value) => { value.moltnet[0]!.cursor = "different"; }], + ["network cursor", (value) => { value.moltnet[0]!.cursor = "2026-10-10T00:00:00Z"; }], ["volume snapshot", (value) => { value.volumes[0]!.snapshot = "different"; }] ]; it.each(changes)("refuses conflicting %s fields", (_name, change) => { @@ -55,7 +55,7 @@ describe("pure rewind record selection", () => { it("compares only inputs being used and retains memory declaring-node identity", () => { const first = record(), second = structuredClone(first); - second.resources[0]!.commit = "c".repeat(40); + second.resources[0] = { id: "repo", kind: "git", commit: "c".repeat(40) }; second.memory.push({ id: "self", declaredBy: "team:other", generation: 9, seq: 88 }); const selected = [{ section: "memory" as const, id: "self", declaredBy: "agent:writer" }]; expect(selectMomentRecord([first, second], "agent:writer", at, selected)).toBe(first); @@ -65,7 +65,7 @@ describe("pure rewind record selection", () => { it("accepts reordered record entries without mutating its inputs", () => { const first = record(); - first.resources.push({ id: "another", kind: "bundle", commit: "d".repeat(40) }); + first.resources.push({ id: "another", kind: "bundle", sha256: `sha256:${"d".repeat(64)}` }); const second = structuredClone(first); second.resources.reverse(); const before = structuredClone([first, second]); @@ -73,9 +73,19 @@ describe("pure rewind record selection", () => { expect([first, second]).toEqual(before); }); + it("refuses conflicting bundle digests even when the declared source has a full commit pin", () => { + const first = record(), second = record(); + first.resources = [{ id: "tools", kind: "bundle", sha256: `sha256:${"a".repeat(64)}` }]; + second.resources = [{ id: "tools", kind: "bundle", sha256: `sha256:${"b".repeat(64)}` }]; + expect(() => selectMomentRecord([first, second], "agent:writer", at, [{ + section: "resources", id: "tools", declaredBy: "agent:writer", overridden: false, declarationBase: "/project", + declaration: { id: "tools", kind: "bundle", mode: "readonly", mount: "./tools", build: { files: { root: ".", ref: "c".repeat(40) } } } + }])).toThrow("Conflicting moment records"); + }); + it("does not compare record fields superseded by a full commit pin or another resource kind", () => { const first = record(), second = structuredClone(first); - second.resources[0]!.commit = "c".repeat(40); + second.resources[0] = { id: "repo", kind: "git", commit: "c".repeat(40) }; second.feeds[0]!.revision = "other"; expect(selectMomentRecord([first, second], "agent:writer", at, [{ section: "resources", id: "repo", declaredBy: "agent:writer", overridden: false, declarationBase: "/project", diff --git a/src/compiler/training/rewind/select.ts b/src/compiler/training/rewind/select.ts index ecccc6ce..18d80f7a 100644 --- a/src/compiler/training/rewind/select.ts +++ b/src/compiler/training/rewind/select.ts @@ -35,11 +35,18 @@ const canonical = (value: unknown): string => JSON.stringify(value, (_key, item: export const matchingMomentRecords = (records: readonly MomentRecord[], agent: string, at: string): MomentRecord[] => records.filter((record) => record.agent === agent && compareInstants(record.wake.at, at) === 0); +/** Replays keep the wake identity but later attempts have different pins. Only the original is relevant. */ +export const originalMomentRecords = (records: readonly MomentRecord[]): MomentRecord[] => { + const first = records.reduce((earliest, entry) => + !earliest || compareInstants(entry.wake.startedAt, earliest.wake.startedAt) < 0 ? entry : earliest, null); + return first ? records.filter((entry) => compareInstants(entry.wake.startedAt, first.wake.startedAt) === 0) : []; +}; + /** Capture time and wake payload are not used to materialize data. Normalize actual used instants. */ export const selectMomentRecord = ( records: MomentRecord[], agent: string, at: string, inputs?: Array ): MomentRecord | null => { - const matches = matchingMomentRecords(records, agent, at); + const matches = originalMomentRecords(matchingMomentRecords(records, agent, at)); if (!matches.length) return null; const includes = (section: InputIdentity["section"], id: string, declaredBy?: string, field?: "resources" | "feeds" | "volumes"): boolean => !inputs || inputs.some((input) => { @@ -51,7 +58,9 @@ export const selectMomentRecord = ( if (field === "feeds") return resource.kind === "volume" && Boolean(resource.feed); if (field === "volumes") return resource.kind === "volume" && !resource.feed; if (resource.kind === "volume") return false; - const ref = resource.kind === "git" ? resource.ref : resource.build?.files?.ref; + // Retained bundle digests take precedence over even a pinned build source. + if (resource.kind === "bundle") return true; + const ref = resource.ref; return !ref || !/^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/u.test(ref); }); const sort = (entries: T[]): T[] => entries.sort((a, b) => canonical(a).localeCompare(canonical(b))); @@ -61,7 +70,7 @@ export const selectMomentRecord = ( ...item, landedAt: normalizeInstant(item.landedAt) }))), memory: sort(record.memory.filter((item) => includes("memory", item.id, item.declaredBy))), - moltnet: sort(record.moltnet.filter((item) => includes("networks", item.network))), + moltnet: sort(record.moltnet.filter((item) => includes("networks", item.network)).map((item) => ({ ...item, cursor: normalizeInstant(item.cursor) }))), volumes: sort(record.volumes.filter((item) => includes("resources", item.id, undefined, "volumes"))) }); if (matches.some((record) => used(record) !== used(matches[0]!))) throw new Error(`Conflicting moment records for ${agent} at ${normalizeInstant(at)}`); diff --git a/src/compiler/training/rewind/snapshotFixtures.test-helper.ts b/src/compiler/training/rewind/snapshotFixtures.test-helper.ts new file mode 100644 index 00000000..36ae8c17 --- /dev/null +++ b/src/compiler/training/rewind/snapshotFixtures.test-helper.ts @@ -0,0 +1,50 @@ +import { chmod, lstat, mkdir, readdir, readlink, writeFile, appendFile, symlink } from "node:fs/promises"; +import path from "node:path"; + +interface SnapshotFixture { + agentSlug: string; agentId: string; rootId: string; snapshot: string; startedAt: string; + files: Record; executionId?: string; + directories?: Record; symlinks?: Record; +} + +/** Recorder-format data only: no imports from the parallel Daimon implementation. */ +export const writeRecordedVolumeSnapshot = async (historyRoot: string, options: SnapshotFixture) => { + const recording = path.join(historyRoot, "recording", options.agentSlug); + const base = path.join(recording, "snapshots", options.rootId); + const directory = path.join(base, options.snapshot), manifestPath = `${directory}.manifest.json`; + const rowsPath = path.join(recording, "wake-moments.jsonl"); + await mkdir(directory, { recursive: true, mode: 0o700 }); + for (const [relative, mode] of Object.entries(options.directories ?? {})) { + await mkdir(path.join(directory, relative), { recursive: true }); await chmod(path.join(directory, relative), mode); + } + for (const [relative, text] of Object.entries(options.files)) { + await mkdir(path.dirname(path.join(directory, relative)), { recursive: true }); + await writeFile(path.join(directory, relative), text); + } + for (const [relative, target] of Object.entries(options.symlinks ?? {})) { + await mkdir(path.dirname(path.join(directory, relative)), { recursive: true }); + await symlink(target, path.join(directory, relative)); + } + const entries: Record = {}; + const visit = async (relative: string): Promise => { + for (const name of await readdir(path.join(directory, relative))) { + const rel = relative ? `${relative}/${name}` : name, file = path.join(directory, rel); + const stat = await lstat(file, { bigint: true }); + const type = stat.isDirectory() ? "directory" : stat.isSymbolicLink() ? "symlink" : "file"; + entries[rel] = { type, mode: Number(stat.mode), size: String(stat.size), source: { + dev: String(stat.dev), ino: String(stat.ino), size: String(stat.size), + mtimeNs: String(stat.mtimeNs), ctimeNs: String(stat.ctimeNs), mode: Number(stat.mode) + }, ...(type === "symlink" ? { target: await readlink(file) } : {}) }; + if (type === "directory") await visit(rel); + } + }; + await visit(""); + const manifest = { version: "noopolis.daimon.wake-moment.v1", id: options.rootId, snapshot: options.snapshot, + started_at: options.startedAt, sourcePath: `/deployment/${options.rootId}`, entries }; + await writeFile(manifestPath, `${JSON.stringify(manifest)}\n`); + await appendFile(rowsPath, `${JSON.stringify({ version: manifest.version, agent_id: options.agentId, + execution_id: options.executionId ?? "execution", started_at: options.startedAt, deliveries: [], + snapshots: [{ id: options.rootId, snapshot: options.snapshot, files: Object.keys(options.files).length, + copied: Object.keys(options.files).length, linked: 0, skipped: 0 }] })}\n`); + return { directory, manifestPath, rowsPath, manifest }; +}; diff --git a/src/compiler/training/rewind/snapshotManifest.ts b/src/compiler/training/rewind/snapshotManifest.ts new file mode 100644 index 00000000..7ac4630c --- /dev/null +++ b/src/compiler/training/rewind/snapshotManifest.ts @@ -0,0 +1,118 @@ +import { createHash } from "node:crypto"; +import { chmod, lstat, mkdir, readFile, readdir, readlink, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { z } from "zod"; +import { trainingInstantSchema } from "../../../manifest/index.js"; +import { inside, safeSegment, type History } from "./history.js"; + +const unsigned = z.string().regex(/^(?:0|[1-9][0-9]*)$/u); +const nanos = z.string().regex(/^-?(?:0|[1-9][0-9]*)$/u); +const mode = z.number().int().min(0).max(0xffff); +const relativePath = z.string().min(1).refine((value) => !value.includes("\\") && !value.includes("\0") + && !path.posix.isAbsolute(value) && !value.split("/").some((part) => !part || part === "." || part === ".."), +"Snapshot paths must be normalized relative POSIX paths"); +const sourceIdentity = z.object({ dev: unsigned, ino: unsigned, size: unsigned, mtimeNs: nanos, ctimeNs: nanos, mode }).strict(); +const entrySchema = z.object({ type: z.enum(["file", "directory", "symlink"]), mode, size: unsigned, + source: sourceIdentity, target: z.string().refine((value) => !value.includes("\0")).optional() }).strict().superRefine((entry, ctx) => { + if ((entry.type === "symlink") !== (entry.target !== undefined)) ctx.addIssue({ code: "custom", message: "Only snapshot symlinks require a target" }); + if (entry.mode !== entry.source.mode || entry.size !== entry.source.size) ctx.addIssue({ code: "custom", message: "Snapshot source identity disagrees with entry metadata" }); + const typeMode = { file: 0o100000, directory: 0o040000, symlink: 0o120000 }[entry.type]; + if ((entry.mode & 0o170000) !== typeMode) ctx.addIssue({ code: "custom", message: "Snapshot mode disagrees with entry type" }); +}); + +/** Daimon's manifest records source identity, not the identity of retained/copied inodes. */ +export const snapshotManifestSchema = z.object({ version: z.literal("noopolis.daimon.wake-moment.v1"), id: z.string().min(1), + snapshot: z.string().min(1), started_at: trainingInstantSchema, + sourcePath: z.string().refine((value) => path.posix.isAbsolute(value) && !value.includes("\0"), "Expected an absolute snapshot sourcePath"), + entries: z.record(relativePath, entrySchema) }).strict().superRefine((manifest, ctx) => { + for (const name of Object.keys(manifest.entries)) { + for (let parent = path.posix.dirname(name); parent !== "."; parent = path.posix.dirname(parent)) { + if (manifest.entries[parent]?.type !== "directory") ctx.addIssue({ code: "custom", message: `Snapshot parent ${parent} must be a recorded directory` }); + } + } +}); +export type SnapshotManifest = z.infer; + +export const snapshotHistoryPath = (agentSlug: string, id: string, snapshot: string): string => + `recording/${safeSegment(agentSlug)}/snapshots/${safeSegment(id === "." || id === ".." ? id.replaceAll(".", "%2E") : id)}/${safeSegment(snapshot)}`; + +export const readSnapshotManifest = async (history: History, relative: string, id: string, snapshot: string): Promise => { + const bytes = await history.text(`${relative}.manifest.json`); + if (bytes === null) return null; + let manifest: SnapshotManifest; + try { manifest = snapshotManifestSchema.parse(JSON.parse(bytes)); } + catch (error) { throw new Error(`Invalid snapshot manifest ${relative}.manifest.json: ${error instanceof Error ? error.message : String(error)}`); } + if (manifest.id !== id || manifest.snapshot !== snapshot) throw new Error(`Snapshot manifest identity mismatch at ${relative}.manifest.json`); + return manifest; +}; + +interface TreeInspection { + retained?: boolean; + readBytes?: (relative: string) => Promise; + expectedBytes?: (relative: string) => Promise; +} +/** No symlink traversal. A separate call inspects the copied tree, independently of the producer. */ +export const snapshotTreeFindings = async (root: string, manifest: SnapshotManifest, options: TreeInspection = {}): Promise => { + if (!(await lstat(root)).isDirectory()) throw new Error("Snapshot tree must be a real directory"); + const findings: string[] = [], found = new Set(); + const visit = async (relative: string): Promise => { + for (const leaf of (await readdir(path.join(root, relative))).sort()) { + const name = relative ? `${relative}/${leaf}` : leaf, file = path.join(root, name); + const stat = await lstat(file, { bigint: true }), expected = manifest.entries[name]; + found.add(name); + const type = stat.isFile() ? "file" : stat.isDirectory() ? "directory" : stat.isSymbolicLink() ? "symlink" : "special"; + if (!expected) findings.push(`Unexpected snapshot entry ${name}`); + else if (type !== expected.type) findings.push(`Snapshot type mismatch for ${name}`); + else { + // Recorder directories are deliberately 0700. Restore source modes while copying. + const wantedMode = options.retained && type === "file" ? expected.mode & 0o777 : expected.mode & 0o7777; + if (!(options.retained && type === "directory") && (Number(stat.mode) & 0o7777) !== wantedMode) findings.push(`Snapshot mode mismatch for ${name}`); + // Directory inode sizes describe source filesystem bookkeeping, not tree content. + if (type !== "directory" && stat.size.toString() !== expected.size) findings.push(`Snapshot size mismatch for ${name}`); + if (type === "symlink" && await readlink(file) !== expected.target) findings.push(`Snapshot symlink target mismatch for ${name}`); + if (type === "file" && options.expectedBytes) { + const actual = options.readBytes ? await options.readBytes(name) : await readFile(file); + const original = await options.expectedBytes(name); + const digest = (bytes: Buffer): string => createHash("sha256").update(bytes).digest("hex"); + if (digest(actual) !== digest(original)) findings.push(`Snapshot digest mismatch for ${name}`); + } + } + if (type === "directory") await visit(name); + } + }; + await visit(""); + for (const name of Object.keys(manifest.entries)) if (!found.has(name)) findings.push(`Missing snapshot entry ${name}`); + return findings; +}; + +export const snapshotBytes = async (history: History, relative: string, name: string): Promise => { + const bytes = await history.bytes(`${relative}/${name}`); + if (bytes === null) throw new Error(`Missing snapshot entry ${name}`); + return bytes; +}; + +/** Retained files may be hard-linked to each other; bundle files are always new copies. */ +export const copySnapshotTree = async (history: History, relative: string, destination: string, manifest: SnapshotManifest): Promise => { + const source = await history.path(relative); + if (!source) throw new Error(`Missing snapshot tree ${relative}`); + const findings = await snapshotTreeFindings(source, manifest, { retained: true }); + if (findings.length) throw new Error(findings.join("; ")); + await mkdir(destination, { recursive: true, mode: 0o700 }); + const entries = Object.entries(manifest.entries).sort(([a], [b]) => a.split("/").length - b.split("/").length || a.localeCompare(b)); + for (const [name, entry] of entries) { + const target = path.join(destination, name); + if (entry.type === "directory") await mkdir(target, { mode: 0o700 }); + else if (entry.type === "symlink") { + if (path.isAbsolute(entry.target!) || !inside(destination, path.resolve(path.dirname(target), entry.target!))) { + throw new Error(`Snapshot symlink escapes root: ${name}`); + } + await symlink(entry.target!, target); + } else { + await writeFile(target, await snapshotBytes(history, relative, name), { flag: "wx" }); + await chmod(target, entry.mode & 0o7777); + } + } + for (const [name, entry] of entries.reverse()) if (entry.type === "directory") await chmod(path.join(destination, name), entry.mode & 0o7777); + const copied = await snapshotTreeFindings(destination, manifest, { expectedBytes: (name) => snapshotBytes(history, relative, name) }); + if (copied.length) throw new Error(copied.join("; ")); +}; diff --git a/src/compiler/training/rewind/types.ts b/src/compiler/training/rewind/types.ts index 76d5f433..99287d7b 100644 --- a/src/compiler/training/rewind/types.ts +++ b/src/compiler/training/rewind/types.ts @@ -13,6 +13,8 @@ export type RewindInput = InputBase & ( export interface InputContext { input: RewindInput; at: string; + agentSlug?: string; + agentId?: string; record: MomentRecord | null; candidateRecords?: readonly MomentRecord[]; history: History | undefined; diff --git a/src/compiler/training/rewind/volumes.test.ts b/src/compiler/training/rewind/volumes.test.ts index cd22c2f2..541797a0 100644 --- a/src/compiler/training/rewind/volumes.test.ts +++ b/src/compiler/training/rewind/volumes.test.ts @@ -1,93 +1,139 @@ -import { createHash } from "node:crypto"; -import { mkdtemp, mkdir, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; +import { chmod, lstat, mkdtemp, mkdir, readFile, readdir, readlink, realpath, rm, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { openHistory } from "./history.js"; -import { momentRecordSchema } from "./contract.js"; +import { momentRecordSchema, type LeakCheck, type MaterializedInput } from "./contract.js"; +import { inspectVolumeBundle } from "./leakInputs.js"; +import { writeRecordedVolumeSnapshot } from "./snapshotFixtures.test-helper.js"; import type { InputContext } from "./types.js"; -import { materializeVolume, replayVolumeSnapshot, selectVolumeSnapshot, type VolumeSnapshot } from "./volumes.js"; +import { materializeVolume } from "./volumes.js"; const at = "2026-10-09T10:00:00Z", before = "2026-10-09T09:00:00Z", future = "2026-10-09T10:00:00.001Z"; -const digest = (bytes: string): string => createHash("sha256").update(bytes).digest("hex"); const temps: string[] = []; afterEach(async () => { await Promise.all(temps.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); }); -const snapshot = (id: string, time: string, parent: string | null, files: VolumeSnapshot["files"]): VolumeSnapshot => - ({ snapshot: id, takenAt: time, parent, files }); -const fixtures = async (snapshots?: VolumeSnapshot[]) => { - const root = await mkdtemp(path.join(os.tmpdir(), "rewind-volumes-")); temps.push(root); - const bank = path.join(root, "volumes/team%3Aorg/state"); await mkdir(path.join(bank, "objects"), { recursive: true }); - if (snapshots) await writeFile(path.join(bank, "snapshots.jsonl"), snapshots.map((item) => JSON.stringify(item)).join("\n") + "\n"); +const fixtures = async () => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "rewind-volumes-"))); temps.push(root); const context: InputContext = { at, record: null, history: await openHistory(root), destination: path.join(root, "out"), + agentSlug: "writer-compiled-slug", agentId: "agent:writer", input: { section: "resources", id: "state", declaredBy: "team:org", declarationBase: root, overridden: false, declaration: { id: "state", kind: "volume", mount: "./state", mode: "mutable" } } }; - return { root, bank, context, object: async (bytes: string) => writeFile(path.join(bank, "objects", digest(bytes)), bytes) }; + const snapshot = (name: string, startedAt: string, files: Record = {}) => writeRecordedVolumeSnapshot(root, + { agentSlug: context.agentSlug!, agentId: context.agentId!, rootId: "state", snapshot: name, startedAt, files }); + return { root, context, snapshot }; }; +const record = (snapshot: string) => momentRecordSchema.parse({ version: "spawnfile.moment-record.v1", agent: "agent:writer", + wake: { at, startedAt: at, executionId: "execution", deliveries: [{ deliveryId: "wake", kind: "message", occurredAt: at }] }, + recordedAt: at, exact: true, resources: [], feeds: [], memory: [], moltnet: [], volumes: [{ id: "state", snapshot }] }); +const report = (): LeakCheck => ({ version: "spawnfile.leak-check.v1", at, ok: false, + checked: { files: 0, gitObjects: 0, memoryEvents: 0, moltnetMessages: 0, moltnetEvents: 0, snapshots: 0 }, findings: [] }); -describe("unfed volume replay", () => { - it("includes equal instants, excludes one millisecond later, and selects an ancestor record exactly", async () => { - const rows = [snapshot("first", before, null, { old: digest("old") }), - snapshot("next", "2026-10-09T12:00:00+02:00", "first", { old: null, "nested/new": digest("new") }), - snapshot("future", future, "next", { later: digest("future") })]; - expect(selectVolumeSnapshot(rows, at).snapshot).toBe("next"); - expect(selectVolumeSnapshot(rows, at, "first").snapshot).toBe("first"); - expect(replayVolumeSnapshot(rows, "next").files).toEqual({ "nested/new": digest("new") }); - const { context, object } = await fixtures(rows); await object("old"); await object("new"); - const result = await materializeVolume(context); - expect(result).toMatchObject({ fidelity: "approximate", basis: "snapshot-at", source: { snapshot: "next" } }); +describe("unfed recorded volume trees", () => { + it("selects by inclusive row started_at, copies a full tree, and uses the named record exactly", async () => { + const { context, snapshot } = await fixtures(); + await snapshot("20261009T080000000Z-first", before, { old: "old" }); + const newest = await snapshot("20261009T070000000Z-newest", "2026-10-09T12:00:00+02:00", { "nested/new": "new" }); + await snapshot("20261009T060000000Z-future", future, { later: "future" }); + expect(await materializeVolume(context)).toMatchObject({ fidelity: "approximate", basis: "snapshot-at", + source: { snapshot: "20261009T070000000Z-newest", agentSlug: "writer-compiled-slug" } }); expect(await readdir(context.destination)).toEqual(["nested"]); expect(await readFile(path.join(context.destination, "nested/new"), "utf8")).toBe("new"); - context.destination += "-record"; - context.record = momentRecordSchema.parse({ version: "spawnfile.moment-record.v1", agent: "agent:writer", - wake: { at, surface: "moltnet", room: "assignment", from: "brass" }, recordedAt: at, - resources: [], feeds: [], memory: [], moltnet: [], volumes: [{ id: "state", snapshot: "first" }] }); - expect(await materializeVolume(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { snapshot: "first" } }); + expect((await lstat(path.join(context.destination, "nested/new"))).ino) + .not.toBe((await lstat(path.join(newest.directory, "nested/new"))).ino); + context.destination += "-record"; context.record = record("20261009T080000000Z-first"); + expect(await materializeVolume(context)).toMatchObject({ fidelity: "exact", basis: "record" }); expect(await readFile(path.join(context.destination, "old"), "utf8")).toBe("old"); }); + it("ignores incomplete and .partial trees and chooses the last row when instants tie", async () => { + const { context, snapshot } = await fixtures(); + await snapshot("first", before, { data: "first" }); + await snapshot("tie", before, { data: "last" }); + const incomplete = await snapshot("incomplete", at, { data: "unfinished" }); + await rm(incomplete.manifestPath); + await snapshot("incoming.partial", at, { data: "partial" }); + expect(await materializeVolume(context)).toMatchObject({ source: { snapshot: "tie" } }); + expect(await readFile(path.join(context.destination, "data"), "utf8")).toBe("last"); + context.record = record("incoming.partial"); context.destination += "-record"; + await expect(materializeVolume(context)).rejects.toThrow(/partial/i); + }); + it("creates an empty overridden volume without history", async () => { const { context } = await fixtures(); context.history = undefined; context.input.overridden = true; expect(await materializeVolume(context)).toMatchObject({ fidelity: "overridden", basis: "override" }); expect(await readdir(context.destination)).toEqual([]); }); - it("refuses unavailable history, future-only snapshots, divergent pins, cycles and missing ancestors", async () => { - const { context } = await fixtures(); + it("refuses missing history, missing rows, future-only snapshots, and absent named snapshots", async () => { + const { context, snapshot } = await fixtures(); await expect(materializeVolume(context)).rejects.toThrow("Missing volume snapshot history"); + await snapshot("future", future); + await expect(materializeVolume(context)).rejects.toThrow("No complete volume snapshot"); + context.record = record("missing"); + await expect(materializeVolume(context)).rejects.toThrow("Recorded volume snapshot missing"); context.history = undefined; await expect(materializeVolume(context)).rejects.toThrow("requires --history"); - expect(() => selectVolumeSnapshot([snapshot("late", future, null, {})], at)).toThrow("No volume snapshot"); - const rows = [snapshot("one", before, null, {}), snapshot("other", at, null, {})]; - expect(() => selectVolumeSnapshot(rows, at, "one")).toThrow("not the latest eligible snapshot or its ancestor"); - expect(() => replayVolumeSnapshot([snapshot("one", at, "missing", {})], "one")).toThrow("Missing volume snapshot ancestor"); - expect(() => replayVolumeSnapshot([snapshot("one", at, "one", {})], "one")).toThrow("Cycle"); - expect(() => replayVolumeSnapshot([snapshot("one", at, null, {}), snapshot("two", before, "one", {})], "two")).toThrow("later than child"); }); - it("breaks same-instant ties only by ancestry and refuses malformed records and unsafe paths", async () => { - const rows = [snapshot("one", at, null, {}), snapshot("two", at, "one", {})]; - expect(selectVolumeSnapshot(rows, at).snapshot).toBe("two"); - expect(() => selectVolumeSnapshot([...rows, snapshot("three", at, null, {})], at)).toThrow("Ambiguous"); - expect(() => replayVolumeSnapshot([rows[0]!, rows[0]!], "one")).toThrow("Duplicate"); - expect(() => replayVolumeSnapshot([snapshot("one", at, null, { a: digest("a"), "a/b": digest("b") })], "one")).toThrow("collision"); - const { context, bank } = await fixtures(); - await writeFile(path.join(bank, "snapshots.jsonl"), JSON.stringify({ ...rows[0], unexpected: true }) + "\n"); - await expect(materializeVolume(context)).rejects.toThrow("snapshots.jsonl:1"); - await writeFile(path.join(bank, "snapshots.jsonl"), JSON.stringify(snapshot("one", at, null, { "../escape": digest("x") })) + "\n"); - await expect(materializeVolume(context)).rejects.toThrow("normalized relative POSIX"); + it("refuses invalid rows with line numbers and mismatched manifest identity", async () => { + const { context, snapshot } = await fixtures(), saved = await snapshot("one", before, { data: "bytes" }); + await writeFile(saved.rowsPath, `${await readFile(saved.rowsPath, "utf8")}{}\n`); + await expect(materializeVolume(context)).rejects.toThrow("wake-moments.jsonl:2"); + await writeFile(saved.rowsPath, (await readFile(saved.rowsPath, "utf8")).split("\n")[0] + "\n"); + await writeFile(saved.manifestPath, JSON.stringify({ ...saved.manifest, id: "other" })); + await expect(materializeVolume(context)).rejects.toThrow("manifest identity"); + }); + + it.each(["extra", "missing", "type", "size", "mode", "unsafe"])("refuses manifest %s mismatches", async (mutation) => { + const { context, snapshot } = await fixtures(), saved = await snapshot("one", before, { data: "bytes" }); + if (mutation === "extra") await writeFile(path.join(saved.directory, "extra"), "extra"); + if (mutation === "missing") await rm(path.join(saved.directory, "data")); + if (mutation === "type") { await rm(path.join(saved.directory, "data")); await mkdir(path.join(saved.directory, "data")); } + if (mutation === "size") await writeFile(path.join(saved.directory, "data"), "shortened"); + if (mutation === "mode") await chmod(path.join(saved.directory, "data"), 0o600); + if (mutation === "unsafe") { + const manifest = JSON.parse(await readFile(saved.manifestPath, "utf8")); + manifest.entries["../escape"] = manifest.entries.data; await writeFile(saved.manifestPath, JSON.stringify(manifest)); + } + await expect(materializeVolume(context)).rejects.toThrow({ + extra: "Unexpected snapshot entry extra", missing: "Missing snapshot entry data", type: "Snapshot type mismatch for data", + size: "Snapshot size mismatch for data", mode: "Snapshot mode mismatch for data", unsafe: "Snapshot paths must be normalized" + }[mutation]!); + }); + + it("restores recorded directory modes without comparing nonportable directory sizes and verifies symlinks", async () => { + const { context, root } = await fixtures(); + const saved = await writeRecordedVolumeSnapshot(root, { agentSlug: context.agentSlug!, agentId: context.agentId!, rootId: "state", + snapshot: "tree", startedAt: before, files: { "nested/data": "bytes" }, directories: { nested: 0o750 }, symlinks: { link: "nested/data" } }); + const manifest = JSON.parse(await readFile(saved.manifestPath, "utf8")); + manifest.entries.nested.size = "999999"; manifest.entries.nested.source.size = "999999"; + await writeFile(saved.manifestPath, JSON.stringify(manifest)); await chmod(path.join(saved.directory, "nested"), 0o700); + await materializeVolume(context); + expect((await lstat(path.join(context.destination, "nested"))).mode & 0o777).toBe(0o750); + expect(await readlink(path.join(context.destination, "link"))).toBe("nested/data"); + manifest.entries.link.target = "other/data"; + await writeFile(saved.manifestPath, JSON.stringify(manifest)); context.destination += "-bad-link"; + await expect(materializeVolume(context)).rejects.toThrow(/symlink target/i); }); - it("verifies every chain object, including deleted bytes, and refuses symlink escape", async () => { - const { context, bank, root, object } = await fixtures([ - snapshot("one", before, null, { old: digest("old") }), snapshot("two", at, "one", { old: null })]); - await expect(materializeVolume(context)).rejects.toThrow("Missing volume snapshot object"); - await object("old"); await writeFile(path.join(bank, "objects", digest("old")), "tampered"); - await expect(materializeVolume(context)).rejects.toThrow("SHA-256 mismatch"); - const outside = await mkdtemp(path.join(os.tmpdir(), "rewind-volume-outside-")); temps.push(outside); - await writeFile(path.join(outside, "object"), "old"); - await rm(path.join(bank, "objects", digest("old"))); - await symlink(path.join(outside, "object"), path.join(bank, "objects", digest("old"))); - await expect(materializeVolume(context)).rejects.toThrow("symlink escape"); - expect(await readdir(root)).not.toContain("out"); + it.each(["bytes", "mode", "extra", "missing", "target"])("independently rejects a bundle %s mutation", async (mutation) => { + const { root, context } = await fixtures(); + await writeRecordedVolumeSnapshot(root, { agentSlug: context.agentSlug!, agentId: context.agentId!, rootId: "state", + snapshot: "one", startedAt: before, files: { data: "bytes", other: "other" }, symlinks: { link: "data" } }); + const result = await materializeVolume(context); + if (mutation === "bytes") await writeFile(path.join(context.destination, "data"), "wrong"); + if (mutation === "mode") await chmod(path.join(context.destination, "data"), 0o600); + if (mutation === "extra") await writeFile(path.join(context.destination, "extra"), "extra"); + if (mutation === "missing") await rm(path.join(context.destination, "other")); + if (mutation === "target") { + await rm(path.join(context.destination, "link")); await symlink("other", path.join(context.destination, "link")); + } + const input: MaterializedInput = { section: "resources", id: "state", declaredBy: "team:org", kind: "volume", path: "out", + fidelity: result.fidelity, basis: result.basis, source: result.source, notes: [], refusal: null }; + const check = report(); await inspectVolumeBundle(root, input, check, context.history); + expect(check.findings).toContainEqual(expect.objectContaining({ class: "snapshot", path: "out", detail: expect.stringMatching({ + bytes: /Snapshot digest mismatch for data/, mode: /Snapshot mode mismatch for data/, extra: /Unexpected snapshot entry extra/, + missing: /Missing snapshot entry other/, target: /Snapshot symlink target mismatch for link/ + }[mutation]!) })); }); }); diff --git a/src/compiler/training/rewind/volumes.ts b/src/compiler/training/rewind/volumes.ts index d0d6ba04..1285a1cb 100644 --- a/src/compiler/training/rewind/volumes.ts +++ b/src/compiler/training/rewind/volumes.ts @@ -1,89 +1,48 @@ -import { createHash } from "node:crypto"; -import { mkdir, writeFile } from "node:fs/promises"; -import path from "node:path"; +import { lstat, mkdir } from "node:fs/promises"; import { z } from "zod"; import { trainingInstantSchema } from "../../../manifest/index.js"; -import { historyKey } from "./history.js"; +import { safeSegment, type History } from "./history.js"; import { compareInstants } from "./instant.js"; +import { copySnapshotTree, readSnapshotManifest, snapshotHistoryPath, type SnapshotManifest } from "./snapshotManifest.js"; import type { InputContext, InputResult } from "./types.js"; -const snapshotId = z.string().min(1); -const relativeFile = z.string().min(1).refine((value) => !value.includes("\\") && !value.includes("\0") - && !path.posix.isAbsolute(value) && !value.split("/").some((part) => !part || part === "." || part === ".."), -"Snapshot file paths must be normalized relative POSIX paths"); -export const volumeSnapshotSchema = z.object({ - snapshot: snapshotId, takenAt: trainingInstantSchema, parent: snapshotId.nullable(), - files: z.record(relativeFile, z.string().regex(/^[a-f0-9]{64}$/u).nullable()) -}).strict(); -export type VolumeSnapshot = z.infer; -export interface ReplayedVolumeSnapshot { - snapshot: string; takenAt: string; files: Record; chain: VolumeSnapshot[]; -} +const text = z.string().min(1), count = z.number().int().nonnegative(); +export const wakeSnapshotRowSchema = z.object({ version: z.literal("noopolis.daimon.wake-moment.v1"), + agent_id: text, execution_id: text, started_at: trainingInstantSchema, + deliveries: z.array(z.object({ acceptance_id: text, delivery_id: text, kind: text, occurred_at: trainingInstantSchema }).strict()), + snapshots: z.array(z.object({ id: text, snapshot: text, files: count, copied: count, linked: count, skipped: count }).strict()), + error: text.optional() }).strict(); +export type WakeSnapshotRow = z.infer; +export interface SelectedVolumeSnapshot { snapshot: string; startedAt: string; manifest: SnapshotManifest; relative: string } -const catalog = (snapshots: VolumeSnapshot[]): Map => { - const result = new Map(); - for (const item of snapshots) { - volumeSnapshotSchema.parse(item); - if (result.has(item.snapshot)) throw new Error(`Duplicate volume snapshot ${item.snapshot}`); - result.set(item.snapshot, item); - } - return result; -}; - -/** Replay metadata only. Both materialization and the independent disk checker can call this. */ -export const replayVolumeSnapshot = (snapshots: VolumeSnapshot[], snapshot: string): ReplayedVolumeSnapshot => { - const entries = catalog(snapshots), seen = new Set(), chain: VolumeSnapshot[] = []; - let next: string | null = snapshot; - while (next !== null) { - if (seen.has(next)) throw new Error(`Cycle in volume snapshot chain at ${next}`); - seen.add(next); - const current = entries.get(next); - if (!current) throw new Error(`Missing volume snapshot ancestor ${next}`); - const descendant = chain.at(-1); - if (descendant && compareInstants(current.takenAt, descendant.takenAt) > 0) { - throw new Error(`Volume snapshot parent ${next} is later than child ${descendant.snapshot}`); - } - chain.push(current); next = current.parent; - } - chain.reverse(); - const files = new Map(); - for (const item of chain) { - for (const [file, digest] of Object.entries(item.files)) { - if (digest === null) files.delete(file); else files.set(file, digest); - } - for (const file of files.keys()) { - const components = file.split("/"); - for (let length = 1; length < components.length; length++) { - const parent = components.slice(0, length).join("/"); - if (files.has(parent)) throw new Error(`Snapshot ${item.snapshot} has a file/directory collision at ${parent}`); - } +/** A published row plus retained directory and manifest is complete; never infer time from its name. */ +export const selectVolumeSnapshot = async (history: History, agentSlug: string, agentId: string, id: string, + at: string, recordId?: string): Promise => { + const rowsPath = `recording/${safeSegment(agentSlug)}/wake-moments.jsonl`; + const rows = await history.jsonl(rowsPath, wakeSnapshotRowSchema); + if (!rows) throw new Error(`Missing volume snapshot history ${rowsPath}`); + const candidates = rows.flatMap((row, index) => row.agent_id !== agentId ? [] : row.snapshots.filter((entry) => entry.id === id) + .map((entry) => ({ row, entry, index }))) + .filter(({ row, entry }) => recordId === undefined ? compareInstants(row.started_at, at) <= 0 : entry.snapshot === recordId) + .sort((a, b) => compareInstants(b.row.started_at, a.row.started_at) || b.index - a.index); + for (const { row, entry } of candidates) { + if (entry.snapshot.endsWith(".partial")) { + if (recordId !== undefined) throw new Error(`Recorded volume snapshot ${recordId} is partial`); + continue; } - } - return { snapshot, takenAt: chain.at(-1)!.takenAt, - files: Object.fromEntries([...files].sort(([a], [b]) => a.localeCompare(b))), chain }; -}; - -export const selectVolumeSnapshot = (snapshots: VolumeSnapshot[], at: string, recordId?: string): VolumeSnapshot => { - const entries = catalog(snapshots); - const candidates = [...entries.values()].filter((item) => compareInstants(item.takenAt, at) <= 0) - .sort((a, b) => compareInstants(b.takenAt, a.takenAt)); - if (!candidates.length) throw new Error("No volume snapshot was taken at or before the moment"); - let selected = candidates[0]!; - for (const candidate of candidates.slice(1)) { - if (compareInstants(candidate.takenAt, selected.takenAt) !== 0) break; - if (replayVolumeSnapshot(snapshots, candidate.snapshot).chain.some((item) => item.snapshot === selected.snapshot)) { - selected = candidate; - } else if (!replayVolumeSnapshot(snapshots, selected.snapshot).chain.some((item) => item.snapshot === candidate.snapshot)) { - throw new Error("Ambiguous volume snapshots taken at the same instant on separate branches"); + if (compareInstants(row.started_at, at) > 0) throw new Error(`Recorded volume snapshot ${entry.snapshot} is later than the moment`); + const relative = snapshotHistoryPath(agentSlug, id, entry.snapshot), tree = await history.path(relative); + const manifest = await readSnapshotManifest(history, relative, id, entry.snapshot); + if (!tree || !manifest) { + if (recordId !== undefined) throw new Error(`Recorded volume snapshot ${recordId} is incomplete`); + continue; } + if (!(await lstat(tree)).isDirectory()) throw new Error(`Snapshot tree must be a real directory: ${relative}`); + if (compareInstants(manifest.started_at, row.started_at) !== 0) throw new Error(`Snapshot manifest started_at disagrees with its wake row: ${relative}`); + return { snapshot: entry.snapshot, startedAt: row.started_at, manifest, relative }; } - const chain = replayVolumeSnapshot(snapshots, selected.snapshot).chain; - if (recordId !== undefined) { - const pinned = chain.find((item) => item.snapshot === recordId); - if (!pinned) throw new Error(`Recorded volume snapshot ${recordId} is not the latest eligible snapshot or its ancestor`); - return pinned; - } - return selected; + if (recordId !== undefined) throw new Error(`Recorded volume snapshot ${recordId} is absent from ${rowsPath}`); + throw new Error("No complete volume snapshot was taken at or before the moment"); }; export const materializeVolume = async (context: InputContext): Promise => { @@ -96,27 +55,10 @@ export const materializeVolume = async (context: InputContext): Promise entry.id === input.id); - const selected = selectVolumeSnapshot(snapshots, at, pin?.snapshot); - const replayed = replayVolumeSnapshot(snapshots, selected.snapshot); - const objects = new Map(); - // Verify every object referenced by the replay, including content later overwritten or deleted. - for (const item of replayed.chain) for (const digest of Object.values(item.files)) { - if (digest === null || objects.has(digest)) continue; - const objectPath = `${base}/objects/${digest}`, bytes = await history.bytes(objectPath); - if (bytes === null) throw new Error(`Missing volume snapshot object ${objectPath}`); - if (createHash("sha256").update(bytes).digest("hex") !== digest) throw new Error(`Volume snapshot object SHA-256 mismatch: ${objectPath}`); - objects.set(digest, bytes); - } - await mkdir(destination, { recursive: true }); - for (const [file, digest] of Object.entries(replayed.files)) { - const target = path.join(destination, file); - await mkdir(path.dirname(target), { recursive: true }); - await writeFile(target, objects.get(digest)!); - } + const selected = await selectVolumeSnapshot(history, context.agentSlug, context.agentId, input.id, at, pin?.snapshot); + await copySnapshotTree(history, selected.relative, destination, selected.manifest); return { fidelity: pin ? "exact" : "approximate", basis: pin ? "record" : "snapshot-at", - source: { snapshot: selected.snapshot }, notes: [], instant: selected.takenAt }; + source: { snapshot: selected.snapshot, agentSlug: context.agentSlug }, notes: [], instant: selected.startedAt }; }; From 6d827df38b9a6e9ad66b82fccdb896be576367fe Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sun, 11 Oct 2026 04:38:27 +0200 Subject: [PATCH 6/9] fix: verify retained feed bytes, bound the trial instant and refuse conflicting record copies --- specs/TRAINING.md | 46 +++++++-- src/compiler/training/rewind/AGENTS.md | 2 + src/compiler/training/rewind/contract.ts | 2 +- src/compiler/training/rewind/feed.test.ts | 37 +++++-- src/compiler/training/rewind/feed.ts | 55 ++++++----- .../training/rewind/feedContent.test.ts | 99 +++++++++++++++++++ src/compiler/training/rewind/feedContent.ts | 36 +++++++ src/compiler/training/rewind/gitProcess.ts | 30 +++--- src/compiler/training/rewind/leakFiles.ts | 6 +- src/compiler/training/rewind/leakInputs.ts | 23 ++++- .../training/rewind/leakProvenance.test.ts | 4 +- .../training/rewind/leakRepositories.test.ts | 8 +- .../training/rewind/materialize.test.ts | 62 +++++++++++- src/compiler/training/rewind/materialize.ts | 57 ++++++++--- src/compiler/training/rewind/recordInput.ts | 16 +-- .../training/rewind/recordedCommit.test.ts | 22 +++-- .../training/rewind/recordingFormat.test.ts | 13 ++- src/compiler/training/rewind/select.test.ts | 28 ++++++ src/compiler/training/rewind/select.ts | 47 ++++----- .../training/rewind/taskClock.test.ts | 11 +++ src/compiler/training/rewind/taskClock.ts | 17 +++- .../training/rewind/timestamps.test.ts | 24 +++++ src/compiler/training/rewind/timestamps.ts | 3 +- 23 files changed, 528 insertions(+), 120 deletions(-) create mode 100644 src/compiler/training/rewind/feedContent.test.ts create mode 100644 src/compiler/training/rewind/feedContent.ts create mode 100644 src/compiler/training/rewind/timestamps.test.ts diff --git a/specs/TRAINING.md b/specs/TRAINING.md index b6ff3866..8348efa4 100644 --- a/specs/TRAINING.md +++ b/specs/TRAINING.md @@ -280,7 +280,10 @@ boundaries. The isolated compatibility reader must be replaced by P3's Moment records match exact compiled `agent` and parsed `wake.at == example.at`. For several attempts of that wake, select the EARLIEST `wake.startedAt` (original turn), even when replay pins differ or append/capture order differs. Conflicting -copies at that same earliest instant still refuse when their used pins differ. +copies at that same earliest instant still refuse when any input's effective +decision differs: usable pin versus fallback, selected pin, or fallback fidelity +given `exact` and input-specific `errors`. The refusal MUST name the input. +Unrelated errors and unusable pin values do not create a conflict. Validate recorded commit IDs in every earliest-attempt copy for each used input, including IDs hidden by declared pin precedence; later replays cannot replace or invalidate the original turn's pins. @@ -370,10 +373,21 @@ commit MUST be verified as a commit object with author AND committer dates `<= effectiveAt`. This single selection guard applies equally to plain, retained, prepared, and included git feeds; an eligible landing time does not make a future source commit eligible. Directory feeds have no git source commit. -Feed bytes come from OPTIONAL retained `trees//` when available. -Otherwise a plain git feed re-extracts the landing's expanded paths at its commit -and MUST verify the revision with the existing feed revision function. A mismatch -refuses. `include` or `prepare` requires retained bytes; rewind MUST NOT run them. +Plain git feed bytes MUST come from re-extraction of the landing's expanded paths +at its selected commit, with revision verification using the existing feed +revision function. When retained `trees//` bytes also exist, they MUST +match that extraction (paths, types, executable bits, symlink targets and bytes); +any mismatch refuses. A missing history repository refuses even when retained +bytes exist. Acceptance MUST independently re-extract from the history repository +at the selected commit and paths and compare the bundle bytes and revision. +`include` or `prepare` requires retained bytes; rewind MUST NOT run them. +Prepared revisions identify source/include/recipe provenance, not output content. +When prepared provenance is retained, its composite revision MUST match, and its +git source revision MUST match the selected commit and paths. Prepared output +MUST always be `approximate`, with the note `prepared output not content-verifiable` +and `source.prepared: true` in the moment input (including record-only pins without +a landing log). The independent acceptance gate enforces that fidelity and note. +All leak-check classes still apply to prepared output. Raw blob extraction ignores repository-local `info/attributes` as well as committed archive attributes; later tags cannot change historical file bytes. Output preserves the agent-visible `trees//`, including re-land suffixes, relative `current` @@ -382,9 +396,10 @@ is the selected landing time, or the selected commit's committer time for an feed-rule approximation. A recorded pin or ambiguous incoming transition may have publication metadata after `effectiveAt`; this does not advance the trial clock or filesystem timestamps. The independent gate reopens its record/window -before allowing such publication metadata. Directory feeds are not refetchable: -retained tree bytes MUST recompute to the recorded revision or refuse (including -prepared directory revisions whose composite identity cannot be reproduced). +before allowing such publication metadata. Plain directory feeds are not +refetchable: retained tree bytes MUST recompute to the recorded revision or refuse. +Prepared directory bytes MUST NOT be compared to their provenance revision as +though it were a directory content digest. Memory uses Mneme's single append-only ledger. Any recorded `generation`, including 0, refuses as unsupported: Mneme has no generations or curation @@ -472,6 +487,14 @@ share `originEpochMs + (realNowEpochMs - anchorEpochMs)`; independently started children MUST NOT reset their clock to the origin. `originEpochMs` uses JavaScript `Date.parse(origin)` millisecond precision. +Both dataset `at` and selected `effectiveAt` MUST be within the inclusive range +`1970-01-01T00:00:00Z` .. `9999-12-31T23:59:59.999Z`, compared as instants before +millisecond truncation. Out-of-range trials MUST refuse before any materialization +or output creation. This range is shared with Daimon's ecosystem task-clock +contract. Historical source timestamps may precede 1970; filesystem stamping and +restoration MUST preserve their negative epochs using `Date` or numeric-string +arguments to `utimes`/`lutimes`, never negative numeric seconds. + Each process MUST have exactly one offset owner. These pure helpers replace `taskClockEnvironment`; their outputs MUST NOT be merged for a single process: @@ -523,6 +546,9 @@ messages/events, and snapshots. Each finding carries `{input, class, path, detail}`; `input` is an input identity or `null` for bundle-level problems. Metadata is captured before payload reads; the checker restores the captured access times after inspection, preserving pre-existing future atimes for refusal. +After restoration and rename, filesystem metadata MUST be checked again on the +final published bundle. A failure removes the payload and records the refusal; +the pre-publication check alone cannot establish the published timestamps. Every git-backed input MUST record its selected full commit ID in `moment.json`'s `inputs[].source.commit`; git feed identity MUST record the same ID. Acceptance MUST independently open the source history repository read-only, @@ -1686,6 +1712,10 @@ The child deadline is Paideia's declared deadline plus five seconds for cleanup. "type": "string", "format": "date-time", "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "prepared": { + "type": "boolean", + "const": true } }, "additionalProperties": false diff --git a/src/compiler/training/rewind/AGENTS.md b/src/compiler/training/rewind/AGENTS.md index 9661e8b6..063c96c7 100644 --- a/src/compiler/training/rewind/AGENTS.md +++ b/src/compiler/training/rewind/AGENTS.md @@ -6,6 +6,7 @@ - `types.ts` defines input identities, declarations and per-kind contexts/results; `index.ts` exports the public materializer and pure selection/cut helpers. - `git.ts` owns resources; `gitProcess.ts`, `gitHistory.ts`, `gitTree.ts`, and `gitArchive.ts` isolate argv execution, containment, blob trees, and raw feed trees; `recordedCommit.ts` verifies full recorded IDs name commit objects without peeling. - `feed.ts`, `memory.ts`, and `volumes.ts` own input reconstruction. `feedLandings.ts` is the sole P3 log compatibility reader (replace with readFeedLandings/feedLandingAt after merge); `snapshotManifest.ts` verifies Daimon full trees; `memorySchemas.ts` validates ledger events. Mneme generations are unsupported; seq:0 is exact empty. +- `feedContent.ts` verifies plain git revisions and re-extracts for independent byte acceptance; retained copies must match. Prepared revisions prove provenance only: always approximate, never compared to a directory content digest. - `moltnet.ts` dispatches stores; `moltnetJson.ts`, `moltnetSqlite.ts`, `moltnetSqliteData.ts`, and `moltnetSchemas.ts` own snapshots, copied databases, SQL decoding, and strict native shapes. - `materialize.ts` stages all inputs and publishes only after the independent disk reader in `leakCheck.ts` accepts them. - `leakFiles.ts`, `leakGit.ts`, and `leakInputs.ts` inspect disk metadata, every git object, and native payloads independently of materialization decisions. `leakProvenance.ts` reopens source repositories read-only using disk commit receipts, checking type and both raw dates without the producer's guard. Discover unexpected `.git` directories/gitfiles and bare repositories throughout every input; only the materialized git resource repository is allowed. @@ -21,6 +22,7 @@ - Validate memory JSONL structure locally; rewind never imports Mneme or ships an index/audit state. Checksum verification is skipped because persisted normalized events cannot recover the original JSON checksum input (`node_modules/@noopolis/mneme/dist/identity/ids.js:15-17`, `dist/store/store.js:249-274`). - Mneme 0.1.1's index is rebuilt from the ledger before every index read, not on open: `node_modules/@noopolis/mneme/dist/runtime/runtime.js:86-87` (embedding recall), `node_modules/@noopolis/mneme/dist/kernel/kernel.js:152-153` (search), and `:207-208` (locate). A ledger-only materialized bank is complete. - Inspect atime and mtime from metadata captured before payload reads, then restore captured access times without hiding existing future atimes. +- Preserve negative source epochs with Date/numeric-string filesystem times; recheck the final renamed bundle. Trial at/effectiveAt must be within 1970 through 9999 before materialization. - Generated inputs spanning repositories refuse: this history contract carries one commit per resource. Never claim that one repository proves another repository's bytes. - `recordedBundle.ts` requires digest-matching retained tars for exact bundles and independently checks extracted bytes; source reconstruction is approximate. `leakFeed.ts` proves late publication metadata against record/window history. - Keep modules under 400 lines, use named/barrel exports, and never import Mneme internals. diff --git a/src/compiler/training/rewind/contract.ts b/src/compiler/training/rewind/contract.ts index 76df3609..7acfae87 100644 --- a/src/compiler/training/rewind/contract.ts +++ b/src/compiler/training/rewind/contract.ts @@ -44,7 +44,7 @@ export const materializedInputSchema = inputIdentitySchema.extend({ fidelity: z.enum(["exact", "approximate", "overridden"]).nullable(), basis: text, source: z.object({ commit: text.optional(), repository: text.optional(), revision: text.optional(), seq: count.optional(), sha256: recordedSha256Schema.optional(), agentSlug: text.optional(), snapshot: text.optional(), cursor: text.optional(), - switchingAt: trainingInstantSchema.optional(), landedAt: trainingInstantSchema.optional() }).strict(), + switchingAt: trainingInstantSchema.optional(), landedAt: trainingInstantSchema.optional(), prepared: z.literal(true).optional() }).strict(), path: text.nullable(), notes: z.array(z.string()), refusal: text.nullable(), build: z.literal("deferred").optional() }).strict(); export type MaterializedInput = z.infer; diff --git a/src/compiler/training/rewind/feed.test.ts b/src/compiler/training/rewind/feed.test.ts index 6c31c23e..d727c024 100644 --- a/src/compiler/training/rewind/feed.test.ts +++ b/src/compiler/training/rewind/feed.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { mkdir, readFile, readdir, readlink, rm, symlink, writeFile } from "node:fs/promises"; import path from "node:path"; +import { createHash } from "node:crypto"; import type { TeamWorkspaceResource } from "../../../manifest/index.js"; import { digestDirectory, feedIdentityFindings, resolveFeedSource } from "../../../volume/index.js"; import { materializeFeed, type FeedLanding } from "./feed.js"; @@ -126,17 +127,36 @@ describe("feed rewind", () => { const { context, resource, directory, landing, log } = await fixture(); resource.feed = { git: { repo: "never-used", ref: { command: ["never-run-ref"] } }, freeze: { timezone: "UTC", after: "08:00" }, include: [{ from: "not-on-disk", to: "assets" }], prepare: { host: true, command: ["never-run-prepare"] } }; - const retained = path.join(directory, "trees", `${landing.revision}.2`); + const prepared = { include: [], prepare: { command: ["never-run-prepare"], image: null, network: null, platform: null }, source_revision: landing.revision }; + const revision = createHash("sha256").update(`prepared\n${JSON.stringify(prepared)}\n`).digest("hex"); + const retained = path.join(directory, "trees", `${revision}.2`); await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "prepared.txt"), "retained\n"); - await log([{ ...landing, tree: `trees/${landing.revision}.2`, source: { kind: "git", repo: "never-used", prepared: { - source_revision: landing.revision, include: [], prepare: { command: ["never-run-prepare"], image: null, network: null, platform: null } - } } }]); - expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "landing-log" }); - expect(await readlink(path.join(context.destination, "current"))).toBe(`trees/${landing.revision}.2`); + await log([{ ...landing, revision, tree: `trees/${revision}.2`, source: { kind: "git", repo: "never-used", prepared } }]); + expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", basis: "landing-log", + notes: ["prepared output not content-verifiable"] }); + expect(await readlink(path.join(context.destination, "current"))).toBe(`trees/${revision}.2`); expect(await readFile(path.join(context.destination, "current/prepared.txt"), "utf8")).toBe("retained\n"); expect(JSON.parse(await readFile(path.join(context.destination, ".spawnfile-feed.json"), "utf8")).source.prepared.prepare.command).toEqual(["never-run-prepare"]); }); + it.each(["unchanged", "changed bytes", "extra file", "missing file", "wrong revision", "missing repository"]) + ("verifies retained plain git feed with %s against commit and paths", async (change) => { + const f = await fixture(); + const revision = change === "wrong revision" ? "0".repeat(64) : f.revision; + const retained = path.join(f.directory, "trees", revision); + await mkdir(path.join(retained, "published"), { recursive: true }); + if (change !== "missing file") await writeFile(path.join(retained, "published/news.txt"), change === "changed bytes" ? "future\n" : "past\n"); + if (change === "extra file") await writeFile(path.join(retained, "extra.txt"), "future\n"); + await f.log([{ ...f.landing, revision, tree: revision }]); + if (change === "missing repository") await rm(f.repository, { recursive: true }); + if (change === "unchanged") { + expect(await materializeFeed(f.context)).toMatchObject({ fidelity: "exact" }); + expect(await readFile(path.join(f.context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); + expect(await readdir(f.context.destination)).not.toContain(".retained-check"); + } else await expect(materializeFeed(f.context)).rejects.toThrow(change === "wrong revision" ? "Feed revision mismatch" + : change === "missing repository" ? "Missing feed source repository" : "Retained git feed bytes disagree with selected commit and paths"); + }); + it("uses retained directory feed landings and starts overridden feeds empty", async () => { const { context, resource, directory, log } = await fixture(); resource.feed = { directory: "never-read" }; @@ -201,8 +221,9 @@ describe("feed rewind", () => { resource.feed = { git: { repo: "never-read" }, freeze: { after: "08:00", timezone: "UTC" } }; await expect(materializeFeed(context)).rejects.toThrow("freeze/waiting"); resource.feed = { git: { repo: "never-read" }, prepare: { host: true, command: ["never-execute"] } }; - await log([{ ...landing, source: { kind: "git", repo: "never-read", prepared: { source_revision: landing.revision, - include: [], prepare: { command: ["never-execute"], image: null, network: null, platform: null } } } }]); + const prepared = { include: [], prepare: { command: ["never-execute"], image: null, network: null, platform: null }, source_revision: landing.revision }; + const revision = createHash("sha256").update(`prepared\n${JSON.stringify(prepared)}\n`).digest("hex"); + await log([{ ...landing, revision, tree: revision, source: { kind: "git", repo: "never-read", prepared } }]); await expect(materializeFeed(context)).rejects.toThrow("retained landed tree bytes"); context.history = undefined; await expect(materializeFeed(context)).rejects.toThrow("--history is required"); diff --git a/src/compiler/training/rewind/feed.ts b/src/compiler/training/rewind/feed.ts index 7cf658a6..7f215260 100644 --- a/src/compiler/training/rewind/feed.ts +++ b/src/compiler/training/rewind/feed.ts @@ -1,13 +1,14 @@ -import { lstat, mkdir, readdir, symlink, writeFile } from "node:fs/promises"; +import { lstat, mkdir, readdir, rm, symlink, writeFile } from "node:fs/promises"; import path from "node:path"; import type { TeamWorkspaceResource } from "../../../manifest/index.js"; import { digestDirectory, expandFeedPaths, expandFeedRefTemplate, FEED_IDENTITY_FILE, FEED_IDENTITY_VERSION, - resolveFeedSource, type FeedExec, type FeedIdentity } from "../../../volume/index.js"; + type FeedIdentity } from "../../../volume/index.js"; +import { assertPreparedFeedRevision, feedGitRevision } from "./feedContent.js"; import { resolveRecordedCommit } from "./recordedCommit.js"; import { feedLandingSchema, readFeedLandingHistory, selectFeedLanding, type FeedLanding } from "./feedLandings.js"; import { exportFeedGitTree } from "./gitArchive.js"; import { assertLocalGitRepository } from "./gitHistory.js"; -import { assertCommitNotFuture, commitAt, rewindGitSync, type SelectedCommit } from "./gitProcess.js"; +import { assertCommitNotFuture, commitAt, type SelectedCommit } from "./gitProcess.js"; import { historyKey } from "./history.js"; import { compareInstants } from "./instant.js"; import type { InputContext, InputResult } from "./types.js"; @@ -15,10 +16,6 @@ import type { InputContext, InputResult } from "./types.js"; export { feedLandingSchema, selectFeedLanding, type FeedLanding } from "./feedLandings.js"; type VolumeFeed = NonNullable["feed"]>; -const revisionExec: FeedExec = (command, args) => { - if (command !== "git" || args[0] !== "-C" || !args[1]) throw new Error("Feed revision may only read a local git repository"); - return rewindGitSync(args[1], args.slice(2)); -}; const feedPaths = (paths: string[] | undefined): string[] | undefined => paths?.length ? paths : undefined; const treeName = (tree: string): string => tree.replace(/^trees\//u, ""); @@ -96,9 +93,9 @@ export const materializeFeed = async (context: InputContext): Promise 0 ? context.at : landing.landed_at, - notes: ambiguous ? ["The trial instant falls inside an ambiguous feed transition; the incoming landing is used."] - : basis === "feed-rule-at" ? ["Commit dates do not establish when a ref advanced; historical freeze holds cannot be reconstructed."] : [] }; + notes: [...(isPrepared ? ["prepared output not content-verifiable"] : []), + ...(ambiguous ? ["The trial instant falls inside an ambiguous feed transition; the incoming landing is used."] + : basis === "feed-rule-at" ? ["Commit dates do not establish when a ref advanced; historical freeze holds cannot be reconstructed."] : [])] }; }; diff --git a/src/compiler/training/rewind/feedContent.test.ts b/src/compiler/training/rewind/feedContent.test.ts new file mode 100644 index 00000000..888ebba4 --- /dev/null +++ b/src/compiler/training/rewind/feedContent.test.ts @@ -0,0 +1,99 @@ +import { mkdir, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { digestDirectory, hostExec } from "../../../volume/index.js"; +import { resolveFeedContent } from "../../../volume/feedPrepare.js"; +import { cleanTrainingProjects } from "../plan/fixtures.test-helper.js"; +import * as feed from "./feed.js"; +import { AT, BEFORE, emptyRecord, jsonl, rewindFixture } from "./fixtures.test-helper.js"; +import { leakCheckTrainingMoment } from "./leakCheck.js"; +import { materializeTrainingMoment, resolveTrainingMoment } from "./materialize.js"; +import { stampTree } from "./timestamps.js"; + +afterEach(cleanTrainingProjects); +afterEach(() => vi.restoreAllMocks()); + +const preparedFixture = async (kind: "git" | "directory", recipe: "include" | "prepare", recordOnly = false) => { + const f = await rewindFixture(), directory = path.dirname(f.feedRepo), sourceDirectory = path.join(f.root, "source"); + await mkdir(sourceDirectory); await writeFile(path.join(sourceDirectory, "source.txt"), "source bytes\n"); + const resolved = resolveFeedContent({ resourceId: "news", volume: directory, volumeName: "news", stateDir: directory, keep: 1, healLimit: 3, + source: kind === "git" ? { kind, repo: f.feedRepo, ref: f.commit, fetch: false } : { kind, directory: sourceDirectory }, + ...(recipe === "include" ? { include: [] } : { prepare: { kind: "host" as const, command: ["never-run-recipe"], timeoutMs: 1 } }) + }, { exec: hostExec }); + const retained = path.join(directory, "trees", resolved.revision); + await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "prepared.txt"), "landed prepared bytes\n"); + expect(digestDirectory(retained)).not.toBe(resolved.revision); + const log = "feeds/team%3Apublication/news/landings.jsonl"; + await jsonl(f.historyDir, log, [{ version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "news", + revision: resolved.revision, tree: `trees/${resolved.revision}`, landed_at: BEFORE, + source: { kind, ...(kind === "git" ? { repo: f.feedRepo } : { directory: sourceDirectory }), prepared: resolved.provenance.prepared }, + ...(kind === "git" ? { commit: f.commit, ref: "main" } : {}) }]); + if (recordOnly) { + const input = resolveTrainingMoment(f.plan, f.project, f.options.example).inputs.find((entry) => entry.id === "news" && entry.section === "resources")!; + if (input.section !== "resources" || input.declaration.kind !== "volume") throw new Error("Expected feed"); + input.declaration.feed = { ...(kind === "git" ? { git: { repo: f.feedRepo } } : { directory: sourceDirectory }), include: [] }; + const record = emptyRecord(); record.feeds = [{ id: "news", revision: resolved.revision, tree: `trees/${resolved.revision}`, landedAt: BEFORE, + ...(kind === "git" ? { commit: f.commit, ref: "main" } : {}) }]; + await jsonl(f.historyDir, "moments/prepared.jsonl", [record]); await rm(path.join(f.historyDir, log)); + } + return f; +}; + +describe("independent feed byte verification", () => { + it.each(["changed bytes", "extra file", "wrong revision", "missing repository"]) + ("rejects a broken plain git producer with %s at acceptance", async (change) => { + const f = await rewindFixture(), correct = feed.materializeFeed; + vi.spyOn(feed, "materializeFeed").mockImplementation(async (context) => { + const result = await correct(context); + if (change === "missing repository") await rm(f.feedRepo, { recursive: true }); + else if (change === "wrong revision") { + const file = path.join(context.destination, ".spawnfile-feed.json"), identity = JSON.parse(await readFile(file, "utf8")); + identity.revision = "0".repeat(64); result.source.revision = identity.revision; + await writeFile(file, JSON.stringify(identity)); + } else await writeFile(path.join(context.destination, "current", change === "changed bytes" ? "news.txt" : "extra.txt"), "future bytes\n"); + return result; + }); + const result = await materializeTrainingMoment(f.options); + expect(result.status).toBe("refused"); + const check = JSON.parse(await readFile(path.join(f.options.out, "leak-check.json"), "utf8")); + expect(check.findings).toContainEqual(expect.objectContaining({ class: "feed-identity", path: "resources/team%3Apublication/news", + detail: change === "missing repository" ? "Missing feed source history repository for content verification" + : change === "wrong revision" ? "Feed revision disagrees with selected commit and paths" : "Git feed bytes disagree with selected commit and paths" })); + expect(check.findings.some((finding: { class: string }) => finding.class === "file-time")).toBe(false); + expect(await readdir(f.options.out)).toEqual(["leak-check.json", "moment.json"]); + }); + + it.each((["git", "directory"] as const).flatMap((kind) => (["include", "prepare"] as const).map((recipe) => ({ kind, recipe })))) + ("accepts recording-format $kind/$recipe provenance with approximate prepared bytes", async ({ kind, recipe }) => { + const f = await preparedFixture(kind, recipe), result = await materializeTrainingMoment(f.options); + expect(result, JSON.stringify(result.inputs)).toMatchObject({ status: "materialized", leakCheck: { ok: true } }); + const input = result.inputs.find((entry) => entry.kind === "feed")!; + expect(input).toMatchObject({ fidelity: "approximate", source: { prepared: true }, notes: ["prepared output not content-verifiable"] }); + expect(await readFile(path.join(f.options.out, input.path!, "current/prepared.txt"), "utf8")).toBe("landed prepared bytes\n"); + await stampTree(f.options.out, BEFORE, AT); + expect((await leakCheckTrainingMoment(f.options.out, { historyDir: f.historyDir })).findings).toEqual([]); + result.inputs.find((entry) => entry.kind === "feed")!.fidelity = "exact"; + await writeFile(path.join(f.options.out, "moment.json"), JSON.stringify(result)); await stampTree(f.options.out, BEFORE, AT); + expect((await leakCheckTrainingMoment(f.options.out, { historyDir: f.historyDir })).findings) + .toContainEqual(expect.objectContaining({ class: "feed-identity", detail: "Prepared feed must be approximate: prepared output not content-verifiable" })); + }); + + it.each(["git", "directory"] as const)("keeps record-only prepared %s bytes approximate when the landing log is absent", async (kind) => { + const f = await preparedFixture(kind, "include", true), result = await materializeTrainingMoment(f.options); + expect(result, JSON.stringify(result.inputs)).toMatchObject({ status: "materialized", leakCheck: { ok: true } }); + expect(result.inputs.find((entry) => entry.kind === "feed")).toMatchObject({ fidelity: "approximate", basis: "record", + source: { prepared: true }, notes: ["prepared output not content-verifiable"] }); + }); + + it("still applies filesystem leak classes to prepared output", async () => { + const f = await preparedFixture("directory", "prepare"), result = await materializeTrainingMoment(f.options); + expect(result.status).toBe("materialized"); + const tree = path.join(f.options.out, result.inputs.find((entry) => entry.kind === "feed")!.path!, "current"); + await mkdir(path.join(tree, ".git")); await writeFile(path.join(tree, "2099-01-01.txt"), "future path"); + await symlink("/outside", path.join(tree, "escape")); await stampTree(f.options.out, BEFORE, AT); + await writeFile(path.join(tree, "prepared.txt"), "changed now"); + const check = await leakCheckTrainingMoment(f.options.out, { historyDir: f.historyDir }); + for (const kind of ["file-time", "git-object", "dated-path", "symlink-escape"]) expect(check.findings) + .toContainEqual(expect.objectContaining({ class: kind, path: expect.stringContaining("resources/team%3Apublication/news/trees/") })); + }); +}); diff --git a/src/compiler/training/rewind/feedContent.ts b/src/compiler/training/rewind/feedContent.ts new file mode 100644 index 00000000..e9c873c9 --- /dev/null +++ b/src/compiler/training/rewind/feedContent.ts @@ -0,0 +1,36 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { digestDirectory, resolveFeedSource, type FeedExec, type FeedIdentity } from "../../../volume/index.js"; +import { exportFeedGitTree } from "./gitArchive.js"; +import { rewindGitSync } from "./gitProcess.js"; + +const revisionExec: FeedExec = (command, args) => { + if (command !== "git" || args[0] !== "-C" || !args[1]) throw new Error("Feed revision may only read a local git repository"); + return rewindGitSync(args[1], args.slice(2)); +}; +export const feedGitRevision = (repo: string, commit: string, ref: string, paths?: string[] | null): string => + resolveFeedSource({ kind: "git", fetch: false, repo, ref: commit, label: ref, paths: paths?.length ? paths : undefined }, + { exec: revisionExec }).revision; + +/** A prepared revision proves the recipe/source identity, never its output bytes. */ +export const assertPreparedFeedRevision = (revision: string, prepared: NonNullable): void => { + const provenance = { include: prepared.include, prepare: prepared.prepare, source_revision: prepared.source_revision }; + const actual = createHash("sha256").update(`prepared\n${JSON.stringify(provenance)}\n`).digest("hex"); + if (actual !== revision) throw new Error("Prepared feed revision disagrees with its provenance"); +}; + +/** Acceptance re-extracts from history; it does not consume any producer-computed content digest. */ +export const verifyPlainFeedTree = async (repo: string, identity: FeedIdentity, directory: string): Promise => { + const source = identity.source; + if (source.kind !== "git") throw new Error("Plain git feed verification requires git provenance"); + if (feedGitRevision(repo, source.commit, source.ref, source.paths) !== identity.revision) { + throw new Error("Feed revision disagrees with selected commit and paths"); + } + const scratch = await mkdtemp(path.join(os.tmpdir(), "spawnfile-feed-check-")); + try { + await exportFeedGitTree(repo, source.commit, scratch, source.paths?.length ? source.paths : undefined); + if (digestDirectory(directory) !== digestDirectory(scratch)) throw new Error("Git feed bytes disagree with selected commit and paths"); + } finally { await rm(scratch, { recursive: true, force: true }); } +}; diff --git a/src/compiler/training/rewind/gitProcess.ts b/src/compiler/training/rewind/gitProcess.ts index 0a4e7a3d..7bd4193c 100644 --- a/src/compiler/training/rewind/gitProcess.ts +++ b/src/compiler/training/rewind/gitProcess.ts @@ -39,16 +39,25 @@ export const rewindGitSync = (repo: string, args: readonly string[]): Buffer => export interface SelectedCommit { commit: string; instant: string } -export const commitInstant = async (repo: string, commit: string): Promise => { - const timestamp = Number((await rewindGit(repo, ["show", "-s", "--format=%ct", commit, "--"])).toString().trim()); - if (!Number.isSafeInteger(timestamp)) throw new Error(`Commit ${commit} has an invalid committer date`); - return new Date(timestamp * 1000).toISOString(); +/** Git's pretty formatter drops negative epochs. Read the signed raw dates instead. */ +const commitDates = async (repo: string, commit: string): Promise<{ author: string; committer: string }> => { + const headers = (await rewindGit(repo, ["cat-file", "commit", commit])).toString().split("\n\n", 1)[0]!; + const date = (role: string): string => { + const match = new RegExp(`^${role} .* (-?\\d+) [+-]\\d{4}$`, "mu").exec(headers); + const seconds = match ? Number(match[1]) : NaN; + if (!Number.isSafeInteger(seconds) || !Number.isFinite(new Date(seconds * 1000).getTime())) { + throw new Error(`Commit ${commit} has an invalid ${role} date`); + } + return new Date(seconds * 1000).toISOString(); + }; + return { author: date("author"), committer: date("committer") }; }; +export const commitInstant = async (repo: string, commit: string): Promise => (await commitDates(repo, commit)).committer; /** Validate the selected source before any copy/export, regardless of the materialized layout. */ export const assertCommitNotFuture = async (repo: string, commit: string, at: string): Promise => { - const dates = (await rewindGit(repo, ["show", "-s", "--format=%at %ct", commit, "--"])).toString().trim().split(" "); - if (dates.length !== 2 || dates.some((value) => !Number.isSafeInteger(Number(value)) || compareInstants(new Date(Number(value) * 1000).toISOString(), at) > 0)) { + const dates = await commitDates(repo, commit); + if (Object.values(dates).some((instant) => compareInstants(instant, at) > 0)) { throw new Error(`Selected commit ${commit} has an author or committer date after ${at}`); } }; @@ -63,12 +72,11 @@ export const resolveRewindCommit = async (repo: string, selector: string): Promi export const commitAt = async (repo: string, selector: string, at: string): Promise => { let head: string; try { head = await resolveRewindCommit(repo, selector); } catch { return null; } - const lines = (await rewindGit(repo, ["log", "--first-parent", "--format=%H %ct", head, "--"])) + const commits = (await rewindGit(repo, ["log", "--first-parent", "--format=%H", head, "--"])) .toString().trim().split("\n").filter(Boolean); - return selectCommitAt(lines.map((line) => { - const [commit, seconds] = line.split(" "); - return { commit: commit!, instant: new Date(Number(seconds) * 1000).toISOString() }; - }), at); + const dated: SelectedCommit[] = []; + for (const commit of commits) dated.push({ commit, instant: await commitInstant(repo, commit) }); + return selectCommitAt(dated, at); }; /** Stable ties preserve the first-parent order returned by git. */ diff --git a/src/compiler/training/rewind/leakFiles.ts b/src/compiler/training/rewind/leakFiles.ts index 9c33cd6e..e6124aa1 100644 --- a/src/compiler/training/rewind/leakFiles.ts +++ b/src/compiler/training/rewind/leakFiles.ts @@ -30,14 +30,16 @@ export const snapshotBundleFiles = async (root: string): Promise): Promise => { for (const [file, stats] of [...files].reverse()) { - await (stats.isSymbolicLink() ? lutimes : utimes)(file, Number(stats.atimeNs / 1_000n) / 1_000_000, Number(stats.mtimeNs / 1_000n) / 1_000_000); + // Numeric strings preserve sub-millisecond precision; negative numeric arguments mean "now" in Node. + await (stats.isSymbolicLink() ? lutimes : utimes)(file, + String(Number(stats.atimeNs / 1_000n) / 1_000_000), String(Number(stats.mtimeNs / 1_000n) / 1_000_000)); } }; /** No symlinks are traversed. Inspection never reads a target outside the bundle. */ export const inspectBundleFiles = async (root: string, inputs: MaterializedInput[], report: LeakCheck, files: Map): Promise => { const at = instantNanos(report.at); - const latestDate = new Date(Date.parse(report.at) + 14 * 3600_000).toISOString().slice(0, 10); + const latestDate = new Date(Math.min(253_402_300_799_999, Date.parse(report.at) + 14 * 3600_000)).toISOString().slice(0, 10); const visit = async (file: string): Promise => { const relative = path.relative(root, file).split(path.sep).join("/") || "."; const owner = inputAtPath(inputs, relative); diff --git a/src/compiler/training/rewind/leakInputs.ts b/src/compiler/training/rewind/leakInputs.ts index 08c1922b..a68dff90 100644 --- a/src/compiler/training/rewind/leakInputs.ts +++ b/src/compiler/training/rewind/leakInputs.ts @@ -3,8 +3,10 @@ import path from "node:path"; import { feedIdentityFindings, type FeedIdentity } from "../../../volume/feedRecord.js"; import type { LeakCheck, LeakFinding, MaterializedInput, MaterializedMoment } from "./contract.js"; import { digestDirectory } from "../../../volume/index.js"; +import { assertPreparedFeedRevision, feedGitRevision, verifyPlainFeedTree } from "./feedContent.js"; +import { assertLocalGitRepository } from "./gitHistory.js"; import { permitsFeedPublicationTime } from "./leakFeed.js"; -import { inside, safeSegment, type History } from "./history.js"; +import { historyKey, inside, safeSegment, type History } from "./history.js"; import { compareInstants } from "./instant.js"; import { diskPath } from "./leakFiles.js"; import { parseMemoryLedger } from "./memory.js"; @@ -65,8 +67,23 @@ export const inspectFeedBundle = async (root: string, input: MaterializedInput, || !inside(path.join(directory, "trees"), await realpath(current))) { addInputFinding(report, input, "feed-identity", "Feed current does not point to its tree inside the volume"); } else if (!(await lstat(await realpath(current))).isDirectory()) addInputFinding(report, input, "feed-identity", "Feed tree must be a directory"); - else if (identity.source.kind === "directory" && digestDirectory(await realpath(current)) !== identity.revision) { - addInputFinding(report, input, "feed-identity", "Directory feed revision disagrees with retained bytes"); + else { + const prepared = identity.source.prepared !== undefined || input.source.prepared === true; + if (prepared && (input.fidelity !== "approximate" || !input.notes.includes("prepared output not content-verifiable"))) { + addInputFinding(report, input, "feed-identity", "Prepared feed must be approximate: prepared output not content-verifiable"); + } + if (identity.source.prepared) assertPreparedFeedRevision(identity.revision, identity.source.prepared); + if (identity.source.kind === "directory" && !prepared && digestDirectory(await realpath(current)) !== identity.revision) { + addInputFinding(report, input, "feed-identity", "Directory feed revision disagrees with retained bytes"); + } + if (identity.source.kind === "git") { + const repository = await history?.repository(`feeds/${historyKey(input)}/repo`); + if (!repository) throw new Error("Missing feed source history repository for content verification"); + await assertLocalGitRepository(repository, history); + if (!prepared) await verifyPlainFeedTree(repository, identity, await realpath(current)); + else if (identity.source.prepared && feedGitRevision(repository, identity.source.commit, identity.source.ref, identity.source.paths) + !== identity.source.prepared.source_revision) throw new Error("Prepared feed source revision disagrees with selected commit and paths"); + } } }; diff --git a/src/compiler/training/rewind/leakProvenance.test.ts b/src/compiler/training/rewind/leakProvenance.test.ts index c01e65ee..5f11a9c6 100644 --- a/src/compiler/training/rewind/leakProvenance.test.ts +++ b/src/compiler/training/rewind/leakProvenance.test.ts @@ -37,7 +37,7 @@ const fixture = async (kind: Kind) => { const revision = resolveFeedSource({ kind: "git", fetch: false, repo: repository, ref: selected }).revision; const directory = path.dirname(repository), retained = path.join(directory, "trees", revision); await mkdir(retained, { recursive: true }); - await writeFile(path.join(retained, "data.txt"), selected === commit ? "past bytes\n" : "future retained bytes\n"); + await writeFile(path.join(retained, "data.txt"), "past bytes\n"); await writeFile(path.join(directory, "landings.jsonl"), JSON.stringify({ version: "spawnfile.volume-feed-landing.v1", resource: "source", volume: "source", source: { kind: "git", repo: repository }, revision, tree: revision, commit: selected, ref: "main", landed_at: BEFORE }) + "\n"); await stampTree(retained, BEFORE, AT); @@ -67,7 +67,7 @@ describe("independent source commit provenance", () => { const guard = vi.spyOn(gitProcess, "assertCommitNotFuture").mockResolvedValue(undefined); const moment = await f.stage(); guard.mockRestore(); expect(moment.inputs[0]).toMatchObject({ fidelity: "exact", source: { commit } }); - expect(await readFile(path.join(f.destination, "current/data.txt"), "utf8")).toBe("future retained bytes\n"); + expect(await readFile(path.join(f.destination, "current/data.txt"), "utf8")).toBe("past bytes\n"); await stampTree(f.bundle, BEFORE, AT); const out = path.join(f.root, "refused"); expect((await finalizeTrainingMoment(f.bundle, out, { historyDir: f.historyDir })).status).toBe("refused"); diff --git a/src/compiler/training/rewind/leakRepositories.test.ts b/src/compiler/training/rewind/leakRepositories.test.ts index 689b0ae2..456ac8e7 100644 --- a/src/compiler/training/rewind/leakRepositories.test.ts +++ b/src/compiler/training/rewind/leakRepositories.test.ts @@ -1,4 +1,4 @@ -import { cp, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { cp, mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { cleanTrainingProjects } from "../plan/fixtures.test-helper.js"; @@ -6,6 +6,7 @@ import { AT, rewindFixture, testGit } from "./fixtures.test-helper.js"; import { leakCheckTrainingMoment } from "./leakCheck.js"; import { finalizeTrainingMoment, materializeTrainingMoment } from "./materialize.js"; import { stampTree } from "./timestamps.js"; +import { digestDirectory } from "../../../volume/index.js"; afterEach(cleanTrainingProjects); const unexpected = "Unexpected .git repository metadata outside the materialized git resource repository"; @@ -19,6 +20,11 @@ describe("repository discovery across every bundle input", () => { await mkdir(retained, { recursive: true }); await cp(fixture.gitRepo, path.join(retained, "nested"), { recursive: true }); await stampTree(retained, AT, AT); + // A plain directory revision can match malicious repository bytes; repository discovery must still reject them. + landing.revision = digestDirectory(retained); landing.tree = `trees/${landing.revision}`; + landing.source = { kind: "directory", directory: "never-read" }; delete landing.commit; delete landing.ref; + await rename(retained, path.join(feedDirectory, landing.tree)); + await writeFile(path.join(feedDirectory, "landings.jsonl"), JSON.stringify(landing) + "\n"); const result = await materializeTrainingMoment(fixture.options); expect(result.status).toBe("refused"); const check = JSON.parse(await readFile(path.join(fixture.options.out, "leak-check.json"), "utf8")); diff --git a/src/compiler/training/rewind/materialize.test.ts b/src/compiler/training/rewind/materialize.test.ts index 1e5881e0..8f533d34 100644 --- a/src/compiler/training/rewind/materialize.test.ts +++ b/src/compiler/training/rewind/materialize.test.ts @@ -1,4 +1,4 @@ -import { cp, lstat, mkdir, readFile, readdir, symlink, writeFile } from "node:fs/promises"; +import { cp, lstat, mkdir, readFile, readdir, rename, symlink, utimes, writeFile } from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { loadTrainingProject, resolveTrainingPlan } from "../plan/index.js"; @@ -7,12 +7,72 @@ import { materializeTrainingMoment, resolveTrainingMoment } from "./materialize. import { leakCheckTrainingMoment } from "./leakCheck.js"; import * as gitMaterializer from "./git.js"; import * as volumeMaterializer from "./volumes.js"; +import { rewindGit } from "./gitProcess.js"; +import { selectTrainingExample } from "./select.js"; import { AT, BEFORE, FUTURE, emptyRecord, jsonl, rewindFixture, testGit } from "./fixtures.test-helper.js"; afterEach(cleanTrainingProjects); afterEach(() => vi.restoreAllMocks()); +vi.mock("node:fs/promises", async () => { + const actual = await vi.importActual("node:fs/promises"); + return { ...actual, rename: vi.fn(actual.rename) }; +}); describe("materializeTrainingMoment", () => { + it.each(["at", "effectiveAt"])("refuses out-of-range %s before creating output or dispatching inputs", async (field) => { + const f = await rewindFixture(), selected = selectTrainingExample(f.plan, f.options.example), original = selected.example; + const dispatch = vi.spyOn(gitMaterializer, "materializeGit"); + for (const instant of ["1969-12-31T23:59:59.999999999Z", "9999-12-31T23:59:59.999000001Z"]) { + const record = emptyRecord(); + if (field === "at") { + original.at = instant; + f.project.datasets[selected.sourcePath]!.value.examples.find((entry) => entry.id === selected.id)!.at = instant; + } + else { record.wake.startedAt = instant; await jsonl(f.historyDir, "moments/range.jsonl", [record]); } + const out = path.join(f.root, "not-created", "out"); + expect(() => resolveTrainingMoment(f.plan, f.project, f.options.example, field === "effectiveAt" ? [record] : [])) + .toThrow(/must be within 1970.*9999/u); + await expect(materializeTrainingMoment({ ...f.options, out })).rejects.toThrow(/must be within 1970.*9999/u); + expect(dispatch).not.toHaveBeenCalled(); + expect(await readdir(f.root)).not.toContain("not-created"); + } + }); + + it("preserves a pre-1970 source commit on the final published bundle", async () => { + const f = await rewindFixture(), old = "1960-01-01T00:00:00Z"; + testGit(f.gitRepo, ["read-tree", f.commit]); await symlink("news.txt", path.join(f.gitRepo, "old-link")); + testGit(f.gitRepo, ["add", "old-link"]); + const tree = testGit(f.gitRepo, ["write-tree"]); + const raw = `tree ${tree}\nauthor Rewind -315619200 +0000\ncommitter Rewind -315619200 +0000\n\nOld source\n`; + // Git's authoring CLI refuses negative dates; retained raw historical objects can still contain them. + const commit = (await rewindGit(f.gitRepo, ["hash-object", "--literally", "-t", "commit", "-w", "--stdin"], Buffer.from(raw))).toString().trim(); + const input = resolveTrainingMoment(f.plan, f.project, f.options.example).inputs.find((entry) => entry.id === "code")!; + if (input.section !== "resources" || input.declaration.kind !== "git") throw new Error("Expected git"); + input.declaration.ref = commit; delete input.declaration.branch; + const result = await materializeTrainingMoment(f.options); + expect(result, JSON.stringify(result.inputs)).toMatchObject({ status: "materialized", leakCheck: { ok: true } }); + const directory = path.join(f.options.out, result.inputs.find((entry) => entry.id === "code")!.path!); + for (const name of ["", "news.txt", "old-link"]) { + const stats = await lstat(path.join(directory, name)); + expect(stats.mtimeMs, name).toBe(Date.parse(old)); expect(stats.atimeMs, name).toBe(Date.parse(old)); + } + expect((await leakCheckTrainingMoment(f.options.out, { historyDir: f.historyDir })).findings).toEqual([]); + expect((await lstat(path.join(directory, "news.txt"))).mtimeMs).toBe(Date.parse(old)); + }); + + it("rejects timestamps changed during rename on the published bundle", async () => { + const f = await rewindFixture(), actual = await vi.importActual("node:fs/promises"); + vi.mocked(rename).mockImplementationOnce(async (from, to) => { + await actual.rename(from, to); + await utimes(path.join(String(to), "resources/team%3Apublication/code/news.txt"), new Date(FUTURE), new Date(BEFORE)); + }); + const result = await materializeTrainingMoment(f.options); + expect(result).toMatchObject({ status: "refused", leakCheck: { ok: false } }); + const check = JSON.parse(await readFile(path.join(f.options.out, "leak-check.json"), "utf8")); + expect(check.findings).toContainEqual(expect.objectContaining({ class: "file-time", path: "resources/team%3Apublication/code/news.txt", + detail: "Access time is later than the moment" })); + expect(await readdir(f.options.out)).toEqual(["leak-check.json", "moment.json"]); + }); it("stages every effective input, verifies disk, and preserves source times", async () => { const fixture = await rewindFixture(); const resolved = resolveTrainingMoment(fixture.plan, fixture.project, example.id); diff --git a/src/compiler/training/rewind/materialize.ts b/src/compiler/training/rewind/materialize.ts index 43a9ab04..c7b4c130 100644 --- a/src/compiler/training/rewind/materialize.ts +++ b/src/compiler/training/rewind/materialize.ts @@ -7,11 +7,12 @@ import { materializeGit } from "./git.js"; import { historyKey, inside, openHistory, type History } from "./history.js"; import { resolveRewindInputs } from "./inputs.js"; import { inspectTrainingMoment, type LeakCheckOptions } from "./leakCheck.js"; +import { inspectBundleFiles, restoreInspectedAccessTimes, snapshotBundleFiles } from "./leakFiles.js"; import { materializeMemory } from "./memory.js"; import { materializeMoltnet } from "./moltnet.js"; import { matchingMomentRecords, originalMomentRecords, selectMomentRecord } from "./select.js"; import { hasRecordPin, recordForInput } from "./recordInput.js"; -import { createTaskClock } from "./taskClock.js"; +import { assertTaskClockInstant, createTaskClock } from "./taskClock.js"; import { restoreAccessTimes, stampTree } from "./timestamps.js"; import type { InputContext, InputResult, RewindInput } from "./types.js"; import { materializeVolume } from "./volumes.js"; @@ -29,9 +30,12 @@ export const resolveTrainingMoment = ( plan: TrainingPlan, project: LoadedTrainingProject, example: string, records: MomentRecord[] = [] ) => { const resolved = resolveRewindInputs(plan, project, example); + assertTaskClockInstant(resolved.at); const candidateRecords = originalMomentRecords(matchingMomentRecords(records, plan.agent.id, resolved.at)); const record = selectMomentRecord(candidateRecords, plan.agent.id, resolved.at, resolved.inputs.filter((input) => !input.overridden)); - return { ...resolved, candidateRecords, record, effectiveAt: record?.wake.startedAt ?? resolved.at }; + const effectiveAt = record?.wake.startedAt ?? resolved.at; + assertTaskClockInstant(effectiveAt); + return { ...resolved, candidateRecords, record, effectiveAt }; }; const reason = (error: unknown): string => error instanceof Error ? error.message : String(error); @@ -89,6 +93,33 @@ export const finalizeTrainingMoment = async (staging: string, out: string, optio await restoreAccessTimes(staging, moment.effectiveAt); await assertEmptyOutput(out); await rename(staging, out); + if (check?.ok) { + // The post-restoration, post-rename payload is the acceptance target. + try { + const files = await snapshotBundleFiles(out); + const published = { ...check, checked: { ...check.checked, files: 0 }, findings: [] as typeof check.findings }; + try { await inspectBundleFiles(out, moment.inputs, published, files); } + finally { await restoreInspectedAccessTimes(files); } + if (published.findings.length) { + check.findings.push(...published.findings); check.ok = false; + moment.leakCheck = { ok: false, findings: check.findings.length }; + for (const finding of published.findings) for (const input of moment.inputs.filter((entry) => !finding.input + || entry.section === finding.input.section && entry.id === finding.input.id && entry.declaredBy === finding.input.declaredBy)) { + input.refusal = `${finding.class}: ${finding.path}: ${finding.detail}`; + } + await discardPayload(out, moment); + await writeFile(path.join(out, "moment.json"), serialize(materializedMomentSchema.parse(moment))); + await stampTree(path.join(out, "moment.json"), moment.effectiveAt, moment.effectiveAt); + } + check.checked.files = published.checked.files; + await writeFile(path.join(out, "leak-check.json"), serialize(check)); + await stampTree(path.join(out, "leak-check.json"), moment.effectiveAt, moment.effectiveAt); + await utimes(out, new Date(moment.effectiveAt), new Date(moment.effectiveAt)); + } catch (error) { + await rm(out, { recursive: true, force: true }); + throw error; + } + } return moment; }; @@ -96,26 +127,28 @@ export const finalizeTrainingMoment = async (staging: string, out: string, optio export const materializeTrainingMoment = async (options: MaterializeTrainingMomentOptions): Promise => { const { plan, project, example } = options; const resolved = resolveRewindInputs(plan, project, example); + assertTaskClockInstant(resolved.at); let out = path.resolve(options.out); await assertEmptyOutput(out); let history: History | undefined, historyError: string | undefined; try { history = await openHistory(options.historyDir); } catch (error) { historyError = `Cannot open history: ${reason(error)}`; } + let record: MomentRecord | null = null, recordError: string | undefined; + let candidateRecords: MomentRecord[] = []; + try { + candidateRecords = matchingMomentRecords(await history?.records() ?? [], plan.agent.id, resolved.at); + record = selectMomentRecord(candidateRecords, plan.agent.id, resolved.at, + resolved.inputs.filter((input) => !input.overridden)); + } catch (error) { recordError = reason(error); } + const attempts = candidateRecords.length; + candidateRecords = originalMomentRecords(candidateRecords); + const effectiveAt = record?.wake.startedAt ?? resolved.at; + assertTaskClockInstant(effectiveAt); await mkdir(path.dirname(out), { recursive: true }); out = path.join(await realpath(path.dirname(out)), path.basename(out)); if (history && (inside(history.root, out) || inside(out, history.root))) throw new Error("Rewind output and history must not overlap"); const staging = await mkdtemp(path.join(path.dirname(out), `.${path.basename(out)}.rewind-`)); try { - let record: MomentRecord | null = null, recordError: string | undefined; - let candidateRecords: MomentRecord[] = []; - try { - candidateRecords = matchingMomentRecords(await history?.records() ?? [], plan.agent.id, resolved.at); - record = selectMomentRecord(candidateRecords, plan.agent.id, resolved.at, - resolved.inputs.filter((input) => !input.overridden)); - } catch (error) { recordError = reason(error); } - const attempts = candidateRecords.length; - candidateRecords = originalMomentRecords(candidateRecords); - const effectiveAt = record?.wake.startedAt ?? resolved.at; const moment: MaterializedMoment = { version: "spawnfile.materialized-moment.v1", status: "materialized", agent: plan.agent.id, example: { id: resolved.selected.id, role: resolved.selected.role, dataset: resolved.selected.dataset }, diff --git a/src/compiler/training/rewind/recordInput.ts b/src/compiler/training/rewind/recordInput.ts index 090cc380..07bd5f74 100644 --- a/src/compiler/training/rewind/recordInput.ts +++ b/src/compiler/training/rewind/recordInput.ts @@ -31,13 +31,15 @@ export const recordInputErrors = (record: MomentRecord, input: InputIdentity): s }; /** Do not discard healthy pins just because another input made the whole record inexact. */ -export const recordForInput = (record: MomentRecord | null, input: RewindInput): MomentRecord | null => +export const recordForInput = (record: MomentRecord | null, input: InputIdentity): MomentRecord | null => record && !recordInputErrors(record, input).length ? record : null; -export const hasRecordPin = (record: MomentRecord | null, input: RewindInput): boolean => { - if (!record) return false; - if (input.section === "memory") return record.memory.some((entry) => entry.id === input.id && entry.declaredBy === input.declaredBy); - if (input.section === "networks") return record.moltnet.some((entry) => entry.network === input.id); - const pins = input.declaration.kind !== "volume" ? record.resources : input.declaration.feed ? record.feeds : record.volumes; - return pins.some((entry) => entry.id === input.id); +export const recordPins = (record: MomentRecord | null, input: InputIdentity | RewindInput) => { + if (!record) return []; + if (input.section === "memory") return record.memory.filter((entry) => entry.id === input.id && entry.declaredBy === input.declaredBy); + if (input.section === "networks") return record.moltnet.filter((entry) => entry.network === input.id); + const pins = !("declaration" in input) ? [...record.resources, ...record.feeds, ...record.volumes] + : input.declaration.kind !== "volume" ? record.resources : input.declaration.feed ? record.feeds : record.volumes; + return pins.filter((entry) => entry.id === input.id); }; +export const hasRecordPin = (record: MomentRecord | null, input: InputIdentity | RewindInput): boolean => recordPins(record, input).length > 0; diff --git a/src/compiler/training/rewind/recordedCommit.test.ts b/src/compiler/training/rewind/recordedCommit.test.ts index 4cbcb17b..3d7c593a 100644 --- a/src/compiler/training/rewind/recordedCommit.test.ts +++ b/src/compiler/training/rewind/recordedCommit.test.ts @@ -8,12 +8,13 @@ import { commitTestGit, gitTestRoot, initTestGit, testGit, testMomentRecord } fr import { History } from "./history.js"; import { resolveRecordedCommit } from "./recordedCommit.js"; import type { InputContext } from "./types.js"; +import { resolveFeedSource } from "../../../volume/index.js"; const AT = "2026-10-09T08:20:31Z", EARLY = "2026-10-09T08:00:00Z", REVISION = "e".repeat(64); const roots: string[] = []; afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); const landing = (commit: string) => ({ version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "org-news", source: { kind: "git", repo: "recorded-repo" }, revision: REVISION, tree: REVISION, commit, ref: "main", paths: [], landed_at: EARLY }); -const feedRecord = (commit: string) => ({ id: "news", revision: REVISION, tree: REVISION, commit, paths: [], landedAt: EARLY }); +const feedRecord = (commit: string, revision = REVISION) => ({ id: "news", revision, tree: revision, commit, paths: [], landedAt: EARLY }); const recordedValue = (field: "resources" | "feeds" | "landing", commit: string): string | undefined => { if (field === "landing") return feedLandingSchema.parse(landing(commit)).commit; const value = momentRecordSchema.parse({ ...testMomentRecord(), [field]: field === "resources" @@ -41,14 +42,15 @@ const retainedFeed = async () => { const directory = path.join(source.root, "history/feeds/agent%3Awriter/news"), repository = path.join(directory, "repo"); await mkdir(directory, { recursive: true }); testGit(source.root, ["clone", "--quiet", "--bare", source.repository, repository]); - const retained = path.join(directory, "trees", REVISION); - await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "prepared.txt"), "retained bytes\n"); + const revision = resolveFeedSource({ kind: "git", repo: repository, ref: source.past, fetch: false }).revision; + const retained = path.join(directory, "trees", revision); + await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "readme.txt"), "recorded bytes\n"); const context: InputContext = { at: AT, destination: path.join(source.root, "feed-output"), history: source.context.history, record: null, input: { section: "resources", declaredBy: "agent:writer", id: "news", declarationBase: source.root, overridden: false, declaration: { id: "news", kind: "volume", mode: "mutable", mount: "./news", name: "org-news", feed: { git: { repo: "never-fetch" }, prepare: { host: true, command: ["never-run"] } } } } }; - const log = async (commit: string) => writeFile(path.join(directory, "landings.jsonl"), `${JSON.stringify(landing(commit))}\n`); - return { ...source, repository, directory, context, log }; + const log = async (commit: string) => writeFile(path.join(directory, "landings.jsonl"), `${JSON.stringify({ ...landing(commit), revision, tree: revision })}\n`); + return { ...source, repository, directory, context, log, revision }; }; describe("recorded commit contracts", () => { @@ -141,11 +143,11 @@ describe("recorded commit validation authority", () => { describe("retained feed recorded commit objects", () => { it("normalizes landing and moment IDs before comparing and reports the commit object", async () => { - const { context, past, log } = await retainedFeed(); + const { context, past, log, revision } = await retainedFeed(); await log(past.toUpperCase()); - context.record = { ...testMomentRecord(), feeds: [feedRecord(past.toUpperCase())] }; + context.record = { ...testMomentRecord(), feeds: [feedRecord(past.toUpperCase(), revision)] }; expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", source: { commit: past } }); - expect(await readFile(path.join(context.destination, "current/prepared.txt"), "utf8")).toBe("retained bytes\n"); + expect(await readFile(path.join(context.destination, "current/readme.txt"), "utf8")).toBe("recorded bytes\n"); }); it.each(["blob", "tree", "tag"] as const)("refuses a retained tree whose recorded commit is a %s", async (type) => { @@ -194,8 +196,8 @@ describe("feed moment records without landing logs", () => { }); it("uses a verified moment commit without a landing log", async () => { - const { context, past } = await fallbackFeed(); - context.record = { ...testMomentRecord(), feeds: [feedRecord(past.toUpperCase())] }; + const { context, past, revision } = await fallbackFeed(); + context.record = { ...testMomentRecord(), feeds: [feedRecord(past.toUpperCase(), revision)] }; expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: past } }); }); diff --git a/src/compiler/training/rewind/recordingFormat.test.ts b/src/compiler/training/rewind/recordingFormat.test.ts index c72367ec..5d0b8ce8 100644 --- a/src/compiler/training/rewind/recordingFormat.test.ts +++ b/src/compiler/training/rewind/recordingFormat.test.ts @@ -1,4 +1,4 @@ -import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { momentRecordSchema } from "./contract.js"; @@ -49,6 +49,17 @@ describe("P3 recording format", () => { }); describe("recording pins in a materialized trial", () => { + it.each([false, true])("refuses conflicting copies instead of choosing the empty pin or nonempty fallback (reverse=%s)", async (reverse) => { + const f = await rewindFixture(); roots.push(f.root); + const healthy = emptyRecord(); healthy.memory = [{ id: "self", declaredBy: "agent:a", seq: 0 }]; + const errored = structuredClone(healthy); errored.exact = false; errored.errors = ["memory self: sequence capture failed"]; + await jsonl(f.historyDir, "memory/agent%3Aa/self/events.jsonl", [memoryEvent(1)]); + await jsonl(f.historyDir, "moments/copies.jsonl", reverse ? [errored, healthy] : [healthy, errored]); + const result = await materializeTrainingMoment(f.options); + expect(result.status).toBe("refused"); + expect(result.inputs.find((input) => input.id === "self")!.refusal).toMatch(/Conflicting moment records.*memory agent:a\/self/u); + expect(await readdir(f.options.out)).toEqual(["moment.json"]); + }); it.each([false, true])("accepts incoming feed publication metadata after the bound with record=%s", async (withRecord) => { const f = await rewindFixture(); roots.push(f.root); const log = "feeds/team%3Apublication/news/landings.jsonl"; diff --git a/src/compiler/training/rewind/select.test.ts b/src/compiler/training/rewind/select.test.ts index e1d55357..58538744 100644 --- a/src/compiler/training/rewind/select.test.ts +++ b/src/compiler/training/rewind/select.test.ts @@ -38,6 +38,34 @@ describe("pure rewind record selection", () => { expect(selectMomentRecord([], "agent:writer", at)).toBeNull(); }); + it.each([false, true])("refuses effective pin/fallback disagreements in either order (reverse=%s)", (reverse) => { + for (const section of ["memory", "resources", "networks"] as const) { + const id = section === "memory" ? "self" : section === "resources" ? "news" : "network"; + const first = record(), second = record(); + second.exact = false; second.errors = [`${section} ${id}: capture failed`]; + const copies = reverse ? [second, first] : [first, second]; + expect(() => selectMomentRecord(copies, "agent:writer", at, [{ section, id, declaredBy: "agent:writer" }])) + .toThrow(new RegExp(`Conflicting moment records.*${section} agent:writer/${id}.*effective pin/fallback`, "u")); + } + }); + + it.each([false, true])("compares fallback fidelity when exact differs (reverse=%s)", (reverse) => { + const first = record(), second = record(); second.exact = false; + const inputs = [{ section: "resources" as const, id: "absent-feed", declaredBy: "agent:writer" }]; + expect(() => selectMomentRecord(reverse ? [second, first] : [first, second], "agent:writer", at, inputs)) + .toThrow(/absent-feed.*effective pin\/fallback/u); + }); + + it("ignores unrelated errors and exact flags for healthy pins, and compares usable rather than errored pins", () => { + const first = record(), second = record(); + second.exact = false; second.errors = ["volume unrelated: failed", "memory self (team:other): failed"]; + const inputs = [{ section: "memory" as const, id: "self", declaredBy: "agent:writer" }]; + expect(selectMomentRecord([first, second], "agent:writer", at, inputs)).toBe(first); + first.exact = false; first.errors = ["memory self: failed"]; + second.errors = ["memory self: another failure"]; second.memory[0]!.seq = 99; + expect(selectMomentRecord([first, second], "agent:writer", at, inputs)).toBe(first); + }); + const changes: Array<[string, (value: MomentRecord) => void]> = [ ["resource commit", (value) => { value.resources[0] = { id: "repo", kind: "git", commit: "c".repeat(40) }; }], ["feed revision", (value) => { value.feeds[0]!.revision = "different"; }], diff --git a/src/compiler/training/rewind/select.ts b/src/compiler/training/rewind/select.ts index 18d80f7a..9cfdfff4 100644 --- a/src/compiler/training/rewind/select.ts +++ b/src/compiler/training/rewind/select.ts @@ -1,6 +1,7 @@ import type { TrainingPlan } from "../plan/index.js"; import type { MomentRecord } from "./contract.js"; import { compareInstants, normalizeInstant } from "./instant.js"; +import { hasRecordPin, recordForInput, recordPins } from "./recordInput.js"; import type { InputIdentity, RewindInput } from "./types.js"; export type DatasetRole = "train" | "validation" | "test"; @@ -48,31 +49,25 @@ export const selectMomentRecord = ( ): MomentRecord | null => { const matches = originalMomentRecords(matchingMomentRecords(records, agent, at)); if (!matches.length) return null; - const includes = (section: InputIdentity["section"], id: string, declaredBy?: string, field?: "resources" | "feeds" | "volumes"): boolean => - !inputs || inputs.some((input) => { - if (input.section !== section || input.id !== id || (declaredBy && input.declaredBy !== declaredBy)) return false; - if (!("declaration" in input)) return true; - if (input.overridden) return false; - if (input.section !== "resources" || !field) return true; - const resource = input.declaration; - if (field === "feeds") return resource.kind === "volume" && Boolean(resource.feed); - if (field === "volumes") return resource.kind === "volume" && !resource.feed; - if (resource.kind === "volume") return false; - // Retained bundle digests take precedence over even a pinned build source. - if (resource.kind === "bundle") return true; - const ref = resource.ref; - return !ref || !/^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/u.test(ref); - }); - const sort = (entries: T[]): T[] => entries.sort((a, b) => canonical(a).localeCompare(canonical(b))); - const used = (record: MomentRecord): string => canonical({ - resources: sort(record.resources.filter((item) => includes("resources", item.id, undefined, "resources"))), - feeds: sort(record.feeds.filter((item) => includes("resources", item.id, undefined, "feeds")).map((item) => ({ - ...item, landedAt: normalizeInstant(item.landedAt) - }))), - memory: sort(record.memory.filter((item) => includes("memory", item.id, item.declaredBy))), - moltnet: sort(record.moltnet.filter((item) => includes("networks", item.network)).map((item) => ({ ...item, cursor: normalizeInstant(item.cursor) }))), - volumes: sort(record.volumes.filter((item) => includes("resources", item.id, undefined, "volumes"))) - }); - if (matches.some((record) => used(record) !== used(matches[0]!))) throw new Error(`Conflicting moment records for ${agent} at ${normalizeInstant(at)}`); + const identities: Array = inputs ?? matches.flatMap((record): InputIdentity[] => [ + ...[...record.resources, ...record.feeds, ...record.volumes].map(({ id }) => ({ section: "resources" as const, id, declaredBy: agent })), + ...record.memory.map(({ id, declaredBy }) => ({ section: "memory" as const, id, declaredBy })), + ...record.moltnet.map(({ network }) => ({ section: "networks" as const, id: network, declaredBy: agent })) + ]); + for (const input of identities) { + if ("overridden" in input && input.overridden) continue; + const declaredPin = "declaration" in input && input.section === "resources" && input.declaration.kind === "git" + && /^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/u.test(input.declaration.ref ?? ""); + const decision = (record: MomentRecord): string => { + const usable = recordForInput(record, input); + const pins = recordPins(usable, input).map((pin) => "landedAt" in pin ? { ...pin, landedAt: normalizeInstant(pin.landedAt) } + : "cursor" in pin ? { ...pin, cursor: normalizeInstant(pin.cursor) } : pin); + return canonical({ pin: declaredPin ? "declared-pin" : pins.sort((a, b) => canonical(a).localeCompare(canonical(b))), + approximateFallback: !record.exact && !hasRecordPin(usable, input) }); + }; + if (matches.some((record) => decision(record) !== decision(matches[0]!))) { + throw new Error(`Conflicting moment records for ${agent} at ${normalizeInstant(at)}: ${input.section} ${input.declaredBy}/${input.id} has different effective pin/fallback decisions`); + } + } return matches[0]!; }; diff --git a/src/compiler/training/rewind/taskClock.test.ts b/src/compiler/training/rewind/taskClock.test.ts index 3dc04c66..ba1f7926 100644 --- a/src/compiler/training/rewind/taskClock.test.ts +++ b/src/compiler/training/rewind/taskClock.test.ts @@ -14,6 +14,17 @@ const vectors: Array<[string, number, string]> = [ ]; describe("training task clock contract", () => { + it.each(["1969-12-31T23:59:59.999999999Z", "1970-01-01T00:00:00+00:01", "9999-12-31T23:59:59.999000001Z", "9999-12-31T23:59:59-00:01"]) + ("refuses the out-of-range trial instant %s in every helper", (origin) => { + expect(() => createTaskClock(origin)).toThrow(/must be within 1970.*9999/u); + for (const helper of helpers) expect(() => helper(origin, 0)).toThrow(/must be within 1970.*9999/u); + }); + + it.each(["1970-01-01T00:00:00Z", "1969-12-31T23:00:00-01:00", "9999-12-31T23:59:59.999Z"]) + ("accepts the inclusive trial boundary %s", (origin) => { + expect(() => createTaskClock(origin)).not.toThrow(); + for (const helper of helpers) expect(() => helper(origin, 0)).not.toThrow(); + }); it("normalizes the origin and gives each process exactly one clock owner", () => { const origin = "2026-10-09T10:20:31.120000000+02:00", anchor = 1_790_000_000_123; expect(createTaskClock(origin)).toEqual({ version: "spawnfile.task-clock.v1", origin: "2026-10-09T08:20:31.12Z" }); diff --git a/src/compiler/training/rewind/taskClock.ts b/src/compiler/training/rewind/taskClock.ts index a98e1efe..b70af7fd 100644 --- a/src/compiler/training/rewind/taskClock.ts +++ b/src/compiler/training/rewind/taskClock.ts @@ -1,12 +1,23 @@ -import { normalizeInstant } from "./instant.js"; +import { instantNanos, normalizeInstant } from "./instant.js"; import { taskClockSchema } from "./contract.js"; +/** Shared ecosystem range applies to trial instants, not historical source timestamps. */ +export const assertTaskClockInstant = (instant: string): void => { + const nanos = instantNanos(instant); + if (nanos < 0n || nanos > 253_402_300_799_999_000_000n) { + throw new Error(`Trial instant ${instant} must be within 1970-01-01T00:00:00Z .. 9999-12-31T23:59:59.999Z`); + } +}; +const taskOrigin = (origin: string): string => { + assertTaskClockInstant(origin); + return normalizeInstant(origin); +}; export const createTaskClock = (origin: string) => taskClockSchema.parse({ - version: "spawnfile.task-clock.v1", origin: normalizeInstant(origin) + version: "spawnfile.task-clock.v1", origin: taskOrigin(origin) }); const clockOrigin = (origin: string, anchorEpochMs: number): string => { if (!Number.isSafeInteger(anchorEpochMs)) throw new Error("Task clock anchorEpochMs must be a safe integer"); - return normalizeInstant(origin); + return taskOrigin(origin); }; /** Top-level Daimon only: its clock stays real and it derives each child's clock environment. */ diff --git a/src/compiler/training/rewind/timestamps.test.ts b/src/compiler/training/rewind/timestamps.test.ts new file mode 100644 index 00000000..5d493c56 --- /dev/null +++ b/src/compiler/training/rewind/timestamps.test.ts @@ -0,0 +1,24 @@ +import { lstat, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { gitTestRoot } from "./git.test-helper.js"; +import { restoreInspectedAccessTimes, snapshotBundleFiles } from "./leakFiles.js"; +import { restoreAccessTimes, stampTree } from "./timestamps.js"; + +const roots: string[] = [], old = "1960-01-01T00:00:00Z", at = "2026-10-09T08:20:31Z"; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); + +it.each(["inspection", "materializer"])("preserves negative file, link and directory epochs during %s restoration", async (helper) => { + const root = await gitTestRoot(); roots.push(root); + await writeFile(path.join(root, "data"), "past"); await symlink("data", path.join(root, "link")); + await stampTree(root, old, at); + const files = await snapshotBundleFiles(root); + await readFile(path.join(root, "data")); await readdir(root); + if (helper === "inspection") await restoreInspectedAccessTimes(files); + else await restoreAccessTimes(root, at); + for (const file of files.keys()) { + const stats = await lstat(file); + expect(stats.atimeMs, file).toBe(Date.parse(old)); + expect(stats.mtimeMs, file).toBe(Date.parse(old)); + } +}); diff --git a/src/compiler/training/rewind/timestamps.ts b/src/compiler/training/rewind/timestamps.ts index a83972a8..dc68f2fd 100644 --- a/src/compiler/training/rewind/timestamps.ts +++ b/src/compiler/training/rewind/timestamps.ts @@ -18,7 +18,8 @@ export const restoreAccessTimes = async (root: string, at: string): Promise => { const stats = await lstat(file); if (stats.isDirectory()) for (const name of await readdir(file)) await visit(path.join(file, name)); - await (stats.isSymbolicLink() ? lutimes : utimes)(file, Math.min(stats.mtimeMs, Date.parse(at)) / 1000, stats.mtimeMs / 1000); + await (stats.isSymbolicLink() ? lutimes : utimes)(file, + new Date(Math.min(stats.mtimeMs, Date.parse(at))), new Date(stats.mtimeMs)); }; await visit(root); }; From 52b73ff867375c8255d1a2497662b680e1bfb079 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sun, 11 Oct 2026 04:40:27 +0200 Subject: [PATCH 7/9] docs: keep the merged recording spec and add the rewind section beside it --- specs/TRAINING.md | 185 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 177 insertions(+), 8 deletions(-) diff --git a/specs/TRAINING.md b/specs/TRAINING.md index 8348efa4..d061106c 100644 --- a/specs/TRAINING.md +++ b/specs/TRAINING.md @@ -155,11 +155,34 @@ appears between the `training-plan-schema` markers below. ## Moment record -The recording PR (`feat/training-recording`, not yet merged) owns the normative -`spawnfile.moment-record.v1` format in `specs/TRAINING.md`, “Moment record” and -“Host histories and retention”. Rewind depends on that PR, not the superseded -format sketch. The consumer-dependent recording text below is carried unchanged -until the recording PR merges; it is the same contract, not a second format. +`spawnfile.moment-record.v1` records one turn attempt of one compiled agent. Recording MUST +be off without an effective `record`, with no recording I/O or artifact changes when +off, except reconciliation owed by an earlier recorded deployment or landing. Recording requires a Daimon runtime whose receipt-bound contract manifest declares +`wakeMoments` version `noopolis.daimon.wake-moment.v1` and consumes `agents[].recording`; +other runtimes or a missing capability fail compile. Enabling it requires a capable +Daimon release and a `runtimes.yaml` / contract-manifest re-pin, never a version guess. + +The compiler emits per-agent `recording: {directory, keepMs, snapshots:[{id,path}]}`. +The directory is `/var/lib/spawnfile/recording/`, a runtime-owned `0700` +exclusive-reattach volume; snapshot paths are the resolved unfed volume backing roots, +not workspace symlinks. Recording stores receive the same deletion guard as memory. + +`spawnfile record sync [path] [--deployment ]` selects a deployment like `usage`, +reads wake rows and memory ledgers in bounded exec chunks, and joins host histories using +the compiled agent facts from the activation entry in effect at each row's `started_at`. +Missing activation entries or agent facts MUST set `exact:false` with a reason, including +when the agent has no resource pins. Today's deployment facts MUST NOT fill historical gaps. +It writes private `0600` records via temporary file, fsync and rename to +`/moments//--.json`. +For project records the state directory is `/deployments//`; for home +records it is `~/.spawnfile/deployments//`, scoped to the same identity as the +sibling resource log. The row hash is the first 12 lowercase hex characters of SHA-256 +over canonical wake-row JSON (object keys sorted recursively, array order preserved). +Existing records are never rewritten. Replays may share both `executionId` and `startedAt`; +different row contents produce distinct attempt filenames, while resync skips the same row. Moment +retention deletes attempts older than `now - keep`, preserving the newest per agent +(including ties). Snapshots remain in the deployment's recording volume; records name +them, and P4 reads them. Non-exact records do not fail sync; operational failures do. All instants MUST use one host clock, shared by its containers, and include an offset or `Z`. Every pin MUST describe state in effect at `wake.startedAt`, not capture time. @@ -195,9 +218,155 @@ A later `recordedAt` MUST NOT advance a pin. `errors: [string]`. REQUIRED `exact` MUST be `false` on any snapshot or pin failure; otherwise it MUST be `true`. Missing or invalid history MUST NOT be guessed. -Recorded commit fields MUST additionally be full 40- or 64-hex object IDs, -normalized to lowercase and verified as commit objects without peeling tags. -Bundles carry only a `sha256:…` digest, never a commit. Unknown fields refuse. +```json +{ + "version": "spawnfile.moment-record.v1", + "agent": "agent:writer", + "wake": { + "at": "2026-10-09T08:20:31.000Z", + "startedAt": "2026-10-09T08:20:31.250Z", + "executionId": "execution-17", + "deliveries": [{ "deliveryId": "moltnet:42", "kind": "message", "occurredAt": "2026-10-09T08:20:31.000Z" }], + "surface": "moltnet", "messageId": "42" + }, + "recordedAt": "2026-10-09T08:20:32.000Z", + "exact": true, + "resources": [ + { "id": "archive", "kind": "git", "commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, + { "id": "tools", "kind": "bundle", "sha256": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" } + ], + "feeds": [{ "id": "news", "revision": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "tree": "trees/cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc.1", "commit": "dddddddddddddddddddddddddddddddddddddddd", "ref": "data/2026-10-09", "paths": ["published"], "landedAt": "2026-10-09T08:00:00.000Z", "period": "2026-10-09" }], + "memory": [{ "id": "self", "declaredBy": "agent:writer", "seq": 42 }], + "moltnet": [{ "network": "news", "cursor": "2026-10-09T08:20:31.250Z", "messageIds": ["42"] }], + "volumes": [{ "id": "edition-state", "snapshot": "incremental-snapshot-17" }] +} +``` + +### Host histories and retention + +`record.keep` bounds landing logs, deploy logs, snapshots and moment records; omission +means **30d**. A shared history MUST use the maximum effective keep among its recording +consumers (all recording agents for the deploy log). History-log pruning MUST keep entries newer +than `now - keep` AND the newest entry at or before that cutoff, even across a long gap. +Lookups are inclusive; equal timestamps use the last appended entry. Before the first +entry, lookup returns no state. Invalid lines MUST be reported alongside valid entries. + +For each fed volume consumed by a recording agent, the host MUST append +`landings.jsonl` beside `landed.json` after each successful land, generation re-land, +retained-tree re-point or link repair. Provenance/period adoption MUST also be logged. +Before ANY replacement of `landed.json` (including revision advances, re-points, +repairs and adoption), the publisher MUST reconcile its previous recording snapshot +against the last log entry and append it if missing, preserving the original instant. +A failed reconciliation MUST prevent replacement. A clean refresh also seeds history +when recording is enabled later. Recorded host state retains the original source and +keep window so a changed source declaration cannot rewrite pending provenance. +The entry MUST follow a successful `landed.json` write and contain: +`{version:"spawnfile.volume-feed-landing.v1", resource, volume, revision, tree, +source, commit?, ref?, paths?, switching_at, landed_at, period?}`. `switching_at` +MUST be sampled immediately BEFORE the `current` rename, including link repair; +`landed_at` remains the post-switch instant and MUST equal +`identity.landed_at`; `period` is the host record's period. `source` MUST identify +`{kind:"git", repo}` or `{kind:"directory", directory}` and preserve any `prepared` +provenance. Git entries MUST carry the resolved commit and expanded requested ref; +`paths`, when present, MUST be the expanded git paths. Commit, paths and revision +permit refetch and verification without retaining trees. Directory feeds are verifiable +by revision digest but **not refetchable**. Prepared feeds also require their recorded +include digests and recipe inputs; a revision does not promise reproducible prepare output. +Lookup inside either endpoint of `[switching_at, landed_at]` MUST return the incoming +entry plus `ambiguous:true`. Outside transition windows, select the most recent +`landed_at <= instant`. Adoption without a content switch uses `switching_at = landed_at`; +old nonrecorded identities seeded on opt-in use the same degenerate window. Recording +history missing a required transition boundary is invalid, never assumed exact. + +When any compiled agent records, project `up` and home deployment publication +(including image `up` and release) MUST append a resource log beside their record: +`/deployments/.resource-log.jsonl` for project records, and +`~/.spawnfile/deployments//resource-log.jsonl` for home records. `deployment_id` +is the absolute canonical record path; lookup MUST require that identity. Independent +outputs with the same deployment name MUST NOT share history or retention. + +Each `{version:"spawnfile.deploy-resources.v1", deployment, deployment_id, activated_at, +activation_exact, activation_note?, deployed_at, report_sha256, resources, recording_agents}` entry MUST +reuse `workspace_git_resources` and `workspace_bundles`, without resolving sources again. +`recording_agents` MUST copy the validated compiled recording facts for every recording +agent: node ID, slug, recording directory, keep window, snapshot IDs, consumed feed IDs +and volume names, git/bundle resource IDs, memory bank IDs with declaring node and ledger +path, and attached Moltnet networks. These facts survive releases that remove or change +inputs. Legacy entries without them remain readable but cannot produce exact moments. +Every resource MUST retain its workspace-visible `id`, compiled `agent_node_ids`, and +`declared_by` node ID when known. Git adds `{kind:"git", commit, sha256, url?}`; bundles +add `{kind:"bundle", sha256, origin?}`. Ownership MUST survive compile and distribution +projection, including equal IDs/digests with different commits. Image pins omit host +paths and source URLs. Unowned legacy history MUST be reported as invalid, never guessed. +With no effective recording agent, compile reports MUST omit `agent_node_ids` and +`declared_by` and preserve the original resource deduplication and serialized shape. + +Lookups and retention MUST use `activated_at`; `deployed_at` is informational record +creation time. Activation ordering is path-specific: + +- Image `up` renames/stops the previous container, verifies it stopped, then samples + `activated_at` immediately before `docker run`. It waits for a later host-clock tick + than the stopped observation, so no old turn can have an equal timestamp. +- Drained release first closes admission and waits for old turns, then calls image `up` + under the home lock; the same stop/verify/activation/start sequence applies. +- Project `up` samples immediately before invoking its launch runner. First deployment + has no recorded predecessor. Replacement does not stop/drain its predecessor and MAY + overlap it: entries MUST set `activation_exact:false` with `activation_note`. Records + without a captured launch boundary (including recovered launches) are likewise inexact. + +If a started candidate is removed on readiness or publication failure, image `up`, +release and project `up` MUST append its pins with `{activated_at, rolled_back_at, +exact:false, activation_exact:false}`. `rolled_back_at` is sampled after candidate +removal, before restoration where applicable. This interval MUST NOT replace the +predecessor's authoritative record. Lookup inside the inclusive interval returns the +candidate with `exact:false`, even if it previously published a successful-looking +snapshot; after the interval, that snapshot is excluded and the predecessor applies +until another deployment activates. Retention MUST preserve intervals crossing the +cutoff and the predecessor needed after rollback. No candidate history is written when +recording is off. A failed candidate cleanup MUST NOT invent a completed rollback. +These interval entries describe execution and removal, not successful publication; +their append follows removal without requiring a new authoritative deployment record. + +The host clock MUST remain monotonic across these operations. Containers MUST share +that clock; remote clock skew invalidates exact reconstruction. A moment arriving before +publication MUST wait for its authoritative deployment record/history, or be inexact. + +The deployment record MUST persist `recording: {keep_ms, memory_mount_ids}` and its +complete `resource_snapshot`, including the hash of the exact persisted UTF-8 report +bytes. Before replacing a deployment record OR its cached report, the publisher MUST +reconcile that snapshot against the last log entry. Reconciliation failure MUST abort +replacement. Recompilation MUST preserve deployment records/history, including clean +compiles; it cannot erase a pending snapshot. Retried publication of the same snapshot +MUST NOT append a duplicate. Turning recording off still reconciles an owed predecessor. +Writers MUST receive predecessor metadata from the lifecycle's existing record read; +only effective recording or a recorded predecessor permits additional record reads. + +Host records and cached deployment reports MUST be published by file fsync, rename, +and parent-directory fsync BEFORE log append. Both logs MUST append each complete JSON +line in one write followed by file and directory fsync. Pruning MUST use an exclusive +0600 temporary file, file fsync, rename and directory fsync under a writer lock. +With recording off and no recorded predecessor, neither log nor its lock may be touched. +Host-record/report writes in that case MUST retain their original write/rename sequence, +permissions and bytes, without added reads, durable writes or fsyncs. A matching last +log line MUST still be file- and directory-fsynced before reconciliation succeeds: +visibility alone does not prove a previous failed append became durable. +A deploy-log failure after publication MUST be reported without rolling back the +container named by the published deployment record. +If record rename succeeds but directory fsync fails, the named container MUST also +remain; no log entry may be appended until the authoritative record is re-synced. + +Spawnfile MUST enforce memory protection from the DEPLOYED record's `recording` +metadata, independent of later compile settings. For recorded state, teardown with volume deletion MUST +refuse a changed report fingerprint, missing protected mount, reduced keep, or dropped +recording/mount protection unless explicitly overridden. `down --volumes --discard-recorded-memory` +is that override; `--force` alone is insufficient. Author-declared shared volume names +remain retained under the existing teardown rule. Internal fixture cleanup MUST apply +the same guard. Spawnfile has no memory-ledger reset/prune command; release prunes only +images. This guarantee covers Spawnfile's deletion paths, not external Docker commands. +Mneme stays independent and its ledger MUST NOT be truncated by recording retention. + +Rewind MUST use records when present; otherwise it is best effort and the trial MUST +be labelled as such. ## Rewind From afb31857beae2ae2dab439829b27136437e4bb6b Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sun, 11 Oct 2026 05:02:29 +0200 Subject: [PATCH 8/9] refactor: consume the merged moment record, landing and deployment history readers --- specs/TRAINING.md | 88 +++++++----- src/compiler/training/rewind/AGENTS.md | 7 +- src/compiler/training/rewind/contract.ts | 34 +---- .../training/rewind/deployedResource.test.ts | 133 ++++++++++++++++++ .../training/rewind/deployedResource.ts | 18 +++ src/compiler/training/rewind/feed.test.ts | 24 ++-- src/compiler/training/rewind/feed.ts | 28 ++-- .../training/rewind/feedContent.test.ts | 2 +- .../training/rewind/feedLandings.test.ts | 50 ------- src/compiler/training/rewind/feedLandings.ts | 51 ------- .../training/rewind/fixtures.test-helper.ts | 4 +- .../training/rewind/git.test-helper.ts | 2 +- src/compiler/training/rewind/git.ts | 20 ++- .../training/rewind/gitArchive.test.ts | 2 +- src/compiler/training/rewind/history.test.ts | 2 +- src/compiler/training/rewind/history.ts | 52 ++++++- .../training/rewind/historyLandings.test.ts | 72 ++++++++++ src/compiler/training/rewind/index.ts | 3 +- src/compiler/training/rewind/leakFeed.ts | 8 +- src/compiler/training/rewind/leakInputs.ts | 3 +- .../training/rewind/leakProvenance.test.ts | 4 +- .../training/rewind/leakProvenance.ts | 2 +- src/compiler/training/rewind/materialize.ts | 3 +- .../training/rewind/moltnetJson.test.ts | 2 +- .../training/rewind/moltnetSqlite.test.ts | 2 +- src/compiler/training/rewind/recordInput.ts | 2 +- .../training/rewind/recordedBundle.ts | 2 +- .../training/rewind/recordedCommit.test.ts | 34 +++-- .../training/rewind/recordedCommit.ts | 2 +- .../training/rewind/recordingFormat.test.ts | 42 +++++- src/compiler/training/rewind/select.test.ts | 2 +- src/compiler/training/rewind/select.ts | 4 +- .../rewind/snapshotFixtures.test-helper.ts | 6 +- .../training/rewind/snapshotManifest.ts | 2 + src/compiler/training/rewind/types.ts | 3 +- src/compiler/training/rewind/volumes.test.ts | 49 +++++-- src/compiler/training/rewind/volumes.ts | 11 +- src/recording/AGENTS.md | 1 + src/recording/index.ts | 1 + src/recording/moment.test.ts | 27 ++++ src/recording/moment.ts | 40 ++++++ src/recording/types.ts | 23 +-- 42 files changed, 577 insertions(+), 290 deletions(-) create mode 100644 src/compiler/training/rewind/deployedResource.test.ts create mode 100644 src/compiler/training/rewind/deployedResource.ts delete mode 100644 src/compiler/training/rewind/feedLandings.test.ts delete mode 100644 src/compiler/training/rewind/feedLandings.ts create mode 100644 src/compiler/training/rewind/historyLandings.test.ts create mode 100644 src/recording/moment.test.ts create mode 100644 src/recording/moment.ts diff --git a/specs/TRAINING.md b/specs/TRAINING.md index d061106c..99a7b4c2 100644 --- a/specs/TRAINING.md +++ b/specs/TRAINING.md @@ -404,11 +404,15 @@ paths in replacement declarations resolve from the dataset file's directory. Every input MUST materialize successfully; present-day data MUST NOT substitute for unavailable history. -The local history import layout is normative: +Records and host logs MUST use the merged [Moment record](#moment-record) and +[Host histories and retention](#host-histories-and-retention) contracts. Rewind +uses their shared parsers and lookup helpers. The local history import layout is: ```text DIR/ - moments/**/*.json # one record per file; .jsonl also accepted + moments//--.json + deployments/.resource-log.jsonl # project deployment (choose one log) + deployments//resource-log.jsonl # home deployment (alternative) bundles/.tar git/// # local repository mirror feeds///landings.jsonl @@ -419,6 +423,7 @@ DIR/ recording//wake-moments.jsonl recording//snapshots/// recording//snapshots//.manifest.json + recording//snapshots//.complete ``` `` MUST be `encodeURIComponent` of the node ID: `agent:writer` @@ -426,33 +431,39 @@ becomes `agent%3Awriter`. `` is the selected compiled node's existin slug, including the compiler's collision handling. `` is the `record.volumes` resource ID. IDs and archive names MUST be safe path segments. Every read MUST remain inside `DIR` lexically and after realpath resolution; -symlink escapes refuse. JSON and every JSONL line are validated strictly; +symlink escapes refuse. JSON and every JSONL line are validated by their owning contract; invalid lines refuse the affected input with file, line number, and reason. -Import copies deployment evidence verbatim: composed moment JSON files; the +Import copies deployment evidence verbatim: the deployment state directory's +`moments/` subtree (including agent slug and hashed filenames); exactly one resource +log from the project output or home `deployments/` location above; the host feed state's `landings.jsonl` beside `landed.json`; retained feed volume `trees/` directories; deployed bundle tars from `container/workspace-bundles/`; -Daimon's persisted `recording//` rows, snapshot trees and manifests; +Daimon's persisted `recording//` rows, snapshot trees, manifests and completion markers; Mneme's `/memory/events.jsonl`; and the network's consistent Moltnet store (including SQLite recovery sidecars when needed). The `git/` repositories and feed `repo/` directories are local mirrors retaining historical commits, not recorder output. Rewind neither fetches nor changes these mirrors. -Feed logs follow the recording PR's `spawnfile.volume-feed-landing.v1` shape: -`{version, resource, volume, revision, tree, source, commit?, ref?, paths?, -switching_at?, landed_at, period?}`. Source identifies `{kind:"git", repo}` or -`{kind:"directory", directory}`, with optional prepared provenance. This reader -accepts older entries without `switching_at`; new recording writes transition -boundaries. The isolated compatibility reader must be replaced by P3's -`readFeedLandings` / `feedLandingAt` after that PR merges. +The log retains its original absolute `deployment_id`; that identity is passed to +`deployResourcesAt` without opening the original path. Its name/location MUST mirror +`deployResourceLogPath(deployment_id)`. Multiple logs, mixed deployment identities, +invalid lines, or ambiguous pins refuse; omit the log if it is unavailable. Resource +lookup is scoped to the exact compiled agent and, when present, `declared_by`. +Moment discovery also accepts nested JSONL imports (one shared-format record per line). +Feed history MUST include the required transition boundary; older entries without it +are invalid under the merged contract and refuse. Moment records match exact compiled `agent` and parsed `wake.at == example.at`. For several attempts of that wake, select the EARLIEST `wake.startedAt` (original turn), even when replay pins differ or append/capture order differs. Conflicting -copies at that same earliest instant still refuse when any input's effective +copies at that same earliest instant (including replays sharing BOTH execution ID +and startedAt but differing in row hash) still refuse when any input's effective decision differs: usable pin versus fallback, selected pin, or fallback fidelity given `exact` and input-specific `errors`. The refusal MUST name the input. -Unrelated errors and unusable pin values do not create a conflict. +Unrelated errors and unusable pin values do not create a conflict. Discovery sorts +paths lexically, so equivalent earliest copies yield a deterministic representative; +row hash, capture order and execution ID MUST NOT resolve differing input decisions. Validate recorded commit IDs in every earliest-attempt copy for each used input, including IDs hidden by declared pin precedence; later replays cannot replace or invalidate the original turn's pins. @@ -460,8 +471,10 @@ or invalidate the original turn's pins. The report retains dataset `at` and adds `effectiveAt = selected wake.startedAt`, or `effectiveAt = at` if unmatched. Task-clock origin, historical selection, and leak bounds MUST use `effectiveAt`. Report the matching count, selected execution -ID, selected start instant, and `earliest-startedAt` choice. All comparisons parse -instants, preserving RFC3339 nanoseconds rather than ordering strings or UUIDs. +ID, selected start instant, and `earliest-startedAt` choice. Record matching and +payload cuts compare parsed instants with RFC3339 nanosecond +precision, never strings or UUIDs. Host landing/deployment selection uses the merged +helpers' millisecond host-clock contract. An `exact:false` record does not invalidate healthy pins: a present pin is used with `exact` fidelity and `record` basis (`record-cursor` for Moltnet), unless its @@ -487,8 +500,15 @@ and explicit overrides can succeed; everything else refuses. Git resources select a full 40- or 64-hex declared commit first (`exact`, `pinned`), otherwise the matching record's resource -commit (`exact`, `record`). Otherwise they walk the declared selector's -first-parent history and choose its newest commit with committer time `<= effectiveAt` +commit (`exact`, `record`). With no usable moment pin, consult the imported +deployment log at `effectiveAt` using `deployResourcesAt`; use the selected commit +with `basis: deploy-log`. Fidelity is `exact` only when `activation_exact` is true +and `exact` is not false; inexact activation boundaries and inclusive rollback +intervals MUST remain `approximate`, with the activation/rollback reason in notes. +The merged helper excludes a rolled-back candidate after its interval and restores +the predecessor. A pin that is present but unavailable in the repository refuses. +Only an absent deployment pin falls back to the declared selector's +first-parent history, choosing its newest commit with committer time `<= effectiveAt` (`approximate`, `selector-at`); an absent selector means `HEAD`. Commit dates do not establish when a commit reached a branch or tag, so this fallback cannot prove historical ref membership. A missing eligible commit refuses. @@ -504,13 +524,17 @@ check also rejects any reachable ancestor with author or committer time after `effectiveAt`. Bundle `build.files` extracts `files.root` at the selected commit and honors `exclude`. `build.dependencies` and `build.generated` retain their source directory or declared inputs at that commit and report `build: deferred`; -trial preparation owns builds. Bundle output MAY be a plain tree. A record pins only its bundle SHA-256. +trial preparation owns builds. Bundle output MAY be a plain tree. A moment record +or, when absent, the deployment log pins a bundle by SHA-256 only. Exact bundle output requires `bundles/.tar` whose actual digest -matches that pin. Validate the archive before extraction: escaping paths, +matches that pin (`record` or `deploy-log` basis). A deploy-log digest retains +the activation/rollback fidelity above even when its bytes verify. A present +deployment bundle pin without a matching archive refuses; it MUST NOT silently +select a different source commit. Validate the archive before extraction: escaping paths, symlinks, hard links, devices, and other special entries refuse. Acceptance reopens the archive and compares output paths, bytes, and executable modes. -Absent or digest-mismatched archives fall back to source reconstruction where -available, always `approximate` even with a pinned source commit; notes explain +For a moment-record bundle pin, absent or digest-mismatched archives fall back to +source reconstruction where available, always `approximate` even with a pinned source commit; notes explain that the recorded SHA-256 could not be reproduced. A prebuilt `source` tar has no historical source fallback and refuses unless overridden. Generated inputs spanning multiple repositories or independently pinned input @@ -619,16 +643,16 @@ every listed wake message MUST survive the cut; otherwise refuse. With no pin, cut at `effectiveAt` (`approximate`, `created-at-cut`). Missing stores refuse unless overridden; UUIDs never order messages. -Unfed volumes consume full point-in-time Daimon trees, not deltas or object -chains. Rows in `recording//wake-moments.jsonl` use -`noopolis.daimon.wake-moment.v1` with `agent_id`, `execution_id`, `started_at`, -`deliveries`, `snapshots:[{id,snapshot,files,copied,linked,skipped}]`, and optional -`error`. Snapshot names are compact `started_at` plus execution ID (for example -`20261009T082031123Z-execution-17`), but time selection MUST use the row's -`started_at`, not that prefix. A record names its exact snapshot (`exact`, -`record`); otherwise use the newest complete snapshot row `<= effectiveAt` -(`approximate`, `snapshot-at`). `.partial` directories never count. Missing -named history or no complete eligible snapshot refuses. +Unfed volumes consume the Daimon wake-moment contract required by +[Moment record](#moment-record), including its nonempty deliveries, snapshot name +constraints, and `.complete` publication marker. Read rows with the shared recording +parser. Trees are full point-in-time copies; time selection MUST use row `started_at`, +not the snapshot name. A record names its exact snapshot (`exact`, `record`); +otherwise use the newest complete snapshot row `<= effectiveAt` +(`approximate`, `snapshot-at`). Both the tree and manifest MUST have a published +sibling `.complete` marker. `.partial` directories never count. +Missing named history or no complete eligible snapshot refuses. Independent +acceptance rechecks the marker, manifest and matching wake row. The sibling manifest contains `{version,id,snapshot,started_at,sourcePath,entries}`. Entries map relative paths to `{type,mode,size,source,target?}`: type is file, diff --git a/src/compiler/training/rewind/AGENTS.md b/src/compiler/training/rewind/AGENTS.md index 063c96c7..b4f9437d 100644 --- a/src/compiler/training/rewind/AGENTS.md +++ b/src/compiler/training/rewind/AGENTS.md @@ -1,11 +1,12 @@ # Training Rewind -- `contract.ts` owns strict moment, materialization, leak and task-clock documents and generated schemas. -- `history.ts` is the only boundary for reading history files, logs and retained bytes; every path must be lexically and physically contained. +- `contract.ts` owns strict materialization, leak and task-clock documents and their generated schemas; the recording barrel owns moment records. +- `history.ts` is the only boundary for reading history files, logs and retained bytes; every path must be lexically and physically contained. Reuse recording wake/moment parsers, volume landing readers, and deployment resource lookup through their barrels. +- `deployedResource.ts` scopes fallback pins to the compiled agent and declaring node. Import one deployment log under `deployments/`, retaining its canonical original identity without reading that external path. Inexact activation/rollback pins remain approximate; a present but unavailable pin never silently becomes selector-at. - `select.ts` and `inputs.ts` resolve original attempts, examples and compiled input identities; `recordInput.ts` isolates per-input recording failures. Dataset `at` matches wake time; `effectiveAt` is the original startedAt and owns all cuts, clocks and leak bounds. - `types.ts` defines input identities, declarations and per-kind contexts/results; `index.ts` exports the public materializer and pure selection/cut helpers. - `git.ts` owns resources; `gitProcess.ts`, `gitHistory.ts`, `gitTree.ts`, and `gitArchive.ts` isolate argv execution, containment, blob trees, and raw feed trees; `recordedCommit.ts` verifies full recorded IDs name commit objects without peeling. -- `feed.ts`, `memory.ts`, and `volumes.ts` own input reconstruction. `feedLandings.ts` is the sole P3 log compatibility reader (replace with readFeedLandings/feedLandingAt after merge); `snapshotManifest.ts` verifies Daimon full trees; `memorySchemas.ts` validates ledger events. Mneme generations are unsupported; seq:0 is exact empty. +- `feed.ts`, `memory.ts`, and `volumes.ts` own input reconstruction. `snapshotManifest.ts` requires Daimon completion markers and verifies full trees; `memorySchemas.ts` validates ledger events. Mneme generations are unsupported; seq:0 is exact empty. - `feedContent.ts` verifies plain git revisions and re-extracts for independent byte acceptance; retained copies must match. Prepared revisions prove provenance only: always approximate, never compared to a directory content digest. - `moltnet.ts` dispatches stores; `moltnetJson.ts`, `moltnetSqlite.ts`, `moltnetSqliteData.ts`, and `moltnetSchemas.ts` own snapshots, copied databases, SQL decoding, and strict native shapes. - `materialize.ts` stages all inputs and publishes only after the independent disk reader in `leakCheck.ts` accepts them. diff --git a/src/compiler/training/rewind/contract.ts b/src/compiler/training/rewind/contract.ts index 7acfae87..9cafb385 100644 --- a/src/compiler/training/rewind/contract.ts +++ b/src/compiler/training/rewind/contract.ts @@ -1,44 +1,12 @@ import { z } from "zod"; +import { recordedSha256Schema } from "../../../recording/index.js"; import { trainingInstantSchema } from "../../../manifest/index.js"; const text = z.string().min(1); const count = z.number().int().nonnegative(); -export const recordedSha256Schema = z.string().regex(/^sha256:[a-f0-9]{64}$/u); -export const recordedCommitSchema = z.string() - .regex(/^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/u, "Expected a full 40- or 64-hex commit object id") - .toLowerCase(); export const inputIdentitySchema = z.object({ section: z.enum(["resources", "memory", "networks"]), id: text, declaredBy: text }).strict(); -const unique = (schema: T, key: (item: z.infer) => string) => - z.array(schema).superRefine((items, context) => { - const keys = new Set(); - items.forEach((item, index) => { - const id = key(item); - if (keys.has(id)) context.addIssue({ code: "custom", message: `Duplicate identity ${id}`, path: [index] }); - keys.add(id); - }); - }); -export const momentRecordSchema = z.object({ - version: z.literal("spawnfile.moment-record.v1"), agent: text, - wake: z.object({ at: trainingInstantSchema, startedAt: trainingInstantSchema, executionId: text, - deliveries: z.array(z.object({ deliveryId: text, kind: text, occurredAt: trainingInstantSchema }).strict()).min(1), - surface: z.literal("moltnet").optional(), room: z.string().optional(), - from: z.string().optional(), messageId: text.optional() }).strict(), - recordedAt: trainingInstantSchema, exact: z.boolean(), errors: z.array(z.string()).optional(), - resources: unique(z.discriminatedUnion("kind", [ - z.object({ id: text, kind: z.literal("git"), commit: recordedCommitSchema, sha256: recordedSha256Schema.optional() }).strict(), - z.object({ id: text, kind: z.literal("bundle"), sha256: recordedSha256Schema }).strict() - ]), (r) => r.id), - feeds: unique(z.object({ id: text, revision: text, tree: text, commit: recordedCommitSchema.optional(), ref: text.optional(), - paths: z.array(text).optional(), landedAt: trainingInstantSchema, period: text.optional() }).strict(), (r) => r.id), - memory: unique(z.object({ id: text, declaredBy: text, seq: count, generation: count.optional() }).strict(), - (r) => JSON.stringify([r.declaredBy, r.id])), - moltnet: unique(z.object({ network: text, cursor: trainingInstantSchema, messageIds: z.array(text).optional() }).strict(), (r) => r.network), - volumes: unique(z.object({ id: text, snapshot: text }).strict(), (r) => r.id) -}).strict(); -export type MomentRecord = z.infer; - export const materializedInputSchema = inputIdentitySchema.extend({ kind: z.enum(["git", "bundle", "feed", "volume", "memory", "moltnet"]), fidelity: z.enum(["exact", "approximate", "overridden"]).nullable(), basis: text, diff --git a/src/compiler/training/rewind/deployedResource.test.ts b/src/compiler/training/rewind/deployedResource.test.ts new file mode 100644 index 00000000..99baae4a --- /dev/null +++ b/src/compiler/training/rewind/deployedResource.test.ts @@ -0,0 +1,133 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import * as deployment from "../../../deployment/index.js"; +import { BundleTarWriter } from "../../workspaceBundleTar.js"; +import { AT, BEFORE, emptyRecord, jsonl, rewindFixture, testGit } from "./fixtures.test-helper.js"; +import { materializeTrainingMoment } from "./materialize.js"; + +const roots: string[] = []; +afterEach(async () => { vi.restoreAllMocks(); await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const DIGEST = `sha256:${"a".repeat(64)}`; +const fixture = async (home = false) => { + const f = await rewindFixture(); roots.push(f.root); + testGit(f.gitRepo, ["checkout", "-B", "main", f.commit]); + await writeFile(path.join(f.gitRepo, "news.txt"), "newer eligible selector\n"); + testGit(f.gitRepo, ["add", "."]); testGit(f.gitRepo, ["commit", "--quiet", "-m", "later"], "2026-10-09T08:00:00Z"); + const later = testGit(f.gitRepo, ["rev-parse", "HEAD"]); + const recordPath = home ? "/original/home/deployments/desk/record.json" : "/original/output/deployments/desk.json"; + const logPath = home ? "deployments/desk/resource-log.jsonl" : "deployments/desk.resource-log.jsonl"; + const entry = (commit = f.commit, fields: Partial = {}): deployment.DeployResources => ({ + version: "spawnfile.deploy-resources.v1", deployment: "desk", deployment_id: recordPath, + deployed_at: "2026-10-10T00:00:00Z", activated_at: BEFORE, activation_exact: true, report_sha256: DIGEST, + resources: [{ id: "code", kind: "git", commit, sha256: DIGEST, agent_node_ids: ["agent:a"], declared_by: "team:publication" }], ...fields + }); + const log = (entries: unknown[]) => jsonl(f.historyDir, logPath, entries); + const run = async () => { + const result = await materializeTrainingMoment(f.options); + return { result, input: result.inputs.find((input) => input.id === "code")! }; + }; + return { ...f, later, entry, log, run, logPath, recordPath }; +}; + +describe("merged deployment resource history in rewind", () => { + it.each([false, true])("uses the deployed commit instead of a newer selector (home=%s)", async (home) => { + const f = await fixture(home), lookup = vi.spyOn(deployment, "deployResourcesAt"); + const entry = f.entry(); await f.log([entry]); + const { result, input } = await f.run(); + expect(result.status, JSON.stringify(result.inputs)).toBe("materialized"); + expect(input).toMatchObject({ fidelity: "exact", basis: "deploy-log", source: { commit: f.commit } }); + expect(await readFile(path.join(f.options.out, input.path!, "news.txt"), "utf8")).toBe("Before the wake\n"); + expect(lookup).toHaveBeenCalledWith([entry], f.recordPath, AT, "agent:a"); + }); + + it.each(["2026-10-09T08:20:30Z", AT, "2026-10-09T08:20:32Z", "2026-10-09T08:20:32.001Z"])("respects inclusive rollback and restoration at %s", async (at) => { + const f = await fixture(); + const activated_at = "2026-10-09T08:20:30Z", rolled_back_at = "2026-10-09T08:20:32Z"; + const candidate = f.entry(f.later, { activated_at, report_sha256: `sha256:${"b".repeat(64)}` }); + await f.log([f.entry(), candidate, { ...candidate, activation_exact: false, exact: false, rolled_back_at }]); + // The dataset wake matches a record; all history must be selected at its effective startedAt. + const record = emptyRecord(); record.wake.startedAt = at; + await jsonl(f.historyDir, "moments/attempt.jsonl", [record]); + const { result, input } = await f.run(), rolledBack = at === "2026-10-09T08:20:32.001Z"; + expect(result.status, JSON.stringify(result.inputs)).toBe("materialized"); + expect(input).toMatchObject({ basis: "deploy-log", fidelity: rolledBack ? "exact" : "approximate", + source: { commit: rolledBack ? f.commit : f.later } }); + if (!rolledBack) expect(input.notes).toContainEqual(expect.stringContaining("rollback interval")); + }); + + it("carries an inexact activation note beside the selected pin", async () => { + const f = await fixture(); await f.log([f.entry(f.commit, { activation_exact: false, activation_note: "Predecessor overlaps launch" })]); + const { result, input } = await f.run(); + expect(result.status).toBe("materialized"); + expect(input).toMatchObject({ basis: "deploy-log", fidelity: "approximate", source: { commit: f.commit }, notes: ["Predecessor overlaps launch"] }); + }); + + it.each(["before activation", "other agent", "other declaring node", "absent resource"])("uses selector-at only when the deploy pin is absent: %s", async (state) => { + const f = await fixture(), entry = f.entry(); + if (state === "before activation") entry.activated_at = "2026-10-09T09:00:00Z"; + if (state === "other agent") entry.resources[0]!.agent_node_ids = ["agent:other"]; + if (state === "other declaring node") entry.resources[0]!.declared_by = "team:other"; + if (state === "absent resource") entry.resources = []; + await f.log([entry]); + expect((await f.run()).input).toMatchObject({ basis: "selector-at", fidelity: "approximate", source: { commit: f.later } }); + }); + + it.each(["moment", "declared"])("preserves %s pin precedence over deployment history", async (kind) => { + const f = await fixture(); await f.log([f.entry(f.later)]); + if (kind === "moment") { + const record = emptyRecord(); record.resources = [{ id: "code", kind: "git", commit: f.commit }]; + await jsonl(f.historyDir, "moments/pin.jsonl", [record]); + } else { + const agent = f.project.compilePlan.nodes.find((node) => node.id === f.plan.agent.id)!.value; + if (agent.kind !== "agent") throw new Error("Expected agent"); + const resource = agent.workspaceResources!.find((item) => item.id === "code")!; + if (resource.kind !== "git") throw new Error("Expected git"); resource.ref = f.commit; + } + expect((await f.run()).input).toMatchObject({ basis: kind === "moment" ? "record" : "pinned", fidelity: "exact", source: { commit: f.commit } }); + }); + + it.each(["malformed", "mixed identities", "multiple logs", "duplicate pin"])("refuses %s instead of guessing selector-at", async (state) => { + const f = await fixture(), entry = f.entry(); + if (state === "duplicate pin") entry.resources.push({ ...entry.resources[0]! }); + await f.log(state === "malformed" ? [entry, {}] : state === "mixed identities" ? [entry, { ...entry, deployment_id: "/other/desk.json" }] : [entry]); + if (state === "multiple logs") await jsonl(f.historyDir, "deployments/other.resource-log.jsonl", [entry]); + const { result, input } = await f.run(); + expect(result.status).toBe("refused"); + expect(input.refusal).toMatch(/Invalid history|deployment.*ambiguous|Ambiguous deployment/u); + expect(await readdir(f.options.out)).toEqual(["moment.json"]); + }); + + it.each([true, false])("restores digest-verified deploy bundle bytes with activation_exact=%s", async (exact) => { + const f = await fixture(), archive = path.join(f.root, "saved.tar"); + const writer = await BundleTarWriter.create(archive); + await writer.begin("tools.txt", 6, 0o644); await writer.data(Buffer.from("tools\n")); await writer.end(); await writer.finish(); + const bytes = await readFile(archive), hex = createHash("sha256").update(bytes).digest("hex"); + await mkdir(path.join(f.historyDir, "bundles")); await writeFile(path.join(f.historyDir, "bundles", `${hex}.tar`), bytes); + const agent = f.project.compilePlan.nodes.find((node) => node.id === f.plan.agent.id)!.value; + if (agent.kind !== "agent") throw new Error("Expected agent"); + const index = agent.workspaceResources!.findIndex((item) => item.id === "code"); + agent.workspaceResources![index] = { scope: agent.workspaceResources![index]!.scope, sharing: "per_agent", id: "code", kind: "bundle", mount: "./code", mode: "readonly", source: "never-read.tar" }; + await f.log([f.entry(f.commit, { activation_exact: exact, resources: [{ id: "code", kind: "bundle", sha256: `sha256:${hex}`, + agent_node_ids: ["agent:a"], declared_by: "team:publication" }] })]); + const { result, input } = await f.run(); + expect(result.status, JSON.stringify(result.inputs)).toBe("materialized"); + expect(input).toMatchObject({ basis: "deploy-log", fidelity: exact ? "exact" : "approximate", source: { sha256: `sha256:${hex}` } }); + expect(await readFile(path.join(f.options.out, input.path!, "tools.txt"), "utf8")).toBe("tools\n"); + }); + + it("refuses an unavailable deployment bundle instead of reconstructing a different selector", async () => { + const f = await fixture(), agent = f.project.compilePlan.nodes.find((node) => node.id === f.plan.agent.id)!.value; + if (agent.kind !== "agent") throw new Error("Expected agent"); + const index = agent.workspaceResources!.findIndex((item) => item.id === "code"), previous = agent.workspaceResources![index]!; + agent.workspaceResources![index] = { scope: previous.scope, sharing: "per_agent", id: "code", kind: "bundle", + mount: "./code", mode: "readonly", build: { files: { root: ".", ref: f.later } } }; + await f.log([f.entry(f.commit, { resources: [{ id: "code", kind: "bundle", sha256: DIGEST, + agent_node_ids: ["agent:a"], declared_by: "team:publication" }] })]); + const { result, input } = await f.run(); + expect(result.status).toBe("refused"); + expect(input.refusal).toContain(`Deployment bundle ${DIGEST} could not be reproduced`); + expect(await readdir(f.options.out)).toEqual(["moment.json"]); + }); +}); diff --git a/src/compiler/training/rewind/deployedResource.ts b/src/compiler/training/rewind/deployedResource.ts new file mode 100644 index 00000000..35b386a0 --- /dev/null +++ b/src/compiler/training/rewind/deployedResource.ts @@ -0,0 +1,18 @@ +import type { InputContext } from "./types.js"; + +/** Deployment IDs remain evidence only: all bytes are read from the contained history mirror. */ +export const deployedResource = async (context: InputContext) => { + const agentId = context.agentId ?? context.record?.agent + ?? (context.input.declaredBy.startsWith("agent:") ? context.input.declaredBy : undefined); + const entry = await context.history?.deploymentResources(context.at, agentId); + if (!entry) return null; + const pins = entry.resources.filter((resource) => resource.id === context.input.id + && (resource.declared_by === undefined || resource.declared_by === context.input.declaredBy)); + if (pins.length > 1) throw new Error(`Ambiguous deployment resource pin for ${context.input.declaredBy}/${context.input.id}`); + const pin = pins[0]; + if (!pin) return null; + const exact = entry.activation_exact && entry.exact !== false; + return { pin, exact, notes: exact ? [] : [entry.activation_note + ?? (entry.rolled_back_at ? "Trial instant falls inside an inexact deployment rollback interval." + : "Deployment activation boundary is inexact; an overlapping predecessor may have served different pins.")] }; +}; diff --git a/src/compiler/training/rewind/feed.test.ts b/src/compiler/training/rewind/feed.test.ts index d727c024..473b06ce 100644 --- a/src/compiler/training/rewind/feed.test.ts +++ b/src/compiler/training/rewind/feed.test.ts @@ -3,8 +3,8 @@ import { mkdir, readFile, readdir, readlink, rm, symlink, writeFile } from "node import path from "node:path"; import { createHash } from "node:crypto"; import type { TeamWorkspaceResource } from "../../../manifest/index.js"; -import { digestDirectory, feedIdentityFindings, resolveFeedSource } from "../../../volume/index.js"; -import { materializeFeed, type FeedLanding } from "./feed.js"; +import { digestDirectory, feedIdentityFindings, resolveFeedSource, type FeedLanding } from "../../../volume/index.js"; +import { materializeFeed } from "./feed.js"; import { History } from "./history.js"; import { commitTestGit, gitTestRoot, initTestGit, testGit, testMomentRecord } from "./git.test-helper.js"; import type { InputContext } from "./types.js"; @@ -21,7 +21,7 @@ const fixture = async () => { const future = await commitTestGit(repository, { "published/news.txt": "future\n" }, LATE); const revision = resolveFeedSource({ kind: "git", fetch: false, repo: repository, ref: past, paths: ["published"] }).revision; const landing: FeedLanding = { version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "org-news", - source: { kind: "git", repo: "https://never-fetch.invalid/source" }, revision, tree: `trees/${revision}`, commit: past, ref: "main", paths: ["published"], landed_at: EARLY }; + source: { kind: "git", repo: "https://never-fetch.invalid/source" }, revision, tree: `trees/${revision}`, commit: past, ref: "main", paths: ["published"], switching_at: EARLY, landed_at: EARLY }; const resource: TeamWorkspaceResource = { id: "news", kind: "volume", mode: "mutable", mount: "./news", name: "org-news", feed: { git: { repo: "https://never-fetch.invalid/source", ref: "main", paths: ["published"], fetch: true } } }; const context: InputContext = { at: AT, destination: path.join(root, "output"), history: new History(path.join(root, "history")), record: null, @@ -38,7 +38,7 @@ describe("feed rewind", () => { testGit(repository, ["commit", "--allow-empty", "-m", "future metadata", `--date=${role === "author" ? LATE : EARLY}`], role === "committer" ? LATE : EARLY); const commit = testGit(repository, ["rev-parse", "HEAD"]); const revision = resolveFeedSource({ kind: "git", fetch: false, repo: repository, ref: commit, paths: ["published"] }).revision; - await log([{ ...landing, commit, revision, tree: revision, landed_at: EARLY }]); + await log([{ ...landing, commit, revision, tree: `trees/${revision}`, switching_at: EARLY, landed_at: EARLY }]); if (bytes === "retained") { const retained = path.join(directory, "trees", revision); await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "future.txt"), "future retained bytes"); @@ -51,8 +51,8 @@ describe("feed rewind", () => { it("selects exact boundary landings and matches offset-equivalent records", async () => { const { context, landing, log, revision } = await fixture(); const selected = { ...landing, landed_at: AT }; - await log([selected, { ...landing, revision: "f".repeat(64), tree: "f".repeat(64), landed_at: "2026-10-09T08:20:31.001Z" }]); - context.record = { ...testMomentRecord(), feeds: [{ id: "news", revision, tree: revision, commit: landing.commit!, ref: "main", paths: ["published"], landedAt: "2026-10-09T10:20:31+02:00" }] }; + await log([selected, { ...landing, revision: "f".repeat(64), tree: `trees/${"f".repeat(64)}`, landed_at: "2026-10-09T08:20:31.001Z" }]); + context.record = { ...testMomentRecord(), feeds: [{ id: "news", revision, tree: `trees/${revision}`, commit: landing.commit!, ref: "main", paths: ["published"], landedAt: "2026-10-09T10:20:31+02:00" }] }; expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: landing.commit, revision }, instant: "2026-10-09T10:20:31+02:00" }); expect(await readlink(path.join(context.destination, "current"))).toBe(`trees/${revision}`); expect(await readFile(path.join(context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); @@ -147,7 +147,7 @@ describe("feed rewind", () => { await mkdir(path.join(retained, "published"), { recursive: true }); if (change !== "missing file") await writeFile(path.join(retained, "published/news.txt"), change === "changed bytes" ? "future\n" : "past\n"); if (change === "extra file") await writeFile(path.join(retained, "extra.txt"), "future\n"); - await f.log([{ ...f.landing, revision, tree: revision }]); + await f.log([{ ...f.landing, revision, tree: `trees/${revision}` }]); if (change === "missing repository") await rm(f.repository, { recursive: true }); if (change === "unchanged") { expect(await materializeFeed(f.context)).toMatchObject({ fidelity: "exact" }); @@ -164,7 +164,7 @@ describe("feed rewind", () => { const revision = digestDirectory(retainedTemp), retained = path.join(directory, "trees", revision); await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "data.txt"), "directory\n"); await log([{ version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "org-news", - source: { kind: "directory", directory: "never-read" }, revision, tree: `trees/${revision}`, landed_at: EARLY }]); + source: { kind: "directory", directory: "never-read" }, revision, tree: `trees/${revision}`, switching_at: EARLY, landed_at: EARLY }]); expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", source: { revision } }); expect(await readFile(path.join(context.destination, "current/data.txt"), "utf8")).toBe("directory\n"); await rm(context.destination, { recursive: true }); context.input.overridden = true; context.history = undefined; @@ -208,7 +208,7 @@ describe("feed rewind", () => { it("refuses revisions which cannot be reproduced from the selected commit", async () => { const { context, landing, log } = await fixture(); - await log([{ ...landing, revision: "0".repeat(64), tree: "0".repeat(64) }]); + await log([{ ...landing, revision: "0".repeat(64), tree: `trees/${"0".repeat(64)}` }]); await expect(materializeFeed(context)).rejects.toThrow("Feed revision mismatch"); }); @@ -223,7 +223,7 @@ describe("feed rewind", () => { resource.feed = { git: { repo: "never-read" }, prepare: { host: true, command: ["never-execute"] } }; const prepared = { include: [], prepare: { command: ["never-execute"], image: null, network: null, platform: null }, source_revision: landing.revision }; const revision = createHash("sha256").update(`prepared\n${JSON.stringify(prepared)}\n`).digest("hex"); - await log([{ ...landing, revision, tree: revision, source: { kind: "git", repo: "never-read", prepared } }]); + await log([{ ...landing, revision, tree: `trees/${revision}`, source: { kind: "git", repo: "never-read", prepared } }]); await expect(materializeFeed(context)).rejects.toThrow("retained landed tree bytes"); context.history = undefined; await expect(materializeFeed(context)).rejects.toThrow("--history is required"); @@ -231,10 +231,10 @@ describe("feed rewind", () => { it("validates every landing line and falls back before the first eligible landing", async () => { const { context, directory, landing, log } = await fixture(); - await log([{ ...landing, landed_at: LATE }]); + await log([{ ...landing, switching_at: LATE, landed_at: LATE }]); expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", basis: "feed-rule-at" }); await rm(context.destination, { recursive: true }); - await writeFile(path.join(directory, "landings.jsonl"), `${JSON.stringify(landing)}\n${JSON.stringify({ ...landing, unknown: true })}\n`); + await writeFile(path.join(directory, "landings.jsonl"), `${JSON.stringify(landing)}\n${JSON.stringify({ ...landing, switching_at: undefined })}\n`); await expect(materializeFeed(context)).rejects.toThrow(/landings.jsonl:2/u); await writeFile(path.join(directory, "landings.jsonl"), ""); expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", basis: "feed-rule-at" }); diff --git a/src/compiler/training/rewind/feed.ts b/src/compiler/training/rewind/feed.ts index 7f215260..26de1d03 100644 --- a/src/compiler/training/rewind/feed.ts +++ b/src/compiler/training/rewind/feed.ts @@ -2,18 +2,17 @@ import { lstat, mkdir, readdir, rm, symlink, writeFile } from "node:fs/promises" import path from "node:path"; import type { TeamWorkspaceResource } from "../../../manifest/index.js"; import { digestDirectory, expandFeedPaths, expandFeedRefTemplate, FEED_IDENTITY_FILE, FEED_IDENTITY_VERSION, - type FeedIdentity } from "../../../volume/index.js"; + feedLandingAt, isFeedLanding, type FeedIdentity, type FeedLanding } from "../../../volume/index.js"; import { assertPreparedFeedRevision, feedGitRevision } from "./feedContent.js"; import { resolveRecordedCommit } from "./recordedCommit.js"; -import { feedLandingSchema, readFeedLandingHistory, selectFeedLanding, type FeedLanding } from "./feedLandings.js"; import { exportFeedGitTree } from "./gitArchive.js"; +import { safeCommittedPath } from "./gitTree.js"; import { assertLocalGitRepository } from "./gitHistory.js"; import { assertCommitNotFuture, commitAt, type SelectedCommit } from "./gitProcess.js"; import { historyKey } from "./history.js"; import { compareInstants } from "./instant.js"; import type { InputContext, InputResult } from "./types.js"; -export { feedLandingSchema, selectFeedLanding, type FeedLanding } from "./feedLandings.js"; type VolumeFeed = NonNullable["feed"]>; const feedPaths = (paths: string[] | undefined): string[] | undefined => paths?.length ? paths : undefined; @@ -54,7 +53,7 @@ export const materializeFeed = async (context: InputContext): Promise => { repository ??= await history.repository(`${key}/repo`); @@ -63,6 +62,7 @@ export const materializeFeed = async (context: InputContext): Promise entry.revision === record.revision && treeName(entry.tree) === treeName(record.tree) && compareInstants(entry.landed_at, record.landedAt) === 0 && entry.commit === recordedCommit) : undefined; - const selected = record ? null : selectFeedLanding(log ?? [], context.at); + const selected = record ? null : feedLandingAt(log, context.at); if (record) { // A wake pin is authoritative even if a later attempt or landing changed the currently selected log row. landing = { version: "spawnfile.volume-feed-landing.v1", resource: input.id, volume: matching?.volume ?? resource.name!, - revision: record.revision, tree: record.tree, landed_at: record.landedAt, + revision: record.revision, tree: `trees/${treeName(record.tree)}`, landed_at: record.landedAt, + switching_at: matching?.switching_at ?? record.landedAt, source: matching?.source ?? (feed.git ? { kind: "git", repo: feed.git.repo } : { kind: "directory", directory: feed.directory! }), ...(recordedCommit ? { commit: recordedCommit, ref: record.ref ?? recordedCommit } : {}), - ...(record.paths ? { paths: record.paths } : {}), ...(record.period ? { period: record.period } : {}), - ...(matching?.switching_at ? { switching_at: matching.switching_at } : {}) }; + ...(record.paths ? { paths: record.paths } : {}), ...(record.period ? { period: record.period } : {}) }; basis = "record"; } else if (selected) { - landing = selected.landing; ambiguous = selected.ambiguous; basis = "landing-log"; + landing = selected; ambiguous = selected.ambiguous === true; basis = "landing-log"; } else { // Refuse unknown historical behavior before touching any declared source or executing any command. if (!feed.git || feed.freeze || typeof feed.git.ref === "object" && feed.git.ref.command) await selectFeedCommit(feed, "", context.at); @@ -95,11 +95,12 @@ export const materializeFeed = async (context: InputContext): Promise 0 ? context.at : landing.landed_at, notes: [...(isPrepared ? ["prepared output not content-verifiable"] : []), diff --git a/src/compiler/training/rewind/feedContent.test.ts b/src/compiler/training/rewind/feedContent.test.ts index 888ebba4..dd1c9959 100644 --- a/src/compiler/training/rewind/feedContent.test.ts +++ b/src/compiler/training/rewind/feedContent.test.ts @@ -25,7 +25,7 @@ const preparedFixture = async (kind: "git" | "directory", recipe: "include" | "p expect(digestDirectory(retained)).not.toBe(resolved.revision); const log = "feeds/team%3Apublication/news/landings.jsonl"; await jsonl(f.historyDir, log, [{ version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "news", - revision: resolved.revision, tree: `trees/${resolved.revision}`, landed_at: BEFORE, + revision: resolved.revision, tree: `trees/${resolved.revision}`, switching_at: BEFORE, landed_at: BEFORE, source: { kind, ...(kind === "git" ? { repo: f.feedRepo } : { directory: sourceDirectory }), prepared: resolved.provenance.prepared }, ...(kind === "git" ? { commit: f.commit, ref: "main" } : {}) }]); if (recordOnly) { diff --git a/src/compiler/training/rewind/feedLandings.test.ts b/src/compiler/training/rewind/feedLandings.test.ts deleted file mode 100644 index e9b2218c..00000000 --- a/src/compiler/training/rewind/feedLandings.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { feedLandingSchema, selectFeedLanding, type FeedLanding } from "./feedLandings.js"; - -const AT = "2026-10-09T08:20:31Z", REVISION = "a".repeat(64); -const landing = (fields: Partial = {}): FeedLanding => ({ - version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "org-news", revision: REVISION, - tree: `trees/${REVISION}.2`, source: { kind: "git", repo: "https://never-fetch.invalid/source" }, - commit: "a".repeat(40), ref: "daily/2026-10-09", paths: ["published"], landed_at: AT, ...fields -}); - -describe("recording feed landing compatibility", () => { - it("accepts snake_case recording rows and directory sources without git fields", () => { - const row = landing({ switching_at: AT, period: "2026-10-09" }); - expect(feedLandingSchema.parse(row)).toEqual(row); - const { commit: _commit, ref: _ref, paths: _paths, ...directory } = row; - expect(feedLandingSchema.parse({ ...directory, source: { kind: "directory", directory: "/deployed/news" } })).toMatchObject({ - source: { kind: "directory", directory: "/deployed/news" } - }); - }); - - it("refuses malformed provenance, paths, tree identities and reversed transitions", () => { - const row = landing(); - for (const invalid of [ - { ...row, landedAt: AT }, { ...row, revision: "b".repeat(64) }, { ...row, tree: `trees/${REVISION}/../escape` }, - { ...row, paths: ["../escape"] }, { ...row, commit: "HEAD" }, { ...row, commit: null }, - { ...row, source: { kind: "git" } }, { ...row, ref: undefined }, - { ...row, switching_at: "2026-10-09T08:20:31.000000001Z" } - ]) expect(() => feedLandingSchema.parse(invalid)).toThrow(); - }); - - it("uses last-appended state for equal instants, including equivalent offsets", () => { - const first = landing(), last = landing({ tree: `trees/${REVISION}.3`, landed_at: "2026-10-09T10:20:31+02:00" }); - expect(selectFeedLanding([first, last], AT)).toEqual({ landing: last, ambiguous: false }); - expect(selectFeedLanding([last, first], AT)).toEqual({ landing: first, ambiguous: false }); - expect(selectFeedLanding([last, first], "2026-10-09T08:20:30.999999999Z")).toBeNull(); - }); - - it.each(["2026-10-09T08:20:30Z", AT, "2026-10-09T08:20:32Z"]) - ("selects the incoming row inside the inclusive transition at %s", (at) => { - const previous = landing({ landed_at: "2026-10-09T08:00:00Z" }); - const incoming = landing({ switching_at: "2026-10-09T08:20:30Z", landed_at: "2026-10-09T08:20:32Z" }); - expect(selectFeedLanding([previous, incoming], at)).toEqual({ landing: incoming, ambiguous: true }); - expect(selectFeedLanding([previous, incoming], "2026-10-09T08:20:32.000000001Z")).toEqual({ landing: incoming, ambiguous: false }); - }); - - it("marks a zero-duration transition ambiguous at the shared boundary", () => { - const row = landing({ switching_at: AT }); - expect(selectFeedLanding([row], "2026-10-09T10:20:31+02:00")).toEqual({ landing: row, ambiguous: true }); - }); -}); diff --git a/src/compiler/training/rewind/feedLandings.ts b/src/compiler/training/rewind/feedLandings.ts deleted file mode 100644 index 975d7c55..00000000 --- a/src/compiler/training/rewind/feedLandings.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { z } from "zod"; -import { trainingInstantSchema } from "../../../manifest/index.js"; -import { recordedCommitSchema } from "./contract.js"; -import type { History } from "./history.js"; -import { compareInstants } from "./instant.js"; - -const text = z.string().min(1), revision = z.string().regex(/^[a-f0-9]{64}$/u); -const plainPath = text.refine((value) => !value.startsWith("/") && !/[\\\0]/u.test(value) - && !value.split("/").some((segment) => !segment || segment === "." || segment === ".." || segment.toLowerCase() === ".git"), "Expected a plain tree-relative path"); -const prepared = z.object({ - source_revision: revision, - include: z.array(z.object({ digest: revision, to: plainPath }).strict()), - prepare: z.object({ command: z.array(text).min(1), image: text.nullable(), network: z.boolean().nullable(), platform: text.nullable() }).strict().nullable() -}).strict(); - -// Recording-format compatibility boundary: replace with readFeedLandings/feedLandingAt after P3 merges. -export const feedLandingSchema = z.object({ - version: z.literal("spawnfile.volume-feed-landing.v1"), resource: text, volume: text, - revision, tree: z.string().regex(/^(?:trees\/)?[a-f0-9]{64}(?:\.[1-9][0-9]*)?$/u), - source: z.discriminatedUnion("kind", [ - z.object({ kind: z.literal("git"), repo: text, prepared: prepared.optional() }).strict(), - z.object({ kind: z.literal("directory"), directory: text, prepared: prepared.optional() }).strict() - ]), - commit: recordedCommitSchema.optional(), ref: text.optional(), paths: z.array(plainPath).optional(), - switching_at: trainingInstantSchema.optional(), landed_at: trainingInstantSchema, period: text.optional() -}).strict().superRefine((value, context) => { - if (!value.tree.replace(/^trees\//u, "").startsWith(value.revision)) context.addIssue({ code: "custom", message: "Landing tree must belong to its revision" }); - if (value.source.kind === "git" && (!value.commit || !value.ref)) context.addIssue({ code: "custom", message: "Git feed landing must record its commit and ref" }); - if (value.source.kind === "directory" && (value.commit !== undefined || value.ref !== undefined || value.paths !== undefined)) { - context.addIssue({ code: "custom", message: "Directory feed landing cannot carry git provenance" }); - } - if (value.switching_at && compareInstants(value.switching_at, value.landed_at) > 0) { - context.addIssue({ code: "custom", message: "Landing transition begins after landed_at" }); - } -}); -export type FeedLanding = z.infer; -export interface FeedLandingSelection { landing: FeedLanding; ambiguous: boolean } - -export const readFeedLandingHistory = async (history: History, key: string, resource: string): Promise => - history.jsonl(`${key}/landings.jsonl`, feedLandingSchema.refine((entry) => entry.resource === resource, - "Landing resource disagrees with the history input")); - -/** Inclusive transition windows take priority; equal-instant state uses the last appended row. */ -export const selectFeedLanding = (landings: readonly FeedLanding[], at: string): FeedLandingSelection | null => { - let transition: FeedLanding | null = null, selected: FeedLanding | null = null; - for (const landing of landings) { - if (landing.switching_at && compareInstants(landing.switching_at, at) <= 0 && compareInstants(at, landing.landed_at) <= 0) transition = landing; - if (compareInstants(landing.landed_at, at) <= 0 && (!selected || compareInstants(landing.landed_at, selected.landed_at) >= 0)) selected = landing; - } - return transition ? { landing: transition, ambiguous: true } : selected ? { landing: selected, ambiguous: false } : null; -}; diff --git a/src/compiler/training/rewind/fixtures.test-helper.ts b/src/compiler/training/rewind/fixtures.test-helper.ts index 9e3662c9..b94bdaba 100644 --- a/src/compiler/training/rewind/fixtures.test-helper.ts +++ b/src/compiler/training/rewind/fixtures.test-helper.ts @@ -6,7 +6,7 @@ import { loadTrainingProject, resolveTrainingPlan } from "../plan/index.js"; import { rewindGitEnvironment } from "./gitProcess.js"; import { resolveFeedSource } from "../../../volume/feedSource.js"; import { writeRecordedVolumeSnapshot } from "./snapshotFixtures.test-helper.js"; -import type { MomentRecord } from "./contract.js"; +import type { MomentRecord } from "../../../recording/index.js"; import type { MemoryLedgerEvent } from "./memorySchemas.js"; export const AT = "2026-10-09T08:20:31Z"; @@ -46,7 +46,7 @@ export const rewindFixture = async () => { testGit(root, ["clone", "--quiet", "--no-local", gitRepo, feedRepo]); const feedSource = resolveFeedSource({ kind: "git", repo: feedRepo, ref: commit, fetch: false }); await jsonl(historyDir, "feeds/team%3Apublication/news/landings.jsonl", [{ version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "news", source: { kind: "git", repo: feedRepo }, revision: feedSource.revision, - tree: `trees/${feedSource.revision}`, commit, ref: "main", landed_at: BEFORE }]); + tree: `trees/${feedSource.revision}`, commit, ref: "main", switching_at: BEFORE, landed_at: BEFORE }]); await writeFile(path.join(gitRepo, "future.txt"), "secret after wake\n"); testGit(gitRepo, ["add", "."]); testGit(gitRepo, ["commit", "--quiet", "-m", "future"], "2026-10-10T00:00:00Z"); const futureCommit = testGit(gitRepo, ["rev-parse", "HEAD"]); diff --git a/src/compiler/training/rewind/git.test-helper.ts b/src/compiler/training/rewind/git.test-helper.ts index de85e592..7fe9df0c 100644 --- a/src/compiler/training/rewind/git.test-helper.ts +++ b/src/compiler/training/rewind/git.test-helper.ts @@ -2,7 +2,7 @@ import { execFileSync } from "node:child_process"; import { mkdir, mkdtemp, realpath, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import type { MomentRecord } from "./contract.js"; +import type { MomentRecord } from "../../../recording/index.js"; import { rewindGitEnvironment } from "./gitProcess.js"; export const gitTestRoot = async (): Promise => realpath(await mkdtemp(path.join(os.tmpdir(), "spawnfile-rewind-git-"))); diff --git a/src/compiler/training/rewind/git.ts b/src/compiler/training/rewind/git.ts index d038e0b3..082ae865 100644 --- a/src/compiler/training/rewind/git.ts +++ b/src/compiler/training/rewind/git.ts @@ -3,6 +3,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import type { TeamWorkspaceResource } from "../../../manifest/index.js"; +import { deployedResource } from "./deployedResource.js"; import { historyKey } from "./history.js"; import { assertLocalGitRepository } from "./gitHistory.js"; import { assertCommitNotFuture, commitAt, commitInstant, resolveRewindCommit, rewindGit } from "./gitProcess.js"; @@ -95,12 +96,18 @@ export const materializeGit = async (context: InputContext): Promise candidate.resources.flatMap((entry) => entry.id === input.id && entry.kind === "git" ? [entry.commit] : [])) ?? []; if (matchingRecord && matchingRecord.kind !== resource.kind) throw new Error(`Moment resource ${input.id} kind disagrees with its declaration`); - const bundlePin = matchingRecord?.kind === "bundle" ? matchingRecord : undefined; + const deployed = !input.overridden && !matchingRecord && !(resource.kind === "git" && pinned) + ? await deployedResource(context) : null; + if (deployed && deployed.pin.kind !== resource.kind) throw new Error(`Deployment resource ${input.id} kind disagrees with its declaration`); + const effectivePin = matchingRecord ?? deployed?.pin; + const bundlePin = effectivePin?.kind === "bundle" ? effectivePin : undefined; if (bundlePin && await materializeRecordedBundle(history, bundlePin.sha256, context.destination)) { - return { fidelity: "exact", basis: "record", source: { sha256: bundlePin.sha256 }, notes: [], instant: context.at }; + return { fidelity: deployed && !deployed.exact ? "approximate" : "exact", basis: deployed ? "deploy-log" : "record", + source: { sha256: bundlePin.sha256 }, notes: deployed?.notes ?? [], instant: context.at }; } + if (deployed && bundlePin) throw new Error(`Deployment bundle ${bundlePin.sha256} could not be reproduced from retained archive bytes`); const bundleNote = bundlePin ? `Recorded bundle ${bundlePin.sha256} could not be reproduced from retained archive bytes.` : undefined; - const gitPin = matchingRecord?.kind === "git" ? matchingRecord : undefined; + const gitPin = effectivePin?.kind === "git" ? effectivePin : undefined; const record = pinned ? undefined : gitPin; if (input.overridden && resource.kind === "bundle" && resource.source) { await extractOverrideBundle(localRepositoryPath(resource.source, input.declarationBase), context.destination, resource.sha256); @@ -127,10 +134,11 @@ export const materializeGit = async (context: InputContext): Promise { let output = context.destination; if (kind === "feed") { const revision = resolveFeedSource({ kind: "git", repo: repository, ref: commit, fetch: false }).revision; - await writeFile(path.join(repository, "../landings.jsonl"), `${JSON.stringify({ version: "spawnfile.volume-feed-landing.v1", resource: "input", volume: "input", source: { kind: "git", repo: repository }, revision, tree: revision, commit, ref: "main", landed_at: BEFORE })}\n`); + await writeFile(path.join(repository, "../landings.jsonl"), `${JSON.stringify({ version: "spawnfile.volume-feed-landing.v1", resource: "input", volume: "input", source: { kind: "git", repo: repository }, revision, tree: `trees/${revision}`, commit, ref: "main", switching_at: BEFORE, landed_at: BEFORE })}\n`); expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", source: { revision, commit } }); output = path.join(output, "current"); } else expect(await materializeGit(context)).toMatchObject({ fidelity: "approximate", source: { commit } }); diff --git a/src/compiler/training/rewind/history.test.ts b/src/compiler/training/rewind/history.test.ts index 26e1fe4d..144a0e88 100644 --- a/src/compiler/training/rewind/history.test.ts +++ b/src/compiler/training/rewind/history.test.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { z } from "zod"; import { emptyRecord } from "./fixtures.test-helper.js"; -import { momentRecordSchema } from "./contract.js"; +import { momentRecordSchema } from "../../../recording/index.js"; import { History, historyKey, openHistory, safeSegment } from "./history.js"; const roots: string[] = []; diff --git a/src/compiler/training/rewind/history.ts b/src/compiler/training/rewind/history.ts index 5c633d97..e87e68a3 100644 --- a/src/compiler/training/rewind/history.ts +++ b/src/compiler/training/rewind/history.ts @@ -1,7 +1,9 @@ import { chmod, lstat, mkdir, readFile, readdir, readlink, realpath, symlink, writeFile } from "node:fs/promises"; import path from "node:path"; import type { z } from "zod"; -import { momentRecordSchema, type MomentRecord } from "./contract.js"; +import { momentRecordSchema, parseMomentRecord, parseWakeMoments, type MomentRecord, type WakeMomentRow } from "../../../recording/index.js"; +import { deployResourceLogPath, deployResourcesAt, deployResourcesSchema, type DeployResources } from "../../../deployment/index.js"; +import { readFeedLandings, type FeedLanding } from "../../../volume/index.js"; export const inside = (root: string, target: string): boolean => { const relative = path.relative(root, target); @@ -65,6 +67,52 @@ export class History { catch (error) { throw new Error(`Invalid history ${relative}:${index + 1}: ${reason(error)}`); } }); } + async feedLandings(key: string, resource: string): Promise { + const relative = `${key}/landings.jsonl`, file = await this.path(relative); + if (!file) return []; + if (!(await lstat(await realpath(file))).isFile()) throw new Error(`History entry must be a regular file: ${relative}`); + const result = readFeedLandings(path.dirname(file)); + if (result.errors.length) throw new Error(`Invalid history ${result.errors.map((error) => `${relative}:${error.replace(/^(?:wake moments )?line /u, "")}`).join("; ")}`); + result.entries.forEach((entry, index) => { + if (entry.resource !== resource) throw new Error(`Invalid history ${relative}:${index + 1}: Landing resource disagrees with the history input`); + }); + return result.entries; + } + async wakeMoments(relative: string): Promise { + const payload = await this.text(relative); + if (payload === null) return null; + const result = parseWakeMoments(payload); + if (result.errors.length) throw new Error(`Invalid history ${result.errors.map((error) => `${relative}:${error.replace(/^(?:wake moments )?line /u, "")}`).join("; ")}`); + return result.entries; + } + /** One imported deployment, preserving its original absolute identity without opening that path. */ + async deploymentResources(at: string, agentId?: string): Promise { + const directory = await this.path("deployments"); + if (!directory) return null; + const logs: string[] = []; + for (const name of (await readdir(directory)).sort()) { + const relative = `deployments/${name}`, entry = await this.path(relative); + if (!entry) throw new Error(`History entry vanished: ${relative}`); + if (name.endsWith(".resource-log.jsonl")) logs.push(relative); + else if ((await lstat(await realpath(entry))).isDirectory() && await this.path(`${relative}/resource-log.jsonl`)) logs.push(`${relative}/resource-log.jsonl`); + } + if (!logs.length) return null; + if (logs.length !== 1) throw new Error("Rewind history must contain one deployment resource log; multiple deployments are ambiguous"); + const relative = logs[0]!, entries = await this.jsonl(relative, deployResourcesSchema) ?? []; + if (!entries.length) return null; + const identities = new Set(entries.map((entry) => entry.deployment_id)); + const recordPath = entries[0]!.deployment_id; + if (identities.size !== 1 || !path.isAbsolute(recordPath) || path.resolve(recordPath) !== recordPath) { + throw new Error(`Invalid history ${relative}: expected one canonical deployment_id`); + } + const originalLog = deployResourceLogPath(recordPath); + const mirror = path.basename(recordPath) === "record.json" + ? `deployments/${path.basename(path.dirname(recordPath))}/${path.basename(originalLog)}` + : `deployments/${path.basename(originalLog)}`; + if (relative !== mirror) throw new Error(`Invalid history ${relative}: deployment_id requires ${mirror}`); + if (!agentId) throw new Error("Deployment resource lookup requires the compiled agent ID"); + return deployResourcesAt(entries, recordPath, at, agentId); + } async records(): Promise { const directory = await this.path("moments"); if (!directory) return []; @@ -80,7 +128,7 @@ export class History { for (const name of (await readdir(file)).sort()) await walk(`${relative}/${name}`); } else if (relative.endsWith(".jsonl")) records.push(...(await this.jsonl(relative, momentRecordSchema) ?? [])); else if (relative.endsWith(".json")) { - try { records.push(momentRecordSchema.parse(JSON.parse((await this.text(relative))!))); } + try { records.push(parseMomentRecord((await this.text(relative))!)); } catch (error) { throw new Error(`Invalid history ${relative}: ${reason(error)}`); } } }; diff --git a/src/compiler/training/rewind/historyLandings.test.ts b/src/compiler/training/rewind/historyLandings.test.ts new file mode 100644 index 00000000..f7719e4e --- /dev/null +++ b/src/compiler/training/rewind/historyLandings.test.ts @@ -0,0 +1,72 @@ +import { mkdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import * as volume from "../../../volume/index.js"; +import { materializeFeed } from "./feed.js"; +import { gitTestRoot } from "./git.test-helper.js"; +import { History } from "./history.js"; + +const roots: string[] = []; +afterEach(async () => { vi.restoreAllMocks(); await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const AT = "2026-10-09T08:20:31Z", BEFORE = "2026-10-09T08:00:00Z"; +const fixture = async () => { + const root = await gitTestRoot(); roots.push(root); + const key = "feeds/agent%3Awriter/news", directory = path.join(root, key), retained = path.join(directory, "retained"); + await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "news.txt"), "past\n"); + const revision = volume.digestDirectory(retained), tree = path.join(directory, "trees", revision); + await mkdir(tree, { recursive: true }); await writeFile(path.join(tree, "news.txt"), "past\n"); + const row: volume.FeedLanding = { version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "news", + revision, tree: `trees/${revision}`, source: { kind: "directory", directory: "/never-read" }, switching_at: BEFORE, landed_at: BEFORE }; + const history = new History(root); + const log = (rows: unknown[]) => writeFile(path.join(directory, "landings.jsonl"), rows.map((value) => JSON.stringify(value)).join("\n") + "\n"); + const materialize = (at = AT) => materializeFeed({ at, history, record: null, destination: path.join(root, "out"), input: { + section: "resources", id: "news", declaredBy: "agent:writer", declarationBase: root, overridden: false, + declaration: { id: "news", kind: "volume", mode: "mutable", mount: "./news", name: "news", feed: { directory: "/never-read" } } + } }); + return { history, key, row, log, materialize }; +}; + +describe("merged feed landing readers in rewind", () => { + it.each(["2026-10-09T08:20:30Z", AT, "2026-10-09T08:20:32Z"])("uses the merged inclusive ambiguity window at %s", async (at) => { + const f = await fixture(), read = vi.spyOn(volume, "readFeedLandings"), lookup = vi.spyOn(volume, "feedLandingAt"); + const incoming = { ...f.row, switching_at: "2026-10-09T08:20:30Z", landed_at: "2026-10-09T08:20:32Z" }; + await f.log([f.row, incoming]); + expect(await f.materialize(at)).toMatchObject({ fidelity: "approximate", basis: "landing-log", + source: { switchingAt: incoming.switching_at, landedAt: incoming.landed_at }, notes: [expect.stringContaining("ambiguous")] }); + expect(read).toHaveBeenCalledOnce(); expect(lookup).toHaveBeenCalledWith([f.row, incoming], at); + }); + + it("selects the newest transition even when an older overlapping window was appended last", async () => { + const f = await fixture(); + const newer = { ...f.row, switching_at: "2026-10-09T08:20:30Z", landed_at: "2026-10-09T08:20:32Z" }; + const older = { ...f.row, switching_at: "2026-10-09T08:20:29Z", landed_at: "2026-10-09T08:20:33Z" }; + await f.log([newer, older]); + expect(await f.materialize()).toMatchObject({ fidelity: "approximate", source: { landedAt: newer.landed_at } }); + }); + + it.each([false, true])("uses the last appended landing outside windows (reverse=%s)", async (reverse) => { + const f = await fixture(), second = { ...f.row, volume: "second", landed_at: "2026-10-09T10:00:00+02:00" }; + const rows = reverse ? [second, f.row] : [f.row, second]; await f.log(rows); + const result = await f.materialize(); + expect(result).toMatchObject({ fidelity: "exact", basis: "landing-log", source: { landedAt: rows[1]!.landed_at } }); + }); + + it("keeps a zero-duration window ambiguous at its exact instant", async () => { + const f = await fixture(); await f.log([f.row]); + expect(await f.materialize(BEFORE)).toMatchObject({ fidelity: "approximate", basis: "landing-log" }); + }); + + it.each(["missing boundary", "reversed boundary", "wrong tree", "wrong resource", "invalid line"])("refuses %s with its line number", async (change) => { + const f = await fixture(); + const bad = { ...f.row, ...(change === "missing boundary" ? { switching_at: undefined } : {}), + ...(change === "reversed boundary" ? { switching_at: AT } : {}), + ...(change === "wrong tree" ? { tree: f.row.revision } : {}), ...(change === "wrong resource" ? { resource: "other" } : {}) }; + await f.log([f.row, change === "invalid line" ? {} : bad]); + await expect(f.materialize()).rejects.toThrow(/landings.jsonl:2:/u); + }); + + it("keeps history containment before invoking the host reader", async () => { + const f = await fixture(); + await expect(f.history.feedLandings("../escape", "news")).rejects.toThrow("escapes root"); + }); +}); diff --git a/src/compiler/training/rewind/index.ts b/src/compiler/training/rewind/index.ts index d07dd1d7..bee15df8 100644 --- a/src/compiler/training/rewind/index.ts +++ b/src/compiler/training/rewind/index.ts @@ -6,7 +6,8 @@ export * from "./leakCheck.js"; export * from "./materialize.js"; export * from "./select.js"; export * from "./taskClock.js"; -export { selectFeedLanding, selectFeedCommit } from "./feed.js"; +export { feedLandingAt } from "../../../volume/index.js"; +export { selectFeedCommit } from "./feed.js"; export { selectCommitAt } from "./gitProcess.js"; export { selectMemoryPrefix, parseMemoryLedger } from "./memory.js"; export { rewindMoltnetJson } from "./moltnetJson.js"; diff --git a/src/compiler/training/rewind/leakFeed.ts b/src/compiler/training/rewind/leakFeed.ts index fd70b5e9..ec8b37cc 100644 --- a/src/compiler/training/rewind/leakFeed.ts +++ b/src/compiler/training/rewind/leakFeed.ts @@ -1,6 +1,5 @@ -import type { FeedIdentity } from "../../../volume/index.js"; +import { feedLandingAt, type FeedIdentity } from "../../../volume/index.js"; import type { MaterializedInput, MaterializedMoment } from "./contract.js"; -import { readFeedLandingHistory, selectFeedLanding } from "./feedLandings.js"; import { historyKey, type History } from "./history.js"; import { compareInstants } from "./instant.js"; import { recordInputErrors } from "./recordInput.js"; @@ -20,9 +19,8 @@ export const permitsFeedPublicationTime = async ( && pin.commit === input.source.commit && compareInstants(pin.landedAt, identity.landed_at) === 0); } if (input.basis !== "landing-log" || input.fidelity !== "approximate") return false; - const selection = selectFeedLanding(await readFeedLandingHistory(history, `feeds/${historyKey(input)}`, input.id) ?? [], moment.effectiveAt); - const landing = selection?.landing; - return Boolean(selection?.ambiguous && landing && landing.revision === identity.revision + const landing = feedLandingAt(await history.feedLandings(`feeds/${historyKey(input)}`, input.id), moment.effectiveAt); + return Boolean(landing?.ambiguous && landing.revision === identity.revision && landing.tree.replace(/^trees\//u, "") === identity.tree.replace(/^trees\//u, "") && landing.commit === input.source.commit && landing.switching_at === input.source.switchingAt && compareInstants(landing.landed_at, identity.landed_at) === 0); diff --git a/src/compiler/training/rewind/leakInputs.ts b/src/compiler/training/rewind/leakInputs.ts index a68dff90..f89b6e87 100644 --- a/src/compiler/training/rewind/leakInputs.ts +++ b/src/compiler/training/rewind/leakInputs.ts @@ -13,7 +13,6 @@ import { parseMemoryLedger } from "./memory.js"; import { inspectMoltnetJson } from "./moltnetJson.js"; import { inspectMoltnetSqlite } from "./moltnetSqlite.js"; import { readSnapshotManifest, snapshotBytes, snapshotHistoryPath, snapshotTreeFindings } from "./snapshotManifest.js"; -import { wakeSnapshotRowSchema } from "./volumes.js"; export const addInputFinding = (report: LeakCheck, input: MaterializedInput, kind: LeakFinding["class"], detail: string, file = input.path!): void => { report.findings.push({ input: { section: input.section, id: input.id, declaredBy: input.declaredBy }, class: kind, path: file, detail }); @@ -111,7 +110,7 @@ export const inspectVolumeBundle = async (root: string, input: MaterializedInput const retained = await history.path(relative); if (!manifest || !retained) throw new Error("Snapshot tree or manifest is missing"); if (compareInstants(manifest.started_at, report.at) > 0) throw new Error("Selected snapshot is later than the moment"); - const rows = await history.jsonl(`recording/${safeSegment(input.source.agentSlug)}/wake-moments.jsonl`, wakeSnapshotRowSchema); + const rows = await history.wakeMoments(`recording/${safeSegment(input.source.agentSlug)}/wake-moments.jsonl`); if (!rows?.some((row) => compareInstants(row.started_at, manifest.started_at) === 0 && row.snapshots.some((entry) => entry.id === input.id && entry.snapshot === input.source.snapshot))) { throw new Error("Selected snapshot has no matching wake row"); diff --git a/src/compiler/training/rewind/leakProvenance.test.ts b/src/compiler/training/rewind/leakProvenance.test.ts index 5f11a9c6..4fb6d192 100644 --- a/src/compiler/training/rewind/leakProvenance.test.ts +++ b/src/compiler/training/rewind/leakProvenance.test.ts @@ -38,8 +38,8 @@ const fixture = async (kind: Kind) => { const directory = path.dirname(repository), retained = path.join(directory, "trees", revision); await mkdir(retained, { recursive: true }); await writeFile(path.join(retained, "data.txt"), "past bytes\n"); - await writeFile(path.join(directory, "landings.jsonl"), JSON.stringify({ version: "spawnfile.volume-feed-landing.v1", resource: "source", volume: "source", source: { kind: "git", repo: repository }, revision, tree: revision, commit: selected, - ref: "main", landed_at: BEFORE }) + "\n"); + await writeFile(path.join(directory, "landings.jsonl"), JSON.stringify({ version: "spawnfile.volume-feed-landing.v1", resource: "source", volume: "source", source: { kind: "git", repo: repository }, revision, tree: `trees/${revision}`, commit: selected, + ref: "main", switching_at: BEFORE, landed_at: BEFORE }) + "\n"); await stampTree(retained, BEFORE, AT); }; const stage = async () => { diff --git a/src/compiler/training/rewind/leakProvenance.ts b/src/compiler/training/rewind/leakProvenance.ts index c38c8e3b..a0968b87 100644 --- a/src/compiler/training/rewind/leakProvenance.ts +++ b/src/compiler/training/rewind/leakProvenance.ts @@ -16,7 +16,7 @@ export const inspectInputProvenance = async ( ): Promise => { const add = (detail: string) => addInputFinding(report, input, "git-object", detail); try { - if (input.kind === "bundle" && input.basis === "record" && input.source.sha256) { + if (input.kind === "bundle" && (input.basis === "record" || input.basis === "deploy-log") && input.source.sha256) { await verifyRecordedBundle(await openHistory(historyDir), input.source.sha256, await diskPath(root, input.path!)); return; } diff --git a/src/compiler/training/rewind/materialize.ts b/src/compiler/training/rewind/materialize.ts index c7b4c130..8e2eaf0a 100644 --- a/src/compiler/training/rewind/materialize.ts +++ b/src/compiler/training/rewind/materialize.ts @@ -1,7 +1,8 @@ import { lstat, mkdir, mkdtemp, readdir, realpath, rename, rm, utimes, writeFile } from "node:fs/promises"; import path from "node:path"; import type { LoadedTrainingProject, TrainingPlan } from "../plan/index.js"; -import { materializedMomentSchema, type MaterializedInput, type MaterializedMoment, type MomentRecord } from "./contract.js"; +import { materializedMomentSchema, type MaterializedInput, type MaterializedMoment } from "./contract.js"; +import { type MomentRecord } from "../../../recording/index.js"; import { materializeFeed } from "./feed.js"; import { materializeGit } from "./git.js"; import { historyKey, inside, openHistory, type History } from "./history.js"; diff --git a/src/compiler/training/rewind/moltnetJson.test.ts b/src/compiler/training/rewind/moltnetJson.test.ts index 5874df73..6c63ecce 100644 --- a/src/compiler/training/rewind/moltnetJson.test.ts +++ b/src/compiler/training/rewind/moltnetJson.test.ts @@ -3,7 +3,7 @@ import path from "node:path"; import os from "node:os"; import { afterEach, describe, expect, it } from "vitest"; import { materializeMoltnet } from "./moltnet.js"; -import { momentRecordSchema } from "./contract.js"; +import { momentRecordSchema } from "../../../recording/index.js"; import { openHistory } from "./history.js"; import { emptyMoltnetSnapshot, inspectMoltnetJson, moltnetMessages, parseMoltnetSnapshot, rewindMoltnetJson } from "./moltnetJson.js"; import type { MoltnetMessage } from "./moltnetSchemas.js"; diff --git a/src/compiler/training/rewind/moltnetSqlite.test.ts b/src/compiler/training/rewind/moltnetSqlite.test.ts index 2303d9dd..ae41ee4f 100644 --- a/src/compiler/training/rewind/moltnetSqlite.test.ts +++ b/src/compiler/training/rewind/moltnetSqlite.test.ts @@ -4,7 +4,7 @@ import path from "node:path"; import os from "node:os"; import { afterEach, describe, expect, it } from "vitest"; import { materializeMoltnet } from "./moltnet.js"; -import { momentRecordSchema } from "./contract.js"; +import { momentRecordSchema } from "../../../recording/index.js"; import { openHistory } from "./history.js"; import { inspectMoltnetSqlite } from "./moltnetSqlite.js"; import type { InputContext } from "./types.js"; diff --git a/src/compiler/training/rewind/recordInput.ts b/src/compiler/training/rewind/recordInput.ts index 07bd5f74..aa2fcec0 100644 --- a/src/compiler/training/rewind/recordInput.ts +++ b/src/compiler/training/rewind/recordInput.ts @@ -1,4 +1,4 @@ -import type { MomentRecord } from "./contract.js"; +import type { MomentRecord } from "../../../recording/index.js"; import type { InputIdentity, RewindInput } from "./types.js"; const sections: Record = { diff --git a/src/compiler/training/rewind/recordedBundle.ts b/src/compiler/training/rewind/recordedBundle.ts index 7d523f86..0b608565 100644 --- a/src/compiler/training/rewind/recordedBundle.ts +++ b/src/compiler/training/rewind/recordedBundle.ts @@ -2,7 +2,7 @@ import { createHash } from "node:crypto"; import { lstat, readFile, readdir } from "node:fs/promises"; import path from "node:path"; import { validateWorkspaceBundleTar } from "../../workspaceBundleArtifacts.js"; -import { recordedSha256Schema } from "./contract.js"; +import { recordedSha256Schema } from "../../../recording/index.js"; import { extractBundleBytes } from "./gitTree.js"; import type { History } from "./history.js"; diff --git a/src/compiler/training/rewind/recordedCommit.test.ts b/src/compiler/training/rewind/recordedCommit.test.ts index 3d7c593a..b946b0e2 100644 --- a/src/compiler/training/rewind/recordedCommit.test.ts +++ b/src/compiler/training/rewind/recordedCommit.test.ts @@ -1,25 +1,29 @@ import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { momentRecordSchema } from "./contract.js"; -import { feedLandingSchema, materializeFeed } from "./feed.js"; +import { momentRecordSchema } from "../../../recording/index.js"; +import { materializeFeed } from "./feed.js"; import { materializeGit } from "./git.js"; import { commitTestGit, gitTestRoot, initTestGit, testGit, testMomentRecord } from "./git.test-helper.js"; import { History } from "./history.js"; import { resolveRecordedCommit } from "./recordedCommit.js"; import type { InputContext } from "./types.js"; -import { resolveFeedSource } from "../../../volume/index.js"; +import { resolveFeedSource, isFeedLanding } from "../../../volume/index.js"; const AT = "2026-10-09T08:20:31Z", EARLY = "2026-10-09T08:00:00Z", REVISION = "e".repeat(64); const roots: string[] = []; afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); -const landing = (commit: string) => ({ version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "org-news", source: { kind: "git", repo: "recorded-repo" }, revision: REVISION, tree: REVISION, commit, ref: "main", paths: [], landed_at: EARLY }); -const feedRecord = (commit: string, revision = REVISION) => ({ id: "news", revision, tree: revision, commit, paths: [], landedAt: EARLY }); +const landing = (commit: string) => ({ version: "spawnfile.volume-feed-landing.v1", resource: "news", volume: "org-news", source: { kind: "git", repo: "recorded-repo" }, revision: REVISION, tree: `trees/${REVISION}`, commit, ref: "main", paths: [], switching_at: EARLY, landed_at: EARLY }); +const feedRecord = (commit: string, revision = REVISION) => ({ id: "news", revision, tree: `trees/${revision}`, commit, paths: [], landedAt: EARLY }); const recordedValue = (field: "resources" | "feeds" | "landing", commit: string): string | undefined => { - if (field === "landing") return feedLandingSchema.parse(landing(commit)).commit; + if (field === "landing") { + const row = landing(commit); + if (!isFeedLanding(row)) throw new Error("Expected a full 40- or 64-hex commit object id in a valid landing"); + return row.commit; + } const value = momentRecordSchema.parse({ ...testMomentRecord(), [field]: field === "resources" ? [{ id: "archive", kind: "git", commit }] - : [{ id: "news", revision: REVISION, tree: REVISION, commit, paths: [], landedAt: EARLY }] }); + : [{ id: "news", revision: REVISION, tree: `trees/${REVISION}`, commit, paths: [], landedAt: EARLY }] }); const entry = value[field][0]!; return "commit" in entry ? entry.commit : undefined; }; @@ -49,12 +53,18 @@ const retainedFeed = async () => { input: { section: "resources", declaredBy: "agent:writer", id: "news", declarationBase: source.root, overridden: false, declaration: { id: "news", kind: "volume", mode: "mutable", mount: "./news", name: "org-news", feed: { git: { repo: "never-fetch" }, prepare: { host: true, command: ["never-run"] } } } } }; - const log = async (commit: string) => writeFile(path.join(directory, "landings.jsonl"), `${JSON.stringify({ ...landing(commit), revision, tree: revision })}\n`); + const log = async (commit: string) => writeFile(path.join(directory, "landings.jsonl"), `${JSON.stringify({ ...landing(commit), revision, tree: `trees/${revision}` })}\n`); return { ...source, repository, directory, context, log, revision }; }; describe("recorded commit contracts", () => { - it.each(["resources", "feeds", "landing"] as const)("normalizes full SHA-1 and SHA-256 %s commits to lowercase", (field) => { + it("requires the merged landing contract's lowercase full commit IDs", () => { + for (const length of [40, 64]) { + expect(recordedValue("landing", "a".repeat(length))).toBe("a".repeat(length)); + expect(() => recordedValue("landing", "A".repeat(length))).toThrow(); + } + }); + it.each(["resources", "feeds"] as const)("normalizes full SHA-1 and SHA-256 %s commits to lowercase", (field) => { for (const length of [40, 64]) expect(recordedValue(field, "AbCDef12".repeat(length / 8))).toBe("abcdef12".repeat(length / 8)); }); @@ -66,7 +76,7 @@ describe("recorded commit contracts", () => { it("directory landing logs omit git commit fields", () => { const { commit: _commit, ref: _ref, paths: _paths, ...entry } = landing("a".repeat(40)); - expect(feedLandingSchema.parse({ ...entry, source: { kind: "directory", directory: "/source" } }).commit).toBeUndefined(); + expect(isFeedLanding({ ...entry, source: { kind: "directory", directory: "/source" } })).toBe(true); }); }); @@ -142,9 +152,9 @@ describe("recorded commit validation authority", () => { }); describe("retained feed recorded commit objects", () => { - it("normalizes landing and moment IDs before comparing and reports the commit object", async () => { + it("normalizes moment IDs before comparing with lowercase landings and reports the commit object", async () => { const { context, past, log, revision } = await retainedFeed(); - await log(past.toUpperCase()); + await log(past); context.record = { ...testMomentRecord(), feeds: [feedRecord(past.toUpperCase(), revision)] }; expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", source: { commit: past } }); expect(await readFile(path.join(context.destination, "current/readme.txt"), "utf8")).toBe("recorded bytes\n"); diff --git a/src/compiler/training/rewind/recordedCommit.ts b/src/compiler/training/rewind/recordedCommit.ts index 80746b81..466fc957 100644 --- a/src/compiler/training/rewind/recordedCommit.ts +++ b/src/compiler/training/rewind/recordedCommit.ts @@ -1,4 +1,4 @@ -import { recordedCommitSchema } from "./contract.js"; +import { recordedCommitSchema } from "../../../recording/index.js"; import { rewindGit } from "./gitProcess.js"; /** Recorded identity names the commit object itself, never a selector or a peeled tag. */ diff --git a/src/compiler/training/rewind/recordingFormat.test.ts b/src/compiler/training/rewind/recordingFormat.test.ts index 5d0b8ce8..dc4fa8be 100644 --- a/src/compiler/training/rewind/recordingFormat.test.ts +++ b/src/compiler/training/rewind/recordingFormat.test.ts @@ -1,7 +1,8 @@ import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { momentRecordSchema } from "./contract.js"; +import { momentRecordSchema, momentFileName, withMomentStore, writeMomentRecord, type WakeMomentRow } from "../../../recording/index.js"; +import * as rewindContract from "./contract.js"; import { AT, BEFORE, emptyRecord, jsonl, memoryEvent, rewindFixture } from "./fixtures.test-helper.js"; import { gitTestRoot } from "./git.test-helper.js"; import { History } from "./history.js"; @@ -18,6 +19,9 @@ const recorded = () => ({ }); describe("P3 recording format", () => { + it("leaves the moment record schema owned only by the recording barrel", () => { + expect(rewindContract).not.toHaveProperty("momentRecordSchema"); + }); it("accepts the recording shape and rejects the superseded sketch", () => { const value = { ...recorded(), exact: false, errors: ["volume state: capture failed"], resources: [{ id: "git", kind: "git", commit: "a".repeat(40) }, { id: "bundle", kind: "bundle", sha256: `sha256:${"b".repeat(64)}` }], @@ -49,6 +53,42 @@ describe("P3 recording format", () => { }); describe("recording pins in a materialized trial", () => { + it.each([false, true])("consumes equal executionId+startedAt replay files and refuses differing decisions=%s", async (conflicting) => { + const f = await rewindFixture(); roots.push(f.root); + const first = emptyRecord(); first.memory = [{ id: "self", declaredBy: "agent:a", seq: 0 }]; + const second = structuredClone(first); + second.wake.deliveries[0]!.deliveryId = "replay-delivery"; + if (conflicting) second.memory[0]!.seq = 1; + const row = (record: typeof first): WakeMomentRow => ({ version: "noopolis.daimon.wake-moment.v1", + agent_id: record.agent, execution_id: record.wake.executionId, started_at: record.wake.startedAt, + deliveries: record.wake.deliveries.map((delivery) => ({ acceptance_id: delivery.deliveryId, + delivery_id: delivery.deliveryId, kind: delivery.kind, occurred_at: delivery.occurredAt })), snapshots: [] }); + const slug = f.project.compilePlan.nodes.find((node) => node.id === f.plan.agent.id)!.slug; + await withMomentStore(path.join(f.historyDir, "record.json"), slug, async (directory) => { + for (const record of [second, first]) expect(await writeMomentRecord(directory, record, row(record))).toBe(true); + expect(await readdir(directory)).toEqual(expect.arrayContaining([momentFileName(row(first)), momentFileName(row(second))])); + }); + expect(momentFileName(row(first))).not.toBe(momentFileName(row(second))); + const result = await materializeTrainingMoment(f.options); + if (conflicting) { + expect(result.status).toBe("refused"); + expect(result.inputs.find((input) => input.kind === "memory")!.refusal).toMatch(/Conflicting moment records.*memory agent:a\/self/u); + expect(await readdir(f.options.out)).toEqual(["moment.json"]); + } else { + expect(result.status, JSON.stringify(result.inputs)).toBe("materialized"); + expect(result.record).toMatchObject({ matched: true, attempts: 2, executionId: first.wake.executionId, selectedStartedAt: AT }); + expect(result.inputs.find((input) => input.kind === "memory")).toMatchObject({ fidelity: "exact", source: { seq: 0 } }); + } + }); + + it.each(["json", "jsonl"])("applies shared recording validation to .%s discovery", async (suffix) => { + const root = await gitTestRoot(); roots.push(root); + const record = { ...recorded(), memory: [{ id: "self", declaredBy: "agent:a", seq: Number.MAX_SAFE_INTEGER + 1 }] }; + await mkdir(path.join(root, "moments/a"), { recursive: true }); + await writeFile(path.join(root, `moments/a/invalid.${suffix}`), `${JSON.stringify(record)}\n`); + await expect(new History(root).records()).rejects.toThrow(/Invalid history moments\/a\/invalid/u); + }); + it.each([false, true])("refuses conflicting copies instead of choosing the empty pin or nonempty fallback (reverse=%s)", async (reverse) => { const f = await rewindFixture(); roots.push(f.root); const healthy = emptyRecord(); healthy.memory = [{ id: "self", declaredBy: "agent:a", seq: 0 }]; diff --git a/src/compiler/training/rewind/select.test.ts b/src/compiler/training/rewind/select.test.ts index 58538744..78bc50d6 100644 --- a/src/compiler/training/rewind/select.test.ts +++ b/src/compiler/training/rewind/select.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import type { TrainingPlan } from "../plan/index.js"; -import type { MomentRecord } from "./contract.js"; +import type { MomentRecord } from "../../../recording/index.js"; import { selectMomentRecord, selectTrainingExample } from "./select.js"; const at = "2026-10-09T08:20:31Z"; diff --git a/src/compiler/training/rewind/select.ts b/src/compiler/training/rewind/select.ts index 9cfdfff4..ea46fad2 100644 --- a/src/compiler/training/rewind/select.ts +++ b/src/compiler/training/rewind/select.ts @@ -1,5 +1,5 @@ import type { TrainingPlan } from "../plan/index.js"; -import type { MomentRecord } from "./contract.js"; +import type { MomentRecord } from "../../../recording/index.js"; import { compareInstants, normalizeInstant } from "./instant.js"; import { hasRecordPin, recordForInput, recordPins } from "./recordInput.js"; import type { InputIdentity, RewindInput } from "./types.js"; @@ -36,7 +36,7 @@ const canonical = (value: unknown): string => JSON.stringify(value, (_key, item: export const matchingMomentRecords = (records: readonly MomentRecord[], agent: string, at: string): MomentRecord[] => records.filter((record) => record.agent === agent && compareInstants(record.wake.at, at) === 0); -/** Replays keep the wake identity but later attempts have different pins. Only the original is relevant. */ +/** Replays can share executionId AND startedAt. Keep every earliest tie for per-input agreement. */ export const originalMomentRecords = (records: readonly MomentRecord[]): MomentRecord[] => { const first = records.reduce((earliest, entry) => !earliest || compareInstants(entry.wake.startedAt, earliest.wake.startedAt) < 0 ? entry : earliest, null); diff --git a/src/compiler/training/rewind/snapshotFixtures.test-helper.ts b/src/compiler/training/rewind/snapshotFixtures.test-helper.ts index 36ae8c17..9184df1b 100644 --- a/src/compiler/training/rewind/snapshotFixtures.test-helper.ts +++ b/src/compiler/training/rewind/snapshotFixtures.test-helper.ts @@ -7,7 +7,7 @@ interface SnapshotFixture { directories?: Record; symlinks?: Record; } -/** Recorder-format data only: no imports from the parallel Daimon implementation. */ +/** Recorder-format data only: no runtime dependency on Daimon. */ export const writeRecordedVolumeSnapshot = async (historyRoot: string, options: SnapshotFixture) => { const recording = path.join(historyRoot, "recording", options.agentSlug); const base = path.join(recording, "snapshots", options.rootId); @@ -42,8 +42,10 @@ export const writeRecordedVolumeSnapshot = async (historyRoot: string, options: const manifest = { version: "noopolis.daimon.wake-moment.v1", id: options.rootId, snapshot: options.snapshot, started_at: options.startedAt, sourcePath: `/deployment/${options.rootId}`, entries }; await writeFile(manifestPath, `${JSON.stringify(manifest)}\n`); + await writeFile(`${directory}.complete`, ""); await appendFile(rowsPath, `${JSON.stringify({ version: manifest.version, agent_id: options.agentId, - execution_id: options.executionId ?? "execution", started_at: options.startedAt, deliveries: [], + execution_id: options.executionId ?? "execution", started_at: options.startedAt, + deliveries: [{ acceptance_id: "acceptance", delivery_id: "delivery", kind: "manual", occurred_at: options.startedAt }], snapshots: [{ id: options.rootId, snapshot: options.snapshot, files: Object.keys(options.files).length, copied: Object.keys(options.files).length, linked: 0, skipped: 0 }] })}\n`); return { directory, manifestPath, rowsPath, manifest }; diff --git a/src/compiler/training/rewind/snapshotManifest.ts b/src/compiler/training/rewind/snapshotManifest.ts index 7ac4630c..0a88a760 100644 --- a/src/compiler/training/rewind/snapshotManifest.ts +++ b/src/compiler/training/rewind/snapshotManifest.ts @@ -37,6 +37,8 @@ export const snapshotHistoryPath = (agentSlug: string, id: string, snapshot: str `recording/${safeSegment(agentSlug)}/snapshots/${safeSegment(id === "." || id === ".." ? id.replaceAll(".", "%2E") : id)}/${safeSegment(snapshot)}`; export const readSnapshotManifest = async (history: History, relative: string, id: string, snapshot: string): Promise => { + // Daimon publishes this marker only after the tree and manifest are durable. + if (await history.bytes(`${relative}.complete`) === null) return null; const bytes = await history.text(`${relative}.manifest.json`); if (bytes === null) return null; let manifest: SnapshotManifest; diff --git a/src/compiler/training/rewind/types.ts b/src/compiler/training/rewind/types.ts index 99287d7b..d3b9c991 100644 --- a/src/compiler/training/rewind/types.ts +++ b/src/compiler/training/rewind/types.ts @@ -1,6 +1,7 @@ import type { MemoryBank, TeamNetwork, TeamWorkspaceResource } from "../../../manifest/index.js"; import type { ResolvedMemoryBank } from "../../types.js"; -import type { MomentRecord, MaterializedInput } from "./contract.js"; +import type { MaterializedInput } from "./contract.js"; +import type { MomentRecord } from "../../../recording/index.js"; import type { History } from "./history.js"; export interface InputIdentity { section: "resources" | "memory" | "networks"; id: string; declaredBy: string } diff --git a/src/compiler/training/rewind/volumes.test.ts b/src/compiler/training/rewind/volumes.test.ts index 541797a0..41343433 100644 --- a/src/compiler/training/rewind/volumes.test.ts +++ b/src/compiler/training/rewind/volumes.test.ts @@ -3,7 +3,8 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { openHistory } from "./history.js"; -import { momentRecordSchema, type LeakCheck, type MaterializedInput } from "./contract.js"; +import { type LeakCheck, type MaterializedInput } from "./contract.js"; +import { momentRecordSchema } from "../../../recording/index.js"; import { inspectVolumeBundle } from "./leakInputs.js"; import { writeRecordedVolumeSnapshot } from "./snapshotFixtures.test-helper.js"; import type { InputContext } from "./types.js"; @@ -29,6 +30,26 @@ const report = (): LeakCheck => ({ version: "spawnfile.leak-check.v1", at, ok: f checked: { files: 0, gitObjects: 0, memoryEvents: 0, moltnetMessages: 0, moltnetEvents: 0, snapshots: 0 }, findings: [] }); describe("unfed recorded volume trees", () => { + it("requires the merged completion marker in both selection and independent acceptance", async () => { + const { root, context, snapshot } = await fixtures(); + const saved = await snapshot("20261009T090000000Z-completed", before, { data: "bytes" }); + const result = await materializeVolume(context); + await rm(`${saved.directory}.complete`); + context.destination += "-incomplete"; + await expect(materializeVolume(context)).rejects.toThrow("No complete volume snapshot"); + const input: MaterializedInput = { section: "resources", id: "state", declaredBy: "team:org", kind: "volume", path: "out", + fidelity: result.fidelity, basis: result.basis, source: result.source, notes: [], refusal: null }; + await expect(inspectVolumeBundle(root, input, report(), context.history)).rejects.toThrow("Snapshot tree or manifest is missing"); + }); + + it("uses the merged row parser's nonempty delivery requirement", async () => { + const { context, snapshot } = await fixtures(); + const saved = await snapshot("20261009T090000000Z-deliveries", before, { data: "bytes" }); + const row = JSON.parse(await readFile(saved.rowsPath, "utf8")); row.deliveries = []; + await writeFile(saved.rowsPath, `${JSON.stringify(row)}\n`); + await expect(materializeVolume(context)).rejects.toThrow(/wake-moments.jsonl:1:/u); + }); + it("selects by inclusive row started_at, copies a full tree, and uses the named record exactly", async () => { const { context, snapshot } = await fixtures(); await snapshot("20261009T080000000Z-first", before, { old: "old" }); @@ -47,14 +68,14 @@ describe("unfed recorded volume trees", () => { it("ignores incomplete and .partial trees and chooses the last row when instants tie", async () => { const { context, snapshot } = await fixtures(); - await snapshot("first", before, { data: "first" }); - await snapshot("tie", before, { data: "last" }); - const incomplete = await snapshot("incomplete", at, { data: "unfinished" }); + await snapshot("20261009T090000000Z-first", before, { data: "first" }); + await snapshot("20261009T090000000Z-tie", before, { data: "last" }); + const incomplete = await snapshot("20261009T090000000Z-incomplete", at, { data: "unfinished" }); await rm(incomplete.manifestPath); - await snapshot("incoming.partial", at, { data: "partial" }); - expect(await materializeVolume(context)).toMatchObject({ source: { snapshot: "tie" } }); + await snapshot("20261009T090000000Z-incoming.partial", at, { data: "partial" }); + expect(await materializeVolume(context)).toMatchObject({ source: { snapshot: "20261009T090000000Z-tie" } }); expect(await readFile(path.join(context.destination, "data"), "utf8")).toBe("last"); - context.record = record("incoming.partial"); context.destination += "-record"; + context.record = record("20261009T090000000Z-incoming.partial"); context.destination += "-record"; await expect(materializeVolume(context)).rejects.toThrow(/partial/i); }); @@ -67,16 +88,16 @@ describe("unfed recorded volume trees", () => { it("refuses missing history, missing rows, future-only snapshots, and absent named snapshots", async () => { const { context, snapshot } = await fixtures(); await expect(materializeVolume(context)).rejects.toThrow("Missing volume snapshot history"); - await snapshot("future", future); + await snapshot("20261009T090000000Z-future", future); await expect(materializeVolume(context)).rejects.toThrow("No complete volume snapshot"); - context.record = record("missing"); - await expect(materializeVolume(context)).rejects.toThrow("Recorded volume snapshot missing"); + context.record = record("20261009T090000000Z-missing"); + await expect(materializeVolume(context)).rejects.toThrow("Recorded volume snapshot 20261009T090000000Z-missing"); context.history = undefined; await expect(materializeVolume(context)).rejects.toThrow("requires --history"); }); it("refuses invalid rows with line numbers and mismatched manifest identity", async () => { - const { context, snapshot } = await fixtures(), saved = await snapshot("one", before, { data: "bytes" }); + const { context, snapshot } = await fixtures(), saved = await snapshot("20261009T090000000Z-one", before, { data: "bytes" }); await writeFile(saved.rowsPath, `${await readFile(saved.rowsPath, "utf8")}{}\n`); await expect(materializeVolume(context)).rejects.toThrow("wake-moments.jsonl:2"); await writeFile(saved.rowsPath, (await readFile(saved.rowsPath, "utf8")).split("\n")[0] + "\n"); @@ -85,7 +106,7 @@ describe("unfed recorded volume trees", () => { }); it.each(["extra", "missing", "type", "size", "mode", "unsafe"])("refuses manifest %s mismatches", async (mutation) => { - const { context, snapshot } = await fixtures(), saved = await snapshot("one", before, { data: "bytes" }); + const { context, snapshot } = await fixtures(), saved = await snapshot("20261009T090000000Z-one", before, { data: "bytes" }); if (mutation === "extra") await writeFile(path.join(saved.directory, "extra"), "extra"); if (mutation === "missing") await rm(path.join(saved.directory, "data")); if (mutation === "type") { await rm(path.join(saved.directory, "data")); await mkdir(path.join(saved.directory, "data")); } @@ -104,7 +125,7 @@ describe("unfed recorded volume trees", () => { it("restores recorded directory modes without comparing nonportable directory sizes and verifies symlinks", async () => { const { context, root } = await fixtures(); const saved = await writeRecordedVolumeSnapshot(root, { agentSlug: context.agentSlug!, agentId: context.agentId!, rootId: "state", - snapshot: "tree", startedAt: before, files: { "nested/data": "bytes" }, directories: { nested: 0o750 }, symlinks: { link: "nested/data" } }); + snapshot: "20261009T090000000Z-tree", startedAt: before, files: { "nested/data": "bytes" }, directories: { nested: 0o750 }, symlinks: { link: "nested/data" } }); const manifest = JSON.parse(await readFile(saved.manifestPath, "utf8")); manifest.entries.nested.size = "999999"; manifest.entries.nested.source.size = "999999"; await writeFile(saved.manifestPath, JSON.stringify(manifest)); await chmod(path.join(saved.directory, "nested"), 0o700); @@ -119,7 +140,7 @@ describe("unfed recorded volume trees", () => { it.each(["bytes", "mode", "extra", "missing", "target"])("independently rejects a bundle %s mutation", async (mutation) => { const { root, context } = await fixtures(); await writeRecordedVolumeSnapshot(root, { agentSlug: context.agentSlug!, agentId: context.agentId!, rootId: "state", - snapshot: "one", startedAt: before, files: { data: "bytes", other: "other" }, symlinks: { link: "data" } }); + snapshot: "20261009T090000000Z-one", startedAt: before, files: { data: "bytes", other: "other" }, symlinks: { link: "data" } }); const result = await materializeVolume(context); if (mutation === "bytes") await writeFile(path.join(context.destination, "data"), "wrong"); if (mutation === "mode") await chmod(path.join(context.destination, "data"), 0o600); diff --git a/src/compiler/training/rewind/volumes.ts b/src/compiler/training/rewind/volumes.ts index 1285a1cb..7eb0278e 100644 --- a/src/compiler/training/rewind/volumes.ts +++ b/src/compiler/training/rewind/volumes.ts @@ -1,25 +1,16 @@ import { lstat, mkdir } from "node:fs/promises"; -import { z } from "zod"; -import { trainingInstantSchema } from "../../../manifest/index.js"; import { safeSegment, type History } from "./history.js"; import { compareInstants } from "./instant.js"; import { copySnapshotTree, readSnapshotManifest, snapshotHistoryPath, type SnapshotManifest } from "./snapshotManifest.js"; import type { InputContext, InputResult } from "./types.js"; -const text = z.string().min(1), count = z.number().int().nonnegative(); -export const wakeSnapshotRowSchema = z.object({ version: z.literal("noopolis.daimon.wake-moment.v1"), - agent_id: text, execution_id: text, started_at: trainingInstantSchema, - deliveries: z.array(z.object({ acceptance_id: text, delivery_id: text, kind: text, occurred_at: trainingInstantSchema }).strict()), - snapshots: z.array(z.object({ id: text, snapshot: text, files: count, copied: count, linked: count, skipped: count }).strict()), - error: text.optional() }).strict(); -export type WakeSnapshotRow = z.infer; export interface SelectedVolumeSnapshot { snapshot: string; startedAt: string; manifest: SnapshotManifest; relative: string } /** A published row plus retained directory and manifest is complete; never infer time from its name. */ export const selectVolumeSnapshot = async (history: History, agentSlug: string, agentId: string, id: string, at: string, recordId?: string): Promise => { const rowsPath = `recording/${safeSegment(agentSlug)}/wake-moments.jsonl`; - const rows = await history.jsonl(rowsPath, wakeSnapshotRowSchema); + const rows = await history.wakeMoments(rowsPath); if (!rows) throw new Error(`Missing volume snapshot history ${rowsPath}`); const candidates = rows.flatMap((row, index) => row.agent_id !== agentId ? [] : row.snapshots.filter((entry) => entry.id === id) .map((entry) => ({ row, entry, index }))) diff --git a/src/recording/AGENTS.md b/src/recording/AGENTS.md index 46916a45..810f075f 100644 --- a/src/recording/AGENTS.md +++ b/src/recording/AGENTS.md @@ -1,5 +1,6 @@ # Moment Recording +- `moment.ts` owns the shared strict moment-record schema, inferred type and parser; rewind consumes the public barrel. - `types.ts`, `parse.ts`, and `compose.ts` own the pure wake-row contract and pin composition. - `sync.ts` joins activation-scoped evidence; `read.ts` reads bounded byte chunks via the usage exec transport. - `history.ts` selects recording facts from the activation log; `deployment.ts` selects the live deployment and resolves feed histories. diff --git a/src/recording/index.ts b/src/recording/index.ts index bb65208c..3bd866df 100644 --- a/src/recording/index.ts +++ b/src/recording/index.ts @@ -1,4 +1,5 @@ export * from "./types.js"; +export * from "./moment.js"; export * from "./compose.js"; export * from "./parse.js"; export * from "./store.js"; diff --git a/src/recording/moment.test.ts b/src/recording/moment.test.ts new file mode 100644 index 00000000..6e249a02 --- /dev/null +++ b/src/recording/moment.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { composeMomentRecord, momentRecordSchema, parseMomentRecord, type ComposeMomentRecordInput, type MomentRecord } from "./index.js"; + +const at = "2026-10-09T08:20:31Z"; +const input = (): ComposeMomentRecordInput => ({ + row: { version: "noopolis.daimon.wake-moment.v1", agent_id: "agent:writer", execution_id: "execution", + started_at: at, deliveries: [{ acceptance_id: "accept", delivery_id: "wake", kind: "manual", occurred_at: at }], snapshots: [] }, + agentNodeId: "agent:writer", deploymentRecordPath: "/retained/deployments/desk.json", recordedAt: at, + deployResources: [], feedLandings: {}, memoryLedgers: {} +}); + +describe("shared moment record contract", () => { + it("round-trips the merged composer's output without a second consumer schema", () => { + const record: MomentRecord = composeMomentRecord(input()); + expect(record.exact).toBe(false); + expect(parseMomentRecord(JSON.stringify(record))).toEqual(record); + }); + + it("accepts only digest bundle pins and bounded memory cursors with optional generation", () => { + const record = { ...composeMomentRecord(input()), resources: [{ id: "tools", kind: "bundle", sha256: `sha256:${"a".repeat(64)}` }], + memory: [{ id: "self", declaredBy: "agent:writer", seq: 0, generation: 0 }] }; + expect(parseMomentRecord(JSON.stringify(record))).toEqual(record); + for (const invalid of [{ ...record, resources: [{ id: "tools", kind: "bundle", commit: "a".repeat(40) }] }, + { ...record, exact: undefined }, { ...record, memory: [{ ...record.memory[0], seq: Number.MAX_SAFE_INTEGER + 1 }] }, + { ...record, wake: { ...record.wake, deliveries: [] } }]) expect(momentRecordSchema.safeParse(invalid).success).toBe(false); + }); +}); diff --git a/src/recording/moment.ts b/src/recording/moment.ts new file mode 100644 index 00000000..2cc2d553 --- /dev/null +++ b/src/recording/moment.ts @@ -0,0 +1,40 @@ +import { z } from "zod"; +import { trainingInstantSchema } from "../manifest/index.js"; +import { MOMENT_RECORD_VERSION } from "./types.js"; + +const text = z.string().min(1); +const count = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER); +export const recordedSha256Schema = z.string().regex(/^sha256:[a-f0-9]{64}$/u); +export const recordedCommitSchema = z.string() + .regex(/^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/u, "Expected a full 40- or 64-hex commit object id") + .toLowerCase(); +const unique = (schema: T, key: (item: z.infer) => string) => + z.array(schema).superRefine((items, context) => { + const keys = new Set(); + items.forEach((item, index) => { + const id = key(item); + if (keys.has(id)) context.addIssue({ code: "custom", message: `Duplicate identity ${id}`, path: [index] }); + keys.add(id); + }); + }); +export const momentRecordSchema = z.object({ + version: z.literal(MOMENT_RECORD_VERSION), agent: text, + wake: z.object({ at: trainingInstantSchema, startedAt: trainingInstantSchema, executionId: text, + deliveries: z.array(z.object({ deliveryId: text, kind: text, occurredAt: trainingInstantSchema }).strict()).min(1), + surface: z.literal("moltnet").optional(), room: z.string().optional(), + from: z.string().optional(), messageId: text.optional() }).strict(), + recordedAt: trainingInstantSchema, exact: z.boolean(), errors: z.array(z.string()).optional(), + resources: unique(z.discriminatedUnion("kind", [ + z.object({ id: text, kind: z.literal("git"), commit: recordedCommitSchema, sha256: recordedSha256Schema.optional() }).strict(), + z.object({ id: text, kind: z.literal("bundle"), sha256: recordedSha256Schema }).strict() + ]), (r) => r.id), + feeds: unique(z.object({ id: text, revision: text, tree: text, commit: recordedCommitSchema.optional(), ref: text.optional(), + paths: z.array(text).optional(), landedAt: trainingInstantSchema, period: text.optional() }).strict(), (r) => r.id), + memory: unique(z.object({ id: text, declaredBy: text, seq: count, generation: count.optional() }).strict(), + (r) => JSON.stringify([r.declaredBy, r.id])), + moltnet: unique(z.object({ network: text, cursor: trainingInstantSchema, messageIds: z.array(text).optional() }).strict(), (r) => r.network), + volumes: unique(z.object({ id: text, snapshot: text }).strict(), (r) => r.id) +}).strict(); +export type MomentRecord = z.infer; + +export const parseMomentRecord = (payload: string): MomentRecord => momentRecordSchema.parse(JSON.parse(payload)); diff --git a/src/recording/types.ts b/src/recording/types.ts index 0fb84463..302322d8 100644 --- a/src/recording/types.ts +++ b/src/recording/types.ts @@ -29,25 +29,4 @@ export interface ComposeMomentRecordInput { recordedAt: string; } -export interface MomentRecord { - version: typeof MOMENT_RECORD_VERSION; - agent: string; - wake: { - at: string; - startedAt: string; - executionId: string; - deliveries: { deliveryId: string; kind: string; occurredAt: string }[]; - surface?: "moltnet"; - messageId?: string; - }; - recordedAt: string; - exact: boolean; - resources: ({ id: string; kind: "git"; commit: string; sha256?: string } - | { id: string; kind: "bundle"; sha256: string })[]; - feeds: { id: string; revision: string; tree: string; commit?: string; ref?: string; - paths?: string[]; landedAt: string; period?: string }[]; - memory: { id: string; declaredBy: string; seq: number }[]; - moltnet: { network: string; cursor: string; messageIds?: string[] }[]; - volumes: { id: string; snapshot: string }[]; - errors?: string[]; -} +export type { MomentRecord } from "./moment.js"; From 164d669bc126e6967b3c19ef156aa1b3dcab98c2 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sun, 11 Oct 2026 05:27:30 +0200 Subject: [PATCH 9/9] fix: downgrade pins from inexact activations and check bundle payloads, not only provenance --- specs/TRAINING.md | 77 ++++++++++++--- src/compiler/training/rewind/AGENTS.md | 2 +- src/compiler/training/rewind/contract.ts | 3 +- .../training/rewind/deployedResource.test.ts | 35 +++++++ src/compiler/training/rewind/feed.test.ts | 34 ++++++- src/compiler/training/rewind/feed.ts | 1 + src/compiler/training/rewind/git.ts | 42 +++++---- .../training/rewind/gitProcess.test.ts | 37 ++++++++ src/compiler/training/rewind/gitProcess.ts | 9 +- .../training/rewind/leakCheck.test.ts | 2 +- src/compiler/training/rewind/leakFiles.ts | 7 +- src/compiler/training/rewind/leakGit.ts | 4 + .../training/rewind/leakGitPayload.test.ts | 94 +++++++++++++++++++ .../training/rewind/leakGitPayload.ts | 50 ++++++++++ src/compiler/training/rewind/leakInputs.ts | 15 ++- .../training/rewind/leakLayout.test.ts | 35 +++++++ src/compiler/training/rewind/leakLayout.ts | 17 ++++ .../training/rewind/leakProvenance.test.ts | 21 ++++- .../training/rewind/leakProvenance.ts | 15 ++- src/compiler/training/rewind/moltnet.ts | 2 +- .../training/rewind/moltnetJson.test.ts | 47 ++++++++++ src/compiler/training/rewind/moltnetJson.ts | 22 ++++- .../training/rewind/moltnetSqlite.test.ts | 60 +++++++++++- src/compiler/training/rewind/moltnetSqlite.ts | 25 ++++- .../training/rewind/moltnetSqliteData.ts | 3 + src/compiler/training/rewind/recordInput.ts | 5 + src/compiler/training/rewind/select.test.ts | 8 ++ src/compiler/training/rewind/select.ts | 4 +- 28 files changed, 618 insertions(+), 58 deletions(-) create mode 100644 src/compiler/training/rewind/gitProcess.test.ts create mode 100644 src/compiler/training/rewind/leakGitPayload.test.ts create mode 100644 src/compiler/training/rewind/leakGitPayload.ts create mode 100644 src/compiler/training/rewind/leakLayout.test.ts create mode 100644 src/compiler/training/rewind/leakLayout.ts diff --git a/specs/TRAINING.md b/specs/TRAINING.md index 99a7b4c2..687b939a 100644 --- a/specs/TRAINING.md +++ b/specs/TRAINING.md @@ -478,7 +478,11 @@ helpers' millisecond host-clock contract. An `exact:false` record does not invalidate healthy pins: a present pin is used with `exact` fidelity and `record` basis (`record-cursor` for Moltnet), unless its -`errors` names that input. Errored pins are ignored, absent pins use fallback +`errors` names that input. A record error containing `Deployment activation is non-exact` +instead downgrades every git/bundle pin used from that record to `approximate`, +preserving `record` basis and the error in notes; those pins are still used. +This covers inexact activations and rollback intervals even though the marker +does not name individual resources. Errored pins are ignored, absent pins use fallback rules, and those inputs are labelled `approximate`, with the record's error text in notes. Explicit section or declaring-node qualifications apply only to that input; unqualified ID errors conservatively apply to every input with that ID. @@ -508,7 +512,8 @@ intervals MUST remain `approximate`, with the activation/rollback reason in note The merged helper excludes a rolled-back candidate after its interval and restores the predecessor. A pin that is present but unavailable in the repository refuses. Only an absent deployment pin falls back to the declared selector's -first-parent history, choosing its newest commit with committer time `<= effectiveAt` +first-parent history, choosing its newest commit by committer time among commits +whose author AND committer times are `<= effectiveAt` (`approximate`, `selector-at`); an absent selector means `HEAD`. Commit dates do not establish when a commit reached a branch or tag, so this fallback cannot prove historical ref membership. A missing eligible commit refuses. @@ -521,13 +526,17 @@ from the selected commit, with one checked-out branch, no remotes, tags, source packed refs, reflogs, `FETCH_HEAD`, or `ORIG_HEAD`. Git commands use argv without a shell and sanitized configuration. History MUST NOT be mutated. The leak check also rejects any reachable ancestor with author or committer time after -`effectiveAt`. Bundle `build.files` extracts `files.root` at the selected commit and honors +`effectiveAt`. Rebuild the checkout index without live stat data; acceptance checks +every index entry's ctime and mtime against `effectiveAt`. Acceptance also compares +the worktree paths, raw bytes, symlink targets, and executable modes against the +selected commit's tree in the output repository, refusing untracked/ignored extras. +Bundle `build.files` extracts `files.root` at the selected commit and honors `exclude`. `build.dependencies` and `build.generated` retain their source directory or declared inputs at that commit and report `build: deferred`; trial preparation owns builds. Bundle output MAY be a plain tree. A moment record or, when absent, the deployment log pins a bundle by SHA-256 only. Exact bundle output requires `bundles/.tar` whose actual digest -matches that pin (`record` or `deploy-log` basis). A deploy-log digest retains +matches that pin (`record` or `deploy-log` basis). A record or deploy-log digest retains the activation/rollback fidelity above even when its bytes verify. A present deployment bundle pin without a matching archive refuses; it MUST NOT silently select a different source commit. Validate the archive before extraction: escaping paths, @@ -537,6 +546,9 @@ For a moment-record bundle pin, absent or digest-mismatched archives fall back t source reconstruction where available, always `approximate` even with a pinned source commit; notes explain that the recorded SHA-256 could not be reproduced. A prebuilt `source` tar has no historical source fallback and refuses unless overridden. +Source-built bundles record their resolved roots, prefixes and exclusions in +`source.trees`. Acceptance re-exports that selection from the source repository +and byte-compares the result, including when a local pin supplies the repository. Generated inputs spanning multiple repositories or independently pinned input refs cannot be represented by this one-repository/one-commit history contract and MUST refuse. Bundle roots whose historical repository prefix cannot be @@ -551,9 +563,11 @@ landing log is retained. Otherwise, lookup inside the inclusive `[switching_at, landed_at]` window selects the incoming entry with `approximate`, `landing-log` and an ambiguous-transition note. Outside windows, choose the newest `landed_at <= effectiveAt`, breaking equal timestamps by last append -(`exact`, `landing-log`). Before the first entry, there is no landed state. +(`exact`, `landing-log`). Before the first entry, there is no landed state: +a nonempty log without an applicable landing refuses unless a matching record +supplies the state or the feed is explicitly overridden. -Without selected recorded state, a plain git feed evaluates its ref template +Without a record pin or a nonempty landing log, a plain git feed evaluates its ref template and paths at `effectiveAt`, including `${date}` in UTC and `${date:}`. Use the primary ref only if it has an eligible first-parent commit; otherwise try the fallback. Its newest eligible commit is `approximate`, `feed-rule-at`. @@ -631,9 +645,14 @@ events after `effectiveAt`. Empty threads/DMs and threads/DMs whose root is afte are dropped. Thread/DM message counts and last-message times are recomputed; attachment cursors are clamped/remapped to retained events. Room removals after `effectiveAt` are undone when removal time and restorable room metadata exist; a missing -historical removal time retains current removal state. Agents and memberships -have no historical log and are retained as now, explicitly disclosed in input -`notes`: even a cursor-exact content cut cannot prove historical membership. +historical removal time retains current removal state. Restoration MUST refuse +when its only available room metadata comes from a future event. +Agents registered after `effectiveAt` and their memberships and delivery cursors +are removed; retained agents' `updated_at` and SQLite migrations' `applied_at` +are clamped to `effectiveAt`. Both store inspectors reject future registration +timestamps; the SQLite inspector also rejects future migration timestamps. +Other registration metadata and memberships have no historical log, explicitly +disclosed in input `notes`: even a cursor-exact content cut cannot prove historical membership. SQLite changes apply only to a copy through `node:sqlite` on the supported Node >=22.19 line; if that API is unavailable, SQLite rewind refuses. RFC3339Nano fractions with omitted trailing zeros MUST be compared as parsed instants. @@ -755,16 +774,23 @@ reopens it read-only. That source repository must remain available for subsequen verification. Directory feeds and digest-pinned or explicitly overridden prebuilt tars have no git provenance. Recorded bundles are independently checked against their digest-pinned archive. This source check is independent of the materialized git object's reachable-history check and MUST NOT reuse the producer's date assertion. +Native payload layouts use exact entry allowlists: a feed volume contains only +`current`, `.spawnfile-feed.json`, and `trees/`; a memory bank contains +only `memory/events.jsonl`; a Moltnet directory contains exactly one of +`state.json` or `moltnet.sqlite`. Bundle-level entries are limited to the input +paths and their directory ancestors plus `moment.json`, `leak-check.json`, and +`task-clock.json`. Unexpected entries refuse even with historical file times. Every finding refuses the moment: | Class | Refusal condition | | --- | --- | +| `bundle-entry` | An entry outside the input paths, their directory ancestors, and the report allowlist is present. | | `file-time` | A file, directory, or symlink atime or mtime is after `effectiveAt`. | | `symlink-escape` | An absolute link or a lexically/physically escaping link resolves outside the bundle. | | `git-object` | Source provenance is missing, inconsistent, not a commit, or has author/committer time after `effectiveAt`; any materialized commit has a future date, any object is unreachable from the selected commit, or refs/remotes/reflogs/fetch metadata exceed the single branch. Any `.git` directory/gitfile or bare repository (`HEAD` + `objects/` + `refs/`) anywhere outside the materialized git resource repository also refuses, including inside feed trees and bundles. | -| `memory-event` | Future ledger events, sequence gaps, invalid prefix ordering, or derived/audit files are present. | -| `moltnet-message` / `moltnet-event` / `moltnet-room` | Retained messages, delivery events, or rooms are after `effectiveAt`. | -| `feed-identity` | Unproved publication time after `effectiveAt`, a directory revision mismatch, or `current` outside its selected tree. | +| `memory-event` | Future ledger events, sequence gaps, invalid prefix ordering, or unexpected bank entries are present. | +| `moltnet-message` / `moltnet-event` / `moltnet-room` | Retained messages, delivery events, rooms, registration timestamps, or migration timestamps are after `effectiveAt`, or the store directory contains unexpected entries. | +| `feed-identity` | Unproved publication time after `effectiveAt`, a directory revision mismatch, `current` outside its selected tree, or unexpected volume entries/retained trees. | | `dated-path` | A path segment contains a calendar date `YYYY-MM-DD` later than the UTC date of `effectiveAt + 14h`, the latest possible local date. | | `snapshot` | An unfed volume differs from its selected full snapshot manifest or retained bytes. | @@ -1880,6 +1906,32 @@ The child deadline is Paideia's declared deadline plus five seconds for cleanup. "minimum": 0, "maximum": 9007199254740991 }, + "trees": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "properties": { + "root": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "exclude": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": [ + "root" + ], + "additionalProperties": false + } + }, "sha256": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" @@ -2132,6 +2184,7 @@ The child deadline is Paideia's declared deadline plus five seconds for cleanup. "class": { "type": "string", "enum": [ + "bundle-entry", "file-time", "symlink-escape", "git-object", diff --git a/src/compiler/training/rewind/AGENTS.md b/src/compiler/training/rewind/AGENTS.md index b4f9437d..03ed56dd 100644 --- a/src/compiler/training/rewind/AGENTS.md +++ b/src/compiler/training/rewind/AGENTS.md @@ -10,7 +10,7 @@ - `feedContent.ts` verifies plain git revisions and re-extracts for independent byte acceptance; retained copies must match. Prepared revisions prove provenance only: always approximate, never compared to a directory content digest. - `moltnet.ts` dispatches stores; `moltnetJson.ts`, `moltnetSqlite.ts`, `moltnetSqliteData.ts`, and `moltnetSchemas.ts` own snapshots, copied databases, SQL decoding, and strict native shapes. - `materialize.ts` stages all inputs and publishes only after the independent disk reader in `leakCheck.ts` accepts them. -- `leakFiles.ts`, `leakGit.ts`, and `leakInputs.ts` inspect disk metadata, every git object, and native payloads independently of materialization decisions. `leakProvenance.ts` reopens source repositories read-only using disk commit receipts, checking type and both raw dates without the producer's guard. Discover unexpected `.git` directories/gitfiles and bare repositories throughout every input; only the materialized git resource repository is allowed. +- `leakFiles.ts`, `leakGit.ts`, and `leakInputs.ts` inspect disk metadata, every git object, and native payloads independently of materialization decisions. `leakGitPayload.ts` compares raw output-repository blobs to the worktree and checks index stat times; `leakLayout.ts` enforces closed native layouts. `leakProvenance.ts` reopens source repositories read-only using disk commit receipts, checking type and both raw dates without the producer's guard and re-exporting bundle `source.trees` for byte comparison. Discover unexpected `.git` directories/gitfiles and bare repositories throughout every input; only the materialized git resource repository is allowed. - `instant.ts` compares parsed nanosecond instants; `timestamps.ts` stamps metadata without following symlinks. - `taskClock.ts` separates top-level harness JSON, libfaketime child offsets, and clock-aware child variables: one offset owner per process. Relative signed seconds preserve milliseconds and the shared anchor; trial preparation supplies LD_PRELOAD and a dot-decimal numeric locale. Timeouts remain real. - Adjacent tests use local temporary repositories and fixtures. Never use Docker, production hosts or network fetches. diff --git a/src/compiler/training/rewind/contract.ts b/src/compiler/training/rewind/contract.ts index 9cafb385..6ac94451 100644 --- a/src/compiler/training/rewind/contract.ts +++ b/src/compiler/training/rewind/contract.ts @@ -11,6 +11,7 @@ export const materializedInputSchema = inputIdentitySchema.extend({ kind: z.enum(["git", "bundle", "feed", "volume", "memory", "moltnet"]), fidelity: z.enum(["exact", "approximate", "overridden"]).nullable(), basis: text, source: z.object({ commit: text.optional(), repository: text.optional(), revision: text.optional(), seq: count.optional(), + trees: z.array(z.object({ root: z.string(), prefix: z.string().optional(), exclude: z.array(text).optional() }).strict()).min(1).optional(), sha256: recordedSha256Schema.optional(), agentSlug: text.optional(), snapshot: text.optional(), cursor: text.optional(), switchingAt: trainingInstantSchema.optional(), landedAt: trainingInstantSchema.optional(), prepared: z.literal(true).optional() }).strict(), path: text.nullable(), notes: z.array(z.string()), refusal: text.nullable(), build: z.literal("deferred").optional() @@ -27,7 +28,7 @@ export const materializedMomentSchema = z.object({ export type MaterializedMoment = z.infer; export const leakFindingSchema = z.object({ input: inputIdentitySchema.nullable(), - class: z.enum(["file-time", "symlink-escape", "git-object", "memory-event", "moltnet-message", + class: z.enum(["bundle-entry", "file-time", "symlink-escape", "git-object", "memory-event", "moltnet-message", "moltnet-event", "moltnet-room", "feed-identity", "dated-path", "snapshot"]), path: text, detail: text }).strict(); export const leakCheckSchema = z.object({ diff --git a/src/compiler/training/rewind/deployedResource.test.ts b/src/compiler/training/rewind/deployedResource.test.ts index 99baae4a..03f95101 100644 --- a/src/compiler/training/rewind/deployedResource.test.ts +++ b/src/compiler/training/rewind/deployedResource.test.ts @@ -6,6 +6,7 @@ import * as deployment from "../../../deployment/index.js"; import { BundleTarWriter } from "../../workspaceBundleTar.js"; import { AT, BEFORE, emptyRecord, jsonl, rewindFixture, testGit } from "./fixtures.test-helper.js"; import { materializeTrainingMoment } from "./materialize.js"; +import { composeMomentRecord } from "../../../recording/index.js"; const roots: string[] = []; afterEach(async () => { vi.restoreAllMocks(); await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); @@ -32,6 +33,40 @@ const fixture = async (home = false) => { }; describe("merged deployment resource history in rewind", () => { + it.each((["git", "bundle"] as const).flatMap((kind) => ["activation", "rollback"].map((boundary) => ({ kind, boundary })))) + ("downgrades a matched $kind record with non-exact $boundary", async ({ kind, boundary }) => { + const f = await fixture(); + let sha256 = DIGEST; + if (kind === "bundle") { + const archive = path.join(f.root, "saved.tar"), writer = await BundleTarWriter.create(archive); + await writer.begin("tools.txt", 6, 0o644); await writer.data(Buffer.from("tools\n")); await writer.end(); await writer.finish(); + const bytes = await readFile(archive), hex = createHash("sha256").update(bytes).digest("hex"); sha256 = `sha256:${hex}`; + await mkdir(path.join(f.historyDir, "bundles")); await writeFile(path.join(f.historyDir, "bundles", `${hex}.tar`), bytes); + const agent = f.project.compilePlan.nodes.find((node) => node.id === f.plan.agent.id)!.value; + if (agent.kind !== "agent") throw new Error("Expected agent"); + const index = agent.workspaceResources!.findIndex((resource) => resource.id === "code"); + agent.workspaceResources![index] = { scope: agent.workspaceResources![index]!.scope, sharing: "per_agent", id: "code", kind, + mount: "./code", mode: "readonly", source: "never-read.tar" }; + } + const entry = f.entry(f.commit, { activation_exact: boundary !== "activation", activation_note: "Predecessor may still serve code", + ...(boundary === "rollback" ? { exact: false, rolled_back_at: "2026-10-09T08:20:32Z" } : {}), + resources: [{ id: "code", kind, sha256, ...(kind === "git" ? { commit: f.commit } : {}), agent_node_ids: ["agent:a"], declared_by: "team:publication" } as deployment.DeployResources["resources"][number]], + recording_agents: [{ node_id: "agent:a", slug: "a", directory: "/recording/a", keep_ms: 1000, + resources: [{ id: "code", kind }], feeds: [], memory: [], networks: [], snapshots: [] }] }); + const record = composeMomentRecord({ agentNodeId: "agent:a", deploymentRecordPath: f.recordPath, deployResources: [entry], + row: { version: "noopolis.daimon.wake-moment.v1", agent_id: "agent:a", execution_id: "execution-17", started_at: AT, + deliveries: [{ acceptance_id: "accepted", delivery_id: "schedule:edition", kind: "schedule", occurred_at: AT }], snapshots: [] }, + feedLandings: {}, memoryLedgers: {}, recordedAt: AT }); + expect(record.resources).toHaveLength(1); + expect(record.errors).toContainEqual(expect.stringContaining("Deployment activation is non-exact")); + await jsonl(f.historyDir, "moments/pin.jsonl", [record]); + const { result, input } = await f.run(); + expect(result.status, JSON.stringify(result.inputs)).toBe("materialized"); + expect(result.record.matched).toBe(true); + expect(input).toMatchObject({ basis: "record", fidelity: "approximate", notes: record.errors, + source: kind === "git" ? { commit: f.commit } : { sha256 } }); + }); + it.each([false, true])("uses the deployed commit instead of a newer selector (home=%s)", async (home) => { const f = await fixture(home), lookup = vi.spyOn(deployment, "deployResourcesAt"); const entry = f.entry(); await f.log([entry]); diff --git a/src/compiler/training/rewind/feed.test.ts b/src/compiler/training/rewind/feed.test.ts index 473b06ce..282a80dd 100644 --- a/src/compiler/training/rewind/feed.test.ts +++ b/src/compiler/training/rewind/feed.test.ts @@ -103,6 +103,16 @@ describe("feed rewind", () => { expect(await readdir(path.join(context.destination, "current"))).toEqual(["published"]); }); + it("steps back past a future-authored feed commit", async () => { + const { context, resource, repository, past } = await fixture(); + testGit(repository, ["checkout", "--detach", past]); + await commitTestGit(repository, { "published/news.txt": "future-authored\n" }, "2026-10-09T08:10:00Z"); + testGit(repository, ["commit", "--amend", "--no-edit", `--date=${LATE}`], "2026-10-09T08:10:00Z"); + resource.feed!.git!.ref = "HEAD"; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", basis: "feed-rule-at", source: { commit: past } }); + expect(await readFile(path.join(context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); + }); + it("expands the Berlin date in both ref and paths near UTC midnight", async () => { const { context, resource, repository } = await fixture(); const commit = await commitTestGit(repository, { "2026-10-10/news.txt": "Berlin tomorrow\n" }, "2026-10-09T22:00:00Z"); @@ -229,11 +239,27 @@ describe("feed rewind", () => { await expect(materializeFeed(context)).rejects.toThrow("--history is required"); }); - it("validates every landing line and falls back before the first eligible landing", async () => { - const { context, directory, landing, log } = await fixture(); + it("refuses before the first landing in a nonempty log unless overridden", async () => { + const { context, landing, log } = await fixture(); await log([{ ...landing, switching_at: LATE, landed_at: LATE }]); - expect(await materializeFeed(context)).toMatchObject({ fidelity: "approximate", basis: "feed-rule-at" }); - await rm(context.destination, { recursive: true }); + await expect(materializeFeed(context)).rejects.toThrow(`Feed news has no landed state at ${AT}; provide an override`); + await expect(readdir(context.destination)).rejects.toThrow(); + context.input.overridden = true; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "overridden", basis: "override" }); + expect(await readdir(context.destination)).toEqual([]); + }); + + it("preserves a matching wake pin before the first retained landing", async () => { + const { context, landing, log } = await fixture(); + await log([{ ...landing, switching_at: LATE, landed_at: LATE }]); + context.record = { ...testMomentRecord(), feeds: [{ id: "news", revision: landing.revision, tree: landing.tree, + commit: landing.commit, ref: landing.ref, paths: landing.paths, landedAt: EARLY }] }; + expect(await materializeFeed(context)).toMatchObject({ fidelity: "exact", basis: "record", source: { commit: landing.commit } }); + expect(await readFile(path.join(context.destination, "current/published/news.txt"), "utf8")).toBe("past\n"); + }); + + it("validates every landing line and falls back for an empty landing log", async () => { + const { context, directory, landing } = await fixture(); await writeFile(path.join(directory, "landings.jsonl"), `${JSON.stringify(landing)}\n${JSON.stringify({ ...landing, switching_at: undefined })}\n`); await expect(materializeFeed(context)).rejects.toThrow(/landings.jsonl:2/u); await writeFile(path.join(directory, "landings.jsonl"), ""); diff --git a/src/compiler/training/rewind/feed.ts b/src/compiler/training/rewind/feed.ts index 26de1d03..e5e33261 100644 --- a/src/compiler/training/rewind/feed.ts +++ b/src/compiler/training/rewind/feed.ts @@ -88,6 +88,7 @@ export const materializeFeed = async (context: InputContext): Promise => { } }; -const materializeBundle = async (resource: BundleResource, context: InputContext, repository: string, commit: string): Promise => { +const materializeBundle = async (resource: BundleResource, context: InputContext, repository: string, commit: string): Promise<{ + trees: NonNullable; deferred: boolean +}> => { const build = resource.build; - if (!build) { await exportGitTree(repository, commit, context.destination); return false; } - if (build.files) { - await exportGitTree(repository, commit, context.destination, { root: await bundlePrefix(build.files.root, context.input.declarationBase), exclude: build.files.exclude }); - return false; - } - if (build.dependencies) { - await exportGitTree(repository, commit, context.destination, { root: await bundlePrefix(build.dependencies.directory, context.input.declarationBase) }); - } else { + const trees: NonNullable = []; + if (!build) trees.push({ root: "" }); + else if (build.files) trees.push({ root: await bundlePrefix(build.files.root, context.input.declarationBase), exclude: build.files.exclude }); + else if (build.dependencies) trees.push({ root: await bundlePrefix(build.dependencies.directory, context.input.declarationBase) }); + else { const origins = await Promise.all(build.generated!.inputs.map(async (input) => { try { return (await rewindGit(path.resolve(context.input.declarationBase, input.root), ["rev-parse", "--show-toplevel"])).toString().trim(); } catch { return null; } // Missing relative roots are mapped against the retained repository below. @@ -82,10 +86,11 @@ const materializeBundle = async (resource: BundleResource, context: InputContext for (const input of build.generated!.inputs) { if (input.ref !== undefined) throw new Error("Generated bundle inputs cannot pin a ref: the deferred command reads the work tree"); const root = await bundlePrefix(input.root, context.input.declarationBase); - await exportGitTree(repository, commit, context.destination, { root, prefix: root, exclude: input.exclude, mergeIdentical: true }); + trees.push({ root, prefix: root, exclude: input.exclude }); } } - return true; + for (const tree of trees) await exportGitTree(repository, commit, context.destination, { ...tree, mergeIdentical: true }); + return { trees, deferred: Boolean(build?.dependencies || build?.generated) }; }; export const materializeGit = async (context: InputContext): Promise => { @@ -93,6 +98,7 @@ export const materializeGit = async (context: InputContext): Promise entry.id === input.id); + const activationErrors = matchingRecord ? recordActivationErrors(context.record) : []; const candidateCommits = input.overridden ? [] : context.candidateRecords?.flatMap((candidate) => candidate.resources.flatMap((entry) => entry.id === input.id && entry.kind === "git" ? [entry.commit] : [])) ?? []; if (matchingRecord && matchingRecord.kind !== resource.kind) throw new Error(`Moment resource ${input.id} kind disagrees with its declaration`); @@ -102,8 +108,8 @@ export const materializeGit = async (context: InputContext): Promise { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const repository = async (): Promise => { + const root = await gitTestRoot(); roots.push(root); + const repo = path.join(root, "repo"); await initTestGit(repo); return repo; +}; + +describe("historical commit selection", () => { + it.each(["author", "committer"] as const)("steps back past a commit with a future %s date", async (role) => { + const repo = await repository(), past = await commitTestGit(repo, { "readme.txt": "past\n" }, EARLY); + await commitTestGit(repo, { "readme.txt": "future\n" }, AT); + testGit(repo, ["commit", "--amend", "--no-edit", `--date=${role === "author" ? LATE : AT}`], role === "committer" ? LATE : AT); + expect(await commitAt(repo, "main", AT)).toMatchObject({ commit: past, instant: "2026-10-09T08:00:00.000Z" }); + }); + + it("has no eligible state when every ancestor is future-authored", async () => { + const repo = await repository(); await commitTestGit(repo, { "readme.txt": "future\n" }, EARLY); + testGit(repo, ["commit", "--amend", "--no-edit", `--date=${LATE}`], EARLY); + expect(await commitAt(repo, "main", AT)).toBeNull(); + }); + + it("filters author and committer dates before choosing the newest eligible candidate", () => { + const eligible = { commit: "eligible", instant: EARLY, authorInstant: AT }; + expect(selectCommitAt([ + { commit: "future-author", instant: AT, authorInstant: LATE }, + { commit: "future-committer", instant: LATE, authorInstant: EARLY }, eligible + ], AT)).toEqual(eligible); + expect(selectCommitAt([{ commit: "equal", instant: AT, authorInstant: "2026-10-09T10:20:31+02:00" }], AT)?.commit).toBe("equal"); + }); +}); diff --git a/src/compiler/training/rewind/gitProcess.ts b/src/compiler/training/rewind/gitProcess.ts index 7bd4193c..bdf3b4f1 100644 --- a/src/compiler/training/rewind/gitProcess.ts +++ b/src/compiler/training/rewind/gitProcess.ts @@ -37,7 +37,7 @@ export const rewindGitSync = (repo: string, args: readonly string[]): Buffer => env: rewindGitEnvironment(), maxBuffer: LIMIT, stdio: ["ignore", "pipe", "pipe"], timeout: 60_000 }); -export interface SelectedCommit { commit: string; instant: string } +export interface SelectedCommit { commit: string; instant: string; authorInstant?: string } /** Git's pretty formatter drops negative epochs. Read the signed raw dates instead. */ const commitDates = async (repo: string, commit: string): Promise<{ author: string; committer: string }> => { @@ -75,11 +75,14 @@ export const commitAt = async (repo: string, selector: string, at: string): Prom const commits = (await rewindGit(repo, ["log", "--first-parent", "--format=%H", head, "--"])) .toString().trim().split("\n").filter(Boolean); const dated: SelectedCommit[] = []; - for (const commit of commits) dated.push({ commit, instant: await commitInstant(repo, commit) }); + for (const commit of commits) { + const dates = await commitDates(repo, commit); + dated.push({ commit, instant: dates.committer, authorInstant: dates.author }); + } return selectCommitAt(dated, at); }; /** Stable ties preserve the first-parent order returned by git. */ export const selectCommitAt = (commits: readonly SelectedCommit[], at: string): SelectedCommit | null => - commits.filter((entry) => compareInstants(entry.instant, at) <= 0) + commits.filter((entry) => compareInstants(entry.instant, at) <= 0 && compareInstants(entry.authorInstant ?? entry.instant, at) <= 0) .reduce((latest, entry) => latest === null || compareInstants(entry.instant, latest.instant) > 0 ? entry : latest, null); diff --git a/src/compiler/training/rewind/leakCheck.test.ts b/src/compiler/training/rewind/leakCheck.test.ts index b1ddf177..87c179ae 100644 --- a/src/compiler/training/rewind/leakCheck.test.ts +++ b/src/compiler/training/rewind/leakCheck.test.ts @@ -95,7 +95,7 @@ const intendedFinding = (name: string, fixture: Fixture, moment: MaterializedMom "reachable commit future committer only": { path: relative("git"), detail: /^Commit [a-f0-9]+ has a future committer date$/u }, "future memory event": { path: relative("memory"), detail: "Future memory event at line 1" }, "memory seq gap": { path: relative("memory"), detail: "Noncontiguous memory sequence at line 1" }, - "memory audit state": { path: relative("memory"), detail: "Derived or audit memory state index.sqlite is forbidden" }, + "memory audit state": { path: relative("memory"), detail: "Unexpected payload entry index.sqlite" }, "future Moltnet message": { path: `${relative("moltnet")}/state.json`, detail: "Message future is later than the moment" }, "future Moltnet event": { path: `${relative("moltnet")}/state.json`, detail: "Attachment event future is later than the moment" }, "future Moltnet room": { path: `${relative("moltnet")}/state.json`, detail: "Room future is later than the moment" }, diff --git a/src/compiler/training/rewind/leakFiles.ts b/src/compiler/training/rewind/leakFiles.ts index e6124aa1..454de73e 100644 --- a/src/compiler/training/rewind/leakFiles.ts +++ b/src/compiler/training/rewind/leakFiles.ts @@ -44,10 +44,15 @@ export const inspectBundleFiles = async (root: string, inputs: MaterializedInput const relative = path.relative(root, file).split(path.sep).join("/") || "."; const owner = inputAtPath(inputs, relative); const input = owner ? { section: owner.section, id: owner.id, declaredBy: owner.declaredBy } : null; - const add = (kind: "file-time" | "symlink-escape" | "dated-path" | "git-object", detail: string) => { + const add = (kind: "bundle-entry" | "file-time" | "symlink-escape" | "dated-path" | "git-object", detail: string) => { report.findings.push({ input, class: kind, path: relative, detail }); }; const stats = files.get(file)!; + if (!owner && relative !== ".") { + const reportFile = ["moment.json", "leak-check.json", "task-clock.json"].includes(relative) && stats.isFile(); + const wrapper = stats.isDirectory() && inputs.some((entry) => entry.path?.startsWith(`${relative}/`)); + if (!reportFile && !wrapper) add("bundle-entry", "Unexpected entry outside the materialized inputs and reports"); + } report.checked.files++; if (stats.mtimeNs > at) add("file-time", "Modification time is later than the moment"); if (stats.atimeNs > at) add("file-time", "Access time is later than the moment"); diff --git a/src/compiler/training/rewind/leakGit.ts b/src/compiler/training/rewind/leakGit.ts index fdff98fc..7e08a3d3 100644 --- a/src/compiler/training/rewind/leakGit.ts +++ b/src/compiler/training/rewind/leakGit.ts @@ -3,6 +3,7 @@ import path from "node:path"; import { rewindGit } from "./gitProcess.js"; import { instantNanos } from "./instant.js"; import type { LeakCheck, MaterializedInput } from "./contract.js"; +import { inspectGitIndexTimes, inspectGitWorktree } from "./leakGitPayload.js"; /** Enumerate every stored object, independently of the materializer's chosen-object list. */ export const inspectGitBundle = async (repository: string, input: MaterializedInput, report: LeakCheck): Promise => { @@ -54,4 +55,7 @@ export const inspectGitBundle = async (repository: string, input: MaterializedIn if (BigInt(match[1]!) * 1_000_000_000n > instantNanos(report.at)) add(`Commit ${oid} has a future ${role} date`); } } + for (const inspect of [() => inspectGitWorktree(repository, commit), () => inspectGitIndexTimes(repository, report.at)]) { + try { await inspect(); } catch (error) { add(error instanceof Error ? error.message : String(error)); } + } }; diff --git a/src/compiler/training/rewind/leakGitPayload.test.ts b/src/compiler/training/rewind/leakGitPayload.test.ts new file mode 100644 index 00000000..63163877 --- /dev/null +++ b/src/compiler/training/rewind/leakGitPayload.test.ts @@ -0,0 +1,94 @@ +import { createHash } from "node:crypto"; +import { chmod, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { LeakCheck, MaterializedInput } from "./contract.js"; +import { materializeCleanRepository } from "./git.js"; +import { commitTestGit, gitTestRoot, initTestGit, testGit } from "./git.test-helper.js"; +import { inspectGitBundle } from "./leakGit.js"; +import { stampTree } from "./timestamps.js"; +import * as gitProcess from "./gitProcess.js"; + +const AT = "2026-10-09T08:20:31Z", BEFORE = "2026-10-09T08:00:00Z"; +const roots: string[] = []; +afterEach(async () => { vi.restoreAllMocks(); await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const fixture = async () => { + const root = await gitTestRoot(); roots.push(root); + const source = path.join(root, "source"), repository = path.join(root, "output"); + await initTestGit(source); + await symlink("data.txt", path.join(source, "link")); + const commit = await commitTestGit(source, { "data.txt": "past bytes", ".gitignore": "ignored.txt\n", "folder/child": "child" }, BEFORE); + await materializeCleanRepository(source, commit, repository); + const input: MaterializedInput = { section: "resources", declaredBy: "agent:writer", id: "code", kind: "git", + fidelity: "exact", basis: "record", source: { commit }, path: "resources/code", notes: [], refusal: null }; + const inspect = async () => { + const report: LeakCheck = { version: "spawnfile.leak-check.v1", at: AT, ok: false, + checked: { files: 0, gitObjects: 0, memoryEvents: 0, moltnetMessages: 0, moltnetEvents: 0, snapshots: 0 }, findings: [] }; + await inspectGitBundle(repository, input, report); + return report.findings; + }; + return { source, repository, inspect }; +}; + +describe("independent git payload inspection", () => { + it("rebuilds the checked-out index without materialization-time stat fields", async () => { + const f = await fixture(); + const debug = testGit(f.repository, ["ls-files", "--debug"]); + const times = [...debug.matchAll(/(?:ctime|mtime): (\d+):(\d+)/gu)]; + expect(times.length).toBeGreaterThan(0); + expect(times.every((match) => match[1] === "0" && match[2] === "0")).toBe(true); + expect(await f.inspect()).toEqual([]); + }); + + it("clears refreshed checkout stat fields before publishing", async () => { + const run = gitProcess.rewindGit; + // Some git versions leave checkout-index stat fields zero already. Populate real + // checkout stats at this boundary so deleting the rebuild cannot silently pass. + const checkout = vi.spyOn(gitProcess, "rewindGit").mockImplementation(async (repo, args, input) => { + const result = await run(repo, args, input); + if (args[0] === "checkout-index") { + await run(repo, ["update-index", "--refresh"]); + const populated = (await run(repo, ["ls-files", "--debug"])).toString(); + expect([...populated.matchAll(/ctime: (\d+):/gu)].some((match) => Number(match[1]) > Date.parse(AT) / 1000)).toBe(true); + } + return result; + }); + const f = await fixture(); checkout.mockRestore(); + expect(await f.inspect()).toEqual([]); + const times = [...testGit(f.repository, ["ls-files", "--debug"]).matchAll(/(?:ctime|mtime): (\d+):(\d+)/gu)]; + expect(times.length).toBeGreaterThan(0); + expect(times.every((match) => match[1] === "0" && match[2] === "0")).toBe(true); + }); + + it.each(["changed", "missing", "untracked", "ignored", "extra-directory", "mode", "symlink", "index-disguised"]) + ("rejects %s worktree content against local ls-tree blobs", async (mutation) => { + const f = await fixture(); + expect(await f.inspect()).toEqual([]); + // Source history is unavailable: this comparison must use the bundle repository itself. + await rm(f.source, { recursive: true }); + if (["changed", "index-disguised"].includes(mutation)) await writeFile(path.join(f.repository, "data.txt"), "later data"); + if (mutation === "index-disguised") { + testGit(f.repository, ["add", "data.txt"]); + testGit(f.repository, ["update-index", "--assume-unchanged", "data.txt"]); + } + if (mutation === "missing") await rm(path.join(f.repository, "data.txt")); + if (mutation === "untracked" || mutation === "ignored") await writeFile(path.join(f.repository, `${mutation}.txt`), "later data"); + if (mutation === "extra-directory") await mkdir(path.join(f.repository, "extra")); + if (mutation === "mode") await chmod(path.join(f.repository, "data.txt"), 0o755); + if (mutation === "symlink") { await rm(path.join(f.repository, "link")); await symlink("folder/child", path.join(f.repository, "link")); } + await stampTree(f.repository, BEFORE, AT); + expect(await f.inspect()).toContainEqual(expect.objectContaining({ class: "git-object", detail: expect.stringMatching(/git worktree/iu) })); + }); + + it.each(["ctime", "mtime"])("rejects future index %s after filesystem timestamps are stamped", async (field) => { + const f = await fixture(), file = path.join(f.repository, ".git/index"); + expect(await f.inspect()).toEqual([]); + const bytes = await readFile(file); + expect(bytes.subarray(0, 4).toString()).toBe("DIRC"); + expect(bytes.readUInt32BE(8)).toBeGreaterThan(0); + bytes.writeUInt32BE(Math.floor(Date.parse(AT) / 1000) + 1, 12 + (field === "ctime" ? 0 : 8)); + createHash("sha1").update(bytes.subarray(0, -20)).digest().copy(bytes, bytes.length - 20); + await writeFile(file, bytes); await stampTree(f.repository, BEFORE, AT); + expect(await f.inspect()).toContainEqual(expect.objectContaining({ class: "git-object", detail: expect.stringContaining(`Git index ${field} is later`) })); + }); +}); diff --git a/src/compiler/training/rewind/leakGitPayload.ts b/src/compiler/training/rewind/leakGitPayload.ts new file mode 100644 index 00000000..4da53c09 --- /dev/null +++ b/src/compiler/training/rewind/leakGitPayload.ts @@ -0,0 +1,50 @@ +import { lstat, readFile, readdir, readlink } from "node:fs/promises"; +import path from "node:path"; +import { rewindGit } from "./gitProcess.js"; +import { safeCommittedPath } from "./gitTree.js"; +import { instantNanos } from "./instant.js"; + +/** Compare against objects in the output repository, never the index or status' cached stat data. */ +export const inspectGitWorktree = async (repository: string, commit: string): Promise => { + const entries = new Map(); + for (const row of (await rewindGit(repository, ["ls-tree", "-rtz", `${commit}^{tree}`])).toString().split("\0").filter(Boolean)) { + const tab = row.indexOf("\t"), name = safeCommittedPath(row.slice(tab + 1)); + const [mode, kind, oid] = row.slice(0, tab).split(" "); + entries.set(name, { mode: mode!, kind: kind!, oid: oid! }); + } + const visit = async (directory: string, prefix = ""): Promise => { + for (const name of await readdir(directory)) { + if (!prefix && name === ".git") continue; + const relative = prefix ? `${prefix}/${name}` : name, file = path.join(directory, name); + const expected = entries.get(relative), stats = await lstat(file); + if (!expected) throw new Error(`Unexpected git worktree entry ${relative}`); + entries.delete(relative); + if (expected.kind === "tree" && stats.isDirectory()) { await visit(file, relative); continue; } + if (expected.kind !== "blob" || !["100644", "100755", "120000"].includes(expected.mode)) { + throw new Error(`Unsupported git worktree entry ${relative}`); + } + const bytes = await rewindGit(repository, ["cat-file", "blob", expected.oid]); + const matches = expected.mode === "120000" + ? stats.isSymbolicLink() && (await readlink(file, { encoding: "buffer" })).equals(bytes) + : stats.isFile() && Boolean(stats.mode & 0o111) === (expected.mode === "100755") && (await readFile(file)).equals(bytes); + if (!matches) throw new Error(`Git worktree entry mismatch: ${relative}`); + } + }; + await visit(repository); + if (entries.size) throw new Error(`Missing git worktree entries: ${[...entries.keys()].join(", ")}`); +}; + +/** NUL terminates each pathname, so path text cannot masquerade as --debug stat metadata. */ +export const inspectGitIndexTimes = async (repository: string, at: string): Promise => { + const debug = (await rewindGit(repository, ["ls-files", "--debug", "-z"])).toString(); + for (let cursor = 0; cursor < debug.length;) { + const end = debug.indexOf("\0", cursor); + const metadata = end < 0 ? null : /^ ctime: (\d+):(\d+)\n mtime: (\d+):(\d+)\n dev: [^\n]+\n uid: [^\n]+\n size: [^\n]+\n/u.exec(debug.slice(end + 1)); + if (!metadata) throw new Error("Invalid git index stat metadata"); + for (const [label, offset] of [["ctime", 1], ["mtime", 3]] as const) { + const nanos = BigInt(metadata[offset]!) * 1_000_000_000n + BigInt(metadata[offset + 1]!); + if (nanos > instantNanos(at)) throw new Error(`Git index ${label} is later than the moment: ${debug.slice(cursor, end)}`); + } + cursor = end + 1 + metadata[0].length; + } +}; diff --git a/src/compiler/training/rewind/leakInputs.ts b/src/compiler/training/rewind/leakInputs.ts index f89b6e87..f17efbd5 100644 --- a/src/compiler/training/rewind/leakInputs.ts +++ b/src/compiler/training/rewind/leakInputs.ts @@ -9,6 +9,7 @@ import { permitsFeedPublicationTime } from "./leakFeed.js"; import { historyKey, inside, safeSegment, type History } from "./history.js"; import { compareInstants } from "./instant.js"; import { diskPath } from "./leakFiles.js"; +import { assertDirectoryEntries } from "./leakLayout.js"; import { parseMemoryLedger } from "./memory.js"; import { inspectMoltnetJson } from "./moltnetJson.js"; import { inspectMoltnetSqlite } from "./moltnetSqlite.js"; @@ -20,15 +21,8 @@ export const addInputFinding = (report: LeakCheck, input: MaterializedInput, kin export const inspectMemoryBundle = async (root: string, input: MaterializedInput, report: LeakCheck): Promise => { const home = await diskPath(root, input.path!); - const visit = async (directory: string): Promise => { - for (const entry of await readdir(directory, { withFileTypes: true })) { - if (/^(?:index\.sqlite(?:$|[-.])|causal|authority-receipts)/u.test(entry.name)) { - addInputFinding(report, input, "memory-event", `Derived or audit memory state ${entry.name} is forbidden`); - } - if (entry.isDirectory()) await visit(path.join(directory, entry.name)); - } - }; - await visit(home); + await assertDirectoryEntries(home, { memory: "directory" }); + await assertDirectoryEntries(path.join(home, "memory"), { "events.jsonl": "file" }); const ledger = parseMemoryLedger(await readFile(await diskPath(root, `${input.path}/memory/events.jsonl`), "utf8")); report.checked.memoryEvents += ledger.events.length; let native = false; @@ -52,6 +46,7 @@ export const inspectFeedBundle = async (root: string, input: MaterializedInput, return; } const identity: FeedIdentity = JSON.parse(await readFile(await diskPath(root, `${input.path}/.spawnfile-feed.json`), "utf8")); + await assertDirectoryEntries(directory, { current: "symlink", ".spawnfile-feed.json": "file", trees: "directory" }); for (const issue of feedIdentityFindings(identity)) addInputFinding(report, input, "feed-identity", issue); if (compareInstants(identity.landed_at, report.at) > 0 && !(moment && await permitsFeedPublicationTime(identity, input, moment, history))) { addInputFinding(report, input, "feed-identity", "Feed landed_at is later than the moment"); @@ -67,6 +62,7 @@ export const inspectFeedBundle = async (root: string, input: MaterializedInput, addInputFinding(report, input, "feed-identity", "Feed current does not point to its tree inside the volume"); } else if (!(await lstat(await realpath(current))).isDirectory()) addInputFinding(report, input, "feed-identity", "Feed tree must be a directory"); else { + await assertDirectoryEntries(path.join(directory, "trees"), { [path.basename(link)]: "directory" }); const prepared = identity.source.prepared !== undefined || input.source.prepared === true; if (prepared && (input.fidelity !== "approximate" || !input.notes.includes("prepared output not content-verifiable"))) { addInputFinding(report, input, "feed-identity", "Prepared feed must be approximate: prepared output not content-verifiable"); @@ -90,6 +86,7 @@ export const inspectMoltnetBundle = async (root: string, input: MaterializedInpu const directory = await diskPath(root, input.path!); const stores = (await readdir(directory)).filter((name) => name === "state.json" || name === "moltnet.sqlite"); if (stores.length !== 1) throw new Error("Expected exactly one Moltnet store"); + await assertDirectoryEntries(directory, { [stores[0]!]: "file" }); const store = await diskPath(root, `${input.path}/${stores[0]}`); const result = stores[0] === "state.json" ? await inspectMoltnetJson(store, report.at) : await inspectMoltnetSqlite(store, report.at); report.checked.moltnetMessages += result.messages; diff --git a/src/compiler/training/rewind/leakLayout.test.ts b/src/compiler/training/rewind/leakLayout.test.ts new file mode 100644 index 00000000..1ce84c66 --- /dev/null +++ b/src/compiler/training/rewind/leakLayout.test.ts @@ -0,0 +1,35 @@ +import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { cleanTrainingProjects } from "../plan/fixtures.test-helper.js"; +import { AT, rewindFixture } from "./fixtures.test-helper.js"; +import { leakCheckTrainingMoment } from "./leakCheck.js"; +import { finalizeTrainingMoment, materializeTrainingMoment } from "./materialize.js"; +import { stampTree } from "./timestamps.js"; + +afterEach(cleanTrainingProjects); +describe("closed rewind payload layouts", () => { + it.each(["root", "wrapper", "memory-root", "memory-ledger", "moltnet", "feed-root", "feed-tree"]) + ("rejects a stamped extra entry at %s", async (location) => { + const f = await rewindFixture(), moment = await materializeTrainingMoment(f.options); + expect(moment.status, JSON.stringify(moment.inputs)).toBe("materialized"); + const input = moment.inputs.find((entry) => entry.kind === (location.startsWith("memory") ? "memory" + : location.startsWith("feed") ? "feed" : "moltnet"))!; + let directory = location === "root" ? f.options.out : location === "wrapper" ? path.join(f.options.out, "resources") + : path.join(f.options.out, input.path!, location === "memory-ledger" ? "memory" : location === "feed-tree" ? "trees" : ""); + if (location === "feed-tree") { + directory = path.join(directory, "a".repeat(64)); await mkdir(directory); + } + await writeFile(path.join(directory, "unrelated.txt"), "post-instant bytes"); + await stampTree(f.options.out, AT, AT); + const report = await leakCheckTrainingMoment(f.options.out, { historyDir: f.historyDir }); + const expectedClass = location === "root" || location === "wrapper" ? "bundle-entry" + : location.startsWith("memory") ? "memory-event" : location.startsWith("feed") ? "feed-identity" : "moltnet-message"; + expect(report.findings).toContainEqual(expect.objectContaining({ class: expectedClass, detail: expect.stringMatching(/Unexpected/u) })); + expect(report.findings.some((finding) => finding.class === "file-time")).toBe(false); + const out = path.join(f.root, "refused"); + expect((await finalizeTrainingMoment(f.options.out, out, { historyDir: f.historyDir })).status).toBe("refused"); + expect(await readdir(out)).toEqual(["leak-check.json", "moment.json"]); + expect(JSON.parse(await readFile(path.join(out, "leak-check.json"), "utf8")).ok).toBe(false); + }); +}); diff --git a/src/compiler/training/rewind/leakLayout.ts b/src/compiler/training/rewind/leakLayout.ts new file mode 100644 index 00000000..ccaba7e5 --- /dev/null +++ b/src/compiler/training/rewind/leakLayout.ts @@ -0,0 +1,17 @@ +import { lstat, readdir } from "node:fs/promises"; +import path from "node:path"; + +/** Closed native layouts cannot carry unrelated retained bytes, even when their dates are stamped. */ +export const assertDirectoryEntries = async ( + directory: string, expected: Record +): Promise => { + const names = await readdir(directory); + for (const name of names) { + if (!Object.hasOwn(expected, name)) throw new Error(`Unexpected payload entry ${name}`); + const stat = await lstat(path.join(directory, name)), kind = expected[name]; + if (!(kind === "file" ? stat.isFile() : kind === "directory" ? stat.isDirectory() : stat.isSymbolicLink())) { + throw new Error(`Payload entry ${name} must be a ${kind}`); + } + } + for (const name of Object.keys(expected)) if (!names.includes(name)) throw new Error(`Missing payload entry ${name}`); +}; diff --git a/src/compiler/training/rewind/leakProvenance.test.ts b/src/compiler/training/rewind/leakProvenance.test.ts index 4fb6d192..ef9b1bc3 100644 --- a/src/compiler/training/rewind/leakProvenance.test.ts +++ b/src/compiler/training/rewind/leakProvenance.test.ts @@ -104,7 +104,7 @@ describe("independent source commit provenance", () => { if (resource.kind === "bundle") resource.build!.files!.ref = f.commit; f.context.history = undefined; const moment = await f.stage(); - expect(moment.inputs[0]!.source).toEqual({ commit: f.commit, repository: f.repository }); + expect(moment.inputs[0]!.source).toMatchObject({ commit: f.commit, repository: f.repository }); expect((await leakCheckTrainingMoment(f.bundle)).findings).toEqual([]); await rm(f.repository, { recursive: true }); expect((await leakCheckTrainingMoment(f.bundle)).findings).toContainEqual(expect.objectContaining({ class: "git-object" })); @@ -119,4 +119,23 @@ describe("independent source commit provenance", () => { expect((await leakCheckTrainingMoment(f.bundle, { historyDir: f.historyDir })).findings) .toContainEqual(expect.objectContaining({ class: "git-object", detail: "Feed source commit disagrees with moment provenance" })); }); + + it.each(["selector-at", "pinned", "dependencies", "generated"])("compares %s bundle payload with historical roots and exclusions", async (selection) => { + const f = await fixture("bundle"); + const commit = await commitTestGit(f.repository, { "package/keep.txt": "kept", "package/omit.txt": "excluded", "other.txt": "other" }, BEFORE); + if (f.context.input.section !== "resources" || f.context.input.declaration.kind !== "bundle") throw new Error("Expected bundle"); + f.context.input.declarationBase = f.repository; + f.context.input.declaration.build = selection === "dependencies" ? { dependencies: { directory: "package", image: "node:22" } } + : selection === "generated" ? { generated: { command: ["never-run"], inputs: [{ root: "package", exclude: ["omit.txt"] }, { root: ".", exclude: ["package/**"] }] } } + : { files: { root: "package", exclude: ["omit.txt"], ...(selection === "pinned" ? { ref: commit } : {}) } }; + const moment = await f.stage(); + expect(moment.inputs[0]!.source.trees).toBeDefined(); + expect((await leakCheckTrainingMoment(f.bundle, { historyDir: f.historyDir })).findings).toEqual([]); + // The gate must inspect bytes despite a valid receipt and producer-stamped times. + await writeFile(path.join(f.destination, selection === "generated" ? "package/keep.txt" : "keep.txt"), "future payload"); + await stampTree(f.bundle, BEFORE, AT); + const check = await leakCheckTrainingMoment(f.bundle, { historyDir: f.historyDir }); + expect(check.findings).toContainEqual(expect.objectContaining({ class: "git-object", detail: "Bundle bytes disagree with selected commit, roots and exclusions" })); + expect(check.findings.some((finding) => finding.class === "file-time")).toBe(false); + }); }); diff --git a/src/compiler/training/rewind/leakProvenance.ts b/src/compiler/training/rewind/leakProvenance.ts index a0968b87..0c40bc1a 100644 --- a/src/compiler/training/rewind/leakProvenance.ts +++ b/src/compiler/training/rewind/leakProvenance.ts @@ -1,4 +1,5 @@ -import { readFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import type { FeedIdentity } from "../../../volume/feedRecord.js"; import type { LeakCheck, MaterializedInput } from "./contract.js"; @@ -9,6 +10,17 @@ import { instantNanos } from "./instant.js"; import { diskPath } from "./leakFiles.js"; import { addInputFinding } from "./leakInputs.js"; import { verifyRecordedBundle } from "./recordedBundle.js"; +import { digestDirectory } from "../../../volume/index.js"; +import { exportGitTree } from "./gitTree.js"; + +const inspectBundleTree = async (repository: string, commit: string, directory: string, input: MaterializedInput): Promise => { + if (!input.source.trees?.length) throw new Error("Missing bundle source tree selection"); + const scratch = await mkdtemp(path.join(os.tmpdir(), "spawnfile-bundle-check-")); + try { + for (const tree of input.source.trees) await exportGitTree(repository, commit, scratch, { ...tree, mergeIdentical: true }); + if (digestDirectory(directory) !== digestDirectory(scratch)) throw new Error("Bundle bytes disagree with selected commit, roots and exclusions"); + } finally { await rm(scratch, { recursive: true, force: true }); } +}; /** Read receipts and source objects afresh; never trust the producer's date guard or stamped files. */ export const inspectInputProvenance = async ( @@ -55,5 +67,6 @@ export const inspectInputProvenance = async ( if (!match) add(`Provenance commit ${commit} has no valid ${role} date`); else if (BigInt(match[1]!) * 1_000_000_000n > instantNanos(report.at)) add(`Provenance commit ${commit} has a future ${role} date`); } + if (input.kind === "bundle") await inspectBundleTree(repository, commit, await diskPath(root, input.path!), input); } catch (error) { add(error instanceof Error ? error.message : String(error)); } }; diff --git a/src/compiler/training/rewind/moltnet.ts b/src/compiler/training/rewind/moltnet.ts index 5679ac31..01c6af61 100644 --- a/src/compiler/training/rewind/moltnet.ts +++ b/src/compiler/training/rewind/moltnet.ts @@ -44,6 +44,6 @@ export const materializeMoltnet = async (context: InputContext): Promise { expect(rewindMoltnetJson(source, at).removed_rooms.assignment).toBe("0001-01-01T00:00:00Z"); }); + it("refuses restoration from future room events even when the embedded creation date is old", () => { + const source = emptyMoltnetSnapshot(network, before), room = source.rooms.assignment!; + delete source.rooms.assignment; source.removed_rooms.assignment = future; + source.attachment_delivery_events = [{ id: "future-room", type: "room.updated", network_id: "news", created_at: future, + room: { ...room, name: "FUTURE ROOM NAME", members: ["future-agent"] } }]; + expect(() => rewindMoltnetJson(source, at)).toThrow("room metadata was not retained"); + source.attachment_delivery_events.push({ id: "old-room", type: "room.created", network_id: "news", created_at: before, room }); + expect(rewindMoltnetJson(source, at).rooms.assignment).toEqual(room); + }); + + it("drops future registrations with their memberships and cursors while clamping retained updates", async () => { + const source = emptyMoltnetSnapshot(network, before); + for (const [id, created_at, updated_at] of [["writer", before, future], ["boundary", at, at], ["later", future, future]]) { + source.agents[id] = { agent_id: id, network_id: "news", actor_uid: id, actor_uri: id, created_at, updated_at }; + } + source.agents.undated = { agent_id: "undated", network_id: "news", actor_uid: "undated", actor_uri: "undated" }; + source.rooms.assignment!.members = ["writer", "boundary", "later", "unregistered"]; + source.direct_messages.dm = [message("dm-old", before, { kind: "dm", dm_id: "dm" })]; + source.direct_members.dm = { writer: {}, boundary: {}, later: {}, unregistered: {} }; + source.attachment_delivery_cursors = { writer: 0, later: 0 }; + const cut = rewindMoltnetJson(source, at); + expect(Object.keys(cut.agents)).toEqual(["writer", "boundary", "undated"]); + expect(cut.agents.writer).toMatchObject({ created_at: before, updated_at: at }); + expect(cut.agents.boundary).toMatchObject({ created_at: at, updated_at: at }); + expect(cut.agents.undated).not.toHaveProperty("updated_at"); + expect(cut.rooms.assignment!.members).toEqual(["writer", "boundary", "unregistered"]); + expect(cut.direct_members.dm).toEqual({ writer: {}, boundary: {}, unregistered: {} }); + expect(cut.attachment_delivery_cursors).toEqual({ writer: 0 }); + expect(source.agents.writer!.updated_at).toBe(future); expect(source.agents.later).toBeDefined(); + expect(source.rooms.assignment!.members).toContain("later"); expect(source.direct_members.dm).toHaveProperty("later"); + const root = await mkdtemp(path.join(os.tmpdir(), "rewind-moltnet-agents-")); temps.push(root); + const file = path.join(root, "state.json"); await writeFile(file, JSON.stringify(cut)); + expect((await inspectMoltnetJson(file, at)).findings).toEqual([]); + }); + + it.each(["created_at", "updated_at"] as const)("independently flags an agent's future %s", async (field) => { + const root = await mkdtemp(path.join(os.tmpdir(), "rewind-moltnet-agent-leak-")); temps.push(root); + const file = path.join(root, "state.json"), source = emptyMoltnetSnapshot(network, before); + source.agents.writer = { agent_id: "writer", network_id: "news", actor_uid: "uid", actor_uri: "uri", + created_at: before, updated_at: before, [field]: future }; + const bytes = JSON.stringify(source); await writeFile(file, bytes); + expect((await inspectMoltnetJson(file, at)).findings).toEqual([ + { class: "moltnet-event", path: file, detail: `Agent writer has a future ${field}` } + ]); + expect(await readFile(file, "utf8")).toBe(bytes); + }); + it("materializes exact cursor and approximate cuts, keeps history unchanged and makes fresh overridden rooms", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "rewind-moltnet-")); temps.push(root); const bank = path.join(root, "moltnet/team%3Aorg/news"); await mkdir(bank, { recursive: true }); diff --git a/src/compiler/training/rewind/moltnetJson.ts b/src/compiler/training/rewind/moltnetJson.ts index b7fba7d3..8a4ed0ba 100644 --- a/src/compiler/training/rewind/moltnetJson.ts +++ b/src/compiler/training/rewind/moltnetJson.ts @@ -45,6 +45,13 @@ const eventFuture = (event: MoltnetEvent, at: string): boolean => [event.created /** Pure cut. Delivery offsets must be remapped after removing interior future events. */ export const rewindMoltnetJson = (source: MoltnetSnapshot, at: string): MoltnetSnapshot => { const snapshot = structuredClone(source); + const futureAgents = new Set(Object.values(source.agents) + .filter((agent) => agent.created_at && compareInstants(agent.created_at, at) > 0).map((agent) => agent.agent_id)); + for (const [key, agent] of Object.entries(snapshot.agents)) { + if (futureAgents.has(agent.agent_id)) { + delete snapshot.agents[key]; delete snapshot.attachment_delivery_cursors[agent.agent_id]; + } else if (agent.updated_at && compareInstants(agent.updated_at, at) > 0) agent.updated_at = at; + } const futureRooms = new Set(Object.values(source.rooms) .filter((room) => compareInstants(room.created_at, at) > 0).map((room) => room.id)); for (const id of futureRooms) { @@ -56,13 +63,18 @@ export const rewindMoltnetJson = (source: MoltnetSnapshot, at: string): MoltnetS const candidates = source.attachment_delivery_events.filter((event) => event.room?.id === id && compareInstants(event.room.created_at, at) <= 0) .sort((a, b) => compareInstants(b.created_at, a.created_at)); - const room = candidates.find((event) => compareInstants(event.created_at, at) <= 0)?.room - ?? candidates[0]?.room; + const room = candidates.find((event) => compareInstants(event.created_at, at) <= 0)?.room; if (!room) throw new Error(`Cannot restore Moltnet room ${id}: removal is after the moment but room metadata was not retained`); snapshot.rooms[id] = structuredClone(room); } delete snapshot.removed_rooms[id]; } + for (const room of Object.values(snapshot.rooms)) { + if (room.members) room.members = room.members.filter((id) => !futureAgents.has(id)); + } + for (const members of Object.values(snapshot.direct_members)) { + for (const id of futureAgents) delete members[id]; + } for (const group of [snapshot.room_messages, snapshot.thread_messages, snapshot.direct_messages]) { for (const [id, messages] of Object.entries(group)) { group[id] = messages.filter((message) => compareInstants(message.created_at, at) <= 0 @@ -131,6 +143,12 @@ export const inspectMoltnetEvent = (event: MoltnetEvent, at: string, path: strin export const inspectMoltnetJson = async (path: string, at: string): Promise => { const snapshot = parseMoltnetSnapshot(JSON.parse(await readFile(path, "utf8"))); const findings: MoltnetFinding[] = []; + for (const agent of Object.values(snapshot.agents)) { + for (const field of ["created_at", "updated_at"] as const) { + if (agent[field] && compareInstants(agent[field], at) > 0) findings.push({ class: "moltnet-event", path, + detail: `Agent ${agent.agent_id} has a future ${field}` }); + } + } const messages = moltnetMessages(snapshot); for (const message of messages) if (compareInstants(message.created_at, at) > 0) { findings.push({ class: "moltnet-message", path, detail: `Message ${message.id} is later than the moment` }); diff --git a/src/compiler/training/rewind/moltnetSqlite.test.ts b/src/compiler/training/rewind/moltnetSqlite.test.ts index ae41ee4f..d322bd4a 100644 --- a/src/compiler/training/rewind/moltnetSqlite.test.ts +++ b/src/compiler/training/rewind/moltnetSqlite.test.ts @@ -76,7 +76,7 @@ describe("Moltnet SQLite rewind", () => { message(database, "dm-regressed", before, { kind: "dm", dm_id: "dm-dropped" }); database.prepare("INSERT INTO artifacts VALUES (?, ?, ?)").run("old-artifact", "old", before); database.prepare("INSERT INTO artifacts VALUES (?, ?, ?)").run("future-artifact", "future-root", future); - for (const agent of ["writer", "brass"]) database.prepare("INSERT INTO agents VALUES (?, 'news', 'uid', 'uri', NULL, 'key', ?, ?)").run(agent, future, future); + for (const agent of ["writer", "brass"]) database.prepare("INSERT INTO agents VALUES (?, 'news', 'uid', 'uri', NULL, 'key', ?, ?)").run(agent, before, future); event(database, "event-old", before); event(database, "event-future", future); event(database, "event-boundary", at); database.prepare("INSERT INTO attachment_delivery_cursors VALUES (?, ?, ?)").run("writer", 2, future); database.prepare("INSERT INTO attachment_delivery_cursors VALUES (?, ?, ?)").run("brass", 3, future); @@ -119,6 +119,64 @@ describe("Moltnet SQLite rewind", () => { } }); + it("drops future registrations and memberships, clamps retained updates and migration times, and preserves history bytes", async () => { + const { database, context, file } = await fixture(); + room(database, "assignment"); dm(database, "dm-kept"); + message(database, "dm-old", before, { kind: "dm", dm_id: "dm-kept" }); + for (const [id, time] of [["writer", before], ["boundary", at], ["later", future]]) { + database.prepare("INSERT INTO agents VALUES (?, 'news', ?, ?, NULL, 'key', ?, ?)").run(id, id, id, time, future); + } + for (const id of ["boundary", "later", "unregistered"]) { + database.prepare("INSERT INTO room_members VALUES ('assignment', ?)").run(id); + database.prepare("INSERT INTO dm_participants VALUES ('dm-kept', ?)").run(id); + } + database.prepare("INSERT INTO attachment_delivery_cursors VALUES ('later', 0, ?)").run(future); + database.exec("CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, name TEXT, applied_at TEXT)"); + for (const [version, time] of [[1, before], [2, at], [3, future]] as const) { + database.prepare("INSERT INTO schema_migrations VALUES (?, 'migration', ?)").run(version, time); + } + database.close(); const sourceBytes = await readFile(file); + await materializeMoltnet(context); + const output = path.join(context.destination, "moltnet.sqlite"), result = new DatabaseSync(output, { readOnly: true }); + try { + expect(result.prepare("SELECT agent_id, created_at, updated_at FROM agents ORDER BY agent_id").all()).toEqual([ + { agent_id: "boundary", created_at: at, updated_at: at }, { agent_id: "writer", created_at: before, updated_at: at } + ]); + expect(result.prepare("SELECT member_id FROM room_members ORDER BY member_id").all()) + .toEqual(["boundary", "unregistered", "writer"].map((member_id) => ({ member_id }))); + expect(result.prepare("SELECT participant_id FROM dm_participants ORDER BY participant_id").all()) + .toEqual(["boundary", "unregistered", "writer"].map((participant_id) => ({ participant_id }))); + expect(result.prepare("SELECT * FROM attachment_delivery_cursors").all()).toEqual([]); + expect(result.prepare("SELECT version, applied_at FROM schema_migrations ORDER BY version").all()).toEqual([ + { version: 1, applied_at: before }, { version: 2, applied_at: at }, { version: 3, applied_at: at } + ]); + } finally { result.close(); } + expect(await readFile(file)).toEqual(sourceBytes); + expect((await inspectMoltnetSqlite(output, at)).findings).toEqual([]); + }); + + it.each(["created_at", "updated_at"] as const)("independently flags a SQLite agent's future %s", async (field) => { + const { database, file } = await fixture(); + database.prepare("INSERT INTO agents VALUES ('writer', 'news', 'uid', 'uri', NULL, 'key', ?, ?)") + .run(field === "created_at" ? future : before, field === "updated_at" ? future : before); + database.close(); const bytes = await readFile(file); + expect((await inspectMoltnetSqlite(file, at)).findings).toEqual([ + { class: "moltnet-event", path: file, detail: `Agent writer has a future ${field}` } + ]); + expect(await readFile(file)).toEqual(bytes); + }); + + it("independently flags future schema migration times", async () => { + const { database, file } = await fixture(); + database.exec("CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, name TEXT, applied_at TEXT)"); + database.prepare("INSERT INTO schema_migrations VALUES (1, 'migration', ?)").run(future); + database.close(); const bytes = await readFile(file); + expect((await inspectMoltnetSqlite(file, at)).findings).toEqual([ + { class: "moltnet-event", path: file, detail: "Schema migration 1 has a future applied_at" } + ]); + expect(await readFile(file)).toEqual(bytes); + }); + it("copies WAL state and independently detects message, room and attachment-event mutations without changing the file", async () => { const { database, context } = await fixture(); database.exec("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0"); diff --git a/src/compiler/training/rewind/moltnetSqlite.ts b/src/compiler/training/rewind/moltnetSqlite.ts index d025b4ad..2411a943 100644 --- a/src/compiler/training/rewind/moltnetSqlite.ts +++ b/src/compiler/training/rewind/moltnetSqlite.ts @@ -2,7 +2,7 @@ import { lstat, rm } from "node:fs/promises"; import type { DatabaseSync } from "node:sqlite"; import { compareInstants } from "./instant.js"; import { inspectMoltnetEvent, latestMoltnetMessage, moltnetMessages, rewindMoltnetJson } from "./moltnetJson.js"; -import { readMoltnetSqlite, sqliteInteger, sqliteText, sqliteJson } from "./moltnetSqliteData.js"; +import { readMoltnetSqlite, readMoltnetSqliteMigrations, sqliteInteger, sqliteText, sqliteJson } from "./moltnetSqliteData.js"; import { moltnetEventSchema, type MoltnetFinding, type MoltnetInspection, type MoltnetMessage } from "./moltnetSchemas.js"; const openSqlite = async (file: string, readOnly = false): Promise => { @@ -26,6 +26,21 @@ export const rewindMoltnetSqlite = async (file: string, at: string): Promise sqliteInteger(row, "seq"))); database.exec("PRAGMA secure_delete=ON; PRAGMA foreign_keys=OFF; BEGIN IMMEDIATE"); try { + for (const [id, agent] of Object.entries(source.agents)) { + if (!cut.agents[id]) { + database.prepare("DELETE FROM agents WHERE agent_id = ?").run(id); + database.prepare("DELETE FROM room_members WHERE member_id = ?").run(id); + database.prepare("DELETE FROM dm_participants WHERE participant_id = ?").run(id); + database.prepare("DELETE FROM attachment_delivery_cursors WHERE agent_id = ?").run(id); + } else if (agent.updated_at !== cut.agents[id].updated_at) { + database.prepare("UPDATE agents SET updated_at = ? WHERE agent_id = ?").run(cut.agents[id].updated_at!, id); + } + } + for (const row of readMoltnetSqliteMigrations(database)) { + if (compareInstants(sqliteText(row, "applied_at"), at) > 0) { + database.prepare("UPDATE schema_migrations SET applied_at = ? WHERE version = ?").run(at, row.version!); + } + } const dropMessage = database.prepare("DELETE FROM messages WHERE id = ?"); for (const message of moltnetMessages(source)) if (!keepMessages.has(message.id)) dropMessage.run(message.id); const dropRoom = database.prepare("DELETE FROM rooms WHERE id = ?"), restoreRoom = database.prepare("UPDATE rooms SET deleted_at = NULL WHERE id = ?"); @@ -95,6 +110,14 @@ export const inspectMoltnetSqlite = async (file: string, at: string): Promise { if (compareInstants(instant, at) > 0) findings.push({ class: kind, path: file, detail }); }; + for (const agent of Object.values(snapshot.agents)) { + for (const field of ["created_at", "updated_at"] as const) { + if (agent[field]) check(agent[field], "moltnet-event", `Agent ${agent.agent_id} has a future ${field}`); + } + } + for (const row of readMoltnetSqliteMigrations(database)) { + check(sqliteText(row, "applied_at"), "moltnet-event", `Schema migration ${row.version} has a future applied_at`); + } for (const message of messages) check(message.created_at, "moltnet-message", `Message ${message.id} is later than the moment`); for (const room of Object.values(snapshot.rooms)) check(room.created_at, "moltnet-room", `Room ${room.id} is later than the moment`); for (const [id, instant] of Object.entries(snapshot.removed_rooms)) check(instant, "moltnet-room", `Room ${id} has a future removal marker`); diff --git a/src/compiler/training/rewind/moltnetSqliteData.ts b/src/compiler/training/rewind/moltnetSqliteData.ts index d9b50434..ff505121 100644 --- a/src/compiler/training/rewind/moltnetSqliteData.ts +++ b/src/compiler/training/rewind/moltnetSqliteData.ts @@ -18,6 +18,9 @@ export const sqliteJson = (row: SqliteRow, key: string): unknown => { try { return JSON.parse(sqliteText(row, key)); } catch (error) { throw new Error(`Invalid Moltnet SQLite ${key}: ${(error as Error).message}`); } }; +export const readMoltnetSqliteMigrations = (database: DatabaseSync): SqliteRow[] => + database.prepare("SELECT name FROM sqlite_schema WHERE name = 'schema_migrations'").get() + ? database.prepare("SELECT version, applied_at FROM schema_migrations").all() : []; const allowedColumns: Record = { rooms: ["id", "network_id", "fqid", "name", "created_at", "deleted_at", "visibility", "write_policy", "federation"], room_members: ["room_id", "member_id"], diff --git a/src/compiler/training/rewind/recordInput.ts b/src/compiler/training/rewind/recordInput.ts index aa2fcec0..9942df6e 100644 --- a/src/compiler/training/rewind/recordInput.ts +++ b/src/compiler/training/rewind/recordInput.ts @@ -12,11 +12,16 @@ const qualifiedPrefix = new RegExp(`(?:^|[^A-Za-z0-9_.-])(${Object.keys(sections const ownerPrefix = new RegExp(`(?:^|[^A-Za-z0-9_.-])(${ownerPattern})[\\s/]+$`, "u"); const ownerSuffix = new RegExp(`^\\s*(?:\\(\\s*|declaredBy\\s*[=:]\\s*|@\\s*)(${ownerPattern})(?=[\\s):,;]|$)`, "u"); +/** Merged recording keeps deployment uncertainty at record level, rather than naming each pin. */ +export const recordActivationErrors = (record: MomentRecord | null): string[] => + (record?.errors ?? []).filter((error) => error.includes("Deployment activation is non-exact")); + /** Errors are free text in P3: honor explicit qualifications; otherwise conservatively match the whole ID. */ export const recordInputErrors = (record: MomentRecord, input: InputIdentity): string[] => { const escaped = input.id.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); const namesInput = new RegExp(`(? { + if (recordActivationErrors(record).includes(error)) return false; let decoded = error; try { decoded = decodeURIComponent(error); } catch { /* A literal percent is valid free text. */ } return [...decoded.matchAll(namesInput)].some((match) => { diff --git a/src/compiler/training/rewind/select.test.ts b/src/compiler/training/rewind/select.test.ts index 78bc50d6..9143738c 100644 --- a/src/compiler/training/rewind/select.test.ts +++ b/src/compiler/training/rewind/select.test.ts @@ -101,6 +101,14 @@ describe("pure rewind record selection", () => { expect([first, second]).toEqual(before); }); + it.each(["git", "bundle"] as const)("refuses tied %s pins with different activation fidelity", (kind) => { + const first = record(), second = structuredClone(first); + first.resources = second.resources = [kind === "git" ? { id: "repo", kind, commit: "a".repeat(40) } + : { id: "repo", kind, sha256: `sha256:${"a".repeat(64)}` }]; + second.exact = false; second.errors = ["Deployment activation is non-exact"]; + expect(() => selectMomentRecord([first, second], "agent:writer", at)).toThrow("different effective pin/fallback decisions"); + }); + it("refuses conflicting bundle digests even when the declared source has a full commit pin", () => { const first = record(), second = record(); first.resources = [{ id: "tools", kind: "bundle", sha256: `sha256:${"a".repeat(64)}` }]; diff --git a/src/compiler/training/rewind/select.ts b/src/compiler/training/rewind/select.ts index ea46fad2..599c5e20 100644 --- a/src/compiler/training/rewind/select.ts +++ b/src/compiler/training/rewind/select.ts @@ -1,7 +1,7 @@ import type { TrainingPlan } from "../plan/index.js"; import type { MomentRecord } from "../../../recording/index.js"; import { compareInstants, normalizeInstant } from "./instant.js"; -import { hasRecordPin, recordForInput, recordPins } from "./recordInput.js"; +import { hasRecordPin, recordActivationErrors, recordForInput, recordPins } from "./recordInput.js"; import type { InputIdentity, RewindInput } from "./types.js"; export type DatasetRole = "train" | "validation" | "test"; @@ -63,6 +63,8 @@ export const selectMomentRecord = ( const pins = recordPins(usable, input).map((pin) => "landedAt" in pin ? { ...pin, landedAt: normalizeInstant(pin.landedAt) } : "cursor" in pin ? { ...pin, cursor: normalizeInstant(pin.cursor) } : pin); return canonical({ pin: declaredPin ? "declared-pin" : pins.sort((a, b) => canonical(a).localeCompare(canonical(b))), + approximateActivation: !declaredPin && pins.some((pin) => "kind" in pin && ["git", "bundle"].includes(pin.kind)) + && recordActivationErrors(usable).length > 0, approximateFallback: !record.exact && !hasRecordPin(usable, input) }); }; if (matches.some((record) => decision(record) !== decision(matches[0]!))) {