From 104c8b098f6d5db3fc28c4c09f17e93c56f2023d Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 8 Nov 2026 23:54:07 +0300 Subject: [PATCH] feat(improve): regression corpus store and growth Co-Authored-By: Claude Opus 5.5 --- api/improve.api.md | 17 +++++ src/improve/corpus.test.ts | 118 ++++++++++++++++++++++++++++++++ src/improve/corpus.ts | 133 +++++++++++++++++++++++++++++++++++++ src/improve/index.ts | 2 + 4 files changed, 270 insertions(+) create mode 100644 src/improve/corpus.test.ts create mode 100644 src/improve/corpus.ts diff --git a/api/improve.api.md b/api/improve.api.md index c29845e6..a75f9d56 100644 --- a/api/improve.api.md +++ b/api/improve.api.md @@ -12,6 +12,9 @@ export function addEntry(dir: string, entry: Omit; + // @public export function admitProtectedCases(options: { harness: Harness; @@ -361,6 +364,17 @@ interface ContextClearedEvent extends AgentEventBase<'context.cleared'> { sessionId: string; } +// @public +export interface CorpusCase { + addedAt: string; + baseline: CaseCount; + id: string; + input: string; + origin: 'protected' | 'fixed'; + revision: string; + traceId: string; +} + // @public export function createShadowHook(options: { readOnly: readonly string[]; @@ -1004,6 +1018,9 @@ export function readAgentHarness(dir: string): Harness; // @public export function readAudit(dir: string): AuditEntry[]; +// @public +export function readCorpus(dir: string): CorpusCase[]; + // @public interface ReasoningBlock { // (undocumented) diff --git a/src/improve/corpus.test.ts b/src/improve/corpus.test.ts new file mode 100644 index 00000000..de66f422 --- /dev/null +++ b/src/improve/corpus.test.ts @@ -0,0 +1,118 @@ +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { ConfigurationError } from '../execution/errors'; +import { addToCorpus, readCorpus, type CorpusCase } from './corpus'; + +let dir: string; + +const item = (traceId: string, origin: CorpusCase['origin'], addedAt: string): CorpusCase => ({ + id: `case-${traceId}`, + traceId, + input: `input for ${traceId}`, + addedAt, + revision: 'a'.repeat(64), + origin, + baseline: { passed: 6, scored: 6 }, +}); + +beforeEach(() => { + dir = mkdtempSync(path.join(tmpdir(), 'lousho-corpus-')); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +describe('corpus', () => { + it('adds and dedupes by traceId', async () => { + await addToCorpus(dir, [item('t1', 'protected', '2026-11-01T00:00:00Z'), item('t2', 'fixed', '2026-11-02T00:00:00Z')], 30); + const after = await addToCorpus(dir, [{ ...item('t1', 'fixed', '2026-11-03T00:00:00Z'), input: 'changed' }, item('t3', 'fixed', '2026-11-03T00:00:00Z')], 30); + expect(after.map((c) => c.traceId)).toEqual(['t1', 't2', 't3']); + expect(after[0]).toMatchObject({ origin: 'protected', input: 'input for t1' }); + }); + + it('dedupes within one batch', async () => { + const after = await addToCorpus(dir, [item('t1', 'fixed', '2026-11-01T00:00:00Z'), item('t1', 'fixed', '2026-11-02T00:00:00Z')], 30); + expect(after).toHaveLength(1); + }); + + it('evicts oldest fixed first', async () => { + await addToCorpus(dir, [item('old', 'fixed', '2026-11-01T00:00:00Z'), item('mid', 'fixed', '2026-11-02T00:00:00Z')], 3); + const after = await addToCorpus(dir, [item('new', 'fixed', '2026-11-03T00:00:00Z'), item('newer', 'fixed', '2026-11-04T00:00:00Z')], 3); + expect(after.map((c) => c.traceId)).toEqual(['mid', 'new', 'newer']); + }); + + it('never evicts protected', async () => { + await addToCorpus(dir, [item('p1', 'protected', '2026-10-01T00:00:00Z'), item('p2', 'protected', '2026-10-02T00:00:00Z')], 2); + const after = await addToCorpus(dir, [item('f1', 'fixed', '2026-11-01T00:00:00Z'), item('p3', 'protected', '2026-11-02T00:00:00Z')], 2); + expect(after.map((c) => c.traceId)).toEqual(['p1', 'p2', 'p3']); + }); + + it('round-trips through disk', async () => { + const written = await addToCorpus(dir, [item('t1', 'protected', '2026-11-01T00:00:00Z')], 30); + expect(readCorpus(dir)).toEqual(written); + expect(JSON.parse(readFileSync(path.join(dir, '.lousho', 'corpus.json'), 'utf8'))).toEqual({ cases: written }); + }); + + it('is empty without a file and refuses a broken one', () => { + expect(readCorpus(dir)).toEqual([]); + mkdirSync(path.join(dir, '.lousho')); + writeFileSync(path.join(dir, '.lousho', 'corpus.json'), '{"cases": [{"id": 1}]}'); + expect(() => readCorpus(dir)).toThrow(ConfigurationError); + writeFileSync(path.join(dir, '.lousho', 'corpus.json'), '{'); + expect(() => readCorpus(dir)).toThrow(/not valid JSON/); + }); + + it('refuses a malformed case and a bad maxCases', async () => { + await expect(addToCorpus(dir, [{ ...item('t1', 'fixed', 'x'), origin: 'other' as 'fixed' }], 30)).rejects.toThrow(ConfigurationError); + await expect(addToCorpus(dir, [item('t1', 'fixed', 'x')], 0)).rejects.toThrow(/maxCases/); + }); + + it('keeps every case from concurrent adds', async () => { + await Promise.all(['a', 'b', 'c', 'd'].map((id) => addToCorpus(dir, [item(id, 'protected', '2026-11-01T00:00:00Z')], 30))); + expect(readCorpus(dir).map((c) => c.traceId).sort()).toEqual(['a', 'b', 'c', 'd']); + }); + + it('refuses a symlinked corpus file or .lousho folder instead of replacing it', async () => { + const outside = mkdtempSync(path.join(tmpdir(), 'lousho-corpus-outside-')); + writeFileSync(path.join(outside, 'corpus.json'), JSON.stringify({ cases: [item('kept', 'protected', '2026-11-01T00:00:00Z')] })); + mkdirSync(path.join(dir, '.lousho')); + symlinkSync(path.join(outside, 'corpus.json'), path.join(dir, '.lousho', 'corpus.json')); + await expect(addToCorpus(dir, [item('t1', 'fixed', '2026-11-02T00:00:00Z')], 30)).rejects.toThrow(/not a regular file/); + rmSync(path.join(dir, '.lousho'), { recursive: true }); + symlinkSync(outside, path.join(dir, '.lousho')); + await expect(addToCorpus(dir, [item('t1', 'fixed', '2026-11-02T00:00:00Z')], 30)).rejects.toThrow(/symlink/); + expect(() => readCorpus(dir)).toThrow(/symlink/); + expect(JSON.parse(readFileSync(path.join(outside, 'corpus.json'), 'utf8')).cases).toHaveLength(1); + rmSync(outside, { recursive: true, force: true }); + }); + + it('refuses a case id that another trace already uses', async () => { + await addToCorpus(dir, [item('t1', 'protected', '2026-11-01T00:00:00Z')], 30); + await expect(addToCorpus(dir, [{ ...item('t2', 'fixed', '2026-11-02T00:00:00Z'), id: 'case-t1' }], 30)).rejects.toThrow(/already used/); + }); + + it('refuses impossible baselines and timestamps', async () => { + for (const bad of [{ passed: -1, scored: 3 }, { passed: 4, scored: 3 }, { passed: 0, scored: 0 }]) { + await expect(addToCorpus(dir, [{ ...item('t1', 'fixed', '2026-11-01T00:00:00Z'), baseline: bad }], 30)).rejects.toThrow(ConfigurationError); + } + await expect(addToCorpus(dir, [item('t1', 'fixed', 'yesterday')], 30)).rejects.toThrow(ConfigurationError); + }); + + it('evicts by time, not by how the timestamp is written', async () => { + await addToCorpus(dir, [item('late', 'fixed', '2026-11-01T10:00:00+05:00'), item('early', 'fixed', '2026-11-01T04:00:00-02:00')], 30); + const after = await addToCorpus(dir, [item('new', 'fixed', '2026-11-02T00:00:00Z')], 2); + expect(after.map((c) => c.traceId)).toEqual(['early', 'new']); + }); + + it('leaves a broken file untouched and no temp file behind', async () => { + mkdirSync(path.join(dir, '.lousho')); + writeFileSync(path.join(dir, '.lousho', 'corpus.json'), 'null'); + await expect(addToCorpus(dir, [item('t1', 'fixed', '2026-11-01T00:00:00Z')], 30)).rejects.toThrow(/list of corpus cases/); + expect(readFileSync(path.join(dir, '.lousho', 'corpus.json'), 'utf8')).toBe('null'); + expect(readdirSync(path.join(dir, '.lousho')).filter((name) => name.endsWith('.tmp'))).toEqual([]); + }); +}); + diff --git a/src/improve/corpus.ts b/src/improve/corpus.ts new file mode 100644 index 00000000..b67fbd9e --- /dev/null +++ b/src/improve/corpus.ts @@ -0,0 +1,133 @@ +import { randomBytes } from 'node:crypto'; +import { closeSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeSync } from 'node:fs'; +import path from 'node:path'; +import { ConfigurationError } from '../execution/errors'; +import { withFileLock } from '../storage/fileLock'; +import type { CaseCount } from './types'; + +/** A case that must keep passing under every later revision. */ +export interface CorpusCase { + /** The case id the gate and the guard use; unique in the corpus. */ + id: string; + /** The trace it was captured from; one corpus case per trace. */ + traceId: string; + /** The recorded input, as captured; never shown to an editor. */ + input: string; + /** When it joined the corpus, as an ISO timestamp. */ + addedAt: string; + /** The revision it was admitted under. */ + revision: string; + /** `protected`: a run that went well; `fixed`: a training case a kept edit fixed. */ + origin: 'protected' | 'fixed'; + /** Its pass count when admitted, which the guard compares against. */ + baseline: CaseCount; +} + +const LOUSHO = '.lousho'; +const FILE = path.join(LOUSHO, 'corpus.json'); +const ORIGINS: readonly string[] = ['protected', 'fixed']; + +const filled = (value: unknown): boolean => typeof value === 'string' && value.trim() !== ''; + +function isCount(value: unknown): value is CaseCount { + const count = value as CaseCount | undefined; + if (!Number.isInteger(count?.passed) || !Number.isInteger(count?.scored)) return false; + return (count as CaseCount).scored >= 1 && (count as CaseCount).passed >= 0 && (count as CaseCount).passed <= (count as CaseCount).scored; +} + +function isCase(value: unknown): value is CorpusCase { + const item = value as CorpusCase | undefined; + if (![item?.id, item?.traceId, item?.revision].every(filled) || typeof item?.input !== 'string') return false; + return Number.isFinite(Date.parse(String(item.addedAt))) && ORIGINS.includes(String(item.origin)) && isCount(item.baseline); +} + +function refuse(message: string, field = 'corpus'): never { + throw new ConfigurationError(message, field); +} + +function guardPaths(dir: string): string { + if (lstatSync(path.join(dir, LOUSHO), { throwIfNoEntry: false })?.isSymbolicLink()) refuse(`${path.join(dir, LOUSHO)} is a symlink; the corpus must live inside the agent directory.`); + const file = path.join(dir, FILE); + const stat = lstatSync(file, { throwIfNoEntry: false }); + if (stat && !stat.isFile()) refuse(`${file} is not a regular file. Fix or delete it.`); + return file; +} + +function duplicate(cases: readonly CorpusCase[]): string | undefined { + const ids = new Set(); + const traces = new Set(); + for (const item of cases) { + if (ids.has(item.id) || traces.has(item.traceId)) return item.id; + ids.add(item.id); + traces.add(item.traceId); + } + return undefined; +} + +/** Reads .lousho/corpus.json; empty when there is none, and throws when it is broken. */ +export function readCorpus(dir: string): CorpusCase[] { + const file = guardPaths(dir); + if (!lstatSync(file, { throwIfNoEntry: false })) return []; + let parsed: unknown; + try { + parsed = JSON.parse(readFileSync(file, 'utf8')); + } catch (error) { + return refuse(`readCorpus: ${file} is not valid JSON (${(error as Error).message}). Fix or delete it.`); + } + const cases = typeof parsed === 'object' && parsed !== null ? (parsed as { cases?: unknown }).cases : undefined; + if (!Array.isArray(cases) || !cases.every(isCase)) return refuse(`readCorpus: ${file} does not hold a list of corpus cases. Fix or delete it.`); + const twice = duplicate(cases); + if (twice !== undefined) refuse(`readCorpus: ${file} holds case ${twice} twice. Fix or delete it.`); + return cases; +} + +function evict(cases: CorpusCase[], maxCases: number): CorpusCase[] { + const fixed = cases.filter((item) => item.origin === 'fixed').sort((a, b) => Date.parse(a.addedAt) - Date.parse(b.addedAt)); + const drop = new Set(fixed.slice(0, Math.max(0, cases.length - maxCases)).map((item) => item.traceId)); + return cases.filter((item) => !drop.has(item.traceId)); +} + +function save(file: string, cases: readonly CorpusCase[]): void { + const temp = path.join(path.dirname(file), `.corpus.json.${process.pid}.${randomBytes(4).toString('hex')}.tmp`); + try { + const fd = openSync(temp, 'wx', 0o600); + try { + writeSync(fd, `${JSON.stringify({ cases }, null, 2)}\n`); + fsyncSync(fd); + } finally { + closeSync(fd); + } + renameSync(temp, file); + } catch (error) { + rmSync(temp, { force: true }); + throw error; + } +} + +function admitted(current: readonly CorpusCase[], cases: readonly CorpusCase[]): CorpusCase[] { + const byTrace = new Map(current.map((item) => [item.traceId, item.id])); + const byId = new Map(current.map((item) => [item.id, item.traceId])); + const added: CorpusCase[] = []; + for (const item of cases) { + if (byTrace.has(item.traceId)) continue; + if (byId.has(item.id)) refuse(`addToCorpus: case id ${item.id} is already used by trace ${byId.get(item.id)}; ids must be unique.`, 'cases'); + byTrace.set(item.traceId, item.id); + byId.set(item.id, item.traceId); + added.push(item); + } + return added; +} + +/** Adds admitted cases, dropping the oldest 'fixed' cases over maxCases (a new case can be the one dropped); protected cases are never evicted automatically. */ +export async function addToCorpus(dir: string, cases: readonly CorpusCase[], maxCases: number): Promise { + if (!Number.isInteger(maxCases) || maxCases < 1) refuse(`addToCorpus: maxCases must be a positive integer, not ${maxCases}.`, 'maxCases'); + if (!cases.every(isCase)) refuse('addToCorpus: every case needs a non-empty id, traceId and revision, an input, an ISO addedAt, an origin and a baseline with 0 <= passed <= scored and scored >= 1.', 'cases'); + guardPaths(dir); + mkdirSync(path.join(dir, LOUSHO), { recursive: true }); + return withFileLock(path.join(dir, `${FILE}.lock`), async () => { + const current = readCorpus(dir); + const next = evict([...current, ...admitted(current, cases)], maxCases); + save(guardPaths(dir), next); + return next; + }); +} diff --git a/src/improve/index.ts b/src/improve/index.ts index ef51773f..6d11887c 100644 --- a/src/improve/index.ts +++ b/src/improve/index.ts @@ -23,3 +23,5 @@ export { loadImproveSettings } from './settings'; export type { ImproveSettings } from './settings'; export { approvePending, listPending, rejectPending, savePending } from './pending'; export type { PendingProposal } from './pending'; +export { addToCorpus, readCorpus } from './corpus'; +export type { CorpusCase } from './corpus';