feat(improve): healOnce, one full self-healing cycle

api/improve.api.md

@@ -190,7 +190,7 @@ export function approvePending(dir: string, id: string, approvedBy: string): Pro
 
 // @public
 export interface AuditEntry {
-    action: 'propose' | 'apply' | 'reject' | 'stale' | 'rollback' | 'retire' | 'guard-pass' | 'guard-fail' | 'unverified' | 'pause' | 'resume' | 'canary-start' | 'canary-end';
+    action: 'propose' | 'apply' | 'reject' | 'stale' | 'rollback' | 'retire' | 'guard-pass' | 'guard-fail' | 'unverified' | 'pause' | 'resume' | 'canary-start' | 'canary-end' | 'reflect-fail';
     actor: 'gate' | 'canary' | 'monitor' | 'guard' | 'scheduler' | `person:${string}`;
     at: string;
     cycle?: string;
@@ -303,14 +303,12 @@ export function casesFromFailures(db: SqlDatabase, filter?: {
     since?: number;
     agent?: string;
     minScore?: number;
+    traceIds?: readonly string[];
+    recurring?: boolean;
 }): CapturedCase[];
 
 // @public
-export function casesFromRecoveries(db: SqlDatabase, filter?: {
-    since?: number;
-    agent?: string;
-    minScore?: number;
-}): CapturedCase[];
+export function casesFromRecoveries(db: SqlDatabase, filter?: Parameters<typeof casesFromFailures>[1]): CapturedCase[];
 
 // @public
 export interface CaseVerdict {
@@ -590,6 +588,54 @@ interface HealActionEvent extends AgentEventBase<'heal.action'> {
     toolName: string;
 }
 
+// @public
+export interface HealCycle {
+    approval: ImproveSettings['approval'];
+    canary?: CanaryReplayResult;
+    endedAt?: string;
+    errors?: string[];
+    from?: string;
+    guard?: GuardCorpusResult;
+    id: string;
+    outcome: 'idle' | 'paused' | 'busy' | 'budget' | 'cooldown' | Exclude<ImproveOutcome, 'proposal'> | 'canary-failed' | 'applied' | 'pending' | 'guard-rolled-back' | 'stale' | 'refused' | 'interrupted';
+    pending?: string;
+    reason: string;
+    result?: ImproveResult;
+    rolledBackTo?: string;
+    startedAt: string;
+    to?: string;
+    tokens: number;
+    trigger: 'failures' | 'score' | 'manual';
+}
+
+// @public
+export function healOnce(options: HealOptions): Promise<HealCycle>;
+
+// @public
+export interface HealOptions {
+    agent?: string;
+    check: CaseCheck;
+    db: SqlDatabase;
+    dir: string;
+    editor: ImproveOptions['editor'];
+    editorHooks?: readonly AgentHook[];
+    editorTimeoutMs?: number;
+    maxTokens?: number;
+    protect?: readonly string[];
+    reflect?: boolean;
+    runCase: (harness: Harness, input: string) => Promise<{
+        answer: string;
+        transcript: string;
+        tokens: number;
+        ended: string;
+    }>;
+    settings: ImproveSettings;
+    signal?: AbortSignal;
+    since?: number;
+    trigger?: HealCycle['trigger'];
+    withRecoveries?: boolean;
+}
+
 // @public
 interface HookContext {
     agentId?: string;
@@ -675,6 +721,7 @@ export interface ImproveOptions {
         model: string;
     };
     editorHooks?: readonly AgentHook[];
+    editorTimeoutMs?: number;
     harness: Harness;
     holdout?: readonly string[];
     library?: Readonly<Record<string, string>>;

docs/cli.md

@@ -18,7 +18,7 @@ Run it with `npx lousho <command>` inside a project that has the SDK installed.
 | `lousho traces [id]` | List the recent runs saved by `fileTraceExporter()` (duration, model and tool calls, tokens, cost), or print one run as a span tree. | [Observability](./observability.md#local-traces) |
 | `lousho build --target=<t> --agent=<spec>` | Build a deployable Node server, Docker image or Cloudflare Worker. | [Deployment](./deployment.md) |
 | `lousho studio` | Launch Agent Forge, the visual dashboard, on one local port. | [Agent Forge](./agent-forge.md) |
-| `lousho improve <path> --check <module>` | Experimental. Propose edits to an agent's instructions and skills from failed runs in your local observe database. Prints a review card and applies nothing. | [Below](#lousho-improve) |
+| `lousho improve <path> --check <module>` | Experimental. Run one self-healing cycle on an agent directory: propose edits to its instructions and skills from failed runs in your local observe database, gate them, replay them on fresh runs, and apply (auto mode) or queue (manual mode) one that passes. `--dry-run` prints the review card and applies nothing. | [Below](#lousho-improve) |
 
 Every command that takes a `<spec>` reads an agent spec file (`.yaml`,
 `.yml` or `.json`, see [Configuration](./configuration.md#agent-spec-files-agentspec)).
@@ -375,9 +375,17 @@ export default (input, answer) => !answer.includes('I cannot help');
 It takes the failed runs in your local observe database (default
 `~/.lousho/observe.db`, or `--db`), replays each recorded input through the
 agent, scores each answer with your check, and prints the review card from
-`formatImproveResult()`. It never changes the agent: no file in the agent
-directory is written, and there is no flag to apply a proposal. Review the card
-and apply what you accept by hand.
+`formatImproveResult()`. On an agent directory it then runs one self-healing
+cycle with `healOnce()`: only failed runs of the current harness revision count,
+a proposal that passes the gate is replayed on fresh runs (the canary), and in
+auto mode (the default) it is written to `instructions.md` and
+`skills/<name>/SKILL.md`, after which the corpus guard replays the protected
+cases and rolls back on a regression. In manual mode it is queued for
+`lousho improve approve`. Every step is audited, and `improve.json`'s daily caps,
+the cooldown after a rollback and the `.lousho/improve.stop` kill switch apply.
+`--dry-run` keeps the old behaviour: every failed run counts, the card is
+printed, and nothing is written. An agent module, or an agent whose instructions
+don't live in `instructions.md`, always runs dry.
 
 - `<path>` is an agent directory or a `.ts`/`.js` agent module whose
   instructions are plain text. Editors may change only the instructions and

docs/improve.md

@@ -346,7 +346,10 @@ npx lousho improve ./my-agent --check ./check.mjs --since 7d --out proposal.md
 It reads cases only from your local observe database, needs a module whose
 default export is your `CaseCheck` (return `failure` to name the check a failed
 answer missed), simulates tools other than loading skills
-unless `--read-only` names them, and prints the review card. It applies nothing.
+unless `--read-only` names them, and prints the review card. On an agent
+directory it runs one self-healing cycle (`healOnce()`): in auto mode a proposal
+that passes the gate and a replay canary is applied and guarded, and in manual
+mode it is queued for approval. `--dry-run` applies nothing.
 `--max-tokens <n>` sets `maxTokens`; it has no default, so without it there is no
 cap. It is checked after each run or editor call, so the last one can go over,
 and a run that hits it ends `budget-exceeded`.

llms-full.txt

@@ -5518,7 +5518,7 @@ Run it with `npx lousho <command>` inside a project that has the SDK installed.
 | `lousho traces [id]` | List the recent runs saved by `fileTraceExporter()` (duration, model and tool calls, tokens, cost), or print one run as a span tree. | [Observability](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/observability.md#local-traces) |
 | `lousho build --target=<t> --agent=<spec>` | Build a deployable Node server, Docker image or Cloudflare Worker. | [Deployment](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/deployment.md) |
 | `lousho studio` | Launch Agent Forge, the visual dashboard, on one local port. | [Agent Forge](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/agent-forge.md) |
-| `lousho improve <path> --check <module>` | Experimental. Propose edits to an agent's instructions and skills from failed runs in your local observe database. Prints a review card and applies nothing. | [Below](#lousho-improve) |
+| `lousho improve <path> --check <module>` | Experimental. Run one self-healing cycle on an agent directory: propose edits to its instructions and skills from failed runs in your local observe database, gate them, replay them on fresh runs, and apply (auto mode) or queue (manual mode) one that passes. `--dry-run` prints the review card and applies nothing. | [Below](#lousho-improve) |
 
 Every command that takes a `<spec>` reads an agent spec file (`.yaml`,
 `.yml` or `.json`, see [Configuration](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/configuration.md#agent-spec-files-agentspec)).
@@ -5875,9 +5875,17 @@ export default (input, answer) => !answer.includes('I cannot help');
 It takes the failed runs in your local observe database (default
 `~/.lousho/observe.db`, or `--db`), replays each recorded input through the
 agent, scores each answer with your check, and prints the review card from
-`formatImproveResult()`. It never changes the agent: no file in the agent
-directory is written, and there is no flag to apply a proposal. Review the card
-and apply what you accept by hand.
+`formatImproveResult()`. On an agent directory it then runs one self-healing
+cycle with `healOnce()`: only failed runs of the current harness revision count,
+a proposal that passes the gate is replayed on fresh runs (the canary), and in
+auto mode (the default) it is written to `instructions.md` and
+`skills/<name>/SKILL.md`, after which the corpus guard replays the protected
+cases and rolls back on a regression. In manual mode it is queued for
+`lousho improve approve`. Every step is audited, and `improve.json`'s daily caps,
+the cooldown after a rollback and the `.lousho/improve.stop` kill switch apply.
+`--dry-run` keeps the old behaviour: every failed run counts, the card is
+printed, and nothing is written. An agent module, or an agent whose instructions
+don't live in `instructions.md`, always runs dry.
 
 - `<path>` is an agent directory or a `.ts`/`.js` agent module whose
   instructions are plain text. Editors may change only the instructions and
@@ -13221,7 +13229,10 @@ npx lousho improve ./my-agent --check ./check.mjs --since 7d --out proposal.md
 It reads cases only from your local observe database, needs a module whose
 default export is your `CaseCheck` (return `failure` to name the check a failed
 answer missed), simulates tools other than loading skills
-unless `--read-only` names them, and prints the review card. It applies nothing.
+unless `--read-only` names them, and prints the review card. On an agent
+directory it runs one self-healing cycle (`healOnce()`): in auto mode a proposal
+that passes the gate and a replay canary is applied and guarded, and in manual
+mode it is queued for approval. `--dry-run` applies nothing.
 `--max-tokens <n>` sets `maxTokens`; it has no default, so without it there is no
 cap. It is checked after each run or editor call, so the last one can go over,
 and a run that hits it ends `budget-exceeded`.

src/cli/improve.test.ts

@@ -9,6 +9,9 @@ import type { IngestEvent } from '../observe/envelope';
 import { ingestBatch, openDb } from '../observe/store';
 import { mockModel, type MockTurn } from '../testing';
 import { defineTool } from '../tools/defineTool';
+import { readAudit } from '../improve/audit';
+import { listPending } from '../improve/pending';
+import { currentRevision } from '../improve/revisions';
 import { parseImproveArgs, runImprove, type ImproveIo } from './improve';
 
 const T0 = Date.now() - 60_000;
@@ -80,6 +83,10 @@ function snapshot(dir: string): Record<string, string> {
 const fixedWhenForbidden = (request: { messages: readonly unknown[] }) => ({ text: JSON.stringify(request.messages).includes('forbidden') ? 'ok, that is forbidden' : 'here is the doc' });
 
 async function improve(args: string[], over: Partial<ImproveIo> = {}) {
+  return run(args.includes('--help') || args[0] === undefined ? args : [...args, '--dry-run'], over);
+}
+
+async function run(args: string[], over: Partial<ImproveIo> = {}) {
   const out = sink();
   const err = sink();
   const io: ImproveIo = {
@@ -203,7 +210,8 @@ describe('lousho improve', () => {
     const { code, stdout } = await improve(['--help']);
     expect(code).toBe(0);
     expect(stdout).toContain('Experimental');
-    expect(stdout).toContain('never changes');
+    expect(stdout).toContain('--dry-run');
+    expect(stdout).not.toContain('never changes');
     expect(stdout).toContain('[--max-tokens n]');
     expect(stdout).toMatch(/--max-tokens n {2,}stop once n tokens are spent/);
     expect(stdout).toContain('no cap');
@@ -541,3 +549,74 @@ describe('lousho improve', () => {
     expect(stderr).toMatch(/pass:p1 was not protected: it passed 0 of 6 scored runs/);
   });
 });
+
+describe('lousho improve, one self-healing cycle', () => {
+  const editorFor = (instructions: string) =>
+    mockModel(
+      [
+        (request) => {
+          if (request.messages.at(-1)?.role === 'tool') return '<notes>Traced to a failed run.</notes>';
+          return { toolCalls: [{ name: 'write_file', args: { path: 'harness/instructions.md', content: instructions } }] };
+        },
+      ],
+      { onExhausted: 'repeat-last' }
+    );
+
+  function stamped(trace: string): IngestEvent[] {
+    const revision = currentRevision(agentDir);
+    return failedTrace(trace, JSON.stringify([{ role: 'user', parts: [{ type: 'text', content: 'can I read the secret doc?' }] }])).map((event) =>
+      event.type === 'observation' && event.data.kind === 'span' ? { ...event, data: { ...event.data, attrs: { ...event.data.attrs, 'lousho.harness.revision': revision } } } : event
+    );
+  }
+
+  beforeEach(() => {
+    const db = openDb(dbPath);
+    ingestBatch(db, [...stamped('s1'), ...stamped('s2')]);
+    db.close();
+  });
+
+  const instructions = () => fs.readFileSync(path.join(agentDir, 'instructions.md'), 'utf8').trim();
+
+  it('auto: applies a fix that passes the gate and the canary, and audits it', async () => {
+    const { code, stdout } = await run([agentDir, '--check', checkPath, '--db', dbPath], { editor: editorFor(FIX) });
+    expect(code).toBe(0);
+    expect(stdout).toMatch(/lousho improve: applied: Applied [0-9a-f]{12}/);
+    expect(instructions()).toBe(FIX);
+    expect(readAudit(agentDir).map((entry) => entry.action)).toEqual(['canary-end', 'apply']);
+  });
+
+  it('manual: queues the fix for a person and writes nothing to the agent', async () => {
+    const { code, stdout } = await run([agentDir, '--check', checkPath, '--db', dbPath, '--approval', 'manual'], { editor: editorFor(FIX) });
+    expect(code).toBe(0);
+    expect(stdout).toMatch(/lousho improve: pending: Queued for approval as [0-9a-f]{12}/);
+    expect(instructions()).toBe('You answer refund questions.');
+    expect(listPending(agentDir)).toHaveLength(1);
+  });
+
+  it('--dry-run writes nothing', async () => {
+    const before = snapshot(root);
+    const { code, stdout } = await run([agentDir, '--check', checkPath, '--db', dbPath, '--dry-run'], { editor: editorFor(FIX) });
+    expect(code).toBe(0);
+    expect(stdout).toContain('# Improve result: proposal');
+    const after = snapshot(root);
+    expect(Object.keys(after).filter((file) => after[file] !== before[file] && !file.startsWith('observe.db'))).toEqual([]);
+  });
+
+  it('a module target is forced to dry run', async () => {
+    const file = path.join(root, 'agent.mjs');
+    fs.writeFileSync(file, "export default { name: 'support', instructions: 'You answer refund questions.' };\n");
+    const { code, stderr } = await run([file, '--check', checkPath, '--db', dbPath], { editor: editorFor(FIX) });
+    expect(code).toBe(0);
+    expect(stderr).toMatch(/this is a dry run, because .*agent module/);
+    expect(fs.existsSync(path.join(root, '.lousho'))).toBe(false);
+  });
+
+  it('an agent whose instructions are not in instructions.md is forced to dry run', async () => {
+    fs.mkdirSync(path.join(agentDir, 'instructions'));
+    fs.writeFileSync(path.join(agentDir, 'instructions', 'claude.md'), 'You answer refund questions for Claude.');
+    const { code, stderr } = await run([agentDir, '--check', checkPath, '--db', dbPath], { editor: editorFor(FIX) });
+    expect(code).toBe(0);
+    expect(stderr).toMatch(/this is a dry run, because .*model-specific instructions/);
+    expect(instructions()).toBe('You answer refund questions.');
+  });
+});