AA08: proposeAgent tool: validate, catalog checks, requirements, card output

apps/agent-forge/server/author/proposal/__tests__/catalogChecks.test.ts

@@ -0,0 +1,211 @@
+import { describe, expect, it } from 'vitest';
+import type { AgentSpecV2 } from 'lousho';
+import type { AuthorToolDeps, CatalogService } from '../../tools/index';
+import type { AuthorTurn } from '../../turnRunner';
+import type { WorkspaceContext } from '../../../workspace/context';
+import type { AgentRepo } from '../../../storage/agentRepo';
+import type { AuthorToolFailure } from '../../../../shared/authorWire';
+import { checkCatalogs, loadCatalogFacts, type CatalogFacts } from '../catalogChecks';
+
+/**
+ * The catalog checks proposeAgent/proposeAgentUpdate run between schema
+ * validation and requirements: every value the draft names that must
+ * exist in Studio's catalogs is collected into one failure (first
+ * problem's code wins), and the per-turn memoization of the facts load.
+ */
+
+function turn(): AuthorTurn {
+  return {
+    sessionId: 'sess-1',
+    requestId: 'req-test',
+    focus: { mode: 'create' },
+    storedTranscript: [],
+    userMessage: { id: 'm-1', role: 'user', text: 'hi', at: '2026-01-01T00:00:00.000Z' },
+    signal: new AbortController().signal,
+    stepsRemaining: () => 12,
+    state: new Map(),
+  };
+}
+
+function spec(over: Partial<AgentSpecV2> = {}): AgentSpecV2 {
+  return {
+    version: 2,
+    name: 'Test agent',
+    description: 'does a thing',
+    model: 'anthropic/claude-sonnet-5-5',
+    instructions: 'do a thing',
+    ...over,
+  };
+}
+
+function facts(over: Partial<CatalogFacts> = {}): CatalogFacts {
+  return {
+    modelIds: new Set(['anthropic/claude-sonnet-5-5', 'openai/gpt-5']),
+    builtinTools: new Set(['web_fetch', 'run_command']),
+    guardrails: new Set(['pii', 'regex']),
+    triggerTypes: new Set(['chat', 'cron', 'webhook', 'channel', 'agent']),
+    agentIds: new Set(['billing-bot', 'support-triage']),
+    ...over,
+  };
+}
+
+function failure(result: AuthorToolFailure | null): AuthorToolFailure {
+  expect(result).not.toBeNull();
+  expect(result!.ok).toBe(false);
+  return result!;
+}
+
+describe('checkCatalogs', () => {
+  it('returns null when every catalog value exists', () => {
+    const ok = spec({
+      fallbackModels: ['openai/gpt-5'],
+      tools: [{ builtin: 'web_fetch' }],
+      guardrails: [{ id: 'g1', name: 'pii' }],
+      triggers: [{ id: 't1', type: 'chat' }, { id: 't2', type: 'cron', cron: '0 9 * * 1', prompt: 'go' }],
+      subagents: { helper: { ref: 'billing-bot' } },
+      handoffs: [{ agent: 'support-triage' }],
+    });
+    expect(checkCatalogs(ok, facts(), 'proposeAgent')).toBeNull();
+  });
+
+  it('UNKNOWN_MODEL when the first problem is a model not in the catalog', () => {
+    const result = failure(checkCatalogs(spec({ model: 'acme/not-real' }), facts(), 'proposeAgent'));
+    expect(result.code).toBe('UNKNOWN_MODEL');
+    expect(result.error).toBe('Some values are not in Studio\'s catalogs: model "acme/not-real" is not in the catalog.');
+  });
+
+  it('flags each fallback model that is not in the catalog', () => {
+    const result = failure(
+      checkCatalogs(spec({ fallbackModels: ['acme/gone-1', 'openai/gpt-5', 'acme/gone-2'] }), facts(), 'proposeAgent')
+    );
+    expect(result.code).toBe('UNKNOWN_MODEL');
+    expect(result.error).toContain('model "acme/gone-1" is not in the catalog');
+    expect(result.error).toContain('model "acme/gone-2" is not in the catalog');
+    expect(result.error).not.toContain('openai/gpt-5');
+  });
+
+  it('UNKNOWN_TOOL for a builtin name the catalog lacks', () => {
+    const result = failure(
+      checkCatalogs(spec({ tools: [{ builtin: 'web_fetch' }, { builtin: 'not-a-tool' }] }), facts(), 'proposeAgent')
+    );
+    expect(result.code).toBe('UNKNOWN_TOOL');
+    expect(result.error).toBe('Some values are not in Studio\'s catalogs: tools[1] builtin "not-a-tool" does not exist.');
+  });
+
+  it('UNKNOWN_GUARDRAIL for a guardrail name the catalog lacks', () => {
+    const result = failure(
+      checkCatalogs(spec({ guardrails: [{ id: 'g1', name: 'pii' }, { id: 'g2', name: 'not-a-guardrail' as never }] }), facts(), 'proposeAgent')
+    );
+    expect(result.code).toBe('UNKNOWN_GUARDRAIL');
+    expect(result.error).toContain('guardrails[1] name "not-a-guardrail" does not exist');
+  });
+
+  it('UNKNOWN_TRIGGER_TYPE for a trigger type the catalog lacks', () => {
+    const result = failure(
+      checkCatalogs(spec({ triggers: [{ id: 't1', type: 'chat' }, { id: 't2', type: 'smoke' as never }] }), facts(), 'proposeAgent')
+    );
+    expect(result.code).toBe('UNKNOWN_TRIGGER_TYPE');
+    expect(result.error).toContain('triggers[1] type "smoke" does not exist');
+  });
+
+  it('UNKNOWN_AGENT_REF for a subagents.<key>.ref that is not a workspace agent', () => {
+    const result = failure(
+      checkCatalogs(spec({ subagents: { helper: { ref: 'billing-bot' }, ghost: { ref: 'gone' } } }), facts(), 'proposeAgent')
+    );
+    expect(result.code).toBe('UNKNOWN_AGENT_REF');
+    expect(result.error).toContain('subagents.ghost.ref agent "gone" does not exist');
+  });
+
+  it('UNKNOWN_AGENT_REF for a handoffs[i].agent that is not a workspace agent', () => {
+    const result = failure(checkCatalogs(spec({ handoffs: [{ agent: 'gone' }] }), facts(), 'proposeAgent'));
+    expect(result.code).toBe('UNKNOWN_AGENT_REF');
+    expect(result.error).toContain('handoffs[0].agent agent "gone" does not exist');
+  });
+
+  it('allows a handoff to one of the spec\'s own subagent keys', () => {
+    const ok = spec({
+      subagents: { helper: { inline: { name: 'Helper', model: 'anthropic/claude-sonnet-5-5', instructions: 'help', description: 'helps' } } },
+      handoffs: [{ agent: 'helper' }],
+    });
+    expect(checkCatalogs(ok, facts(), 'proposeAgent')).toBeNull();
+  });
+
+  it('the first problem in check order picks the code; every problem is listed', () => {
+    const result = failure(
+      checkCatalogs(spec({ model: 'acme/not-real', tools: [{ builtin: 'not-a-tool' }] }), facts(), 'proposeAgent')
+    );
+    expect(result.code).toBe('UNKNOWN_MODEL');
+    expect(result.error).toContain('model "acme/not-real" is not in the catalog');
+    expect(result.error).toContain('tools[0] builtin "not-a-tool" does not exist');
+    expect(result.userMessage).toBe("The draft referred to something Studio doesn't have. The Author is fixing it.");
+    expect(result.next).toBe(
+      'Call listModels, listTools, listGuardrails, listTriggerTypes or listAgents and copy exact values, then call proposeAgent again.'
+    );
+  });
+
+  it('lists at most the first 10 problems', () => {
+    const many = spec({ tools: Array.from({ length: 12 }, (_, i) => ({ builtin: `tool-${i}` })) });
+    const result = failure(checkCatalogs(many, facts(), 'proposeAgent'));
+    expect(result.code).toBe('UNKNOWN_TOOL');
+    expect(result.error).toContain('tools[9] builtin "tool-9" does not exist');
+    expect(result.error).not.toContain('tools[10]');
+    expect(result.error).not.toContain('tool-11');
+  });
+});
+
+describe('loadCatalogFacts', () => {
+  function deps(agentIds: string[], catalog: Partial<CatalogService> = {}): AuthorToolDeps {
+    return {
+      workspace: async () =>
+        ({ repo: { listIds: () => agentIds } as unknown as AgentRepo }) as WorkspaceContext,
+      catalog: {
+        listModels: () => [{ id: 'anthropic/claude-sonnet-5-5' }] as never,
+        listTools: () => [{ name: 'web_fetch' }] as never,
+        listGuardrails: () => [{ name: 'pii' }] as never,
+        listTriggerTypes: () => [{ type: 'chat' }] as never,
+        listSkills: async () => [],
+        listKits: async () => [],
+        listRegistryMcpServers: async () => [],
+        ...catalog,
+      },
+      templates: { list: async () => [] },
+      probeMcp: async () => ({ ok: true, tools: [] }),
+      docsDir: '/nonexistent-docs',
+      reviewDraft: async () => ({ pass: true }),
+      requirements: async () => [],
+    };
+  }
+
+  it('reads ids from the catalog service and the workspace repo', async () => {
+    const loaded = await loadCatalogFacts(turn(), deps(['billing-bot']));
+    expect(loaded).toEqual({
+      modelIds: new Set(['anthropic/claude-sonnet-5-5']),
+      builtinTools: new Set(['web_fetch']),
+      guardrails: new Set(['pii']),
+      triggerTypes: new Set(['chat']),
+      agentIds: new Set(['billing-bot']),
+    });
+  });
+
+  it('memoizes per turn: a second call never re-reads the catalog', async () => {
+    let modelCalls = 0;
+    const t = turn();
+    const d = deps(['a'], {
+      listModels: () => {
+        modelCalls += 1;
+        return [{ id: 'anthropic/claude-sonnet-5-5' }] as never;
+      },
+    });
+    const first = await loadCatalogFacts(t, d);
+    const second = await loadCatalogFacts(t, d);
+    expect(modelCalls).toBe(1);
+    expect(second).toBe(first);
+  });
+
+  it('does not share the memo across turns', async () => {
+    const loaded = await loadCatalogFacts(turn(), deps(['a']));
+    const other = await loadCatalogFacts(turn(), deps(['b']));
+    expect([...loaded.agentIds]).toEqual(['a']);
+    expect([...other.agentIds]).toEqual(['b']);
+  });
+});

apps/agent-forge/server/author/proposal/catalogChecks.ts

@@ -0,0 +1,162 @@
+/**
+ * The catalog checks a propose tool runs between schema validation and
+ * requirements (research §4.1 steps 4-8): every value the draft names that
+ * must exist in Studio's catalogs - its models, builtin tools, guardrails,
+ * trigger types and agent references - is verified against
+ * {@link CatalogFacts}, and all problems come back in ONE failure so the
+ * model can fix them in a single retry instead of one round-trip each.
+ *
+ * The schema already rejects malformed values (a `{ builtin }` the spec
+ * grammar does not know never reaches here); these checks are about
+ * membership in the live catalogs the `deps.catalog`/`deps.workspace`
+ * seams report, which the prompt lists and the model copies ids from.
+ *
+ * {@link loadCatalogFacts} reads the catalogs once per turn (memoized in
+ * `turn.state['catalogFacts']`): the model may propose several drafts in
+ * a turn and the catalogs cannot change mid-turn.
+ */
+import type { AgentSpecV2 } from 'lousho';
+import type { AuthorFailureCode, AuthorToolFailure } from '../../../shared/authorWire';
+import type { AuthorTurn } from '../turnRunner';
+import { toolFailure } from '../tools/failure';
+import type { AuthorToolDeps } from '../tools/index';
+
+/** The catalog membership sets the checks run against, loaded once per turn. */
+export interface CatalogFacts {
+  /** 'provider/model' ids the model catalog knows. */
+  modelIds: ReadonlySet<string>;
+  /** Builtin tool names `listTools` lists. */
+  builtinTools: ReadonlySet<string>;
+  /** Guardrail names `listGuardrails` lists. */
+  guardrails: ReadonlySet<string>;
+  /** Trigger types `listTriggerTypes` lists. */
+  triggerTypes: ReadonlySet<string>;
+  /** Agent ids in the session's workspace (`repo.listIds()`). */
+  agentIds: ReadonlySet<string>;
+}
+
+/** The `turn.state` key the memoized {@link CatalogFacts} promise lives under. */
+const CATALOG_FACTS_KEY = 'catalogFacts';
+
+/**
+ * The turn's {@link CatalogFacts}: read once and memoized in
+ * `turn.state['catalogFacts']` (the promise itself is cached, so two
+ * racing calls share the load). Models, tools, guardrails and trigger
+ * types come from `deps.catalog`; agent ids from the session's workspace.
+ */
+export function loadCatalogFacts(turn: AuthorTurn, deps: AuthorToolDeps): Promise<CatalogFacts> {
+  const existing = turn.state.get(CATALOG_FACTS_KEY) as Promise<CatalogFacts> | undefined;
+  if (existing !== undefined) return existing;
+  const promise = (async (): Promise<CatalogFacts> => {
+    const [{ repo }, models, tools, guardrails, triggers] = await Promise.all([
+      deps.workspace(),
+      deps.catalog.listModels(),
+      deps.catalog.listTools(),
+      deps.catalog.listGuardrails(),
+      deps.catalog.listTriggerTypes(),
+    ]);
+    return {
+      modelIds: new Set(models.map((model) => model.id)),
+      builtinTools: new Set(tools.map((tool) => tool.name)),
+      guardrails: new Set(guardrails.map((guardrail) => guardrail.name)),
+      triggerTypes: new Set(triggers.map((trigger) => trigger.type)),
+      agentIds: new Set(repo.listIds()),
+    };
+  })();
+  turn.state.set(CATALOG_FACTS_KEY, promise);
+  return promise;
+}
+
+/** One catalog problem: the failure code it picks when it is the first found, and its model-facing text. */
+interface CatalogProblem {
+  code: AuthorFailureCode;
+  text: string;
+}
+
+/** `model` and each `fallbackModels[i]` must be ids the model catalog knows. */
+function modelProblems(spec: AgentSpecV2, facts: CatalogFacts): CatalogProblem[] {
+  return [spec.model, ...(spec.fallbackModels ?? [])]
+    .filter((model) => !facts.modelIds.has(model))
+    .map((model) => ({ code: 'UNKNOWN_MODEL' as const, text: `model "${model}" is not in the catalog` }));
+}
+
+/** Each `tools[i].builtin` must be a builtin the tool catalog knows. */
+function toolProblems(spec: AgentSpecV2, facts: CatalogFacts): CatalogProblem[] {
+  const problems: CatalogProblem[] = [];
+  (spec.tools ?? []).forEach((ref, i) => {
+    if ('builtin' in ref && !facts.builtinTools.has(ref.builtin)) {
+      problems.push({ code: 'UNKNOWN_TOOL', text: `tools[${i}] builtin "${ref.builtin}" does not exist` });
+    }
+  });
+  return problems;
+}
+
+/** Each `guardrails[i].name` must be a guardrail the catalog knows. */
+function guardrailProblems(spec: AgentSpecV2, facts: CatalogFacts): CatalogProblem[] {
+  const problems: CatalogProblem[] = [];
+  (spec.guardrails ?? []).forEach((entry, i) => {
+    if (!facts.guardrails.has(entry.name)) {
+      problems.push({ code: 'UNKNOWN_GUARDRAIL', text: `guardrails[${i}] name "${entry.name}" does not exist` });
+    }
+  });
+  return problems;
+}
+
+/** Each `triggers[i].type` must be a trigger type the catalog knows. */
+function triggerProblems(spec: AgentSpecV2, facts: CatalogFacts): CatalogProblem[] {
+  const problems: CatalogProblem[] = [];
+  (spec.triggers ?? []).forEach((entry, i) => {
+    if (!facts.triggerTypes.has(entry.type)) {
+      problems.push({ code: 'UNKNOWN_TRIGGER_TYPE', text: `triggers[${i}] type "${entry.type}" does not exist` });
+    }
+  });
+  return problems;
+}
+
+/**
+ * Agent references must name workspace agents: each `subagents.<key>.ref`,
+ * and each `handoffs[i].agent` unless it names one of the spec's own
+ * `subagents` keys (a handoff to an inline subagent is legal).
+ */
+function agentRefProblems(spec: AgentSpecV2, facts: CatalogFacts): CatalogProblem[] {
+  const problems: CatalogProblem[] = [];
+  const subagentKeys = new Set(Object.keys(spec.subagents ?? {}));
+  for (const [key, entry] of Object.entries(spec.subagents ?? {})) {
+    if ('ref' in entry && !facts.agentIds.has(entry.ref)) {
+      problems.push({ code: 'UNKNOWN_AGENT_REF', text: `subagents.${key}.ref agent "${entry.ref}" does not exist` });
+    }
+  }
+  (spec.handoffs ?? []).forEach((entry, i) => {
+    if (!subagentKeys.has(entry.agent) && !facts.agentIds.has(entry.agent)) {
+      problems.push({ code: 'UNKNOWN_AGENT_REF', text: `handoffs[${i}].agent agent "${entry.agent}" does not exist` });
+    }
+  });
+  return problems;
+}
+
+/** The most problems one failure lists; the code is still the first problem's. */
+const MAX_LISTED_PROBLEMS = 10;
+
+/**
+ * All of `spec`'s catalog problems in one {@link AuthorToolFailure}, or
+ * `null` when every value is known. Problems are collected in check order
+ * (models, tools, guardrails, triggers, agent refs); the first problem's
+ * code is the failure's code and at most ten problem texts are listed.
+ */
+export function checkCatalogs(spec: AgentSpecV2, facts: CatalogFacts, toolName: string): AuthorToolFailure | null {
+  const problems = [
+    ...modelProblems(spec, facts),
+    ...toolProblems(spec, facts),
+    ...guardrailProblems(spec, facts),
+    ...triggerProblems(spec, facts),
+    ...agentRefProblems(spec, facts),
+  ];
+  if (problems.length === 0) return null;
+  const listed = problems.slice(0, MAX_LISTED_PROBLEMS).map((problem) => problem.text).join('; ');
+  return toolFailure(
+    problems[0].code,
+    `Some values are not in Studio's catalogs: ${listed}.`,
+    "The draft referred to something Studio doesn't have. The Author is fixing it.",
+    `Call listModels, listTools, listGuardrails, listTriggerTypes or listAgents and copy exact values, then call ${toolName} again.`
+  );
+}