diff --git a/apps/agent-forge/server/author/proposal/__tests__/catalogChecks.test.ts b/apps/agent-forge/server/author/proposal/__tests__/catalogChecks.test.ts new file mode 100644 index 00000000..d61007c2 --- /dev/null +++ b/apps/agent-forge/server/author/proposal/__tests__/catalogChecks.test.ts @@ -0,0 +1,211 @@ +import { describe, expect, it } from 'vitest'; +import type { AgentSpecV2 } from 'lousho'; +import type { AuthorToolDeps, CatalogService } from '../../tools/index'; +import type { AuthorTurn } from '../../turnRunner'; +import type { WorkspaceContext } from '../../../workspace/context'; +import type { AgentRepo } from '../../../storage/agentRepo'; +import type { AuthorToolFailure } from '../../../../shared/authorWire'; +import { checkCatalogs, loadCatalogFacts, type CatalogFacts } from '../catalogChecks'; + +/** + * The catalog checks proposeAgent/proposeAgentUpdate run between schema + * validation and requirements: every value the draft names that must + * exist in Studio's catalogs is collected into one failure (first + * problem's code wins), and the per-turn memoization of the facts load. + */ + +function turn(): AuthorTurn { + return { + sessionId: 'sess-1', + requestId: 'req-test', + focus: { mode: 'create' }, + storedTranscript: [], + userMessage: { id: 'm-1', role: 'user', text: 'hi', at: '2026-01-01T00:00:00.000Z' }, + signal: new AbortController().signal, + stepsRemaining: () => 12, + state: new Map(), + }; +} + +function spec(over: Partial = {}): AgentSpecV2 { + return { + version: 2, + name: 'Test agent', + description: 'does a thing', + model: 'anthropic/claude-sonnet-5-5', + instructions: 'do a thing', + ...over, + }; +} + +function facts(over: Partial = {}): CatalogFacts { + return { + modelIds: new Set(['anthropic/claude-sonnet-5-5', 'openai/gpt-5']), + builtinTools: new Set(['web_fetch', 'run_command']), + guardrails: new Set(['pii', 'regex']), + triggerTypes: new Set(['chat', 'cron', 'webhook', 'channel', 'agent']), + agentIds: new Set(['billing-bot', 'support-triage']), + ...over, + }; +} + +function failure(result: AuthorToolFailure | null): AuthorToolFailure { + expect(result).not.toBeNull(); + expect(result!.ok).toBe(false); + return result!; +} + +describe('checkCatalogs', () => { + it('returns null when every catalog value exists', () => { + const ok = spec({ + fallbackModels: ['openai/gpt-5'], + tools: [{ builtin: 'web_fetch' }], + guardrails: [{ id: 'g1', name: 'pii' }], + triggers: [{ id: 't1', type: 'chat' }, { id: 't2', type: 'cron', cron: '0 9 * * 1', prompt: 'go' }], + subagents: { helper: { ref: 'billing-bot' } }, + handoffs: [{ agent: 'support-triage' }], + }); + expect(checkCatalogs(ok, facts(), 'proposeAgent')).toBeNull(); + }); + + it('UNKNOWN_MODEL when the first problem is a model not in the catalog', () => { + const result = failure(checkCatalogs(spec({ model: 'acme/not-real' }), facts(), 'proposeAgent')); + expect(result.code).toBe('UNKNOWN_MODEL'); + expect(result.error).toBe('Some values are not in Studio\'s catalogs: model "acme/not-real" is not in the catalog.'); + }); + + it('flags each fallback model that is not in the catalog', () => { + const result = failure( + checkCatalogs(spec({ fallbackModels: ['acme/gone-1', 'openai/gpt-5', 'acme/gone-2'] }), facts(), 'proposeAgent') + ); + expect(result.code).toBe('UNKNOWN_MODEL'); + expect(result.error).toContain('model "acme/gone-1" is not in the catalog'); + expect(result.error).toContain('model "acme/gone-2" is not in the catalog'); + expect(result.error).not.toContain('openai/gpt-5'); + }); + + it('UNKNOWN_TOOL for a builtin name the catalog lacks', () => { + const result = failure( + checkCatalogs(spec({ tools: [{ builtin: 'web_fetch' }, { builtin: 'not-a-tool' }] }), facts(), 'proposeAgent') + ); + expect(result.code).toBe('UNKNOWN_TOOL'); + expect(result.error).toBe('Some values are not in Studio\'s catalogs: tools[1] builtin "not-a-tool" does not exist.'); + }); + + it('UNKNOWN_GUARDRAIL for a guardrail name the catalog lacks', () => { + const result = failure( + checkCatalogs(spec({ guardrails: [{ id: 'g1', name: 'pii' }, { id: 'g2', name: 'not-a-guardrail' as never }] }), facts(), 'proposeAgent') + ); + expect(result.code).toBe('UNKNOWN_GUARDRAIL'); + expect(result.error).toContain('guardrails[1] name "not-a-guardrail" does not exist'); + }); + + it('UNKNOWN_TRIGGER_TYPE for a trigger type the catalog lacks', () => { + const result = failure( + checkCatalogs(spec({ triggers: [{ id: 't1', type: 'chat' }, { id: 't2', type: 'smoke' as never }] }), facts(), 'proposeAgent') + ); + expect(result.code).toBe('UNKNOWN_TRIGGER_TYPE'); + expect(result.error).toContain('triggers[1] type "smoke" does not exist'); + }); + + it('UNKNOWN_AGENT_REF for a subagents..ref that is not a workspace agent', () => { + const result = failure( + checkCatalogs(spec({ subagents: { helper: { ref: 'billing-bot' }, ghost: { ref: 'gone' } } }), facts(), 'proposeAgent') + ); + expect(result.code).toBe('UNKNOWN_AGENT_REF'); + expect(result.error).toContain('subagents.ghost.ref agent "gone" does not exist'); + }); + + it('UNKNOWN_AGENT_REF for a handoffs[i].agent that is not a workspace agent', () => { + const result = failure(checkCatalogs(spec({ handoffs: [{ agent: 'gone' }] }), facts(), 'proposeAgent')); + expect(result.code).toBe('UNKNOWN_AGENT_REF'); + expect(result.error).toContain('handoffs[0].agent agent "gone" does not exist'); + }); + + it('allows a handoff to one of the spec\'s own subagent keys', () => { + const ok = spec({ + subagents: { helper: { inline: { name: 'Helper', model: 'anthropic/claude-sonnet-5-5', instructions: 'help', description: 'helps' } } }, + handoffs: [{ agent: 'helper' }], + }); + expect(checkCatalogs(ok, facts(), 'proposeAgent')).toBeNull(); + }); + + it('the first problem in check order picks the code; every problem is listed', () => { + const result = failure( + checkCatalogs(spec({ model: 'acme/not-real', tools: [{ builtin: 'not-a-tool' }] }), facts(), 'proposeAgent') + ); + expect(result.code).toBe('UNKNOWN_MODEL'); + expect(result.error).toContain('model "acme/not-real" is not in the catalog'); + expect(result.error).toContain('tools[0] builtin "not-a-tool" does not exist'); + expect(result.userMessage).toBe("The draft referred to something Studio doesn't have. The Author is fixing it."); + expect(result.next).toBe( + 'Call listModels, listTools, listGuardrails, listTriggerTypes or listAgents and copy exact values, then call proposeAgent again.' + ); + }); + + it('lists at most the first 10 problems', () => { + const many = spec({ tools: Array.from({ length: 12 }, (_, i) => ({ builtin: `tool-${i}` })) }); + const result = failure(checkCatalogs(many, facts(), 'proposeAgent')); + expect(result.code).toBe('UNKNOWN_TOOL'); + expect(result.error).toContain('tools[9] builtin "tool-9" does not exist'); + expect(result.error).not.toContain('tools[10]'); + expect(result.error).not.toContain('tool-11'); + }); +}); + +describe('loadCatalogFacts', () => { + function deps(agentIds: string[], catalog: Partial = {}): AuthorToolDeps { + return { + workspace: async () => + ({ repo: { listIds: () => agentIds } as unknown as AgentRepo }) as WorkspaceContext, + catalog: { + listModels: () => [{ id: 'anthropic/claude-sonnet-5-5' }] as never, + listTools: () => [{ name: 'web_fetch' }] as never, + listGuardrails: () => [{ name: 'pii' }] as never, + listTriggerTypes: () => [{ type: 'chat' }] as never, + listSkills: async () => [], + listKits: async () => [], + listRegistryMcpServers: async () => [], + ...catalog, + }, + templates: { list: async () => [] }, + probeMcp: async () => ({ ok: true, tools: [] }), + docsDir: '/nonexistent-docs', + reviewDraft: async () => ({ pass: true }), + requirements: async () => [], + }; + } + + it('reads ids from the catalog service and the workspace repo', async () => { + const loaded = await loadCatalogFacts(turn(), deps(['billing-bot'])); + expect(loaded).toEqual({ + modelIds: new Set(['anthropic/claude-sonnet-5-5']), + builtinTools: new Set(['web_fetch']), + guardrails: new Set(['pii']), + triggerTypes: new Set(['chat']), + agentIds: new Set(['billing-bot']), + }); + }); + + it('memoizes per turn: a second call never re-reads the catalog', async () => { + let modelCalls = 0; + const t = turn(); + const d = deps(['a'], { + listModels: () => { + modelCalls += 1; + return [{ id: 'anthropic/claude-sonnet-5-5' }] as never; + }, + }); + const first = await loadCatalogFacts(t, d); + const second = await loadCatalogFacts(t, d); + expect(modelCalls).toBe(1); + expect(second).toBe(first); + }); + + it('does not share the memo across turns', async () => { + const loaded = await loadCatalogFacts(turn(), deps(['a'])); + const other = await loadCatalogFacts(turn(), deps(['b'])); + expect([...loaded.agentIds]).toEqual(['a']); + expect([...other.agentIds]).toEqual(['b']); + }); +}); diff --git a/apps/agent-forge/server/author/proposal/catalogChecks.ts b/apps/agent-forge/server/author/proposal/catalogChecks.ts new file mode 100644 index 00000000..b17e537e --- /dev/null +++ b/apps/agent-forge/server/author/proposal/catalogChecks.ts @@ -0,0 +1,162 @@ +/** + * The catalog checks a propose tool runs between schema validation and + * requirements (research §4.1 steps 4-8): every value the draft names that + * must exist in Studio's catalogs - its models, builtin tools, guardrails, + * trigger types and agent references - is verified against + * {@link CatalogFacts}, and all problems come back in ONE failure so the + * model can fix them in a single retry instead of one round-trip each. + * + * The schema already rejects malformed values (a `{ builtin }` the spec + * grammar does not know never reaches here); these checks are about + * membership in the live catalogs the `deps.catalog`/`deps.workspace` + * seams report, which the prompt lists and the model copies ids from. + * + * {@link loadCatalogFacts} reads the catalogs once per turn (memoized in + * `turn.state['catalogFacts']`): the model may propose several drafts in + * a turn and the catalogs cannot change mid-turn. + */ +import type { AgentSpecV2 } from 'lousho'; +import type { AuthorFailureCode, AuthorToolFailure } from '../../../shared/authorWire'; +import type { AuthorTurn } from '../turnRunner'; +import { toolFailure } from '../tools/failure'; +import type { AuthorToolDeps } from '../tools/index'; + +/** The catalog membership sets the checks run against, loaded once per turn. */ +export interface CatalogFacts { + /** 'provider/model' ids the model catalog knows. */ + modelIds: ReadonlySet; + /** Builtin tool names `listTools` lists. */ + builtinTools: ReadonlySet; + /** Guardrail names `listGuardrails` lists. */ + guardrails: ReadonlySet; + /** Trigger types `listTriggerTypes` lists. */ + triggerTypes: ReadonlySet; + /** Agent ids in the session's workspace (`repo.listIds()`). */ + agentIds: ReadonlySet; +} + +/** The `turn.state` key the memoized {@link CatalogFacts} promise lives under. */ +const CATALOG_FACTS_KEY = 'catalogFacts'; + +/** + * The turn's {@link CatalogFacts}: read once and memoized in + * `turn.state['catalogFacts']` (the promise itself is cached, so two + * racing calls share the load). Models, tools, guardrails and trigger + * types come from `deps.catalog`; agent ids from the session's workspace. + */ +export function loadCatalogFacts(turn: AuthorTurn, deps: AuthorToolDeps): Promise { + const existing = turn.state.get(CATALOG_FACTS_KEY) as Promise | undefined; + if (existing !== undefined) return existing; + const promise = (async (): Promise => { + const [{ repo }, models, tools, guardrails, triggers] = await Promise.all([ + deps.workspace(), + deps.catalog.listModels(), + deps.catalog.listTools(), + deps.catalog.listGuardrails(), + deps.catalog.listTriggerTypes(), + ]); + return { + modelIds: new Set(models.map((model) => model.id)), + builtinTools: new Set(tools.map((tool) => tool.name)), + guardrails: new Set(guardrails.map((guardrail) => guardrail.name)), + triggerTypes: new Set(triggers.map((trigger) => trigger.type)), + agentIds: new Set(repo.listIds()), + }; + })(); + turn.state.set(CATALOG_FACTS_KEY, promise); + return promise; +} + +/** One catalog problem: the failure code it picks when it is the first found, and its model-facing text. */ +interface CatalogProblem { + code: AuthorFailureCode; + text: string; +} + +/** `model` and each `fallbackModels[i]` must be ids the model catalog knows. */ +function modelProblems(spec: AgentSpecV2, facts: CatalogFacts): CatalogProblem[] { + return [spec.model, ...(spec.fallbackModels ?? [])] + .filter((model) => !facts.modelIds.has(model)) + .map((model) => ({ code: 'UNKNOWN_MODEL' as const, text: `model "${model}" is not in the catalog` })); +} + +/** Each `tools[i].builtin` must be a builtin the tool catalog knows. */ +function toolProblems(spec: AgentSpecV2, facts: CatalogFacts): CatalogProblem[] { + const problems: CatalogProblem[] = []; + (spec.tools ?? []).forEach((ref, i) => { + if ('builtin' in ref && !facts.builtinTools.has(ref.builtin)) { + problems.push({ code: 'UNKNOWN_TOOL', text: `tools[${i}] builtin "${ref.builtin}" does not exist` }); + } + }); + return problems; +} + +/** Each `guardrails[i].name` must be a guardrail the catalog knows. */ +function guardrailProblems(spec: AgentSpecV2, facts: CatalogFacts): CatalogProblem[] { + const problems: CatalogProblem[] = []; + (spec.guardrails ?? []).forEach((entry, i) => { + if (!facts.guardrails.has(entry.name)) { + problems.push({ code: 'UNKNOWN_GUARDRAIL', text: `guardrails[${i}] name "${entry.name}" does not exist` }); + } + }); + return problems; +} + +/** Each `triggers[i].type` must be a trigger type the catalog knows. */ +function triggerProblems(spec: AgentSpecV2, facts: CatalogFacts): CatalogProblem[] { + const problems: CatalogProblem[] = []; + (spec.triggers ?? []).forEach((entry, i) => { + if (!facts.triggerTypes.has(entry.type)) { + problems.push({ code: 'UNKNOWN_TRIGGER_TYPE', text: `triggers[${i}] type "${entry.type}" does not exist` }); + } + }); + return problems; +} + +/** + * Agent references must name workspace agents: each `subagents..ref`, + * and each `handoffs[i].agent` unless it names one of the spec's own + * `subagents` keys (a handoff to an inline subagent is legal). + */ +function agentRefProblems(spec: AgentSpecV2, facts: CatalogFacts): CatalogProblem[] { + const problems: CatalogProblem[] = []; + const subagentKeys = new Set(Object.keys(spec.subagents ?? {})); + for (const [key, entry] of Object.entries(spec.subagents ?? {})) { + if ('ref' in entry && !facts.agentIds.has(entry.ref)) { + problems.push({ code: 'UNKNOWN_AGENT_REF', text: `subagents.${key}.ref agent "${entry.ref}" does not exist` }); + } + } + (spec.handoffs ?? []).forEach((entry, i) => { + if (!subagentKeys.has(entry.agent) && !facts.agentIds.has(entry.agent)) { + problems.push({ code: 'UNKNOWN_AGENT_REF', text: `handoffs[${i}].agent agent "${entry.agent}" does not exist` }); + } + }); + return problems; +} + +/** The most problems one failure lists; the code is still the first problem's. */ +const MAX_LISTED_PROBLEMS = 10; + +/** + * All of `spec`'s catalog problems in one {@link AuthorToolFailure}, or + * `null` when every value is known. Problems are collected in check order + * (models, tools, guardrails, triggers, agent refs); the first problem's + * code is the failure's code and at most ten problem texts are listed. + */ +export function checkCatalogs(spec: AgentSpecV2, facts: CatalogFacts, toolName: string): AuthorToolFailure | null { + const problems = [ + ...modelProblems(spec, facts), + ...toolProblems(spec, facts), + ...guardrailProblems(spec, facts), + ...triggerProblems(spec, facts), + ...agentRefProblems(spec, facts), + ]; + if (problems.length === 0) return null; + const listed = problems.slice(0, MAX_LISTED_PROBLEMS).map((problem) => problem.text).join('; '); + return toolFailure( + problems[0].code, + `Some values are not in Studio's catalogs: ${listed}.`, + "The draft referred to something Studio doesn't have. The Author is fixing it.", + `Call listModels, listTools, listGuardrails, listTriggerTypes or listAgents and copy exact values, then call ${toolName} again.` + ); +} diff --git a/apps/agent-forge/server/author/proposal/review.ts b/apps/agent-forge/server/author/proposal/review.ts new file mode 100644 index 00000000..e69d1ab8 --- /dev/null +++ b/apps/agent-forge/server/author/proposal/review.ts @@ -0,0 +1,44 @@ +/** + * The pluggable draft review the propose tools run before showing a card + * (PLAN §8.4). This module holds only the contract: `AuthorToolDeps.reviewDraft` + * is injected so AA12 can plug the real review-model pass in behind the + * same call, and `passReview` - the default `createApp` wires until then - + * always lets the draft through. + * + * A rejected verdict is a returned failure (`pass: false` carries the + * `AuthorToolFailure` the tool hands back to the model), never a throw - + * the review bouncing a draft is data, not an error. + */ +import type { AgentSpecV2 } from 'lousho'; +import type { AuthorToolFailure, Requirement, SpecChange } from '../../../shared/authorWire'; +import type { AuthorTurn } from '../turnRunner'; +import type { ProposalIdentity } from './turnState'; + +/** Everything a review needs to judge the draft a propose call built. */ +export interface ReviewInput { + /** The running turn (state access for AA12's per-identity round cap). */ + turn: AuthorTurn; + /** What the proposal is about. */ + identity: ProposalIdentity; + /** Which propose tool is asking, for failure copy. */ + toolName: 'proposeAgent' | 'proposeAgentUpdate'; + /** The complete spec a save would write. */ + spec: AgentSpecV2; + /** The update diff (`[]` for creates). */ + changes: SpecChange[]; + /** The spec's readiness rows, already computed. */ + requirements: Requirement[]; +} + +/** The review's answer: let the card through, or return this failure to the model. */ +export type ReviewVerdict = { pass: true } | { pass: false; failure: AuthorToolFailure }; + +/** The review as a dependency: one verdict per input, never throws. */ +export type ReviewDraft = (input: ReviewInput) => Promise; + +/** + * The review every turn gets until AA12 lands the real one: always + * `{ pass: true }`. `createApp` wires it as `AuthorToolDeps.reviewDraft`'s + * default so `proposeAgent` already runs the full pipeline. + */ +export const passReview: ReviewDraft = async () => ({ pass: true }); diff --git a/apps/agent-forge/server/author/proposal/turnState.ts b/apps/agent-forge/server/author/proposal/turnState.ts new file mode 100644 index 00000000..d5f17e0b --- /dev/null +++ b/apps/agent-forge/server/author/proposal/turnState.ts @@ -0,0 +1,55 @@ +/** + * The per-turn proposal record (PLAN §8.3): which identity keys the + * Author proposed, discarded or saved THIS turn, plus the review-round + * counter AA12's draft review uses. The durable store is the session + * transcript; this map only covers what the not-yet-persisted current + * turn did, so a save call in the same turn can answer "that card is + * still unanswered" and the review can cap its rounds. + * + * The state hangs off `turn.state` under `'proposals'` and is created + * lazily by {@link proposalTurnState}, so a turn that never proposes + * pays nothing. + */ +import type { ProposalOutput } from '../../../shared/authorWire'; +import type { AuthorTurn } from '../turnRunner'; + +/** What a proposal is about: a new agent, or the agent `agentId` an update patches. */ +export type ProposalIdentity = { kind: 'create' } | { kind: 'update'; agentId: string }; + +/** + * The key `ProposalTurnState` maps/sets are indexed by: `'create'` for a + * new-agent card, `'update:'` for an update card. One open card + * per identity per turn - proposing again replaces the entry. + */ +export function identityKey(identity: ProposalIdentity): string { + return identity.kind === 'create' ? 'create' : `update:${identity.agentId}`; +} + +/** The proposal bookkeeping one turn accumulates. */ +export interface ProposalTurnState { + /** The newest card proposed THIS turn, by identity key. */ + proposed: Map; + /** Identity keys `discardPendingChanges` cleared this turn (AA11). */ + discarded: Set; + /** Identity keys saved this turn (AA10b). */ + saved: Set; + /** Review rounds used per identity key this turn (AA12 caps at 3). */ + reviewRounds: Map; +} + +/** The `turn.state` key the {@link ProposalTurnState} lives under. */ +const PROPOSALS_KEY = 'proposals'; + +/** + * The turn's {@link ProposalTurnState}, created on first use and kept in + * `turn.state['proposals']` - so every tool of the same turn (propose, + * save, discard, review) shares one record. + */ +export function proposalTurnState(turn: AuthorTurn): ProposalTurnState { + let state = turn.state.get(PROPOSALS_KEY) as ProposalTurnState | undefined; + if (state === undefined) { + state = { proposed: new Map(), discarded: new Set(), saved: new Set(), reviewRounds: new Map() }; + turn.state.set(PROPOSALS_KEY, state); + } + return state; +} diff --git a/apps/agent-forge/server/author/tools/__tests__/describeCoverage.test.ts b/apps/agent-forge/server/author/tools/__tests__/describeCoverage.test.ts index d3e2e193..e1875aa5 100644 --- a/apps/agent-forge/server/author/tools/__tests__/describeCoverage.test.ts +++ b/apps/agent-forge/server/author/tools/__tests__/describeCoverage.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect } from 'vitest'; import type { AuthorTurn } from '../../turnRunner'; +import { authorDraftSchema, authorPatchSchema } from '../../draft/schema'; import { buildAuthorTools, type AuthorToolDeps, type CatalogService } from '../index'; import type { WorkspaceContext } from '../../../workspace/context'; import type { AgentRepo } from '../../../storage/agentRepo'; @@ -12,7 +13,10 @@ import type { VersionStore } from '../../../storage/versionStore'; * optionals/defaults/effects and descends objects, arrays, unions and * record value types; a field may put its description anywhere on its own * wrapper chain, but container children (elements, options, record - * values) each need one too. + * values) each need one too. AA07's spec-derived draft/patch schemas are + * leaf boundaries: their per-key describes are pinned by + * draft/__tests__/schema.test.ts and their inner nodes are the SDK's own + * `agentSpecV2Schema`, which carries no describes by design. */ const TURN: AuthorTurn = { @@ -44,6 +48,8 @@ const DEPS: AuthorToolDeps = { templates: { list: async () => [] }, probeMcp: async () => ({ ok: true, tools: [] }), docsDir: '/nonexistent-docs', + reviewDraft: async () => ({ pass: true }), + requirements: async () => [], }; /** The zod internals the walk reads (zod 3 `_def` fields, structurally typed). */ @@ -114,11 +120,36 @@ function children(schema: ZodLike): [string, ZodLike][] { } } +/** + * Whether `schema` unwraps to one of AA07's spec-derived draft/patch + * schemas. Those are leaf boundaries for this walk: their top-level keys + * all carry FIELD_DESCRIPTIONS (asserted by draft/__tests__/schema.test.ts) + * and their inner nodes are the SDK's `agentSpecV2Schema` itself, which + * ships no `.describe()`s - the propose tools' describe contract lives at + * the draft/patch field and its direct keys. Identity is by shape: a + * `.describe()` clone shares the same `_def.shape` result. + */ +function isSpecBoundary(schema: ZodLike): boolean { + let current: ZodLike | undefined = schema; + while (current !== undefined) { + const def = current._def; + if (def === undefined) return false; + if (def.typeName === 'ZodObject') { + const shape = def.shape?.(); + return shape === authorDraftSchema.shape || shape === authorPatchSchema.shape; + } + current = wrapped(def); + } + return false; +} + function missingDescriptions(root: ZodLike): string[] { const missing: string[] = []; const walk = (schema: ZodLike, path: string, isRoot: boolean): void => { // The root input object needs no describe of its own; every other node does. if (!isRoot && !described(schema)) missing.push(path === '' ? '' : path); + // Spec-derived draft/patch schemas are leaf boundaries (see isSpecBoundary). + if (isSpecBoundary(schema)) return; for (const [key, child] of children(schema)) { walk(child, path === '' ? key : `${path}.${key}`, false); } diff --git a/apps/agent-forge/server/author/tools/__tests__/proposeAgent.test.ts b/apps/agent-forge/server/author/tools/__tests__/proposeAgent.test.ts new file mode 100644 index 00000000..70d7ddd0 --- /dev/null +++ b/apps/agent-forge/server/author/tools/__tests__/proposeAgent.test.ts @@ -0,0 +1,209 @@ +import { mkdirSync, mkdtempSync, readdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import * as path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { testToolContext } from 'lousho/testing'; +import type { AgentSpecV2 } from 'lousho'; +import type { AuthorToolDeps, CatalogService } from '../index'; +import type { AuthorTurn } from '../../turnRunner'; +import type { WorkspaceContext } from '../../../workspace/context'; +import type { AgentRepo } from '../../../storage/agentRepo'; +import type { AuthorToolFailure, ProposalOutput } from '../../../../shared/authorWire'; +import { proposalTurnState } from '../../proposal/turnState'; +import { passReview } from '../../proposal/review'; +import { AUTHOR_TOOL_FACTORIES, buildAuthorTools } from '../index'; +import type { AuthorDraft } from '../../draft/schema'; + +/** + * proposeAgent (AA08): draft -> spec validation, catalog checks, the + * requirements dep, the pluggable review, the per-turn proposal record, + * and the card-shaped ProposalOutput. The tool writes nothing - a real + * .lousho/agents dir stays byte-identical. + */ + +const DRAFT: AuthorDraft = { + name: 'Support triage', + description: 'Routes support mail to the right queue.', + model: 'mock/mock-1', + instructions: 'You triage support mail and pick the queue.', +}; + +function turn(): AuthorTurn { + return { + sessionId: 'sess-1', + requestId: 'req-test', + focus: { mode: 'create' }, + storedTranscript: [], + userMessage: { id: 'm-1', role: 'user', text: 'hi', at: '2026-01-01T00:00:00.000Z' }, + signal: new AbortController().signal, + stepsRemaining: () => 12, + state: new Map(), + }; +} + +function fakeCatalog(over: Partial = {}): CatalogService { + return { + listModels: () => [{ id: 'mock/mock-1' }, { id: 'anthropic/claude-sonnet-5-5' }] as never, + listTools: () => [{ name: 'web_fetch' }] as never, + listGuardrails: () => [], + listTriggerTypes: () => [{ type: 'chat' }, { type: 'cron' }] as never, + listSkills: async () => [], + listKits: async () => [], + listRegistryMcpServers: async () => [], + ...over, + }; +} + +function fakeDeps(over: Partial = {}): AuthorToolDeps { + return { + workspace: async () => + ({ repo: { listIds: () => [] } as unknown as AgentRepo }) as WorkspaceContext, + catalog: fakeCatalog(), + templates: { list: async () => [] }, + probeMcp: async () => ({ ok: true, tools: [] }), + docsDir: '/nonexistent-docs', + reviewDraft: passReview, + requirements: async () => [], + ...over, + }; +} + +function proposeTool(t: AuthorTurn, deps: AuthorToolDeps) { + const tool = buildAuthorTools(t, deps).find( + (candidate) => ((candidate as { name?: string }).name ?? candidate.displayName) === 'proposeAgent' + )!; + return tool; +} + +async function call(tool: ReturnType, args: Record, toolCallId = 'call_a') { + return tool.execute!(args as never, testToolContext({ toolCallId })); +} + +describe('proposeAgent', () => { + it('is registered in AUTHOR_TOOL_FACTORIES', () => { + const names = AUTHOR_TOOL_FACTORIES.map((factory) => factory(turn(), fakeDeps())).map( + (tool) => (tool as { name?: string }).name ?? tool.displayName + ); + expect(names).toContain('proposeAgent'); + }); + + it('returns a create ProposalOutput keyed on the toolCallId, writing nothing', async () => { + const root = mkdtempSync(path.join(tmpdir(), 'aa08-')); + mkdirSync(path.join(root, '.lousho', 'agents'), { recursive: true }); + const before = readdirSync(root, { recursive: true }); + const t = turn(); + const requirements = [{ id: 'provider:mock', kind: 'provider_key', status: 'ok', label: 'Mock provider' }]; + const tool = proposeTool(t, fakeDeps({ requirements: async () => requirements as never })); + const output = (await call(tool, { draft: DRAFT })) as ProposalOutput; + + expect(output.ok).toBe(true); + expect(output.proposalId).toBe('call_a'); + expect(output.kind).toBe('create'); + expect(output.agentId).toBeUndefined(); + expect(output.enabled).toBe(false); + expect(output.changes).toEqual([]); + expect(output.carriedOver).toBe(false); + expect(output.carriedFields).toEqual([]); + expect(output.baseSpecHash).toBeNull(); + expect(output.requirements).toEqual(requirements); + const spec = output.spec as AgentSpecV2; + expect(spec.version).toBe(2); + expect(spec.name).toBe('Support triage'); + expect(spec.triggers?.some((trigger) => trigger.type === 'chat')).toBe(true); + // Nothing was written anywhere under .lousho/. + expect(readdirSync(root, { recursive: true })).toEqual(before); + }); + + it('records the card under the create identity in the turn state', async () => { + const t = turn(); + const output = (await call(proposeTool(t, fakeDeps()), { draft: DRAFT })) as ProposalOutput; + expect(proposalTurnState(t).proposed.get('create')).toBe(output); + }); + + it('one UNKNOWN_MODEL failure lists both an unknown model and an unknown tool', async () => { + const t = turn(); + // 'web_fetch' is schema-valid but absent from this fake catalog's tool list, + // and 'acme/not-real' is a well-formed provider/model the model catalog lacks. + const deps = fakeDeps({ catalog: fakeCatalog({ listTools: () => [] }) }); + const output = (await call(proposeTool(t, deps), { + draft: { ...DRAFT, model: 'acme/not-real', tools: [{ builtin: 'web_fetch' }] }, + })) as AuthorToolFailure; + expect(output.ok).toBe(false); + expect(output.code).toBe('UNKNOWN_MODEL'); + expect(output.error).toContain('model "acme/not-real" is not in the catalog'); + expect(output.error).toContain('tools[0] builtin "web_fetch" does not exist'); + expect(output.next).toContain('proposeAgent'); + expect(proposalTurnState(t).proposed.size).toBe(0); + }); + + it('a failing review returns its failure and leaves proposed empty', async () => { + const t = turn(); + const reviewFailure: AuthorToolFailure = { + ok: false, + code: 'DRAFT_REVIEW', + error: 'The draft would not run: no tool can read mail.', + userMessage: 'The Author is still working on the draft.', + }; + const output = (await call( + proposeTool(t, fakeDeps({ reviewDraft: async () => ({ pass: false, failure: reviewFailure }) })), + { draft: DRAFT } + )) as AuthorToolFailure; + expect(output).toBe(reviewFailure); + expect(proposalTurnState(t).proposed.size).toBe(0); + }); + + it('passes spec, changes and requirements to the review', async () => { + const t = turn(); + let seen: unknown; + const requirements = [{ id: 'mcp:gh', kind: 'mcp', status: 'unknown', label: 'MCP server gh' }]; + const tool = proposeTool( + t, + fakeDeps({ + requirements: async () => requirements as never, + reviewDraft: async (input) => { + seen = input; + return { pass: true }; + }, + }) + ); + const output = (await call(tool, { draft: DRAFT })) as ProposalOutput; + expect(output.ok).toBe(true); + const input = seen as { turn: AuthorTurn; spec: AgentSpecV2 }; + expect(input.turn).toBe(t); + expect(seen).toMatchObject({ + identity: { kind: 'create' }, + toolName: 'proposeAgent', + changes: [], + requirements, + }); + expect(input.spec.name).toBe('Support triage'); + }); + + it('invalid draft -> DRAFT_INVALID with path: message text', async () => { + const t = turn(); + const output = (await call(proposeTool(t, fakeDeps()), { + draft: { ...DRAFT, name: '!!!', description: '' }, + })) as AuthorToolFailure; + expect(output.ok).toBe(false); + expect(output.code).toBe('DRAFT_INVALID'); + expect(output.error).toContain('description'); + expect(output.next).toContain('proposeAgent'); + expect(proposalTurnState(t).proposed.size).toBe(0); + }); + + it('a requirements dep that throws still yields a card with requirements []', async () => { + const t = turn(); + const tool = proposeTool( + t, + fakeDeps({ + requirements: async () => { + throw new Error('requirements exploded'); + }, + }) + ); + const output = (await call(tool, { draft: DRAFT })) as ProposalOutput; + expect(output.ok).toBe(true); + expect(output.requirements).toEqual([]); + expect(proposalTurnState(t).proposed.get('create')).toBe(output); + }); +}); diff --git a/apps/agent-forge/server/author/tools/__tests__/readTools.test.ts b/apps/agent-forge/server/author/tools/__tests__/readTools.test.ts index 8611014f..a90ff443 100644 --- a/apps/agent-forge/server/author/tools/__tests__/readTools.test.ts +++ b/apps/agent-forge/server/author/tools/__tests__/readTools.test.ts @@ -82,6 +82,8 @@ function fakeDeps(over: Partial = {}): AuthorToolDeps { templates: { list: async () => [] }, probeMcp: async () => ({ ok: true, tools: [] }), docsDir: '/nonexistent-docs', + reviewDraft: async () => ({ pass: true }), + requirements: async () => [], ...over, }; } @@ -107,15 +109,15 @@ function failure(value: unknown): AuthorToolFailure { } describe('buildAuthorTools', () => { - it('returns the 13 read tools sorted by name, every one in AUTHOR_TOOL_NAMES', () => { + it('returns the 14 tools sorted by name, every one in AUTHOR_TOOL_NAMES', () => { const built = buildAuthorTools(TURN, fakeDeps()); const names = built.map(nameOf); - expect(names).toHaveLength(13); + expect(names).toHaveLength(14); expect(names).toEqual([...names].sort((a, b) => a.localeCompare(b))); for (const name of names) { expect(AUTHOR_TOOL_NAMES as readonly string[]).toContain(name); } - expect(AUTHOR_TOOL_FACTORIES).toHaveLength(13); + expect(AUTHOR_TOOL_FACTORIES).toHaveLength(14); }); it('builds the same list for different turns (the tool surface never changes mid-session)', () => { diff --git a/apps/agent-forge/server/author/tools/__tests__/searchDocs.test.ts b/apps/agent-forge/server/author/tools/__tests__/searchDocs.test.ts index c81d8e41..10ec385a 100644 --- a/apps/agent-forge/server/author/tools/__tests__/searchDocs.test.ts +++ b/apps/agent-forge/server/author/tools/__tests__/searchDocs.test.ts @@ -42,6 +42,8 @@ function deps(docsDir: string): AuthorToolDeps { templates: { list: async () => [] }, probeMcp: async () => ({ ok: true, tools: [] }), docsDir, + reviewDraft: async () => ({ pass: true }), + requirements: async () => [], }; } diff --git a/apps/agent-forge/server/author/tools/index.ts b/apps/agent-forge/server/author/tools/index.ts index eed6536c..c012b3b2 100644 --- a/apps/agent-forge/server/author/tools/index.ts +++ b/apps/agent-forge/server/author/tools/index.ts @@ -14,7 +14,8 @@ * is D17's catalog, `probeMcp` is D11's probe as a function, and * `docsDir` is the bundled SDK docs folder `searchDocs` reads. */ -import type { ToolDescriptor } from 'lousho'; +import type { AgentSpecV2, ToolDescriptor } from 'lousho'; +import type { Requirement } from '../../../shared/authorWire'; import type { CatalogGuardrail, CatalogKit, @@ -27,11 +28,13 @@ import type { } from '../../../shared/wireTypes'; import type { McpProbeResult } from '../../routes/mcpProbe'; import type { WorkspaceContext } from '../../workspace/context'; +import type { ReviewDraft } from '../proposal/review'; import type { AuthorTurn } from '../turnRunner'; import { readToolFactories } from './readTools'; import { probeMcpServerTool } from './mcpProbeTool'; import { searchDocsTool } from './searchDocsTool'; import { openPageTool } from './openPageTool'; +import { proposeAgentTool } from './proposeAgentTool'; /** * A registry-index item that names an MCP server (`type: 'mcp'`, @@ -84,6 +87,18 @@ export interface AuthorToolDeps { probeMcp: (server: McpServerSpecV2) => Promise; /** The bundled SDK docs folder `searchDocs` indexes (`/docs`, resolved in server/index.ts). */ docsDir: string; + /** + * The draft review a propose tool runs before showing its card (PLAN + * §8.4): AA12's review-model pass; `createApp` wires `passReview` (the + * always-approve default) until then. + */ + reviewDraft: ReviewDraft; + /** + * The spec's readiness rows for the proposal card (AA13's + * `computeRequirements`, wired per turn to the session's workspace + * root). Advisory - the propose tools degrade a throwing dep to `[]`. + */ + requirements: (spec: AgentSpecV2) => Promise; } /** Builds one tool for one turn - the factories close over `turn` (and `deps`). */ @@ -100,6 +115,7 @@ export const AUTHOR_TOOL_FACTORIES: readonly AuthorToolFactory[] = [ probeMcpServerTool, searchDocsTool, openPageTool, + proposeAgentTool, ]; /** The call name a `ToolDescriptor` carries (`defineTool`/loader results), else its display name. */ diff --git a/apps/agent-forge/server/author/tools/proposeAgentTool.ts b/apps/agent-forge/server/author/tools/proposeAgentTool.ts new file mode 100644 index 00000000..5a97c27a --- /dev/null +++ b/apps/agent-forge/server/author/tools/proposeAgentTool.ts @@ -0,0 +1,65 @@ +/** + * The Author's `proposeAgent` tool (PLAN §8.2, AA08): shows the user a + * proposal card for a NEW agent and writes nothing. `proposalId` is the + * call's own `toolCallId` - the transcript is the proposal store, so the + * card the panel renders is the tool result itself. + * + * The pipeline (research §4.1): `draftToSpec` validates the draft against + * `agentSpecV2Schema` (and appends the chat-trigger backstop), then the + * catalog checks refuse ids Studio does not have (one failure listing + * every problem), then the requirements dep computes the card's readiness + * rows (advisory; `[]` on error), then the injected review gets its say + * (`passReview` until AA12). Only then is the output recorded under the + * 'create' identity in {@link proposalTurnState} and returned - a failed + * step leaves `proposed` untouched. + */ +import { defineTool } from 'lousho'; +import { z } from 'zod'; +import type { ProposalOutput } from '../../../shared/authorWire'; +import { authorDraftSchema } from '../draft/schema'; +import { draftInvalidFailure, draftToSpec } from '../draft/toSpec'; +import { checkCatalogs, loadCatalogFacts } from '../proposal/catalogChecks'; +import { identityKey, proposalTurnState } from '../proposal/turnState'; +import { guardedToolCall } from './failure'; +import type { AuthorToolFactory } from './index'; + +/** AA08's propose tool for new agents; updates are `proposeAgentUpdate` (AA09). */ +export const proposeAgentTool: AuthorToolFactory = (turn, deps) => + defineTool({ + name: 'proposeAgent', + description: + 'Show the user a proposal card for a NEW agent. Nothing is saved. After calling it, end your turn and wait for the user to confirm or press Apply.', + input: z.object({ draft: authorDraftSchema.describe('The complete new agent. Fields you leave out stay unset.') }), + execute: ({ draft }, ctx) => + guardedToolCall('proposeAgent', async () => { + const built = draftToSpec(draft); + if (!built.ok) return draftInvalidFailure('proposeAgent', built.issues); + const catalogFailure = checkCatalogs(built.spec, await loadCatalogFacts(turn, deps), 'proposeAgent'); + if (catalogFailure !== null) return catalogFailure; + // Requirements are advisory: a dep failure means an empty list, never a failed call. + const requirements = await deps.requirements(built.spec).catch(() => []); + const verdict = await deps.reviewDraft({ + turn, + identity: { kind: 'create' }, + toolName: 'proposeAgent', + spec: built.spec, + changes: [], + requirements, + }); + if (!verdict.pass) return verdict.failure; + const output: ProposalOutput = { + ok: true, + proposalId: ctx.toolCallId, + kind: 'create', + spec: built.spec, + changes: [], + requirements, + carriedOver: false, + carriedFields: [], + baseSpecHash: null, + enabled: false, + }; + proposalTurnState(turn).proposed.set(identityKey({ kind: 'create' }), output); + return output; + }), + }); diff --git a/apps/agent-forge/server/author/wiring.ts b/apps/agent-forge/server/author/wiring.ts index 2cc36e34..057503fc 100644 --- a/apps/agent-forge/server/author/wiring.ts +++ b/apps/agent-forge/server/author/wiring.ts @@ -22,6 +22,8 @@ import type { AuthorSessionStore } from './sessionStore'; import { buildAuthorSystemPrompt, type AuthorPromptCatalogs } from './prompt/index'; import { buildAuthorTools, type AuthorToolDeps } from './tools/index'; import { createAuthorCatalogService } from './tools/catalogService'; +import { passReview } from './proposal/review'; +import { computeRequirements, requirementDeps } from './requirements'; /** * The D09 catalog functions (`/catalog/*` reads the same SDK calls) @@ -83,11 +85,15 @@ export function createAuthorRunner(deps: CreateAuthorRunnerDeps): AuthorTurnRunn resolveSecret: (name: string) => deps.secrets.getSecret(name), ...(deps.mcpConnect ? { connect: deps.mcpConnect } : {}), }; - const toolDeps: Omit = { + // `workspace` (and `requirements`, which reads under the workspace + // root) is bound per turn below. `reviewDraft` is AA12's seam; + // passReview is the always-approve default until it lands. + const toolDeps: Omit = { catalog: createAuthorCatalogService({ secretsStore: deps.secrets }), templates: deps.templates, probeMcp: (server) => probeMcpServer(server, probeDeps), docsDir: deps.docsDir, + reviewDraft: passReview, }; const workspaceForTurn = async (turn: AuthorTurn): Promise => { const session = await deps.sessions.get(turn.sessionId); @@ -102,7 +108,15 @@ export function createAuthorRunner(deps: CreateAuthorRunnerDeps): AuthorTurnRunn settings: deps.settings, secrets: deps.secrets, systemPrompt: () => authorPrompt, - buildTools: (turn) => buildAuthorTools(turn, { ...toolDeps, workspace: () => workspaceForTurn(turn) }), + buildTools: (turn) => + buildAuthorTools(turn, { + ...toolDeps, + workspace: () => workspaceForTurn(turn), + // AA13's readiness rows, resolved against the session's workspace + // root per call (fileExist checks stay inside that root). + requirements: async (spec) => + computeRequirements(spec, requirementDeps((await workspaceForTurn(turn)).ref.root, deps.secrets)), + }), // Stand-in until AA07 lands the real turnContextInput. turnContextInput: async (s, now) => ({ focus: s.focus, openProposals: [], now }), ...(deps.providerFor ? { providerFor: deps.providerFor } : {}), diff --git a/apps/agent-forge/shared/authorWire.ts b/apps/agent-forge/shared/authorWire.ts index 6d8dc6d0..df1473a6 100644 --- a/apps/agent-forge/shared/authorWire.ts +++ b/apps/agent-forge/shared/authorWire.ts @@ -9,12 +9,19 @@ * module instead of re-declaring them. */ +import type { AgentSpecV2 } from 'lousho'; +import type { Requirement, SpecChange } from './wireTypes'; + // The readiness row AA13's `computeRequirements` returns and the Author's // proposal cards carry: defined next to `RequirementKind` in wireTypes.ts // (where `AgentRecord.requirements` reads it), re-exported here so Author // code has one import site per the PLAN §8.5 contract. export type { Requirement } from './wireTypes'; +// The diff rows a proposal card for an UPDATE renders: D10's `diffSpecs` +// output type, re-exported for the same one-import-site reason. +export type { SpecChange } from './wireTypes'; + /** What the Author is working on this session. */ export type AuthorFocus = { mode: 'create' } | { mode: 'edit'; agentId: string }; @@ -155,6 +162,45 @@ export type AuthorClientMessage = | { type: 'chat.stop'; requestId: string } | { type: 'chat.resume'; requestId: string }; +/** Whether a proposal card creates a new agent or updates an existing one. */ +export type ProposalKind = 'create' | 'update'; + +/** + * The `ok: true` output of `proposeAgent`/`proposeAgentUpdate` - the card + * the Author panel renders (PLAN §8.3). `proposalId` is the propose call's + * `toolCallId`: the transcript is the proposal store, so nothing is + * written anywhere else. `spec` is exactly what a save would write; + * `changes` is the diff a card shows for an update (`[]` for a create, + * which shows the full config); `requirements` is the advisory readiness + * list. `carriedOver`/`carriedFields` report an update that folded an + * earlier unapplied card (AA09; always false/[] for creates), + * `baseSpecHash` pins the saved spec an update patched, and `enabled` is + * the agent's current switch state (`false` for creates: a new agent + * saves disabled). + */ +export interface ProposalOutput { + ok: true; + /** The propose call's `toolCallId` - the card's id in the transcript. */ + proposalId: string; + kind: ProposalKind; + /** Set on updates only: the agent the patch applies to. */ + agentId?: string; + /** The complete spec a save would write. */ + spec: AgentSpecV2; + /** The update diff; `[]` for creates (the card shows the full config instead). */ + changes: SpecChange[]; + /** The spec's readiness rows (`computeRequirements`, AA13) - advisory, never a block. */ + requirements: Requirement[]; + /** Whether an earlier unapplied card for the same agent was folded in (AA09). */ + carriedOver: boolean; + /** The fields an earlier folded card contributed (AA09); `[]` for creates. */ + carriedFields: string[]; + /** Updates: `specHash` of the saved spec the patch applied to; creates: `null`. */ + baseSpecHash: string | null; + /** Creates: `false`; updates: the agent's `meta.enabled`. */ + enabled: boolean; +} + /** * Every event one Author turn produces, in order. `AuthorTurnRunner` * emits them on its `'event'` channel as `(sessionId, event)`; the