From 2090fe5437d120d0eaf8957160de6014dc28d8df Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Tue, 3 Nov 2026 05:18:41 +0300 Subject: [PATCH] feat(studio): AA08 proposeAgent tool: validate, catalog checks, requirements, card output proposeAgent({ draft }) validates a new-agent draft against AgentSpec v2 (draftToSpec), checks the live catalogs (models, builtin tools, guardrails, trigger types, agent refs) in one failure listing all problems, computes advisory requirements, runs the pluggable review (passReview until the real review lands), records the card in per-turn proposal state keyed by identity, and returns a ProposalOutput the panel renders. proposalId is the call's toolCallId; nothing is written. Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proposal/__tests__/catalogChecks.test.ts | 211 ++++++++++++++++++ .../server/author/proposal/catalogChecks.ts | 162 ++++++++++++++ .../server/author/proposal/review.ts | 44 ++++ .../server/author/proposal/turnState.ts | 55 +++++ .../tools/__tests__/describeCoverage.test.ts | 33 ++- .../tools/__tests__/proposeAgent.test.ts | 209 +++++++++++++++++ .../author/tools/__tests__/readTools.test.ts | 8 +- .../author/tools/__tests__/searchDocs.test.ts | 2 + apps/agent-forge/server/author/tools/index.ts | 18 +- .../server/author/tools/proposeAgentTool.ts | 65 ++++++ apps/agent-forge/server/author/wiring.ts | 18 +- apps/agent-forge/shared/authorWire.ts | 46 ++++ 12 files changed, 864 insertions(+), 7 deletions(-) create mode 100644 apps/agent-forge/server/author/proposal/__tests__/catalogChecks.test.ts create mode 100644 apps/agent-forge/server/author/proposal/catalogChecks.ts create mode 100644 apps/agent-forge/server/author/proposal/review.ts create mode 100644 apps/agent-forge/server/author/proposal/turnState.ts create mode 100644 apps/agent-forge/server/author/tools/__tests__/proposeAgent.test.ts create mode 100644 apps/agent-forge/server/author/tools/proposeAgentTool.ts diff --git a/apps/agent-forge/server/author/proposal/__tests__/catalogChecks.test.ts b/apps/agent-forge/server/author/proposal/__tests__/catalogChecks.test.ts new file mode 100644 index 00000000..d61007c2 --- /dev/null +++ b/apps/agent-forge/server/author/proposal/__tests__/catalogChecks.test.ts @@ -0,0 +1,211 @@ +import { describe, expect, it } from 'vitest'; +import type { AgentSpecV2 } from 'lousho'; +import type { AuthorToolDeps, CatalogService } from '../../tools/index'; +import type { AuthorTurn } from '../../turnRunner'; +import type { WorkspaceContext } from '../../../workspace/context'; +import type { AgentRepo } from '../../../storage/agentRepo'; +import type { AuthorToolFailure } from '../../../../shared/authorWire'; +import { checkCatalogs, loadCatalogFacts, type CatalogFacts } from '../catalogChecks'; + +/** + * The catalog checks proposeAgent/proposeAgentUpdate run between schema + * validation and requirements: every value the draft names that must + * exist in Studio's catalogs is collected into one failure (first + * problem's code wins), and the per-turn memoization of the facts load. + */ + +function turn(): AuthorTurn { + return { + sessionId: 'sess-1', + requestId: 'req-test', + focus: { mode: 'create' }, + storedTranscript: [], + userMessage: { id: 'm-1', role: 'user', text: 'hi', at: '2026-01-01T00:00:00.000Z' }, + signal: new AbortController().signal, + stepsRemaining: () => 12, + state: new Map(), + }; +} + +function spec(over: Partial = {}): AgentSpecV2 { + return { + version: 2, + name: 'Test agent', + description: 'does a thing', + model: 'anthropic/claude-sonnet-5-5', + instructions: 'do a thing', + ...over, + }; +} + +function facts(over: Partial = {}): CatalogFacts { + return { + modelIds: new Set(['anthropic/claude-sonnet-5-5', 'openai/gpt-5']), + builtinTools: new Set(['web_fetch', 'run_command']), + guardrails: new Set(['pii', 'regex']), + triggerTypes: new Set(['chat', 'cron', 'webhook', 'channel', 'agent']), + agentIds: new Set(['billing-bot', 'support-triage']), + ...over, + }; +} + +function failure(result: AuthorToolFailure | null): AuthorToolFailure { + expect(result).not.toBeNull(); + expect(result!.ok).toBe(false); + return result!; +} + +describe('checkCatalogs', () => { + it('returns null when every catalog value exists', () => { + const ok = spec({ + fallbackModels: ['openai/gpt-5'], + tools: [{ builtin: 'web_fetch' }], + guardrails: [{ id: 'g1', name: 'pii' }], + triggers: [{ id: 't1', type: 'chat' }, { id: 't2', type: 'cron', cron: '0 9 * * 1', prompt: 'go' }], + subagents: { helper: { ref: 'billing-bot' } }, + handoffs: [{ agent: 'support-triage' }], + }); + expect(checkCatalogs(ok, facts(), 'proposeAgent')).toBeNull(); + }); + + it('UNKNOWN_MODEL when the first problem is a model not in the catalog', () => { + const result = failure(checkCatalogs(spec({ model: 'acme/not-real' }), facts(), 'proposeAgent')); + expect(result.code).toBe('UNKNOWN_MODEL'); + expect(result.error).toBe('Some values are not in Studio\'s catalogs: model "acme/not-real" is not in the catalog.'); + }); + + it('flags each fallback model that is not in the catalog', () => { + const result = failure( + checkCatalogs(spec({ fallbackModels: ['acme/gone-1', 'openai/gpt-5', 'acme/gone-2'] }), facts(), 'proposeAgent') + ); + expect(result.code).toBe('UNKNOWN_MODEL'); + expect(result.error).toContain('model "acme/gone-1" is not in the catalog'); + expect(result.error).toContain('model "acme/gone-2" is not in the catalog'); + expect(result.error).not.toContain('openai/gpt-5'); + }); + + it('UNKNOWN_TOOL for a builtin name the catalog lacks', () => { + const result = failure( + checkCatalogs(spec({ tools: [{ builtin: 'web_fetch' }, { builtin: 'not-a-tool' }] }), facts(), 'proposeAgent') + ); + expect(result.code).toBe('UNKNOWN_TOOL'); + expect(result.error).toBe('Some values are not in Studio\'s catalogs: tools[1] builtin "not-a-tool" does not exist.'); + }); + + it('UNKNOWN_GUARDRAIL for a guardrail name the catalog lacks', () => { + const result = failure( + checkCatalogs(spec({ guardrails: [{ id: 'g1', name: 'pii' }, { id: 'g2', name: 'not-a-guardrail' as never }] }), facts(), 'proposeAgent') + ); + expect(result.code).toBe('UNKNOWN_GUARDRAIL'); + expect(result.error).toContain('guardrails[1] name "not-a-guardrail" does not exist'); + }); + + it('UNKNOWN_TRIGGER_TYPE for a trigger type the catalog lacks', () => { + const result = failure( + checkCatalogs(spec({ triggers: [{ id: 't1', type: 'chat' }, { id: 't2', type: 'smoke' as never }] }), facts(), 'proposeAgent') + ); + expect(result.code).toBe('UNKNOWN_TRIGGER_TYPE'); + expect(result.error).toContain('triggers[1] type "smoke" does not exist'); + }); + + it('UNKNOWN_AGENT_REF for a subagents..ref that is not a workspace agent', () => { + const result = failure( + checkCatalogs(spec({ subagents: { helper: { ref: 'billing-bot' }, ghost: { ref: 'gone' } } }), facts(), 'proposeAgent') + ); + expect(result.code).toBe('UNKNOWN_AGENT_REF'); + expect(result.error).toContain('subagents.ghost.ref agent "gone" does not exist'); + }); + + it('UNKNOWN_AGENT_REF for a handoffs[i].agent that is not a workspace agent', () => { + const result = failure(checkCatalogs(spec({ handoffs: [{ agent: 'gone' }] }), facts(), 'proposeAgent')); + expect(result.code).toBe('UNKNOWN_AGENT_REF'); + expect(result.error).toContain('handoffs[0].agent agent "gone" does not exist'); + }); + + it('allows a handoff to one of the spec\'s own subagent keys', () => { + const ok = spec({ + subagents: { helper: { inline: { name: 'Helper', model: 'anthropic/claude-sonnet-5-5', instructions: 'help', description: 'helps' } } }, + handoffs: [{ agent: 'helper' }], + }); + expect(checkCatalogs(ok, facts(), 'proposeAgent')).toBeNull(); + }); + + it('the first problem in check order picks the code; every problem is listed', () => { + const result = failure( + checkCatalogs(spec({ model: 'acme/not-real', tools: [{ builtin: 'not-a-tool' }] }), facts(), 'proposeAgent') + ); + expect(result.code).toBe('UNKNOWN_MODEL'); + expect(result.error).toContain('model "acme/not-real" is not in the catalog'); + expect(result.error).toContain('tools[0] builtin "not-a-tool" does not exist'); + expect(result.userMessage).toBe("The draft referred to something Studio doesn't have. The Author is fixing it."); + expect(result.next).toBe( + 'Call listModels, listTools, listGuardrails, listTriggerTypes or listAgents and copy exact values, then call proposeAgent again.' + ); + }); + + it('lists at most the first 10 problems', () => { + const many = spec({ tools: Array.from({ length: 12 }, (_, i) => ({ builtin: `tool-${i}` })) }); + const result = failure(checkCatalogs(many, facts(), 'proposeAgent')); + expect(result.code).toBe('UNKNOWN_TOOL'); + expect(result.error).toContain('tools[9] builtin "tool-9" does not exist'); + expect(result.error).not.toContain('tools[10]'); + expect(result.error).not.toContain('tool-11'); + }); +}); + +describe('loadCatalogFacts', () => { + function deps(agentIds: string[], catalog: Partial = {}): AuthorToolDeps { + return { + workspace: async () => + ({ repo: { listIds: () => agentIds } as unknown as AgentRepo }) as WorkspaceContext, + catalog: { + listModels: () => [{ id: 'anthropic/claude-sonnet-5-5' }] as never, + listTools: () => [{ name: 'web_fetch' }] as never, + listGuardrails: () => [{ name: 'pii' }] as never, + listTriggerTypes: () => [{ type: 'chat' }] as never, + listSkills: async () => [], + listKits: async () => [], + listRegistryMcpServers: async () => [], + ...catalog, + }, + templates: { list: async () => [] }, + probeMcp: async () => ({ ok: true, tools: [] }), + docsDir: '/nonexistent-docs', + reviewDraft: async () => ({ pass: true }), + requirements: async () => [], + }; + } + + it('reads ids from the catalog service and the workspace repo', async () => { + const loaded = await loadCatalogFacts(turn(), deps(['billing-bot'])); + expect(loaded).toEqual({ + modelIds: new Set(['anthropic/claude-sonnet-5-5']), + builtinTools: new Set(['web_fetch']), + guardrails: new Set(['pii']), + triggerTypes: new Set(['chat']), + agentIds: new Set(['billing-bot']), + }); + }); + + it('memoizes per turn: a second call never re-reads the catalog', async () => { + let modelCalls = 0; + const t = turn(); + const d = deps(['a'], { + listModels: () => { + modelCalls += 1; + return [{ id: 'anthropic/claude-sonnet-5-5' }] as never; + }, + }); + const first = await loadCatalogFacts(t, d); + const second = await loadCatalogFacts(t, d); + expect(modelCalls).toBe(1); + expect(second).toBe(first); + }); + + it('does not share the memo across turns', async () => { + const loaded = await loadCatalogFacts(turn(), deps(['a'])); + const other = await loadCatalogFacts(turn(), deps(['b'])); + expect([...loaded.agentIds]).toEqual(['a']); + expect([...other.agentIds]).toEqual(['b']); + }); +}); diff --git a/apps/agent-forge/server/author/proposal/catalogChecks.ts b/apps/agent-forge/server/author/proposal/catalogChecks.ts new file mode 100644 index 00000000..b17e537e --- /dev/null +++ b/apps/agent-forge/server/author/proposal/catalogChecks.ts @@ -0,0 +1,162 @@ +/** + * The catalog checks a propose tool runs between schema validation and + * requirements (research §4.1 steps 4-8): every value the draft names that + * must exist in Studio's catalogs - its models, builtin tools, guardrails, + * trigger types and agent references - is verified against + * {@link CatalogFacts}, and all problems come back in ONE failure so the + * model can fix them in a single retry instead of one round-trip each. + * + * The schema already rejects malformed values (a `{ builtin }` the spec + * grammar does not know never reaches here); these checks are about + * membership in the live catalogs the `deps.catalog`/`deps.workspace` + * seams report, which the prompt lists and the model copies ids from. + * + * {@link loadCatalogFacts} reads the catalogs once per turn (memoized in + * `turn.state['catalogFacts']`): the model may propose several drafts in + * a turn and the catalogs cannot change mid-turn. + */ +import type { AgentSpecV2 } from 'lousho'; +import type { AuthorFailureCode, AuthorToolFailure } from '../../../shared/authorWire'; +import type { AuthorTurn } from '../turnRunner'; +import { toolFailure } from '../tools/failure'; +import type { AuthorToolDeps } from '../tools/index'; + +/** The catalog membership sets the checks run against, loaded once per turn. */ +export interface CatalogFacts { + /** 'provider/model' ids the model catalog knows. */ + modelIds: ReadonlySet; + /** Builtin tool names `listTools` lists. */ + builtinTools: ReadonlySet; + /** Guardrail names `listGuardrails` lists. */ + guardrails: ReadonlySet; + /** Trigger types `listTriggerTypes` lists. */ + triggerTypes: ReadonlySet; + /** Agent ids in the session's workspace (`repo.listIds()`). */ + agentIds: ReadonlySet; +} + +/** The `turn.state` key the memoized {@link CatalogFacts} promise lives under. */ +const CATALOG_FACTS_KEY = 'catalogFacts'; + +/** + * The turn's {@link CatalogFacts}: read once and memoized in + * `turn.state['catalogFacts']` (the promise itself is cached, so two + * racing calls share the load). Models, tools, guardrails and trigger + * types come from `deps.catalog`; agent ids from the session's workspace. + */ +export function loadCatalogFacts(turn: AuthorTurn, deps: AuthorToolDeps): Promise { + const existing = turn.state.get(CATALOG_FACTS_KEY) as Promise | undefined; + if (existing !== undefined) return existing; + const promise = (async (): Promise => { + const [{ repo }, models, tools, guardrails, triggers] = await Promise.all([ + deps.workspace(), + deps.catalog.listModels(), + deps.catalog.listTools(), + deps.catalog.listGuardrails(), + deps.catalog.listTriggerTypes(), + ]); + return { + modelIds: new Set(models.map((model) => model.id)), + builtinTools: new Set(tools.map((tool) => tool.name)), + guardrails: new Set(guardrails.map((guardrail) => guardrail.name)), + triggerTypes: new Set(triggers.map((trigger) => trigger.type)), + agentIds: new Set(repo.listIds()), + }; + })(); + turn.state.set(CATALOG_FACTS_KEY, promise); + return promise; +} + +/** One catalog problem: the failure code it picks when it is the first found, and its model-facing text. */ +interface CatalogProblem { + code: AuthorFailureCode; + text: string; +} + +/** `model` and each `fallbackModels[i]` must be ids the model catalog knows. */ +function modelProblems(spec: AgentSpecV2, facts: CatalogFacts): CatalogProblem[] { + return [spec.model, ...(spec.fallbackModels ?? [])] + .filter((model) => !facts.modelIds.has(model)) + .map((model) => ({ code: 'UNKNOWN_MODEL' as const, text: `model "${model}" is not in the catalog` })); +} + +/** Each `tools[i].builtin` must be a builtin the tool catalog knows. */ +function toolProblems(spec: AgentSpecV2, facts: CatalogFacts): CatalogProblem[] { + const problems: CatalogProblem[] = []; + (spec.tools ?? []).forEach((ref, i) => { + if ('builtin' in ref && !facts.builtinTools.has(ref.builtin)) { + problems.push({ code: 'UNKNOWN_TOOL', text: `tools[${i}] builtin "${ref.builtin}" does not exist` }); + } + }); + return problems; +} + +/** Each `guardrails[i].name` must be a guardrail the catalog knows. */ +function guardrailProblems(spec: AgentSpecV2, facts: CatalogFacts): CatalogProblem[] { + const problems: CatalogProblem[] = []; + (spec.guardrails ?? []).forEach((entry, i) => { + if (!facts.guardrails.has(entry.name)) { + problems.push({ code: 'UNKNOWN_GUARDRAIL', text: `guardrails[${i}] name "${entry.name}" does not exist` }); + } + }); + return problems; +} + +/** Each `triggers[i].type` must be a trigger type the catalog knows. */ +function triggerProblems(spec: AgentSpecV2, facts: CatalogFacts): CatalogProblem[] { + const problems: CatalogProblem[] = []; + (spec.triggers ?? []).forEach((entry, i) => { + if (!facts.triggerTypes.has(entry.type)) { + problems.push({ code: 'UNKNOWN_TRIGGER_TYPE', text: `triggers[${i}] type "${entry.type}" does not exist` }); + } + }); + return problems; +} + +/** + * Agent references must name workspace agents: each `subagents..ref`, + * and each `handoffs[i].agent` unless it names one of the spec's own + * `subagents` keys (a handoff to an inline subagent is legal). + */ +function agentRefProblems(spec: AgentSpecV2, facts: CatalogFacts): CatalogProblem[] { + const problems: CatalogProblem[] = []; + const subagentKeys = new Set(Object.keys(spec.subagents ?? {})); + for (const [key, entry] of Object.entries(spec.subagents ?? {})) { + if ('ref' in entry && !facts.agentIds.has(entry.ref)) { + problems.push({ code: 'UNKNOWN_AGENT_REF', text: `subagents.${key}.ref agent "${entry.ref}" does not exist` }); + } + } + (spec.handoffs ?? []).forEach((entry, i) => { + if (!subagentKeys.has(entry.agent) && !facts.agentIds.has(entry.agent)) { + problems.push({ code: 'UNKNOWN_AGENT_REF', text: `handoffs[${i}].agent agent "${entry.agent}" does not exist` }); + } + }); + return problems; +} + +/** The most problems one failure lists; the code is still the first problem's. */ +const MAX_LISTED_PROBLEMS = 10; + +/** + * All of `spec`'s catalog problems in one {@link AuthorToolFailure}, or + * `null` when every value is known. Problems are collected in check order + * (models, tools, guardrails, triggers, agent refs); the first problem's + * code is the failure's code and at most ten problem texts are listed. + */ +export function checkCatalogs(spec: AgentSpecV2, facts: CatalogFacts, toolName: string): AuthorToolFailure | null { + const problems = [ + ...modelProblems(spec, facts), + ...toolProblems(spec, facts), + ...guardrailProblems(spec, facts), + ...triggerProblems(spec, facts), + ...agentRefProblems(spec, facts), + ]; + if (problems.length === 0) return null; + const listed = problems.slice(0, MAX_LISTED_PROBLEMS).map((problem) => problem.text).join('; '); + return toolFailure( + problems[0].code, + `Some values are not in Studio's catalogs: ${listed}.`, + "The draft referred to something Studio doesn't have. The Author is fixing it.", + `Call listModels, listTools, listGuardrails, listTriggerTypes or listAgents and copy exact values, then call ${toolName} again.` + ); +} diff --git a/apps/agent-forge/server/author/proposal/review.ts b/apps/agent-forge/server/author/proposal/review.ts new file mode 100644 index 00000000..e69d1ab8 --- /dev/null +++ b/apps/agent-forge/server/author/proposal/review.ts @@ -0,0 +1,44 @@ +/** + * The pluggable draft review the propose tools run before showing a card + * (PLAN §8.4). This module holds only the contract: `AuthorToolDeps.reviewDraft` + * is injected so AA12 can plug the real review-model pass in behind the + * same call, and `passReview` - the default `createApp` wires until then - + * always lets the draft through. + * + * A rejected verdict is a returned failure (`pass: false` carries the + * `AuthorToolFailure` the tool hands back to the model), never a throw - + * the review bouncing a draft is data, not an error. + */ +import type { AgentSpecV2 } from 'lousho'; +import type { AuthorToolFailure, Requirement, SpecChange } from '../../../shared/authorWire'; +import type { AuthorTurn } from '../turnRunner'; +import type { ProposalIdentity } from './turnState'; + +/** Everything a review needs to judge the draft a propose call built. */ +export interface ReviewInput { + /** The running turn (state access for AA12's per-identity round cap). */ + turn: AuthorTurn; + /** What the proposal is about. */ + identity: ProposalIdentity; + /** Which propose tool is asking, for failure copy. */ + toolName: 'proposeAgent' | 'proposeAgentUpdate'; + /** The complete spec a save would write. */ + spec: AgentSpecV2; + /** The update diff (`[]` for creates). */ + changes: SpecChange[]; + /** The spec's readiness rows, already computed. */ + requirements: Requirement[]; +} + +/** The review's answer: let the card through, or return this failure to the model. */ +export type ReviewVerdict = { pass: true } | { pass: false; failure: AuthorToolFailure }; + +/** The review as a dependency: one verdict per input, never throws. */ +export type ReviewDraft = (input: ReviewInput) => Promise; + +/** + * The review every turn gets until AA12 lands the real one: always + * `{ pass: true }`. `createApp` wires it as `AuthorToolDeps.reviewDraft`'s + * default so `proposeAgent` already runs the full pipeline. + */ +export const passReview: ReviewDraft = async () => ({ pass: true }); diff --git a/apps/agent-forge/server/author/proposal/turnState.ts b/apps/agent-forge/server/author/proposal/turnState.ts new file mode 100644 index 00000000..d5f17e0b --- /dev/null +++ b/apps/agent-forge/server/author/proposal/turnState.ts @@ -0,0 +1,55 @@ +/** + * The per-turn proposal record (PLAN §8.3): which identity keys the + * Author proposed, discarded or saved THIS turn, plus the review-round + * counter AA12's draft review uses. The durable store is the session + * transcript; this map only covers what the not-yet-persisted current + * turn did, so a save call in the same turn can answer "that card is + * still unanswered" and the review can cap its rounds. + * + * The state hangs off `turn.state` under `'proposals'` and is created + * lazily by {@link proposalTurnState}, so a turn that never proposes + * pays nothing. + */ +import type { ProposalOutput } from '../../../shared/authorWire'; +import type { AuthorTurn } from '../turnRunner'; + +/** What a proposal is about: a new agent, or the agent `agentId` an update patches. */ +export type ProposalIdentity = { kind: 'create' } | { kind: 'update'; agentId: string }; + +/** + * The key `ProposalTurnState` maps/sets are indexed by: `'create'` for a + * new-agent card, `'update:'` for an update card. One open card + * per identity per turn - proposing again replaces the entry. + */ +export function identityKey(identity: ProposalIdentity): string { + return identity.kind === 'create' ? 'create' : `update:${identity.agentId}`; +} + +/** The proposal bookkeeping one turn accumulates. */ +export interface ProposalTurnState { + /** The newest card proposed THIS turn, by identity key. */ + proposed: Map; + /** Identity keys `discardPendingChanges` cleared this turn (AA11). */ + discarded: Set; + /** Identity keys saved this turn (AA10b). */ + saved: Set; + /** Review rounds used per identity key this turn (AA12 caps at 3). */ + reviewRounds: Map; +} + +/** The `turn.state` key the {@link ProposalTurnState} lives under. */ +const PROPOSALS_KEY = 'proposals'; + +/** + * The turn's {@link ProposalTurnState}, created on first use and kept in + * `turn.state['proposals']` - so every tool of the same turn (propose, + * save, discard, review) shares one record. + */ +export function proposalTurnState(turn: AuthorTurn): ProposalTurnState { + let state = turn.state.get(PROPOSALS_KEY) as ProposalTurnState | undefined; + if (state === undefined) { + state = { proposed: new Map(), discarded: new Set(), saved: new Set(), reviewRounds: new Map() }; + turn.state.set(PROPOSALS_KEY, state); + } + return state; +} diff --git a/apps/agent-forge/server/author/tools/__tests__/describeCoverage.test.ts b/apps/agent-forge/server/author/tools/__tests__/describeCoverage.test.ts index d3e2e193..e1875aa5 100644 --- a/apps/agent-forge/server/author/tools/__tests__/describeCoverage.test.ts +++ b/apps/agent-forge/server/author/tools/__tests__/describeCoverage.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect } from 'vitest'; import type { AuthorTurn } from '../../turnRunner'; +import { authorDraftSchema, authorPatchSchema } from '../../draft/schema'; import { buildAuthorTools, type AuthorToolDeps, type CatalogService } from '../index'; import type { WorkspaceContext } from '../../../workspace/context'; import type { AgentRepo } from '../../../storage/agentRepo'; @@ -12,7 +13,10 @@ import type { VersionStore } from '../../../storage/versionStore'; * optionals/defaults/effects and descends objects, arrays, unions and * record value types; a field may put its description anywhere on its own * wrapper chain, but container children (elements, options, record - * values) each need one too. + * values) each need one too. AA07's spec-derived draft/patch schemas are + * leaf boundaries: their per-key describes are pinned by + * draft/__tests__/schema.test.ts and their inner nodes are the SDK's own + * `agentSpecV2Schema`, which carries no describes by design. */ const TURN: AuthorTurn = { @@ -44,6 +48,8 @@ const DEPS: AuthorToolDeps = { templates: { list: async () => [] }, probeMcp: async () => ({ ok: true, tools: [] }), docsDir: '/nonexistent-docs', + reviewDraft: async () => ({ pass: true }), + requirements: async () => [], }; /** The zod internals the walk reads (zod 3 `_def` fields, structurally typed). */ @@ -114,11 +120,36 @@ function children(schema: ZodLike): [string, ZodLike][] { } } +/** + * Whether `schema` unwraps to one of AA07's spec-derived draft/patch + * schemas. Those are leaf boundaries for this walk: their top-level keys + * all carry FIELD_DESCRIPTIONS (asserted by draft/__tests__/schema.test.ts) + * and their inner nodes are the SDK's `agentSpecV2Schema` itself, which + * ships no `.describe()`s - the propose tools' describe contract lives at + * the draft/patch field and its direct keys. Identity is by shape: a + * `.describe()` clone shares the same `_def.shape` result. + */ +function isSpecBoundary(schema: ZodLike): boolean { + let current: ZodLike | undefined = schema; + while (current !== undefined) { + const def = current._def; + if (def === undefined) return false; + if (def.typeName === 'ZodObject') { + const shape = def.shape?.(); + return shape === authorDraftSchema.shape || shape === authorPatchSchema.shape; + } + current = wrapped(def); + } + return false; +} + function missingDescriptions(root: ZodLike): string[] { const missing: string[] = []; const walk = (schema: ZodLike, path: string, isRoot: boolean): void => { // The root input object needs no describe of its own; every other node does. if (!isRoot && !described(schema)) missing.push(path === '' ? '' : path); + // Spec-derived draft/patch schemas are leaf boundaries (see isSpecBoundary). + if (isSpecBoundary(schema)) return; for (const [key, child] of children(schema)) { walk(child, path === '' ? key : `${path}.${key}`, false); } diff --git a/apps/agent-forge/server/author/tools/__tests__/proposeAgent.test.ts b/apps/agent-forge/server/author/tools/__tests__/proposeAgent.test.ts new file mode 100644 index 00000000..70d7ddd0 --- /dev/null +++ b/apps/agent-forge/server/author/tools/__tests__/proposeAgent.test.ts @@ -0,0 +1,209 @@ +import { mkdirSync, mkdtempSync, readdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import * as path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { testToolContext } from 'lousho/testing'; +import type { AgentSpecV2 } from 'lousho'; +import type { AuthorToolDeps, CatalogService } from '../index'; +import type { AuthorTurn } from '../../turnRunner'; +import type { WorkspaceContext } from '../../../workspace/context'; +import type { AgentRepo } from '../../../storage/agentRepo'; +import type { AuthorToolFailure, ProposalOutput } from '../../../../shared/authorWire'; +import { proposalTurnState } from '../../proposal/turnState'; +import { passReview } from '../../proposal/review'; +import { AUTHOR_TOOL_FACTORIES, buildAuthorTools } from '../index'; +import type { AuthorDraft } from '../../draft/schema'; + +/** + * proposeAgent (AA08): draft -> spec validation, catalog checks, the + * requirements dep, the pluggable review, the per-turn proposal record, + * and the card-shaped ProposalOutput. The tool writes nothing - a real + * .lousho/agents dir stays byte-identical. + */ + +const DRAFT: AuthorDraft = { + name: 'Support triage', + description: 'Routes support mail to the right queue.', + model: 'mock/mock-1', + instructions: 'You triage support mail and pick the queue.', +}; + +function turn(): AuthorTurn { + return { + sessionId: 'sess-1', + requestId: 'req-test', + focus: { mode: 'create' }, + storedTranscript: [], + userMessage: { id: 'm-1', role: 'user', text: 'hi', at: '2026-01-01T00:00:00.000Z' }, + signal: new AbortController().signal, + stepsRemaining: () => 12, + state: new Map(), + }; +} + +function fakeCatalog(over: Partial = {}): CatalogService { + return { + listModels: () => [{ id: 'mock/mock-1' }, { id: 'anthropic/claude-sonnet-5-5' }] as never, + listTools: () => [{ name: 'web_fetch' }] as never, + listGuardrails: () => [], + listTriggerTypes: () => [{ type: 'chat' }, { type: 'cron' }] as never, + listSkills: async () => [], + listKits: async () => [], + listRegistryMcpServers: async () => [], + ...over, + }; +} + +function fakeDeps(over: Partial = {}): AuthorToolDeps { + return { + workspace: async () => + ({ repo: { listIds: () => [] } as unknown as AgentRepo }) as WorkspaceContext, + catalog: fakeCatalog(), + templates: { list: async () => [] }, + probeMcp: async () => ({ ok: true, tools: [] }), + docsDir: '/nonexistent-docs', + reviewDraft: passReview, + requirements: async () => [], + ...over, + }; +} + +function proposeTool(t: AuthorTurn, deps: AuthorToolDeps) { + const tool = buildAuthorTools(t, deps).find( + (candidate) => ((candidate as { name?: string }).name ?? candidate.displayName) === 'proposeAgent' + )!; + return tool; +} + +async function call(tool: ReturnType, args: Record, toolCallId = 'call_a') { + return tool.execute!(args as never, testToolContext({ toolCallId })); +} + +describe('proposeAgent', () => { + it('is registered in AUTHOR_TOOL_FACTORIES', () => { + const names = AUTHOR_TOOL_FACTORIES.map((factory) => factory(turn(), fakeDeps())).map( + (tool) => (tool as { name?: string }).name ?? tool.displayName + ); + expect(names).toContain('proposeAgent'); + }); + + it('returns a create ProposalOutput keyed on the toolCallId, writing nothing', async () => { + const root = mkdtempSync(path.join(tmpdir(), 'aa08-')); + mkdirSync(path.join(root, '.lousho', 'agents'), { recursive: true }); + const before = readdirSync(root, { recursive: true }); + const t = turn(); + const requirements = [{ id: 'provider:mock', kind: 'provider_key', status: 'ok', label: 'Mock provider' }]; + const tool = proposeTool(t, fakeDeps({ requirements: async () => requirements as never })); + const output = (await call(tool, { draft: DRAFT })) as ProposalOutput; + + expect(output.ok).toBe(true); + expect(output.proposalId).toBe('call_a'); + expect(output.kind).toBe('create'); + expect(output.agentId).toBeUndefined(); + expect(output.enabled).toBe(false); + expect(output.changes).toEqual([]); + expect(output.carriedOver).toBe(false); + expect(output.carriedFields).toEqual([]); + expect(output.baseSpecHash).toBeNull(); + expect(output.requirements).toEqual(requirements); + const spec = output.spec as AgentSpecV2; + expect(spec.version).toBe(2); + expect(spec.name).toBe('Support triage'); + expect(spec.triggers?.some((trigger) => trigger.type === 'chat')).toBe(true); + // Nothing was written anywhere under .lousho/. + expect(readdirSync(root, { recursive: true })).toEqual(before); + }); + + it('records the card under the create identity in the turn state', async () => { + const t = turn(); + const output = (await call(proposeTool(t, fakeDeps()), { draft: DRAFT })) as ProposalOutput; + expect(proposalTurnState(t).proposed.get('create')).toBe(output); + }); + + it('one UNKNOWN_MODEL failure lists both an unknown model and an unknown tool', async () => { + const t = turn(); + // 'web_fetch' is schema-valid but absent from this fake catalog's tool list, + // and 'acme/not-real' is a well-formed provider/model the model catalog lacks. + const deps = fakeDeps({ catalog: fakeCatalog({ listTools: () => [] }) }); + const output = (await call(proposeTool(t, deps), { + draft: { ...DRAFT, model: 'acme/not-real', tools: [{ builtin: 'web_fetch' }] }, + })) as AuthorToolFailure; + expect(output.ok).toBe(false); + expect(output.code).toBe('UNKNOWN_MODEL'); + expect(output.error).toContain('model "acme/not-real" is not in the catalog'); + expect(output.error).toContain('tools[0] builtin "web_fetch" does not exist'); + expect(output.next).toContain('proposeAgent'); + expect(proposalTurnState(t).proposed.size).toBe(0); + }); + + it('a failing review returns its failure and leaves proposed empty', async () => { + const t = turn(); + const reviewFailure: AuthorToolFailure = { + ok: false, + code: 'DRAFT_REVIEW', + error: 'The draft would not run: no tool can read mail.', + userMessage: 'The Author is still working on the draft.', + }; + const output = (await call( + proposeTool(t, fakeDeps({ reviewDraft: async () => ({ pass: false, failure: reviewFailure }) })), + { draft: DRAFT } + )) as AuthorToolFailure; + expect(output).toBe(reviewFailure); + expect(proposalTurnState(t).proposed.size).toBe(0); + }); + + it('passes spec, changes and requirements to the review', async () => { + const t = turn(); + let seen: unknown; + const requirements = [{ id: 'mcp:gh', kind: 'mcp', status: 'unknown', label: 'MCP server gh' }]; + const tool = proposeTool( + t, + fakeDeps({ + requirements: async () => requirements as never, + reviewDraft: async (input) => { + seen = input; + return { pass: true }; + }, + }) + ); + const output = (await call(tool, { draft: DRAFT })) as ProposalOutput; + expect(output.ok).toBe(true); + const input = seen as { turn: AuthorTurn; spec: AgentSpecV2 }; + expect(input.turn).toBe(t); + expect(seen).toMatchObject({ + identity: { kind: 'create' }, + toolName: 'proposeAgent', + changes: [], + requirements, + }); + expect(input.spec.name).toBe('Support triage'); + }); + + it('invalid draft -> DRAFT_INVALID with path: message text', async () => { + const t = turn(); + const output = (await call(proposeTool(t, fakeDeps()), { + draft: { ...DRAFT, name: '!!!', description: '' }, + })) as AuthorToolFailure; + expect(output.ok).toBe(false); + expect(output.code).toBe('DRAFT_INVALID'); + expect(output.error).toContain('description'); + expect(output.next).toContain('proposeAgent'); + expect(proposalTurnState(t).proposed.size).toBe(0); + }); + + it('a requirements dep that throws still yields a card with requirements []', async () => { + const t = turn(); + const tool = proposeTool( + t, + fakeDeps({ + requirements: async () => { + throw new Error('requirements exploded'); + }, + }) + ); + const output = (await call(tool, { draft: DRAFT })) as ProposalOutput; + expect(output.ok).toBe(true); + expect(output.requirements).toEqual([]); + expect(proposalTurnState(t).proposed.get('create')).toBe(output); + }); +}); diff --git a/apps/agent-forge/server/author/tools/__tests__/readTools.test.ts b/apps/agent-forge/server/author/tools/__tests__/readTools.test.ts index 8611014f..a90ff443 100644 --- a/apps/agent-forge/server/author/tools/__tests__/readTools.test.ts +++ b/apps/agent-forge/server/author/tools/__tests__/readTools.test.ts @@ -82,6 +82,8 @@ function fakeDeps(over: Partial = {}): AuthorToolDeps { templates: { list: async () => [] }, probeMcp: async () => ({ ok: true, tools: [] }), docsDir: '/nonexistent-docs', + reviewDraft: async () => ({ pass: true }), + requirements: async () => [], ...over, }; } @@ -107,15 +109,15 @@ function failure(value: unknown): AuthorToolFailure { } describe('buildAuthorTools', () => { - it('returns the 13 read tools sorted by name, every one in AUTHOR_TOOL_NAMES', () => { + it('returns the 14 tools sorted by name, every one in AUTHOR_TOOL_NAMES', () => { const built = buildAuthorTools(TURN, fakeDeps()); const names = built.map(nameOf); - expect(names).toHaveLength(13); + expect(names).toHaveLength(14); expect(names).toEqual([...names].sort((a, b) => a.localeCompare(b))); for (const name of names) { expect(AUTHOR_TOOL_NAMES as readonly string[]).toContain(name); } - expect(AUTHOR_TOOL_FACTORIES).toHaveLength(13); + expect(AUTHOR_TOOL_FACTORIES).toHaveLength(14); }); it('builds the same list for different turns (the tool surface never changes mid-session)', () => { diff --git a/apps/agent-forge/server/author/tools/__tests__/searchDocs.test.ts b/apps/agent-forge/server/author/tools/__tests__/searchDocs.test.ts index c81d8e41..10ec385a 100644 --- a/apps/agent-forge/server/author/tools/__tests__/searchDocs.test.ts +++ b/apps/agent-forge/server/author/tools/__tests__/searchDocs.test.ts @@ -42,6 +42,8 @@ function deps(docsDir: string): AuthorToolDeps { templates: { list: async () => [] }, probeMcp: async () => ({ ok: true, tools: [] }), docsDir, + reviewDraft: async () => ({ pass: true }), + requirements: async () => [], }; } diff --git a/apps/agent-forge/server/author/tools/index.ts b/apps/agent-forge/server/author/tools/index.ts index eed6536c..c012b3b2 100644 --- a/apps/agent-forge/server/author/tools/index.ts +++ b/apps/agent-forge/server/author/tools/index.ts @@ -14,7 +14,8 @@ * is D17's catalog, `probeMcp` is D11's probe as a function, and * `docsDir` is the bundled SDK docs folder `searchDocs` reads. */ -import type { ToolDescriptor } from 'lousho'; +import type { AgentSpecV2, ToolDescriptor } from 'lousho'; +import type { Requirement } from '../../../shared/authorWire'; import type { CatalogGuardrail, CatalogKit, @@ -27,11 +28,13 @@ import type { } from '../../../shared/wireTypes'; import type { McpProbeResult } from '../../routes/mcpProbe'; import type { WorkspaceContext } from '../../workspace/context'; +import type { ReviewDraft } from '../proposal/review'; import type { AuthorTurn } from '../turnRunner'; import { readToolFactories } from './readTools'; import { probeMcpServerTool } from './mcpProbeTool'; import { searchDocsTool } from './searchDocsTool'; import { openPageTool } from './openPageTool'; +import { proposeAgentTool } from './proposeAgentTool'; /** * A registry-index item that names an MCP server (`type: 'mcp'`, @@ -84,6 +87,18 @@ export interface AuthorToolDeps { probeMcp: (server: McpServerSpecV2) => Promise; /** The bundled SDK docs folder `searchDocs` indexes (`/docs`, resolved in server/index.ts). */ docsDir: string; + /** + * The draft review a propose tool runs before showing its card (PLAN + * §8.4): AA12's review-model pass; `createApp` wires `passReview` (the + * always-approve default) until then. + */ + reviewDraft: ReviewDraft; + /** + * The spec's readiness rows for the proposal card (AA13's + * `computeRequirements`, wired per turn to the session's workspace + * root). Advisory - the propose tools degrade a throwing dep to `[]`. + */ + requirements: (spec: AgentSpecV2) => Promise; } /** Builds one tool for one turn - the factories close over `turn` (and `deps`). */ @@ -100,6 +115,7 @@ export const AUTHOR_TOOL_FACTORIES: readonly AuthorToolFactory[] = [ probeMcpServerTool, searchDocsTool, openPageTool, + proposeAgentTool, ]; /** The call name a `ToolDescriptor` carries (`defineTool`/loader results), else its display name. */ diff --git a/apps/agent-forge/server/author/tools/proposeAgentTool.ts b/apps/agent-forge/server/author/tools/proposeAgentTool.ts new file mode 100644 index 00000000..5a97c27a --- /dev/null +++ b/apps/agent-forge/server/author/tools/proposeAgentTool.ts @@ -0,0 +1,65 @@ +/** + * The Author's `proposeAgent` tool (PLAN §8.2, AA08): shows the user a + * proposal card for a NEW agent and writes nothing. `proposalId` is the + * call's own `toolCallId` - the transcript is the proposal store, so the + * card the panel renders is the tool result itself. + * + * The pipeline (research §4.1): `draftToSpec` validates the draft against + * `agentSpecV2Schema` (and appends the chat-trigger backstop), then the + * catalog checks refuse ids Studio does not have (one failure listing + * every problem), then the requirements dep computes the card's readiness + * rows (advisory; `[]` on error), then the injected review gets its say + * (`passReview` until AA12). Only then is the output recorded under the + * 'create' identity in {@link proposalTurnState} and returned - a failed + * step leaves `proposed` untouched. + */ +import { defineTool } from 'lousho'; +import { z } from 'zod'; +import type { ProposalOutput } from '../../../shared/authorWire'; +import { authorDraftSchema } from '../draft/schema'; +import { draftInvalidFailure, draftToSpec } from '../draft/toSpec'; +import { checkCatalogs, loadCatalogFacts } from '../proposal/catalogChecks'; +import { identityKey, proposalTurnState } from '../proposal/turnState'; +import { guardedToolCall } from './failure'; +import type { AuthorToolFactory } from './index'; + +/** AA08's propose tool for new agents; updates are `proposeAgentUpdate` (AA09). */ +export const proposeAgentTool: AuthorToolFactory = (turn, deps) => + defineTool({ + name: 'proposeAgent', + description: + 'Show the user a proposal card for a NEW agent. Nothing is saved. After calling it, end your turn and wait for the user to confirm or press Apply.', + input: z.object({ draft: authorDraftSchema.describe('The complete new agent. Fields you leave out stay unset.') }), + execute: ({ draft }, ctx) => + guardedToolCall('proposeAgent', async () => { + const built = draftToSpec(draft); + if (!built.ok) return draftInvalidFailure('proposeAgent', built.issues); + const catalogFailure = checkCatalogs(built.spec, await loadCatalogFacts(turn, deps), 'proposeAgent'); + if (catalogFailure !== null) return catalogFailure; + // Requirements are advisory: a dep failure means an empty list, never a failed call. + const requirements = await deps.requirements(built.spec).catch(() => []); + const verdict = await deps.reviewDraft({ + turn, + identity: { kind: 'create' }, + toolName: 'proposeAgent', + spec: built.spec, + changes: [], + requirements, + }); + if (!verdict.pass) return verdict.failure; + const output: ProposalOutput = { + ok: true, + proposalId: ctx.toolCallId, + kind: 'create', + spec: built.spec, + changes: [], + requirements, + carriedOver: false, + carriedFields: [], + baseSpecHash: null, + enabled: false, + }; + proposalTurnState(turn).proposed.set(identityKey({ kind: 'create' }), output); + return output; + }), + }); diff --git a/apps/agent-forge/server/author/wiring.ts b/apps/agent-forge/server/author/wiring.ts index 2cc36e34..057503fc 100644 --- a/apps/agent-forge/server/author/wiring.ts +++ b/apps/agent-forge/server/author/wiring.ts @@ -22,6 +22,8 @@ import type { AuthorSessionStore } from './sessionStore'; import { buildAuthorSystemPrompt, type AuthorPromptCatalogs } from './prompt/index'; import { buildAuthorTools, type AuthorToolDeps } from './tools/index'; import { createAuthorCatalogService } from './tools/catalogService'; +import { passReview } from './proposal/review'; +import { computeRequirements, requirementDeps } from './requirements'; /** * The D09 catalog functions (`/catalog/*` reads the same SDK calls) @@ -83,11 +85,15 @@ export function createAuthorRunner(deps: CreateAuthorRunnerDeps): AuthorTurnRunn resolveSecret: (name: string) => deps.secrets.getSecret(name), ...(deps.mcpConnect ? { connect: deps.mcpConnect } : {}), }; - const toolDeps: Omit = { + // `workspace` (and `requirements`, which reads under the workspace + // root) is bound per turn below. `reviewDraft` is AA12's seam; + // passReview is the always-approve default until it lands. + const toolDeps: Omit = { catalog: createAuthorCatalogService({ secretsStore: deps.secrets }), templates: deps.templates, probeMcp: (server) => probeMcpServer(server, probeDeps), docsDir: deps.docsDir, + reviewDraft: passReview, }; const workspaceForTurn = async (turn: AuthorTurn): Promise => { const session = await deps.sessions.get(turn.sessionId); @@ -102,7 +108,15 @@ export function createAuthorRunner(deps: CreateAuthorRunnerDeps): AuthorTurnRunn settings: deps.settings, secrets: deps.secrets, systemPrompt: () => authorPrompt, - buildTools: (turn) => buildAuthorTools(turn, { ...toolDeps, workspace: () => workspaceForTurn(turn) }), + buildTools: (turn) => + buildAuthorTools(turn, { + ...toolDeps, + workspace: () => workspaceForTurn(turn), + // AA13's readiness rows, resolved against the session's workspace + // root per call (fileExist checks stay inside that root). + requirements: async (spec) => + computeRequirements(spec, requirementDeps((await workspaceForTurn(turn)).ref.root, deps.secrets)), + }), // Stand-in until AA07 lands the real turnContextInput. turnContextInput: async (s, now) => ({ focus: s.focus, openProposals: [], now }), ...(deps.providerFor ? { providerFor: deps.providerFor } : {}), diff --git a/apps/agent-forge/shared/authorWire.ts b/apps/agent-forge/shared/authorWire.ts index 6d8dc6d0..df1473a6 100644 --- a/apps/agent-forge/shared/authorWire.ts +++ b/apps/agent-forge/shared/authorWire.ts @@ -9,12 +9,19 @@ * module instead of re-declaring them. */ +import type { AgentSpecV2 } from 'lousho'; +import type { Requirement, SpecChange } from './wireTypes'; + // The readiness row AA13's `computeRequirements` returns and the Author's // proposal cards carry: defined next to `RequirementKind` in wireTypes.ts // (where `AgentRecord.requirements` reads it), re-exported here so Author // code has one import site per the PLAN §8.5 contract. export type { Requirement } from './wireTypes'; +// The diff rows a proposal card for an UPDATE renders: D10's `diffSpecs` +// output type, re-exported for the same one-import-site reason. +export type { SpecChange } from './wireTypes'; + /** What the Author is working on this session. */ export type AuthorFocus = { mode: 'create' } | { mode: 'edit'; agentId: string }; @@ -155,6 +162,45 @@ export type AuthorClientMessage = | { type: 'chat.stop'; requestId: string } | { type: 'chat.resume'; requestId: string }; +/** Whether a proposal card creates a new agent or updates an existing one. */ +export type ProposalKind = 'create' | 'update'; + +/** + * The `ok: true` output of `proposeAgent`/`proposeAgentUpdate` - the card + * the Author panel renders (PLAN §8.3). `proposalId` is the propose call's + * `toolCallId`: the transcript is the proposal store, so nothing is + * written anywhere else. `spec` is exactly what a save would write; + * `changes` is the diff a card shows for an update (`[]` for a create, + * which shows the full config); `requirements` is the advisory readiness + * list. `carriedOver`/`carriedFields` report an update that folded an + * earlier unapplied card (AA09; always false/[] for creates), + * `baseSpecHash` pins the saved spec an update patched, and `enabled` is + * the agent's current switch state (`false` for creates: a new agent + * saves disabled). + */ +export interface ProposalOutput { + ok: true; + /** The propose call's `toolCallId` - the card's id in the transcript. */ + proposalId: string; + kind: ProposalKind; + /** Set on updates only: the agent the patch applies to. */ + agentId?: string; + /** The complete spec a save would write. */ + spec: AgentSpecV2; + /** The update diff; `[]` for creates (the card shows the full config instead). */ + changes: SpecChange[]; + /** The spec's readiness rows (`computeRequirements`, AA13) - advisory, never a block. */ + requirements: Requirement[]; + /** Whether an earlier unapplied card for the same agent was folded in (AA09). */ + carriedOver: boolean; + /** The fields an earlier folded card contributed (AA09); `[]` for creates. */ + carriedFields: string[]; + /** Updates: `specHash` of the saved spec the patch applied to; creates: `null`. */ + baseSpecHash: string | null; + /** Creates: `false`; updates: the agent's `meta.enabled`. */ + enabled: boolean; +} + /** * Every event one Author turn produces, in order. `AuthorTurnRunner` * emits them on its `'event'` channel as `(sessionId, event)`; the