diff --git a/app/src/components/channels/chat-transcript.tsx b/app/src/components/channels/chat-transcript.tsx index 56d12b8c0..d6ac4cd2f 100644 --- a/app/src/components/channels/chat-transcript.tsx +++ b/app/src/components/channels/chat-transcript.tsx @@ -52,7 +52,12 @@ import { splitChatOrigin } from "./chat-origin"; import { markdownComponents } from "@/lib/markdown"; import { EASE_OUT, ENTRANCE_SECONDS } from "@/lib/motion"; import { readToolName } from "@/lib/plugins/tool-name"; -import { asText, forDisplay, REFUSAL_MARKER } from "@/lib/plugins/tool-result"; +import { + asText, + forDisplay, + REFUSAL_MARKER, + toolResultFailed, +} from "@/lib/plugins/tool-result"; import { readTeamBotConsent } from "@/lib/team-bots"; import { cn } from "@/lib/utils"; import { VOICE_CHAT_ACTIVITY, type VoiceChatEntry } from "@/lib/voice/archive"; @@ -1355,6 +1360,7 @@ function ServerToolLine({ name, result }: { name: string; result?: string }) { */ const answer = result === undefined ? undefined : asText(result); const refused = answer?.startsWith(REFUSAL_MARKER) ?? false; + const failed = !refused && toolResultFailed(result); /* * The marker is for this component, not for the reader. Left in, a refusal reads "Blocked" in the * label and then "Refused." again in the first two words of the body, which is the same fact three @@ -1370,6 +1376,7 @@ function ServerToolLine({ name, result }: { name: string; result?: string }) { {...(detail ? { detail } : {})} label={label} refused={refused} + failed={failed} running={result === undefined} > {body ? ( diff --git a/app/src/components/channels/tool-line.tsx b/app/src/components/channels/tool-line.tsx index 643907f1c..1e13bb3c3 100644 --- a/app/src/components/channels/tool-line.tsx +++ b/app/src/components/channels/tool-line.tsx @@ -43,7 +43,11 @@ export function ToolLine({ }`} > - {refused ? "Blocked" : failed ? `${label}, didn't work` : label} + {refused + ? `Blocked: ${label}` + : failed + ? `${label}, didn't work` + : label} {detail ? {detail} : null} diff --git a/app/src/lib/plugins/tool-result.ts b/app/src/lib/plugins/tool-result.ts index d6827a512..88f628104 100644 --- a/app/src/lib/plugins/tool-result.ts +++ b/app/src/lib/plugins/tool-result.ts @@ -116,3 +116,19 @@ export function saidItWentAhead(result: unknown, marker: string): boolean { if (typeof result !== "string") return false; return asText(result).startsWith(marker); } + +/** + * Whether a server-side tool reached a terminal failure rather than returning a result. + * + * These prefixes are written by the two server tool doors after the distinction between policy + * refusal and execution failure has already been made. A refusal has its own marker and must not + * be collapsed into this state. + */ +export function toolResultFailed(result: string | undefined): boolean { + if (result === undefined) return false; + const answer = asText(result); + return ( + answer.startsWith("The vendor reported an error:") || + answer.startsWith("That tool could not be called") + ); +} diff --git a/app/tests/tool-result.test.ts b/app/tests/tool-result.test.ts index a700f2064..e2e9b0eec 100644 --- a/app/tests/tool-result.test.ts +++ b/app/tests/tool-result.test.ts @@ -4,6 +4,7 @@ import { asText, forDisplay, saidItWentAhead, + toolResultFailed, } from "../src/lib/plugins/tool-result"; /** @@ -119,3 +120,24 @@ describe("the markers the server and the transcript both use", () => { expect(saidItWentAhead(undefined, HANDED_OVER)).toBe(true); }); }); + +describe("telling a failed tool from a successful one", () => { + test("vendor errors are failures, encoded or not", () => { + const answer = "The vendor reported an error:\\npermission denied"; + expect(toolResultFailed(answer)).toBe(true); + expect(toolResultFailed(JSON.stringify(answer))).toBe(true); + }); + + test("runtime/deployment call failures are failures", () => { + expect(toolResultFailed("That tool could not be called.")).toBe(true); + expect( + toolResultFailed("That tool could not be called: connection reset"), + ).toBe(true); + }); + + test("ordinary results, refusals, and running calls are not failures", () => { + expect(toolResultFailed("Found 12 results.")).toBe(false); + expect(toolResultFailed("Refused. The rule says no.")).toBe(false); + expect(toolResultFailed(undefined)).toBe(false); + }); +});