diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 8a06dddeeec1..ca60d0bebfa0 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -3579,6 +3579,70 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect.each([ + { failure: "authentication_failed", apiErrorStatus: undefined, opensFreshProcess: true }, + { failure: "server_error", apiErrorStatus: undefined, opensFreshProcess: false }, + { failure: "authentication_failed", apiErrorStatus: 403, opensFreshProcess: false }, + ] as const)( + "after a turn fails with $failure (API status: $apiErrorStatus), the next turn opens a fresh process: $opensFreshProcess", + ({ failure, apiErrorStatus, opensFreshProcess }) => + Effect.gen(function* () { + let closeCount = 0; + const harness = yield* makeWakeHarnessWithOptions({ + freshQueueOnReopen: true, + close: () => + Effect.sync(() => { + closeCount += 1; + }), + }); + const now = yield* DateTime.now; + const startTurn = (attempt: string, text: string) => + harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make(attempt), + text, + attachments: [], + }), + ); + + yield* startTurn("attempt-claude-relogin-1", "Continue."); + yield* Queue.offerAll(harness.sdkMessages, [ + makeAssistantErrorFrame({ uuid: "00000000-0000-4000-8000-000000000611", error: failure }), + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000612", + result: "API Error", + ...(apiErrorStatus === undefined ? {} : { apiErrorStatus }), + terminalReason: "api_error", + isError: true, + }), + ]); + const failed = yield* Queue.take(harness.terminalReceipts); + assert.equal(failed.status, "failed"); + + yield* startTurn("attempt-claude-relogin-2", "Try again."); + assert.equal(harness.processQueues.length, opensFreshProcess ? 2 : 1); + assert.equal(closeCount, opensFreshProcess ? 1 : 0); + if (opensFreshProcess) { + // The fresh process resumes the same Claude conversation. + assert.equal(harness.getOpenedOptions()?.resume, WAKE_NATIVE_SESSION); + assert.isUndefined(harness.getOpenedOptions()?.sessionId); + } + yield* Queue.offer( + harness.processQueues.at(-1)!, + makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000613", result: "Done." }), + ); + const completed = yield* Queue.take(harness.terminalReceipts); + assert.equal(completed.status, "completed"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + it.effect.each([ { recovered: false, expected: "Claude usage limit reached" }, { recovered: true, expected: "Claude gave up after repeated API errors" }, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts index ec3ec5b05246..21efea9cf8f6 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts @@ -526,6 +526,7 @@ function makeReplayQueryRunner( ): AsyncGenerator { while (true) { throwIfFailed(); + if (signal.aborted) return; const entry = transcript.entries[cursor]; if (entry === undefined) { @@ -807,6 +808,7 @@ function makeReplayQueryRunner( return { open: (input) => { + abortReplayIterators(); assertNextOutboundFrame(makeClaudeQueryOpenFrame(input)); return { messages: Stream.fromAsyncIterable(replayMessagesWithGateCleanup(input.options), (cause) => diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index be276af5f0ed..d27862c97ac9 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -481,7 +481,7 @@ export class ClaudeBackgroundWorkBlocksQueryReplacementError extends Schema.Tagg {}, ) { override get message(): string { - return "Claude is still running background agents or commands, and this model or setting change would end them. Wait for them to finish, or press Stop, then send the message again."; + return "Claude is still running background agents or commands, and starting a new Claude process would end them. Wait for them to finish, or press Stop, then send the message again."; } } @@ -3104,6 +3104,8 @@ interface ClaudeLiveQueryContext { readonly openedPermissionMode: PermissionMode; permissionMode: PermissionMode; stopping: boolean; + // A fresh process picks up a new login after authentication fails. + signedOut: boolean; readonly subagentsFromEarlierProcesses: ReadonlySet; } @@ -7888,13 +7890,21 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( resultFailure?.class === "usage_limit" ? { ...resultFailure, resetAt: resetAt ?? resultFailure.resetAt ?? null } : resultFailure; + const status = interrupted + ? "interrupted" + : limitedCompaction !== null + ? "failed" + : terminalStatusFromResult(message, failureHint); + if ( + status === "failed" && + context.authenticationFailureMessage !== undefined && + !permissionDenied + ) { + liveQuery.signedOut = true; + } yield* finalizeActiveTurn({ context, - status: interrupted - ? "interrupted" - : limitedCompaction !== null - ? "failed" - : terminalStatusFromResult(message, failureHint), + status, completedAt, result: message, ...(terminalFailure === null ? {} : { failure: terminalFailure }), @@ -8452,6 +8462,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( if ( existing !== null && existing.nativeThreadId === nativeThreadId && + !existing.signedOut && existing.queryPolicyKey === queryPolicyKey && !existing.stopping && existing.selection.queryIdentity !== compiledSelection.queryIdentity && @@ -8473,6 +8484,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( if ( existing !== null && existing.nativeThreadId === nativeThreadId && + !existing.signedOut && existing.queryPolicyKey === queryPolicyKey && existing.selection.queryIdentity === compiledSelection.queryIdentity ) { @@ -8587,6 +8599,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( openedPermissionMode: queryOptions.permissionMode, permissionMode: queryOptions.permissionMode, stopping: false, + signedOut: false, subagentsFromEarlierProcesses: new Set( [...(yield* Ref.get(sessionSubagentsByTaskId)).values()].filter( (subagent) => subagent.task.status === "running", diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/claude_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/claude_transcript.ndjson index 2597109378ee..8c4553f98d19 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/claude_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/claude_transcript.ndjson @@ -4,6 +4,7 @@ {"type":"emit_inbound","label":"system:init:1","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.183","cwd":"/tmp/claude-replay-claude_result_is_error","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"aa401aa4-0000-4000-8000-000000000001","session_id":"aa401aa4-e597-4e40-8e40-1e401e401e40"}} {"type":"emit_inbound","label":"assistant:synthetic_auth_error","frame":{"type":"assistant","message":{"id":"7f28a402-6d4c-491e-abbb-2b3c231a06f0","container":null,"model":"","role":"assistant","stop_details":null,"stop_reason":"stop_sequence","stop_sequence":"","type":"message","usage":{"input_tokens":0,"output_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":null,"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":null,"iterations":null,"speed":null},"content":[{"type":"text","text":"Failed to authenticate. API Error: 401 Invalid authentication credentials"}],"context_management":null},"parent_tool_use_id":null,"session_id":"aa401aa4-e597-4e40-8e40-1e401e401e40","uuid":"aa401aa4-0000-4000-8000-000000000002","error":"authentication_failed"}} {"type":"emit_inbound","label":"result:1_is_error","frame":{"type":"result","subtype":"success","is_error":true,"api_error_status":401,"duration_ms":2116,"duration_api_ms":0,"num_turns":1,"result":"Failed to authenticate. API Error: 401 Invalid authentication credentials","stop_reason":"stop_sequence","session_id":"aa401aa4-e597-4e40-8e40-1e401e401e40","total_cost_usd":0,"usage":{"input_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":0,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"modelUsage":{},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"aa401aa4-0000-4000-8000-000000000003"}} +{"type":"expect_outbound","label":"query.open:resume","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"settings":{"showThinkingSummaries":true},"resume":"aa401aa4-e597-4e40-8e40-1e401e401e40"}}} {"type":"expect_outbound","label":"prompt.offer:2","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Try again now that auth is back."},"parent_tool_use_id":null}}} {"type":"emit_inbound","label":"assistant:recovered","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_01iserrorfixture1","type":"message","role":"assistant","content":[{"type":"text","text":"claude result is_error fixture recovered"}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":1,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"aa401aa4-e597-4e40-8e40-1e401e401e40","uuid":"aa401aa4-0000-4000-8000-000000000004"}} {"type":"emit_inbound","label":"result:2_success","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":4000,"duration_api_ms":3500,"num_turns":1,"result":"claude result is_error fixture recovered","stop_reason":"end_turn","session_id":"aa401aa4-e597-4e40-8e40-1e401e401e40","total_cost_usd":0.001,"usage":{"input_tokens":2,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":2,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":1,"output_tokens":1,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":2,"outputTokens":2,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.001,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"aa401aa4-0000-4000-8000-000000000005"}} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/output.ts index fe3167b22cbc..9bd33d54bdd4 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/output.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/output.ts @@ -13,7 +13,7 @@ import { const AUTH_ERROR_TEXT = "Failed to authenticate. API Error: 401 Invalid authentication credentials"; const AUTH_FAILURE_TEXT = - "Claude could not authenticate. For subscription login, run `claude auth login` on this environment's machine, then start a new thread. For API-key authentication, check this instance's configured credentials."; + "Claude could not authenticate. For subscription login, run `claude auth login` on this environment's machine, then send the message again. For API-key authentication, check this instance's configured credentials."; export function assertClaudeResultIsErrorOutput( result: OrchestratorV2ScenarioResult, diff --git a/apps/server/src/provider/Drivers/ClaudeHome.test.ts b/apps/server/src/provider/Drivers/ClaudeHome.test.ts index fb1caf753f86..107b146f8584 100644 --- a/apps/server/src/provider/Drivers/ClaudeHome.test.ts +++ b/apps/server/src/provider/Drivers/ClaudeHome.test.ts @@ -73,7 +73,7 @@ it.layer(NodeServices.layer)("ClaudeHome", (it) => { const message = claudeSignedOutMessage({ configDir, cwd: "/synthetic/project" }); expect(message).toContain(`CLAUDE_CONFIG_DIR set to "${configDir}"`); expect(message).not.toContain("CLAUDE_CONFIG_DIR="); - expect(message).toContain("then start a new thread"); + expect(message).toContain("then send the message again"); }); it.effect("separates capability probes by cwd", () => diff --git a/apps/server/src/provider/Drivers/ClaudeHome.ts b/apps/server/src/provider/Drivers/ClaudeHome.ts index 0c7908b0264c..6bca313a14f6 100644 --- a/apps/server/src/provider/Drivers/ClaudeHome.ts +++ b/apps/server/src/provider/Drivers/ClaudeHome.ts @@ -97,5 +97,5 @@ export const claudeSignedOutMessage = (input: { input.configDir !== undefined ? ` from ${quotePath(input.cwd)}, with CLAUDE_CONFIG_DIR set to ${quotePath(input.configDir)}` : ""; - return `Claude could not authenticate. For subscription login, run \`claude auth login\` on this environment's machine${configuration}, then start a new thread. For API-key authentication, check this instance's configured credentials.`; + return `Claude could not authenticate. For subscription login, run \`claude auth login\` on this environment's machine${configuration}, then send the message again. For API-key authentication, check this instance's configured credentials.`; };