diff --git a/.github/triage/PLAYBOOK.md b/.github/triage/PLAYBOOK.md index 39bf3ea01052..32def6bc0c16 100644 --- a/.github/triage/PLAYBOOK.md +++ b/.github/triage/PLAYBOOK.md @@ -59,8 +59,9 @@ different code depending on it: Then work from evidence, not assumption. In rough order of value: -- The server log and the trace file (`server.trace.ndjson`) around the time of the - problem. Recent failures usually leave a trail here. +- The trace file (`server.trace.ndjson`) around the time of the problem, plus the + service log or desktop backend logs from the context file if they exist. Recent + failures usually leave a trail here. - The provider event log, for problems with claude/codex/cursor sessions. - The SQLite database. Read it freely, but only write when a write is necessary to fix the problem the user described, and get their explicit permission diff --git a/apps/desktop/src/backend/DesktopBackendConfiguration.ts b/apps/desktop/src/backend/DesktopBackendConfiguration.ts index b3bec8830ec6..c2012299802d 100644 --- a/apps/desktop/src/backend/DesktopBackendConfiguration.ts +++ b/apps/desktop/src/backend/DesktopBackendConfiguration.ts @@ -118,6 +118,9 @@ const WSL_FORWARDED_ENV_NAMES = [ "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL", + "OTEL_TRACES_EXPORTER", + "OTEL_METRICS_EXPORTER", + "OTEL_LOGS_EXPORTER", ] as const; const WSL_SERVER_SYSTEM_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; diff --git a/apps/mobile/src/connection/environment-cache-store.ts b/apps/mobile/src/connection/environment-cache-store.ts index 44a1d3114532..6e56524770b6 100644 --- a/apps/mobile/src/connection/environment-cache-store.ts +++ b/apps/mobile/src/connection/environment-cache-store.ts @@ -1,6 +1,7 @@ import { ConnectionPersistenceError, EnvironmentCacheStore, + encodeShellSnapshotForCache, } from "@t3tools/client-runtime/platform"; import { type EnvironmentId, @@ -56,7 +57,6 @@ const StoredVcsRefs = Schema.Struct({ const decodeStoredShellSnapshot = Schema.decodeUnknownEffect( Schema.fromJsonString(StoredShellSnapshot), ); -const encodeStoredShellSnapshot = Schema.encodeEffect(Schema.fromJsonString(StoredShellSnapshot)); const decodeStoredThreadSnapshot = Schema.decodeUnknownEffect( Schema.fromJsonString(StoredThreadSnapshot), ); @@ -136,11 +136,18 @@ export const make = Effect.fn("MobileEnvironmentCacheStore.make")(function* () { }).pipe(Effect.tap(() => Effect.promise(() => projectFaviconDatabaseCache.hydrate()))), ), saveShell: Effect.fn("MobileEnvironmentCache.saveShell")(function* (environmentId, snapshot) { - const payload = yield* encodeStoredShellSnapshot({ - schemaVersion: SHELL_SNAPSHOT_CACHE_SCHEMA_VERSION, - environmentId, - snapshot, - }).pipe(Effect.mapError((cause) => persistenceError("save-shell", cause))); + const encodedSnapshot = yield* encodeShellSnapshotForCache(snapshot).pipe( + Effect.mapError((cause) => persistenceError("save-shell", cause)), + ); + const payload = yield* Effect.try({ + try: () => + JSON.stringify({ + schemaVersion: SHELL_SNAPSHOT_CACHE_SCHEMA_VERSION, + environmentId, + snapshot: encodedSnapshot, + } satisfies typeof StoredShellSnapshot.Encoded), + catch: (cause) => persistenceError("save-shell", cause), + }); yield* database .saveCache(environmentId, "shell", "snapshot", SHELL_SNAPSHOT_CACHE_SCHEMA_VERSION, payload) .pipe(Effect.mapError(mapDatabaseError("save-shell"))); diff --git a/apps/mobile/src/features/connection/ConnectionFormField.test.tsx b/apps/mobile/src/features/connection/ConnectionFormField.test.tsx new file mode 100644 index 000000000000..343d44cf376b --- /dev/null +++ b/apps/mobile/src/features/connection/ConnectionFormField.test.tsx @@ -0,0 +1,42 @@ +import type { ReactNode } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it, vi } from "vite-plus/test"; + +vi.mock("react-native", () => ({ + View: ({ children }: { children: ReactNode }) =>
{children}
, +})); +vi.mock("../../components/AppText", () => ({ + AppText: ({ + accessibilityElementsHidden, + importantForAccessibility, + children, + }: { + accessibilityElementsHidden?: boolean; + importantForAccessibility?: string; + children: ReactNode; + }) => ( + + {children} + + ), + AppTextInput: ({ accessibilityLabel }: { accessibilityLabel?: string }) => ( + + ), +})); + +import { ConnectionFormField } from "./ConnectionFormField"; + +describe("ConnectionFormField accessibility", () => { + it.each(["Host", "Pairing code"])("exposes %s once as the input name", (label) => { + const markup = renderToStaticMarkup( + , + ); + + expect(markup).toContain(``); + expect(markup).toContain(``); + }); +}); diff --git a/apps/mobile/src/features/connection/ConnectionFormField.tsx b/apps/mobile/src/features/connection/ConnectionFormField.tsx index 0d5e21148e54..b492dd9ce43c 100644 --- a/apps/mobile/src/features/connection/ConnectionFormField.tsx +++ b/apps/mobile/src/features/connection/ConnectionFormField.tsx @@ -3,7 +3,7 @@ import { View } from "react-native"; import { AppText, AppTextInput, type AppTextInputProps } from "../../components/AppText"; import { cn } from "../../lib/cn"; -type ConnectionFormFieldProps = Omit & { +type ConnectionFormFieldProps = Omit & { readonly label: string; readonly className?: string; }; @@ -12,12 +12,16 @@ type ConnectionFormFieldProps = Omit & { export function ConnectionFormField({ label, className, ...inputProps }: ConnectionFormFieldProps) { return ( - + {label} diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index e32240748861..14beefd6d178 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -679,10 +679,12 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread const isSplitLayout = layoutVariant === "split"; const contentMaxWidth = isSplitLayout ? CHAT_CONTENT_MAX_WIDTH : undefined; const workspaceContentWidth = useWorkspaceContentWidth(); + // Clearing animated width can retain the unfolded width after Android resumes folded. + // Assign both layouts explicitly so the dock always follows its current parent. const composerWidthStyle = useAnimatedStyle(() => isSplitLayout && workspaceContentWidth !== null - ? { width: workspaceContentWidth.value, right: undefined } - : { width: undefined, right: 0 }, + ? { width: workspaceContentWidth.value } + : { width: "100%" }, ); const selectedInstanceId = props.selectedThread.modelSelection.instanceId; useStreamingHaptics(props.selectedThread.id, props.selectedThreadFeed); @@ -964,7 +966,7 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread {/* No paddingTop here: the overlay's measured height becomes the list's bottom inset, so any padding above the pill/composer diff --git a/apps/mobile/src/features/threads/ThreadFeed.tsx b/apps/mobile/src/features/threads/ThreadFeed.tsx index 20848cd2ad10..83b0bef9c022 100644 --- a/apps/mobile/src/features/threads/ThreadFeed.tsx +++ b/apps/mobile/src/features/threads/ThreadFeed.tsx @@ -2477,7 +2477,10 @@ export const ThreadFeed = memo(function ThreadFeed(props: ThreadFeedProps) { // content-inset override. Seed the fresh instance synchronously with the // current overlay height before the scroll integration's next reaction; // on Android the declarative contentInset floor covers this same window. - const listMountKey = `${feedThreadKey}:${presentedFeed.length === 0 ? "empty" : "filled"}`; + // The thinking row a running thread shows while its messages load is not + // content: the list must still remount, and so open at the end, when they + // arrive. + const listMountKey = `${feedThreadKey}:${presentedFeed.some((entry) => entry.type !== "thinking") ? "filled" : "empty"}`; useLayoutEffect(() => { const bottom = props.contentInsetEndAdjustment.value; if (bottom > 0) { diff --git a/apps/mobile/src/features/threads/thread-list-v2-items.tsx b/apps/mobile/src/features/threads/thread-list-v2-items.tsx index eb089b95060a..09ffe404ee79 100644 --- a/apps/mobile/src/features/threads/thread-list-v2-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-v2-items.tsx @@ -1028,27 +1028,25 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { )} {pr ? ( - {pr.kind === "stack" || pr.others > 0 ? ( - - ) : null} + - {pr.kind === "stack" || pr.others > 0 ? pr.label : `#${pr.label}`} + {pr.label} ) : null} diff --git a/apps/mobile/src/features/threads/threadListV2.ts b/apps/mobile/src/features/threads/threadListV2.ts index 9a876c899ac4..8c4dd0b0f097 100644 --- a/apps/mobile/src/features/threads/threadListV2.ts +++ b/apps/mobile/src/features/threads/threadListV2.ts @@ -14,6 +14,7 @@ import { sortActiveThreadsByOrderKey, resolveSettledThreadTimestamp, sortPinnedThreadsByOrderKey, + sortSettledThreads, } from "@t3tools/client-runtime/state/thread-sort"; import type { EnvironmentId, ProjectId } from "@t3tools/contracts"; @@ -606,11 +607,7 @@ export function buildThreadListV2Items(input: { : orderedSnoozed.filter( (thread) => `${thread.environmentId}:${thread.id}` === selectedThreadKey, ); - const orderedSettled = [...settled].sort( - (left, right) => - parseTimestampMs(resolveSettledThreadTimestamp(right) ?? "") - - parseTimestampMs(resolveSettledThreadTimestamp(left) ?? ""), - ); + const orderedSettled = sortSettledThreads(settled); const settledLimit = input.settledLimit ?? Number.POSITIVE_INFINITY; const pagedSettled = orderedSettled.length > settledLimit ? orderedSettled.slice(0, settledLimit) : orderedSettled; diff --git a/apps/mobile/src/features/usage/UsageRouteScreen.tsx b/apps/mobile/src/features/usage/UsageRouteScreen.tsx index 2d5e5398bc51..4582c3135690 100644 --- a/apps/mobile/src/features/usage/UsageRouteScreen.tsx +++ b/apps/mobile/src/features/usage/UsageRouteScreen.tsx @@ -1,6 +1,7 @@ import { ScreenScrollView as ScrollView } from "../../components/ScreenScrollView"; import { EnvironmentId, USAGE_CONTRACT_VERSION } from "@t3tools/contracts"; import { type RouteProp, useIsFocused, useNavigation, useRoute } from "@react-navigation/native"; +import { cursorKeychainAccessEnvironments } from "@t3tools/client-runtime/state/usage"; import { isCompatibleUsageContractVersion, isModelCostUnknown, @@ -15,6 +16,7 @@ import { formatHourShort, formatPercent, formatTokens, + formatUsageContractMismatch, formatUsd, makeWindow, } from "@t3tools/shared/usageFormat"; @@ -101,9 +103,7 @@ export function UsageRouteScreen() { ); const isFocused = useIsFocused(); const limits = useRefreshLimits(selectedEnvironmentIds, isFocused && tab === "limits"); - const cursorAccessEnvironments = selectedEnvironments.filter((environment) => - environment.summary?.sources.some((source) => source.action === "enableCursorKeychain"), - ); + const cursorAccessEnvironments = cursorKeychainAccessEnvironments(selectedEnvironments); const refreshAfterCursorEnable = () => { void refresh(); void limits.refreshAfterEnable(); @@ -753,7 +753,12 @@ function usageEnvironmentStatus(environment: EnvironmentUsageStatus): string { environment.summary && !isCompatibleUsageContractVersion(environment.summary.contractVersion, USAGE_CONTRACT_VERSION) ) { - return "Older server · excluded from usage totals"; + return formatUsageContractMismatch(environment.label, { + direction: + environment.summary.contractVersion < USAGE_CONTRACT_VERSION + ? "serverBehind" + : "clientBehind", + }); } if (!environment.isConnected) return environment.summary ? "Disconnected · showing saved usage" : "Waiting for connection…"; diff --git a/apps/mobile/src/state/usage.ts b/apps/mobile/src/state/usage.ts index d49c26a40a44..c5895d353fef 100644 --- a/apps/mobile/src/state/usage.ts +++ b/apps/mobile/src/state/usage.ts @@ -16,7 +16,7 @@ import { type UsageSummary, type UsageSummaryInput, } from "@t3tools/contracts"; -import { refreshUsage } from "@t3tools/client-runtime/state/usage"; +import { needsCursorKeychainAccess, refreshUsage } from "@t3tools/client-runtime/state/usage"; import { mergeUsage, type EnvironmentUsage, type MergedUsage } from "@t3tools/shared/usageMerge"; import * as Option from "effect/Option"; import { AsyncResult, Atom } from "effect/unstable/reactivity"; @@ -33,6 +33,7 @@ export interface EnvironmentUsageStatus { readonly isConnected: boolean; readonly error: string | null; readonly summary: UsageSummary | null; + readonly needsCursorKeychainAccess: boolean; } /** @@ -50,13 +51,18 @@ const usageByWindowAtom = Atom.family((windowKey: string) => const statuses: EnvironmentUsageStatus[] = []; for (const [environmentId, presentation] of presentations) { const result = get(serverEnvironment.usageSummary({ environmentId, input })); + const summary = Option.getOrNull(AsyncResult.value(result)); statuses.push({ environmentId, label: presentation.entry.target.label, isPending: result.waiting, isConnected: presentation.connection.phase === "connected", error: result._tag === "Failure" ? "This environment could not report usage." : null, - summary: Option.getOrNull(AsyncResult.value(result)), + summary, + needsCursorKeychainAccess: needsCursorKeychainAccess( + summary, + get(serverEnvironment.providersValueAtom(environmentId)), + ), }); } return statuses; diff --git a/apps/server/integration/OrchestrationEngineHarness.integration.ts b/apps/server/integration/OrchestrationEngineHarness.integration.ts index 1e116a91fea7..5e3b4c6e4afe 100644 --- a/apps/server/integration/OrchestrationEngineHarness.integration.ts +++ b/apps/server/integration/OrchestrationEngineHarness.integration.ts @@ -420,6 +420,7 @@ export const makeOrchestrationIntegrationHarness = ( Layer.provideMerge( Layer.succeed(AgentAwarenessRelay.AgentAwarenessRelay, { publishThread: () => Effect.void, + requestCatchUp: () => Effect.void, start: () => Effect.void, }), ), diff --git a/apps/server/package.json b/apps/server/package.json index afa2ca18fdfc..105e4d0d96c3 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -31,7 +31,7 @@ "@opencode-ai/sdk": "^1.3.15", "diff": "8.0.3", "effect": "catalog:", - "node-pty": "^1.1.0", + "node-pty": "^1.2.0-beta.15", "stream-chain": "^4.2.5", "stream-json": "3.6.0", "yaml": "catalog:", diff --git a/apps/server/src/auth/dpop.ts b/apps/server/src/auth/dpop.ts index a841f4fd444c..5e026f584273 100644 --- a/apps/server/src/auth/dpop.ts +++ b/apps/server/src/auth/dpop.ts @@ -18,6 +18,9 @@ import { } from "./EnvironmentAuth.ts"; import * as ServerSecretStore from "./ServerSecretStore.ts"; +/** Secret store name prefix of DPoP replay markers. The server prunes expired ones. */ +export const DPOP_REPLAY_MARKER_PREFIX = "dpop-proof-"; + export const mapDpopFailureReason = (code: DpopVerificationFailureCodeType): DpopFailureReason => { switch (code) { case "time_window": @@ -96,7 +99,7 @@ export const verifyRequestDpopProof = (input: { ); yield* secretStore .create( - `dpop-proof-${replayKey}`, + `${DPOP_REPLAY_MARKER_PREFIX}${replayKey}`, new TextEncoder().encode( [ `thumbprint=${result.thumbprint}`, diff --git a/apps/server/src/auth/replayMarkers.test.ts b/apps/server/src/auth/replayMarkers.test.ts new file mode 100644 index 000000000000..76c31b0162bd --- /dev/null +++ b/apps/server/src/auth/replayMarkers.test.ts @@ -0,0 +1,92 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as TestClock from "effect/testing/TestClock"; + +import * as ServerConfig from "../config.ts"; +import { pruneExpiredReplayMarkers, REPLAY_MARKER_MAX_AGE } from "./replayMarkers.ts"; +import * as ServerSecretStore from "./ServerSecretStore.ts"; + +// Every secret name the server stores today. The last three stand for names +// built from an id at runtime. +const REAL_SECRET_NAMES = [ + "server-signing-key", + "asset-access-signing-key", + "cloud-cli-oauth-token", + "cloud-cli-desired-link", + "cloud-link-ed25519-key-pair", + "cloud-link-ed25519-private-key", + "cloud-link-ed25519-public-key", + "cloud-mint-ed25519-public-key", + "cloud-endpoint-runtime-config", + "cloud-endpoint-confirmed-origin", + "cloud-linked-user-id", + "cloud-relay-url", + "cloud-relay-issuer", + "cloud-relay-environment-credential", + "cloud-publish-agent-activity", + "provider-env-Y29kZXg-T1BFTkFJX0FQSV9LRVk", + "provider-auth-0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0", + "usage-limit-source-aHVi", +]; + +it.layer(NodeServices.layer)("replayMarkers", (it) => { + it.effect("prunes only replay markers older than the max age", () => + Effect.gen(function* () { + const secretStore = yield* ServerSecretStore.ServerSecretStore; + const { secretsDir } = yield* ServerConfig.ServerConfig; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const now = DateTime.makeUnsafe("2026-01-01T00:00:00Z"); + const setAge = (fileName: string, age: Duration.Duration) => { + const mtime = DateTime.toDateUtc(DateTime.subtractDuration(now, age)); + return fileSystem.utimes(path.join(secretsDir, fileName), mtime, mtime); + }; + const writeAged = (name: string, age: Duration.Duration) => + secretStore + .create(name, Uint8Array.from([1])) + .pipe(Effect.andThen(setAge(`${name}.bin`, age))); + + const justExpired = Duration.sum(REPLAY_MARKER_MAX_AGE, Duration.seconds(1)); + const expiredMarkers = [ + "dpop-proof-old", + "cloud-mint-jti-old", + "cloud-mint-nonce-old", + "cloud-health-jti-old", + "cloud-health-nonce-old", + ]; + for (const name of expiredMarkers) yield* writeAged(name, justExpired); + yield* writeAged("dpop-proof-at-max-age", REPLAY_MARKER_MAX_AGE); + for (const name of REAL_SECRET_NAMES) yield* writeAged(name, Duration.days(30)); + const pendingSetFile = "dpop-proof-pending.bin.0000.tmp"; + yield* fileSystem.writeFile(path.join(secretsDir, pendingSetFile), Uint8Array.from([1])); + yield* setAge(pendingSetFile, Duration.days(30)); + yield* TestClock.setTime(DateTime.toEpochMillis(now)); + + yield* pruneExpiredReplayMarkers(); + + const remaining = yield* fileSystem.readDirectory(secretsDir); + assert.deepStrictEqual( + remaining.toSorted(), + [ + ...REAL_SECRET_NAMES.map((name) => `${name}.bin`), + "dpop-proof-at-max-age.bin", + pendingSetFile, + ].toSorted(), + ); + }).pipe( + Effect.provide( + ServerSecretStore.layer.pipe( + Layer.provideMerge( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-replay-markers-test-" }), + ), + ), + ), + ), + ); +}); diff --git a/apps/server/src/auth/replayMarkers.ts b/apps/server/src/auth/replayMarkers.ts new file mode 100644 index 000000000000..67b56b722334 --- /dev/null +++ b/apps/server/src/auth/replayMarkers.ts @@ -0,0 +1,77 @@ +import * as Clock from "effect/Clock"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schedule from "effect/Schedule"; + +import { CLOUD_REPLAY_MARKER_PREFIXES } from "../cloud/http.ts"; +import * as ServerConfig from "../config.ts"; +import { forkParked } from "../serverActivation.ts"; +import { DPOP_REPLAY_MARKER_PREFIX } from "./dpop.ts"; + +const REPLAY_MARKER_PREFIXES = [DPOP_REPLAY_MARKER_PREFIX, ...CLOUD_REPLAY_MARKER_PREFIXES]; + +/** + * How long a replay marker stays on disk. A marker only matters while its proof + * can pass the time check (about 5 minutes for DPoP, 7 for cloud proofs). After + * that, the time check rejects a replay by itself. The sweep and the time check + * both use the wall clock, so a pruned marker can let a replay through only if + * the clock moves back by almost a day, or if the filesystem stamps mtimes almost + * a day behind. Markers are files, so a restart does not reset them. + */ +export const REPLAY_MARKER_MAX_AGE = Duration.days(1); + +/** + * Deletes replay markers whose mtime is older than `REPLAY_MARKER_MAX_AGE`. + * `ServerSecretStore` saves each secret as `.bin`, so only + * `*.bin` names match. Other secrets and the `*.bin..tmp` + * files that `set` writes are never touched. + */ +export const pruneExpiredReplayMarkers = Effect.fn("replayMarkers.pruneExpired")(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { secretsDir } = yield* ServerConfig.ServerConfig; + const cutoff = (yield* Clock.currentTimeMillis) - Duration.toMillis(REPLAY_MARKER_MAX_AGE); + const markers = (yield* fileSystem.readDirectory(secretsDir)).filter( + (name) => + name.endsWith(".bin") && REPLAY_MARKER_PREFIXES.some((prefix) => name.startsWith(prefix)), + ); + // `partition` visits every marker, so one locked file does not stop the sweep. + const [failures, removed] = yield* Effect.partition(markers, (name) => { + const markerPath = path.join(secretsDir, name); + return fileSystem.stat(markerPath).pipe( + Effect.flatMap((info) => + Option.exists(info.mtime, (mtime) => mtime.getTime() < cutoff) + ? fileSystem.remove(markerPath).pipe(Effect.as(true)) + : Effect.succeed(false), + ), + Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed(false)), + ); + }); + yield* Effect.annotateCurrentSpan({ + "replay_markers.matched": markers.length, + "replay_markers.removed": removed.filter(Boolean).length, + "replay_markers.failed": failures.length, + }); + if (failures.length > 0) { + yield* Effect.logWarning("Failed to prune some replay markers", { + failed: failures.length, + cause: failures[0], + }); + } +}); + +/** Prunes expired replay markers after server activation, then every hour. */ +export const layer = Layer.effectDiscard( + forkParked( + pruneExpiredReplayMarkers().pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to prune expired replay markers", { cause }), + ), + Effect.repeat(Schedule.spaced(Duration.hours(1))), + ), + ), +); diff --git a/apps/server/src/bin.ts b/apps/server/src/bin.ts index 0538b94fcde9..e30870aac873 100644 --- a/apps/server/src/bin.ts +++ b/apps/server/src/bin.ts @@ -24,6 +24,7 @@ import { serviceLauncherCommand } from "./cli/serviceLauncher.ts"; import { servicePreflightCommand } from "./cli/servicePreflight.ts"; import { sshHelperCommand } from "./cli/sshHelper.ts"; import { themeCommand } from "./cli/theme.ts"; +import { traceCommand } from "./cli/trace.ts"; import { triageCommand } from "./cli/triage.ts"; const CliRuntimeLayer = Layer.mergeAll(NodeServices.layer, NetService.layer); @@ -71,6 +72,7 @@ export const makeCli = ({ cloudEnabled = hasCloudPublicConfig } = {}) => servicePreflightCommand, sshHelperCommand, themeCommand, + traceCommand, triageCommand, cloudEnabled ? connectCommand : connectUnavailableCommand, ]), diff --git a/apps/server/src/checkpointing/CheckpointDiffQuery.test.ts b/apps/server/src/checkpointing/CheckpointDiffQuery.test.ts index 7fa6065109c3..6df7d7534130 100644 --- a/apps/server/src/checkpointing/CheckpointDiffQuery.test.ts +++ b/apps/server/src/checkpointing/CheckpointDiffQuery.test.ts @@ -84,6 +84,7 @@ describe("CheckpointDiffQuery.layer", () => { getShellSnapshot: () => Effect.die("CheckpointDiffQuery should not request the orchestration shell snapshot"), getDeletedWorktreeThreads: () => Effect.die("unused"), + listThreadsWithPullRequests: () => Effect.die("unused"), getArchivedShellSnapshot: () => Effect.die("CheckpointDiffQuery should not request archived shell snapshots"), getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }), @@ -201,6 +202,7 @@ describe("CheckpointDiffQuery.layer", () => { getShellSnapshot: () => Effect.die("CheckpointDiffQuery should not request the orchestration shell snapshot"), getDeletedWorktreeThreads: () => Effect.die("unused"), + listThreadsWithPullRequests: () => Effect.die("unused"), getArchivedShellSnapshot: () => Effect.die("CheckpointDiffQuery should not request archived shell snapshots"), getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }), @@ -293,6 +295,7 @@ describe("CheckpointDiffQuery.layer", () => { getShellSnapshot: () => Effect.die("CheckpointDiffQuery should not request the orchestration shell snapshot"), getDeletedWorktreeThreads: () => Effect.die("unused"), + listThreadsWithPullRequests: () => Effect.die("unused"), getArchivedShellSnapshot: () => Effect.die("CheckpointDiffQuery should not request archived shell snapshots"), getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }), @@ -370,6 +373,7 @@ describe("CheckpointDiffQuery.layer", () => { getShellSnapshot: () => Effect.die("CheckpointDiffQuery should not request the orchestration shell snapshot"), getDeletedWorktreeThreads: () => Effect.die("unused"), + listThreadsWithPullRequests: () => Effect.die("unused"), getArchivedShellSnapshot: () => Effect.die("CheckpointDiffQuery should not request archived shell snapshots"), getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }), @@ -432,6 +436,7 @@ describe("CheckpointDiffQuery.layer", () => { getShellSnapshot: () => Effect.die("CheckpointDiffQuery should not request the orchestration shell snapshot"), getDeletedWorktreeThreads: () => Effect.die("unused"), + listThreadsWithPullRequests: () => Effect.die("unused"), getArchivedShellSnapshot: () => Effect.die("CheckpointDiffQuery should not request archived shell snapshots"), getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }), diff --git a/apps/server/src/cli/config.test.ts b/apps/server/src/cli/config.test.ts index ef7cdd578899..42932b927008 100644 --- a/apps/server/src/cli/config.test.ts +++ b/apps/server/src/cli/config.test.ts @@ -508,7 +508,6 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => { resolved.terminalLogsDir, resolved.attachmentsDir, resolved.worktreesDir, - path.dirname(resolved.serverLogPath), path.dirname(resolved.serverTracePath), ]) { expect(yield* fs.exists(directory)).toBe(true); diff --git a/apps/server/src/cli/config.ts b/apps/server/src/cli/config.ts index d724730c953f..62461e286748 100644 --- a/apps/server/src/cli/config.ts +++ b/apps/server/src/cli/config.ts @@ -82,16 +82,22 @@ const tailscaleServePortFlag = Flag.Int("tailscale-serve-port").pipe( Flag.optional, ); +// Trace file location, shared by the server and `t3 trace summary`. +export const traceFileConfig = Config.String("T3CODE_TRACE_FILE").pipe( + Config.option, + Config.map(Option.getOrUndefined), +); +export const traceMaxFilesConfig = Config.Int("T3CODE_TRACE_MAX_FILES").pipe( + Config.withDefault(10), +); + const EnvServerConfig = Config.all({ logLevel: Config.LogLevel("T3CODE_LOG_LEVEL").pipe(Config.withDefault("Info")), traceMinLevel: Config.LogLevel("T3CODE_TRACE_MIN_LEVEL").pipe(Config.withDefault("Info")), traceTimingEnabled: Config.Boolean("T3CODE_TRACE_TIMING_ENABLED").pipe(Config.withDefault(true)), - traceFile: Config.String("T3CODE_TRACE_FILE").pipe( - Config.option, - Config.map(Option.getOrUndefined), - ), + traceFile: traceFileConfig, traceMaxBytes: Config.Int("T3CODE_TRACE_MAX_BYTES").pipe(Config.withDefault(10 * 1024 * 1024)), - traceMaxFiles: Config.Int("T3CODE_TRACE_MAX_FILES").pipe(Config.withDefault(10)), + traceMaxFiles: traceMaxFilesConfig, traceBatchWindowMs: Config.Int("T3CODE_TRACE_BATCH_WINDOW_MS").pipe(Config.withDefault(1_000)), otlpTracesUrl: Config.String("T3CODE_OTLP_TRACES_URL").pipe( Config.option, diff --git a/apps/server/src/cli/trace.test.ts b/apps/server/src/cli/trace.test.ts new file mode 100644 index 000000000000..b81a3ed53f25 --- /dev/null +++ b/apps/server/src/cli/trace.test.ts @@ -0,0 +1,100 @@ +import { assert, it } from "@effect/vitest"; + +import { makeTraceSpanSummary } from "./trace.ts"; + +const MINUTE_MS = 60_000; + +function span(name: string, durationMs: number, endMs: number, exitTag = "Success") { + return JSON.stringify({ + type: "effect-span", + name, + traceId: "trace", + spanId: "span", + durationMs, + endTimeUnixNano: String(BigInt(endMs) * 1_000_000n), + exit: { _tag: exitTag, cause: "cause" }, + }); +} + +function browserSpan(name: string, status: { code: string; message?: string }) { + return JSON.stringify({ + type: "otlp-span", + name, + durationMs: 1, + endTimeUnixNano: "1000000", + status, + }); +} + +it("reports count, rate, percentiles, and exits per span name", () => { + // Ten `refresh` spans of 1..10 ms end over minutes 0..9, and one `probe` + // span ends at minute 10, so the recorded window is 10 minutes. + const refreshes = Array.from({ length: 10 }, (_, index) => + span( + "refresh", + index + 1, + index * MINUTE_MS, + index === 0 ? "Interrupted" : index === 1 ? "Failure" : "Success", + ), + ); + const summarizer = makeTraceSpanSummary(); + [...refreshes, span("probe", 2_500, 10 * MINUTE_MS)].forEach(summarizer.addLine); + const summary = summarizer.finish(); + + assert.strictEqual(summary.spanCount, 11); + assert.strictEqual(summary.minutes, 10); + assert.deepStrictEqual(summary.spans, [ + { + name: "refresh", + count: 10, + perMinute: 1, + p50Ms: 5, + p90Ms: 9, + maxMs: 10, + interrupted: 1, + failures: 1, + }, + { + name: "probe", + count: 1, + perMinute: 0.1, + p50Ms: 2_500, + p90Ms: 2_500, + maxMs: 2_500, + interrupted: 0, + failures: 0, + }, + ]); +}); + +it("drops spans that ended before the window and counts unreadable lines", () => { + const summarizer = makeTraceSpanSummary(MINUTE_MS); + [ + span("old", 1, 0), + "", + "{not json", + JSON.stringify({ name: "no-duration" }), + // Ends past the largest Date, so the report could not print it. + JSON.stringify({ name: "far-future", durationMs: 1, endTimeUnixNano: "9".repeat(22) }), + span("recent", 4, 5 * MINUTE_MS), + ].forEach(summarizer.addLine); + const summary = summarizer.finish(); + + assert.strictEqual(summary.skippedLineCount, 3); + assert.deepStrictEqual( + summary.spans.map((entry) => [entry.name, entry.count, entry.perMinute]), + [["recent", 1, undefined]], + ); +}); + +it("reads failures and interrupts of browser spans from their OTLP status", () => { + const summarizer = makeTraceSpanSummary(); + [ + browserSpan("render", { code: "2", message: "boom" }), + browserSpan("render", { code: "1", message: "Interrupted" }), + browserSpan("render", { code: "1" }), + ].forEach(summarizer.addLine); + + const [render] = summarizer.finish().spans; + assert.deepStrictEqual([render?.count, render?.interrupted, render?.failures], [3, 1, 1]); +}); diff --git a/apps/server/src/cli/trace.ts b/apps/server/src/cli/trace.ts new file mode 100644 index 000000000000..61a209471d6e --- /dev/null +++ b/apps/server/src/cli/trace.ts @@ -0,0 +1,208 @@ +/** + * `t3 trace summary` - per-span counts, rates, and latency percentiles from + * the local server trace file and its rotated backups. It reads the files + * directly, so it works while the server is stalled or stopped. + */ +import { PositiveInt } from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Config from "effect/Config"; +import * as Console from "effect/Console"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import { Command, Flag } from "effect/unstable/cli"; + +import * as ServerConfig from "../config.ts"; +import { streamTraceFileLines, toRotatedTracePaths } from "../diagnostics/TraceDiagnostics.ts"; +import { resolveBaseDir } from "../os-jank.ts"; +import { baseDirFlag, DurationFromString, traceFileConfig, traceMaxFilesConfig } from "./config.ts"; + +// Only the fields the summary needs. Other record fields are ignored. +const decodeTraceSpanLine = Schema.decodeUnknownOption( + Schema.fromJsonString( + Schema.Struct({ + name: Schema.String, + durationMs: Schema.Finite, + endTimeUnixNano: Schema.FiniteFromString, + // Server (`effect-span`) records. + exit: Schema.optional(Schema.Struct({ _tag: Schema.String })), + // Browser (`otlp-span`) records. Effect's OTLP tracer writes code "2" for + // errors and code "1" with message "Interrupted" for interrupts. + status: Schema.optional( + Schema.Struct({ + code: Schema.optional(Schema.String), + message: Schema.optional(Schema.String), + }), + ), + }), + ), +); + +/** + * Groups trace NDJSON by span name. Call `addLine` once per line as the files + * stream in, then `finish` for the summary. Spans that ended before `sinceMs` + * are left out. Rates are per minute between the first and last span end, + * since spans are written when they end. + */ +export function makeTraceSpanSummary(sinceMs = -Infinity) { + // Keep each span's duration (8 bytes, a few MB for the default 110 MB of + // rotated traces) for exact percentiles. A bounded sketch would save little + // and make p50 and p90 approximate. + const byName = new Map(); + let spanCount = 0; + let skippedLineCount = 0; + let firstEndMs = Infinity; + let lastEndMs = -Infinity; + + const addLine = (line: string) => { + if (line.trim().length === 0) return; + const span = Option.getOrUndefined(decodeTraceSpanLine(line)); + if (span === undefined) { + skippedLineCount += 1; + return; + } + const endMs = span.endTimeUnixNano / 1_000_000; + // The report prints end times as dates, so skip ones outside the Date range. + if (Option.isNone(DateTime.make(endMs))) { + skippedLineCount += 1; + return; + } + if (endMs < sinceMs) return; + + spanCount += 1; + firstEndMs = Math.min(firstEndMs, endMs); + lastEndMs = Math.max(lastEndMs, endMs); + const stats = byName.get(span.name) ?? { durations: [], interrupted: 0, failures: 0 }; + stats.durations.push(span.durationMs); + if (span.exit?._tag === "Interrupted" || span.status?.message === "Interrupted") { + stats.interrupted += 1; + } + if (span.exit?._tag === "Failure" || span.status?.code === "2") stats.failures += 1; + byName.set(span.name, stats); + }; + + const finish = () => { + const minutes = (lastEndMs - firstEndMs) / 60_000; + const spans = [...byName] + .map(([name, { durations, interrupted, failures }]) => { + const sorted = durations.toSorted((left, right) => left - right); + // Nearest-rank percentile. + const percentile = (p: number) => sorted[Math.ceil(p * sorted.length) - 1]!; + return { + name, + count: sorted.length, + perMinute: minutes > 0 ? sorted.length / minutes : undefined, + p50Ms: percentile(0.5), + p90Ms: percentile(0.9), + maxMs: sorted[sorted.length - 1]!, + interrupted, + failures, + }; + }) + .toSorted((left, right) => right.count - left.count || left.name.localeCompare(right.name)); + + return { spanCount, skippedLineCount, firstEndMs, lastEndMs, minutes, spans }; + }; + + return { addLine, finish }; +} + +const formatMs = (ms: number) => + ms < 1_000 ? `${Math.round(ms)}ms` : `${(ms / 1_000).toFixed(1)}s`; + +function formatTraceSummary( + summary: ReturnType["finish"]>, + limit: number, +) { + const header = ["span", "count", "/min", "p50", "p90", "max", "interrupted", "failed"]; + const rows = summary.spans + .slice(0, limit) + .map((span) => [ + span.name, + String(span.count), + span.perMinute === undefined + ? "-" + : span.perMinute < 0.1 + ? "<0.1" + : span.perMinute.toFixed(1), + formatMs(span.p50Ms), + formatMs(span.p90Ms), + formatMs(span.maxMs), + String(span.interrupted), + String(span.failures), + ]); + const table = [header, ...rows]; + const widths = header.map((_, column) => Math.max(...table.map((row) => row[column]!.length))); + const formatIso = (ms: number) => DateTime.formatIso(DateTime.makeUnsafe(ms)); + return [ + `${summary.spanCount} spans ended from ${formatIso(summary.firstEndMs)} to ${formatIso(summary.lastEndMs)} (${summary.minutes.toFixed(1)} min).`, + ...(summary.skippedLineCount > 0 + ? [`Skipped ${summary.skippedLineCount} lines that are not spans.`] + : []), + "", + ...table.map((row) => + row + .map((cell, column) => + column === 0 ? cell.padEnd(widths[column]!) : cell.padStart(widths[column]!), + ) + .join(" "), + ), + ...(summary.spans.length > limit + ? ["", `${summary.spans.length - limit} more span names. Use --limit to show more.`] + : []), + ].join("\n"); +} + +const traceSummaryCommand = Command.make("summary", { + baseDir: baseDirFlag, + since: Flag.String("since").pipe( + Flag.withSchema(DurationFromString), + Flag.withDescription("Only count spans that ended in this window, for example 30m or 2h."), + Flag.optional, + ), + limit: Flag.Int("limit").pipe( + Flag.withSchema(PositiveInt), + Flag.withDescription("Number of span names to show, busiest first."), + Flag.withDefault(25), + ), +}).pipe( + Command.withDescription("Summarize the local server trace file: counts, rates, and latency."), + Command.withHandler( + Effect.fn("cli.trace.summary")(function* (flags) { + const fs = yield* FileSystem.FileSystem; + // T3CODE_TRACE_FILE, else the userdata trace file for --base-dir or + // T3CODE_HOME. Implicit dev runs write elsewhere; set T3CODE_TRACE_FILE. + const envHome = yield* Config.String("T3CODE_HOME").pipe(Config.option); + const baseDir = yield* resolveBaseDir( + Option.getOrUndefined(Option.orElse(flags.baseDir, () => envHome)), + ); + const traceFilePath = + (yield* traceFileConfig) ?? + (yield* ServerConfig.deriveServerPaths(baseDir, undefined)).serverTracePath; + const sinceMs = Option.isSome(flags.since) + ? (yield* Clock.currentTimeMillis) - Duration.toMillis(flags.since.value) + : undefined; + const summarizer = makeTraceSpanSummary(sinceMs); + yield* Effect.forEach( + toRotatedTracePaths(traceFilePath, yield* traceMaxFilesConfig), + (path) => streamTraceFileLines(fs, path, summarizer.addLine), + { discard: true }, + ); + const summary = summarizer.finish(); + + yield* Console.log( + summary.spanCount === 0 + ? `No spans found in ${traceFilePath} or its rotated files${sinceMs === undefined ? "" : " in that window"}.${summary.skippedLineCount > 0 ? ` Skipped ${summary.skippedLineCount} lines that are not spans.` : ""}` + : formatTraceSummary(summary, flags.limit), + ); + }), + ), +); + +export const traceCommand = Command.make("trace").pipe( + Command.withDescription("Inspect the local server trace file."), + Command.withSubcommands([traceSummaryCommand]), +); diff --git a/apps/server/src/cli/triage.ts b/apps/server/src/cli/triage.ts index c621d331c404..b408e3550112 100644 --- a/apps/server/src/cli/triage.ts +++ b/apps/server/src/cli/triage.ts @@ -27,6 +27,7 @@ import * as Schema from "effect/Schema"; import { Command, Flag } from "effect/unstable/cli"; import packageJson from "../../package.json" with { type: "json" }; +import * as BootService from "../cloud/bootService.ts"; import * as ServerConfig from "../config.ts"; import { resolveBaseDir } from "../os-jank.ts"; import { isProcessAlive, readPersistedServerRuntimeState } from "../serverRuntimeState.ts"; @@ -201,7 +202,11 @@ export const triageCommand = Command.make("triage", { dbPath: paths.dbPath, settingsPath: paths.settingsPath, logsDir: paths.logsDir, - serverLogPath: paths.serverLogPath, + // The server writes no log file of its own. Service installs and the + // desktop app capture its output. The glob covers every desktop backend + // (such as WSL) and rotated copies; names come from DesktopObservability.ts. + serviceLogPath: path.join(paths.logsDir, BootService.BOOT_SERVICE_LOG_FILE), + desktopBackendLogGlob: path.join(paths.logsDir, "server-child*.log*"), serverTracePath: paths.serverTracePath, providerEventLogPath: paths.providerEventLogPath, terminalLogsDir: paths.terminalLogsDir, diff --git a/apps/server/src/cli/triagePrompt.test.ts b/apps/server/src/cli/triagePrompt.test.ts index bf1ac5dbbe5e..fe65bf0dc444 100644 --- a/apps/server/src/cli/triagePrompt.test.ts +++ b/apps/server/src/cli/triagePrompt.test.ts @@ -52,7 +52,8 @@ it("context file carries every path the playbook depends on", () => { dbPath: "/home/u/.t3/userdata/state.sqlite", settingsPath: "/home/u/.t3/userdata/settings.json", logsDir: "/home/u/.t3/userdata/logs", - serverLogPath: "/home/u/.t3/userdata/logs/server.log", + serviceLogPath: "/home/u/.t3/userdata/logs/boot-service.log", + desktopBackendLogGlob: "/home/u/.t3/userdata/logs/server-child*.log*", serverTracePath: "/home/u/.t3/userdata/logs/server.trace.ndjson", providerEventLogPath: "/home/u/.t3/userdata/logs/provider/events.log", terminalLogsDir: "/home/u/.t3/userdata/logs/terminals", @@ -63,6 +64,8 @@ it("context file carries every path the playbook depends on", () => { }); assert.include(context, "/home/u/.t3/userdata/state.sqlite"); assert.include(context, "/home/u/.t3/userdata/logs/server.trace.ndjson"); + assert.include(context, "/home/u/.t3/userdata/logs/boot-service.log"); + assert.include(context, "/home/u/.t3/userdata/logs/server-child*.log*"); assert.include(context, "/home/u/.t3/userdata/logs/provider/events.log"); assert.include(context, "/home/u/.t3/userdata/secrets"); assert.include(context, "/home/u/.t3/source"); diff --git a/apps/server/src/cli/triagePrompt.ts b/apps/server/src/cli/triagePrompt.ts index c2b93a1840a1..1df712854263 100644 --- a/apps/server/src/cli/triagePrompt.ts +++ b/apps/server/src/cli/triagePrompt.ts @@ -71,8 +71,9 @@ different code depending on it: Then work from evidence, not assumption. In rough order of value: -- The server log and the trace file (\`server.trace.ndjson\`) around the time of the - problem. Recent failures usually leave a trail here. +- The trace file (\`server.trace.ndjson\`) around the time of the problem, plus the + service log or desktop backend logs from the context file if they exist. Recent + failures usually leave a trail here. - The provider event log, for problems with claude/codex/cursor sessions. - The SQLite database. Read it freely, but only write when a write is necessary to fix the problem the user described, and get their explicit permission @@ -176,7 +177,8 @@ export interface TriageContextInput { readonly dbPath: string; readonly settingsPath: string; readonly logsDir: string; - readonly serverLogPath: string; + readonly serviceLogPath: string; + readonly desktopBackendLogGlob: string; readonly serverTracePath: string; readonly providerEventLogPath: string; readonly terminalLogsDir: string; @@ -205,7 +207,8 @@ Generated by \`t3 triage\` at ${input.generatedAt}. - Database (SQLite; write only with the user's explicit permission): ${input.paths.dbPath} - Settings: ${input.paths.settingsPath} - Logs dir: ${input.paths.logsDir} -- Server log: ${input.paths.serverLogPath} +- Service log (systemd/launchd service installs only): ${input.paths.serviceLogPath} +- Desktop backend logs (glob; one file per backend plus rotated copies; written only when a backend crashes or fails to start): ${input.paths.desktopBackendLogGlob} - Server trace (ndjson): ${input.paths.serverTracePath} - Provider event log: ${input.paths.providerEventLogPath} - Terminal logs: ${input.paths.terminalLogsDir} diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 0a57ed822af4..fc8fc1549a6b 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -43,6 +43,8 @@ const BOOT_SERVICE_UNIT_FILE = `${BOOT_SERVICE_NAME}.service`; const BOOT_SERVICE_LAUNCHD_LABEL = "com.t3tools.t3code.service"; const BOOT_SERVICE_PLIST_FILE = `${BOOT_SERVICE_LAUNCHD_LABEL}.plist`; const BOOT_SERVICE_UNIT_ENV = "T3_BOOT_SERVICE_UNIT"; +/** File in the logs dir that receives the service's stdout and stderr. `t3 triage` points agents at it. */ +export const BOOT_SERVICE_LOG_FILE = "boot-service.log"; /** systemd expands `%` specifiers, including in unquoted append-log paths. */ function escapeSystemdSpecifiers(value: string): string { @@ -599,7 +601,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { environmentPath, }); const unitPath = detectedManager?.unitPath ?? ""; - const logPath = path.join(input.logsDir, "boot-service.log"); + const logPath = path.join(input.logsDir, BOOT_SERVICE_LOG_FILE); const statePath = path.join(input.baseDir, "runtime", SERVICE_STATE_FILE); const restartPendingPath = path.join(input.baseDir, "runtime", SERVICE_RESTART_PENDING_FILE); const runtimePaths = pinnedRuntimePaths(path, input.baseDir, input.cliVersion, platform); diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 64a512a33aba..1d6b78e56cef 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -33,6 +33,7 @@ import { type ServiceUpdateRecord, } from "./serviceProtocol.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts"; import * as CliTokenManager from "./CliTokenManager.ts"; import { @@ -81,6 +82,12 @@ const storeFailure = (tag: "AlreadyExists" | "PermissionDenied") => }); const unusedSecretStoreOperation = () => Effect.die("unused secret-store operation"); +// Linking wakes the awareness relay; these tests do not run it. +const idleAwarenessRelay = AgentAwarenessRelay.AgentAwarenessRelay.of({ + publishThread: () => Effect.void, + requestCatchUp: () => Effect.void, + start: () => Effect.void, +}); const decodeManagedTunnelRecoveryRegistration = Schema.decodeUnknownEffect( Schema.fromJsonString(RelayManagedEndpointRecoveryRegistrationRequest), ); @@ -262,6 +269,7 @@ describe("reconcileDesiredCloudLink", () => { HttpClient.HttpClient, HttpClient.make(() => unusedSecretStoreOperation()), ), + Effect.provideService(AgentAwarenessRelay.AgentAwarenessRelay, idleAwarenessRelay), Effect.provide(NodeServices.layer), ), ); @@ -375,6 +383,7 @@ describe("releaseManagedTunnelOnShutdown", () => { (effect: Effect.Effect) => effect.pipe( Effect.provideService(ServerSecretStore.ServerSecretStore, harness.store), + Effect.provideService(AgentAwarenessRelay.AgentAwarenessRelay, idleAwarenessRelay), Effect.provideService( ServerEnvironment.ServerEnvironment, ServerEnvironment.ServerEnvironment.of({ diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index b4c366a500c1..a7e3de8c7283 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -69,6 +69,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import { requireEnvironmentScope } from "../auth/http.ts"; import * as ServerConfig from "../config.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; import { SERVICE_STATE_FILE, @@ -104,6 +105,13 @@ const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-"; const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-"; const CLOUD_HEALTH_NONCE_PREFIX = "cloud-health-nonce-"; const CLOUD_HEALTH_JTI_PREFIX = "cloud-health-jti-"; +/** Secret store name prefixes of cloud replay markers. The server prunes expired ones. */ +export const CLOUD_REPLAY_MARKER_PREFIXES = [ + CLOUD_MINT_NONCE_PREFIX, + CLOUD_MINT_JTI_PREFIX, + CLOUD_HEALTH_NONCE_PREFIX, + CLOUD_HEALTH_JTI_PREFIX, +] as const; const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60; const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60; // The desktop app stops its backends within seconds of writing the marker. @@ -404,6 +412,7 @@ interface CloudHttpDependencies { readonly environmentAuth: EnvironmentAuth.EnvironmentAuth["Service"]; readonly cliTokenManager: CliTokenManager.CloudCliTokenManager["Service"]; readonly httpClient: HttpClient.HttpClient; + readonly awarenessRelay: AgentAwarenessRelay.AgentAwarenessRelay["Service"]; } const cloudHttpDependencies = Effect.gen(function* () { @@ -414,6 +423,7 @@ const cloudHttpDependencies = Effect.gen(function* () { environmentAuth: yield* EnvironmentAuth.EnvironmentAuth, cliTokenManager: yield* CliTokenManager.CloudCliTokenManager, httpClient: yield* HttpClient.HttpClient, + awarenessRelay: yield* AgentAwarenessRelay.AgentAwarenessRelay, } satisfies CloudHttpDependencies; }); @@ -661,6 +671,7 @@ const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(fu CLOUD_MINT_PUBLIC_KEY, stringToBytes(payload.cloudMintPublicKey), ); + yield* dependencies.awarenessRelay.requestCatchUp(); if (payload.endpointRuntime) { const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime); yield* dependencies.secrets.set( @@ -1342,6 +1353,7 @@ const cloudPreferencesHandler = Effect.fn("environment.cloud.preferences")( PUBLISH_AGENT_ACTIVITY_SECRET, stringToBytes(String(payload.publishAgentActivity)), ); + yield* dependencies.awarenessRelay.requestCatchUp(); return yield* readCloudLinkState(dependencies); }, Effect.catchIf( diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts index 74ffde8efef0..d8dad5ae4d24 100644 --- a/apps/server/src/config.ts +++ b/apps/server/src/config.ts @@ -44,7 +44,6 @@ export interface ServerDerivedPaths { /** Screenshots the agent asks the collaborative browser to keep for the user. */ readonly browserArtifactsDir: string; readonly logsDir: string; - readonly serverLogPath: string; readonly serverTracePath: string; readonly providerLogsDir: string; readonly providerEventLogPath: string; @@ -154,7 +153,6 @@ export const deriveServerPaths = Effect.fn(function* ( attachmentsDir, browserArtifactsDir: join(stateDir, "browser-artifacts"), logsDir, - serverLogPath: join(logsDir, "server.log"), serverTracePath: join(logsDir, "server.trace.ndjson"), providerLogsDir, providerEventLogPath: join(providerLogsDir, "events.log"), diff --git a/apps/server/src/diagnostics/TraceDiagnostics.test.ts b/apps/server/src/diagnostics/TraceDiagnostics.test.ts index 70bb4dc815c3..3b86841b99ba 100644 --- a/apps/server/src/diagnostics/TraceDiagnostics.test.ts +++ b/apps/server/src/diagnostics/TraceDiagnostics.test.ts @@ -7,6 +7,7 @@ import * as Logger from "effect/Logger"; import * as Option from "effect/Option"; import * as PlatformError from "effect/PlatformError"; import * as References from "effect/References"; +import * as Stream from "effect/Stream"; import * as TraceDiagnostics from "./TraceDiagnostics.ts"; @@ -40,72 +41,78 @@ function record(input: { }); } +const traceFilePath = "/tmp/server.trace.ndjson"; +const readAt = DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"); + +/** Aggregates whole lines in memory, the way diagnostics read traces before streaming. */ +function aggregateLines(lines: ReadonlyArray) { + const aggregator = TraceDiagnostics.makeTraceDiagnosticsAggregator(); + lines.forEach(aggregator.addLine); + return aggregator.finish({ + traceFilePath, + scannedFilePaths: TraceDiagnostics.toRotatedTracePaths(traceFilePath, 1), + readAt, + }); +} + +/** Reads the trace file and one rotated backup through a fake file system. */ +function readTraces(fileSystem: Partial) { + return TraceDiagnostics.readTraceDiagnostics({ traceFilePath, maxFiles: 1, readAt }).pipe( + Effect.provide(TraceDiagnostics.layer.pipe(Layer.provide(FileSystem.layerNoop(fileSystem)))), + ); +} + describe("TraceDiagnostics", () => { it.effect("aggregates failures, slow spans, log levels, and parse errors", () => Effect.sync(() => { - const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({ - traceFilePath: "/tmp/server.trace.ndjson", - readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"), - slowSpanThresholdMs: 1_000, - files: [ - { - path: "/tmp/server.trace.ndjson.1", - text: [ - record({ - name: "server.getConfig", - traceId: "trace-a", - spanId: "span-a", - startMs: 1_000, - durationMs: 50, - }), - "not-json", - ].join("\n"), - }, - { - path: "/tmp/server.trace.ndjson", - text: [ - record({ - name: "orchestration.dispatch", - traceId: "trace-b", - spanId: "span-b", - startMs: 2_000, - durationMs: 1_500, - exit: { _tag: "Failure", cause: "Provider crashed" }, - events: [ - { - name: "provider failed", - timeUnixNano: ns(3_400), - attributes: { "effect.logLevel": "Error" }, - }, - ], - }), - record({ - name: "orchestration.dispatch", - traceId: "trace-c", - spanId: "span-c", - startMs: 4_000, - durationMs: 250, - exit: { _tag: "Failure", cause: "Provider crashed" }, - }), - record({ - name: "git.status", - traceId: "trace-d", - spanId: "span-d", - startMs: 5_000, - durationMs: 25, - exit: { _tag: "Interrupted", cause: "Interrupted" }, - events: [ - { - name: "status delayed", - timeUnixNano: ns(5_010), - attributes: { "effect.logLevel": "Warning" }, - }, - ], - }), - ].join("\n"), - }, - ], - }); + const diagnostics = aggregateLines([ + record({ + name: "server.getConfig", + traceId: "trace-a", + spanId: "span-a", + startMs: 1_000, + durationMs: 50, + }), + "not-json", + record({ + name: "orchestration.dispatch", + traceId: "trace-b", + spanId: "span-b", + startMs: 2_000, + durationMs: 1_500, + exit: { _tag: "Failure", cause: "Provider crashed" }, + events: [ + { + name: "provider failed", + timeUnixNano: ns(3_400), + attributes: { "effect.logLevel": "Error" }, + }, + ], + }), + record({ + name: "orchestration.dispatch", + traceId: "trace-c", + spanId: "span-c", + startMs: 4_000, + durationMs: 250, + exit: { _tag: "Failure", cause: "Provider crashed" }, + }), + record({ + name: "git.status", + traceId: "trace-d", + spanId: "span-d", + startMs: 5_000, + durationMs: 25, + exit: { _tag: "Interrupted", cause: "Interrupted" }, + events: [ + { + name: "status delayed", + timeUnixNano: ns(5_010), + attributes: { "effect.logLevel": "Warning" }, + }, + ], + }), + ]); assert.equal(diagnostics.recordCount, 4); assert.equal(DateTime.formatIso(diagnostics.readAt), "2026-05-05T10:00:00.000Z"); @@ -139,47 +146,110 @@ describe("TraceDiagnostics", () => { ); it.effect("returns a not-found diagnostic when no files are available", () => - Effect.sync(() => { - const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({ - traceFilePath: "/tmp/missing.trace.ndjson", - readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"), - files: [], - }); + Effect.gen(function* () { + const diagnostics = yield* readTraces({}); assert.equal(diagnostics.recordCount, 0); assert.equal(Option.getOrUndefined(diagnostics.error)?.kind, "trace-file-not-found"); }), ); - it.effect("preserves full failure causes and log messages", () => - Effect.sync(() => { - const longCause = `VcsProcessSpawnError: ${"missing executable ".repeat(80)}`.trim(); - const longMessage = `provider warning: ${"retrying command ".repeat(80)}`.trim(); - const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({ - traceFilePath: "/tmp/server.trace.ndjson", - readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"), - files: [ - { - path: "/tmp/server.trace.ndjson", - text: record({ - name: "VcsProcess.run", - traceId: "trace-long", - spanId: "span-long", + it.effect("streams rotated files into the same result as reading them whole", () => + Effect.gen(function* () { + // CRLF and LF endings plus multi-byte text, served one byte per chunk so + // chunks split lines, line endings, and characters. + const files = new Map([ + [ + `${traceFilePath}.1`, + [ + record({ + name: "server.getConfig", + traceId: "trace-a", + spanId: "span-a", startMs: 1_000, + durationMs: 50, + }), + "not-json", + record({ + name: "orchestration.dispatch", + traceId: "trace-b", + spanId: "span-b", + startMs: 2_000, + durationMs: 1_500, + exit: { _tag: "Failure", cause: "Provider crashed: café 🔥" }, + }), + "", + ].join("\r\n"), + ], + [ + traceFilePath, + [ + record({ + name: "git.status", + traceId: "trace-c", + spanId: "span-c", + startMs: 3_000, durationMs: 25, - exit: { _tag: "Failure", cause: longCause }, + exit: { _tag: "Interrupted", cause: "Interrupted" }, events: [ { - name: longMessage, - timeUnixNano: ns(1_010), + name: "status delayed ⏳", + timeUnixNano: ns(3_010), attributes: { "effect.logLevel": "Warning" }, }, ], }), - }, + "", + record({ + name: "orchestration.dispatch", + traceId: "trace-d", + spanId: "span-d", + startMs: 4_000, + durationMs: 250, + exit: { _tag: "Failure", cause: "Provider crashed: café 🔥" }, + }), + ].join("\n"), ], + ]); + const encoder = new TextEncoder(); + + const diagnostics = yield* readTraces({ + stream: (path) => + Stream.fromIterable( + Array.from(encoder.encode(files.get(path)), (byte) => Uint8Array.of(byte)), + ), }); + assert.equal(diagnostics.recordCount, 4); + assert.deepStrictEqual( + diagnostics, + aggregateLines([...files.values()].flatMap((text) => text.split(/\r?\n/))), + ); + }), + ); + + it.effect("preserves full failure causes and log messages", () => + Effect.sync(() => { + const longCause = `VcsProcessSpawnError: ${"missing executable ".repeat(80)}`.trim(); + const longMessage = `provider warning: ${"retrying command ".repeat(80)}`.trim(); + const diagnostics = aggregateLines([ + record({ + name: "VcsProcess.run", + traceId: "trace-long", + spanId: "span-long", + startMs: 1_000, + durationMs: 25, + exit: { _tag: "Failure", cause: longCause }, + events: [ + { + name: longMessage, + timeUnixNano: ns(1_010), + attributes: { "effect.logLevel": "Warning" }, + }, + ], + }), + ]); + assert.equal(diagnostics.latestFailures[0]?.cause, longCause); assert.equal(diagnostics.commonFailures[0]?.cause, longCause); assert.equal(diagnostics.latestWarningAndErrorLogs[0]?.message, longMessage); @@ -188,45 +258,34 @@ describe("TraceDiagnostics", () => { it.effect("keeps loaded trace data when one rotated trace file fails to read", () => Effect.gen(function* () { - const traceFilePath = "/tmp/server.trace.ndjson"; const readFailure = PlatformError.systemError({ _tag: "PermissionDenied", module: "FileSystem", - method: "readFileString", + method: "open", description: "permission denied", pathOrDescriptor: `${traceFilePath}.1`, }); - const fileSystemLayer = FileSystem.layerNoop({ - readFileString: (path) => - path === `${traceFilePath}.1` - ? Effect.fail(readFailure) - : Effect.succeed( - record({ - name: "server.getConfig", - traceId: "trace-a", - spanId: "span-a", - startMs: 1_000, - durationMs: 50, - }), - ), - }); const logAnnotations: Array> = []; const logger = Logger.make((options) => { logAnnotations.push({ ...options.fiber.getRef(References.CurrentLogAnnotations) }); }); - const diagnostics = yield* TraceDiagnostics.readTraceDiagnostics({ - traceFilePath, - maxFiles: 1, - readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"), - }).pipe( - Effect.provide( - Layer.mergeAll( - TraceDiagnostics.layer.pipe(Layer.provide(fileSystemLayer)), - Logger.layer([logger], { mergeWithExisting: false }), - ), - ), - ); + const diagnostics = yield* readTraces({ + stream: (path) => + path === `${traceFilePath}.1` + ? Stream.fail(readFailure) + : Stream.make( + new TextEncoder().encode( + record({ + name: "server.getConfig", + traceId: "trace-a", + spanId: "span-a", + startMs: 1_000, + durationMs: 50, + }), + ), + ), + }).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false }))); assert.equal(diagnostics.recordCount, 1); assert.equal( @@ -251,32 +310,43 @@ describe("TraceDiagnostics", () => { }), ); - it.effect("keeps only the slowest span occurrences while aggregating large inputs", () => + it.effect("keeps only the top spans, failures, and warning logs from large inputs", () => Effect.sync(() => { - const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({ - traceFilePath: "/tmp/server.trace.ndjson", - readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"), - files: [ - { - path: "/tmp/server.trace.ndjson", - text: Array.from({ length: 25 }, (_, index) => - record({ - name: `span-${index}`, - traceId: `trace-${index}`, - spanId: `span-${index}`, - startMs: index * 1_000, - durationMs: index, - }), - ).join("\n"), - }, - ], - }); + // Shuffled, so some older records arrive after the lists are full. + const indexes = Array.from({ length: 30 }, (_, step) => (step * 7) % 30); + const diagnostics = aggregateLines( + indexes.map((index) => + record({ + name: `span-${index}`, + traceId: `trace-${index}`, + spanId: `span-${index}`, + startMs: index * 1_000, + durationMs: index, + exit: { _tag: "Failure", cause: "Provider crashed" }, + events: [ + { + name: `warning ${index}`, + timeUnixNano: ns(index * 1_000), + attributes: { "effect.logLevel": "Warning" }, + }, + ], + }), + ), + ); + const newestTwenty = Array.from({ length: 20 }, (_, rank) => `trace-${29 - rank}`); - assert.equal(diagnostics.recordCount, 25); - assert.equal(diagnostics.slowestSpans.length, 10); + assert.equal(diagnostics.recordCount, 30); assert.deepStrictEqual( diagnostics.slowestSpans.map((span) => span.durationMs), - [24, 23, 22, 21, 20, 19, 18, 17, 16, 15], + [29, 28, 27, 26, 25, 24, 23, 22, 21, 20], + ); + assert.deepStrictEqual( + diagnostics.latestFailures.map((failure) => failure.traceId), + newestTwenty, + ); + assert.deepStrictEqual( + diagnostics.latestWarningAndErrorLogs.map((log) => log.traceId), + newestTwenty, ); }), ); diff --git a/apps/server/src/diagnostics/TraceDiagnostics.ts b/apps/server/src/diagnostics/TraceDiagnostics.ts index ca5552058f3f..6dbc4afdec93 100644 --- a/apps/server/src/diagnostics/TraceDiagnostics.ts +++ b/apps/server/src/diagnostics/TraceDiagnostics.ts @@ -16,6 +16,7 @@ import * as Option from "effect/Option"; import * as PlatformError from "effect/PlatformError"; import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; interface TraceRecordLike { readonly name?: unknown; @@ -63,16 +64,6 @@ export class TraceDiagnostics extends Context.Service< } >()("t3/diagnostics/TraceDiagnostics") {} -interface TraceDiagnosticsInput { - readonly traceFilePath: string; - readonly files: ReadonlyArray<{ readonly path: string; readonly text: string }>; - readonly scannedFilePaths?: ReadonlyArray; - readonly slowSpanThresholdMs?: number; - readonly readAt: DateTime.Utc; - readonly error?: TraceDiagnosticsErrorSummary; - readonly partialFailure?: boolean; -} - interface TraceDiagnosticsErrorSummary { readonly kind: ServerTraceDiagnosticsErrorKind; readonly message: string; @@ -81,7 +72,12 @@ interface TraceDiagnosticsErrorSummary { const DEFAULT_SLOW_SPAN_THRESHOLD_MS = 1_000; const TOP_LIMIT = 10; const RECENT_LIMIT = 20; -function toRotatedTracePaths(traceFilePath: string, maxFiles: number): ReadonlyArray { + +/** The trace file and its rotated backups, oldest first. */ +export function toRotatedTracePaths( + traceFilePath: string, + maxFiles: number, +): ReadonlyArray { const backupCount = Math.max(0, Math.floor(maxFiles)); const backups = Array.from( { length: backupCount }, @@ -168,44 +164,50 @@ function isNotFoundError(error: PlatformError.PlatformError): boolean { return error.reason._tag === "NotFound"; } -function insertBoundedSlowestSpan( - slowestSpans: ServerTraceDiagnosticsSpanOccurrence[], - span: ServerTraceDiagnosticsSpanOccurrence, +/** + * Adds `item` to `items`, which stays sorted by `order` and holds at most + * `limit` entries. Same result as a stable sort and slice over every item, but + * memory stays bounded however many items stream in. + */ +function insertBounded( + items: A[], + item: A, + limit: number, + order: (left: A, right: A) => number, ): void { - if ( - slowestSpans.length >= TOP_LIMIT && - span.durationMs <= slowestSpans[slowestSpans.length - 1]!.durationMs - ) { + if (items.length >= limit && order(item, items[items.length - 1]!) >= 0) { return; } - slowestSpans.push(span); - slowestSpans.sort((left, right) => right.durationMs - left.durationMs); - if (slowestSpans.length > TOP_LIMIT) { - slowestSpans.length = TOP_LIMIT; + items.push(item); + items.sort(order); + if (items.length > limit) { + items.length = limit; } } -export function aggregateTraceDiagnostics( - input: TraceDiagnosticsInput, -): ServerTraceDiagnosticsResult { - const readAt = input.readAt; - const slowSpanThresholdMs = input.slowSpanThresholdMs ?? DEFAULT_SLOW_SPAN_THRESHOLD_MS; - const scannedFilePaths = input.scannedFilePaths ?? input.files.map((file) => file.path); - if (input.files.length === 0) { - return makeEmptyDiagnostics({ - traceFilePath: input.traceFilePath, - scannedFilePaths, - readAt, - slowSpanThresholdMs, - error: input.error ?? { - kind: "trace-file-not-found", - message: "No local trace files were found.", - }, - ...(input.partialFailure ? { partialFailure: true } : {}), - }); - } - +const slowestFirst = ( + left: ServerTraceDiagnosticsSpanOccurrence, + right: ServerTraceDiagnosticsSpanOccurrence, +) => right.durationMs - left.durationMs; + +const latestEndedFirst = ( + left: ServerTraceDiagnosticsRecentFailure, + right: ServerTraceDiagnosticsRecentFailure, +) => DateTime.toEpochMillis(right.endedAt) - DateTime.toEpochMillis(left.endedAt); + +const latestSeenFirst = ( + left: ServerTraceDiagnosticsLogEvent, + right: ServerTraceDiagnosticsLogEvent, +) => DateTime.toEpochMillis(right.seenAt) - DateTime.toEpochMillis(left.seenAt); + +/** + * Folds trace NDJSON into diagnostics. Call `addLine` once per line as the + * rotated files stream in, then `finish` for the result. + */ +export function makeTraceDiagnosticsAggregator( + slowSpanThresholdMs = DEFAULT_SLOW_SPAN_THRESHOLD_MS, +) { let parseErrorCount = 0; let recordCount = 0; let failureCount = 0; @@ -224,182 +226,184 @@ export function aggregateTraceDiagnostics( const latestWarningAndErrorLogs: ServerTraceDiagnosticsLogEvent[] = []; const logLevelCounts: Record = {}; - for (const file of input.files) { - const lines = file.text.split(/\r?\n/); - for (const line of lines) { - if (line.trim().length === 0) continue; - - let parsed: unknown; - try { - parsed = JSON.parse(line); - } catch { - parseErrorCount += 1; - continue; - } + const addLine = (line: string) => { + if (line.trim().length === 0) return; - if (!isRecordObject(parsed)) { - parseErrorCount += 1; - continue; - } + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + parseErrorCount += 1; + return; + } - const name = toStringValue(parsed.name); - const traceId = toStringValue(parsed.traceId); - const spanId = toStringValue(parsed.spanId); - const durationMs = toNumberValue(parsed.durationMs); - const endedAt = unixNanoToDateTime(parsed.endTimeUnixNano); - const startedAt = unixNanoToDateTime(parsed.startTimeUnixNano); + if (!isRecordObject(parsed)) { + parseErrorCount += 1; + return; + } - if (!name || !traceId || !spanId || durationMs === null || !endedAt) { - parseErrorCount += 1; - continue; - } + const name = toStringValue(parsed.name); + const traceId = toStringValue(parsed.traceId); + const spanId = toStringValue(parsed.spanId); + const durationMs = toNumberValue(parsed.durationMs); + const endedAt = unixNanoToDateTime(parsed.endTimeUnixNano); + const startedAt = unixNanoToDateTime(parsed.startTimeUnixNano); - recordCount += 1; - firstSpanAt = - startedAt && (firstSpanAt === null || DateTime.isLessThan(startedAt, firstSpanAt)) - ? startedAt - : firstSpanAt; - lastSpanAt = - lastSpanAt === null || DateTime.isGreaterThan(endedAt, lastSpanAt) ? endedAt : lastSpanAt; - - const exitTag = readExitTag(parsed.exit); - const isFailure = exitTag === "Failure"; - const isInterrupted = exitTag === "Interrupted"; - if (isFailure) failureCount += 1; - if (isInterrupted) interruptionCount += 1; - - const spanSummary = spansByName.get(name) ?? { - count: 0, - failureCount: 0, - totalDurationMs: 0, - maxDurationMs: 0, - }; - spanSummary.count += 1; - spanSummary.totalDurationMs += durationMs; - spanSummary.maxDurationMs = Math.max(spanSummary.maxDurationMs, durationMs); - if (isFailure) spanSummary.failureCount += 1; - spansByName.set(name, spanSummary); - - const spanItem = { name, durationMs, endedAt, traceId, spanId }; - if (durationMs >= slowSpanThresholdMs) { - slowSpanCount += 1; - } - insertBoundedSlowestSpan(slowestSpans, spanItem); - - if (isFailure) { - const cause = readExitCause(parsed.exit); - latestFailures.push({ ...spanItem, cause }); - - const failureKey = `${name}\0${cause}`; - const existing = failuresByKey.get(failureKey); - const isLatestFailure = !existing || DateTime.isGreaterThan(endedAt, existing.lastSeenAt); - failuresByKey.set(failureKey, { - name, - cause, - count: (existing?.count ?? 0) + 1, - lastSeenAt: isLatestFailure ? endedAt : existing!.lastSeenAt, - traceId: isLatestFailure ? traceId : existing!.traceId, - spanId: isLatestFailure ? spanId : existing!.spanId, - }); - } + if (!name || !traceId || !spanId || durationMs === null || !endedAt) { + parseErrorCount += 1; + return; + } - if (Array.isArray(parsed.events)) { - for (const rawEvent of parsed.events) { - if (!isTraceEvent(rawEvent)) continue; - const attributes = readEventAttributes(rawEvent); - const level = toStringValue(attributes["effect.logLevel"]); - if (!level) continue; - - logLevelCounts[level] = (logLevelCounts[level] ?? 0) + 1; - const normalizedLevel = level.toLowerCase(); - if ( - normalizedLevel !== "warning" && - normalizedLevel !== "warn" && - normalizedLevel !== "error" && - normalizedLevel !== "fatal" - ) { - continue; - } - - const seenAt = unixNanoToDateTime(rawEvent.timeUnixNano) ?? endedAt; - const message = toStringValue(rawEvent.name)?.trim() ?? "Log event"; - latestWarningAndErrorLogs.push({ - spanName: name, - level, - message, - seenAt, - traceId, - spanId, - }); + recordCount += 1; + firstSpanAt = + startedAt && (firstSpanAt === null || DateTime.isLessThan(startedAt, firstSpanAt)) + ? startedAt + : firstSpanAt; + lastSpanAt = + lastSpanAt === null || DateTime.isGreaterThan(endedAt, lastSpanAt) ? endedAt : lastSpanAt; + + const exitTag = readExitTag(parsed.exit); + const isFailure = exitTag === "Failure"; + const isInterrupted = exitTag === "Interrupted"; + if (isFailure) failureCount += 1; + if (isInterrupted) interruptionCount += 1; + + const spanSummary = spansByName.get(name) ?? { + count: 0, + failureCount: 0, + totalDurationMs: 0, + maxDurationMs: 0, + }; + spanSummary.count += 1; + spanSummary.totalDurationMs += durationMs; + spanSummary.maxDurationMs = Math.max(spanSummary.maxDurationMs, durationMs); + if (isFailure) spanSummary.failureCount += 1; + spansByName.set(name, spanSummary); + + const spanItem = { name, durationMs, endedAt, traceId, spanId }; + if (durationMs >= slowSpanThresholdMs) { + slowSpanCount += 1; + } + insertBounded(slowestSpans, spanItem, TOP_LIMIT, slowestFirst); + + if (isFailure) { + const cause = readExitCause(parsed.exit); + insertBounded(latestFailures, { ...spanItem, cause }, RECENT_LIMIT, latestEndedFirst); + + const failureKey = `${name}\0${cause}`; + const existing = failuresByKey.get(failureKey); + const isLatestFailure = !existing || DateTime.isGreaterThan(endedAt, existing.lastSeenAt); + failuresByKey.set(failureKey, { + name, + cause, + count: (existing?.count ?? 0) + 1, + lastSeenAt: isLatestFailure ? endedAt : existing!.lastSeenAt, + traceId: isLatestFailure ? traceId : existing!.traceId, + spanId: isLatestFailure ? spanId : existing!.spanId, + }); + } + + if (Array.isArray(parsed.events)) { + for (const rawEvent of parsed.events) { + if (!isTraceEvent(rawEvent)) continue; + const attributes = readEventAttributes(rawEvent); + const level = toStringValue(attributes["effect.logLevel"]); + if (!level) continue; + + logLevelCounts[level] = (logLevelCounts[level] ?? 0) + 1; + const normalizedLevel = level.toLowerCase(); + if ( + normalizedLevel !== "warning" && + normalizedLevel !== "warn" && + normalizedLevel !== "error" && + normalizedLevel !== "fatal" + ) { + continue; } + + const seenAt = unixNanoToDateTime(rawEvent.timeUnixNano) ?? endedAt; + const message = toStringValue(rawEvent.name)?.trim() ?? "Log event"; + insertBounded( + latestWarningAndErrorLogs, + { spanName: name, level, message, seenAt, traceId, spanId }, + RECENT_LIMIT, + latestSeenFirst, + ); } } - } - - const topSpansByCount: ServerTraceDiagnosticsSpanSummary[] = [...spansByName.entries()] - .map(([name, span]) => ({ - name, - count: span.count, - failureCount: span.failureCount, - totalDurationMs: span.totalDurationMs, - averageDurationMs: span.count > 0 ? span.totalDurationMs / span.count : 0, - maxDurationMs: span.maxDurationMs, - })) - .toSorted((left, right) => right.count - left.count || right.maxDurationMs - left.maxDurationMs) - .slice(0, TOP_LIMIT); + }; - return { - traceFilePath: input.traceFilePath, - scannedFilePaths, - readAt, - recordCount, - parseErrorCount, - firstSpanAt: Option.fromNullishOr(firstSpanAt), - lastSpanAt: Option.fromNullishOr(lastSpanAt), - failureCount, - interruptionCount, - slowSpanThresholdMs, - slowSpanCount, - logLevelCounts, - topSpansByCount, - slowestSpans, - commonFailures: [...failuresByKey.values()] - .toSorted( - (left, right) => - right.count - left.count || - DateTime.toEpochMillis(right.lastSeenAt) - DateTime.toEpochMillis(left.lastSeenAt), - ) - .slice(0, TOP_LIMIT), - latestFailures: latestFailures - .toSorted( - (left, right) => - DateTime.toEpochMillis(right.endedAt) - DateTime.toEpochMillis(left.endedAt), - ) - .slice(0, RECENT_LIMIT), - latestWarningAndErrorLogs: latestWarningAndErrorLogs + const finish = (input: { + readonly traceFilePath: string; + readonly scannedFilePaths: ReadonlyArray; + readonly readAt: DateTime.Utc; + readonly error?: TraceDiagnosticsErrorSummary; + readonly partialFailure?: boolean; + }): ServerTraceDiagnosticsResult => { + const topSpansByCount: ServerTraceDiagnosticsSpanSummary[] = [...spansByName.entries()] + .map(([name, span]) => ({ + name, + count: span.count, + failureCount: span.failureCount, + totalDurationMs: span.totalDurationMs, + averageDurationMs: span.count > 0 ? span.totalDurationMs / span.count : 0, + maxDurationMs: span.maxDurationMs, + })) .toSorted( - (left, right) => DateTime.toEpochMillis(right.seenAt) - DateTime.toEpochMillis(left.seenAt), + (left, right) => right.count - left.count || right.maxDurationMs - left.maxDurationMs, ) - .slice(0, RECENT_LIMIT), - partialFailure: input.partialFailure ? Option.some(true) : Option.none(), - error: Option.fromNullishOr(input.error), + .slice(0, TOP_LIMIT); + + return { + traceFilePath: input.traceFilePath, + scannedFilePaths: input.scannedFilePaths, + readAt: input.readAt, + recordCount, + parseErrorCount, + firstSpanAt: Option.fromNullishOr(firstSpanAt), + lastSpanAt: Option.fromNullishOr(lastSpanAt), + failureCount, + interruptionCount, + slowSpanThresholdMs, + slowSpanCount, + logLevelCounts, + topSpansByCount, + slowestSpans, + commonFailures: [...failuresByKey.values()] + .toSorted( + (left, right) => + right.count - left.count || + DateTime.toEpochMillis(right.lastSeenAt) - DateTime.toEpochMillis(left.lastSeenAt), + ) + .slice(0, TOP_LIMIT), + latestFailures, + latestWarningAndErrorLogs, + partialFailure: input.partialFailure ? Option.some(true) : Option.none(), + error: Option.fromNullishOr(input.error), + }; }; -} -type TraceFileReadResult = - | { readonly _tag: "Loaded"; readonly path: string; readonly text: string } - | { readonly _tag: "Missing"; readonly path: string }; + return { addLine, finish }; +} -function readTraceFile( +/** + * Feeds each line of one trace file to `onLine`, streaming so only one chunk of + * text is in memory at a time. Succeeds with false when the file does not exist. + */ +export function streamTraceFileLines( fileSystem: FileSystem.FileSystem, path: string, -): Effect.Effect { - return fileSystem.readFileString(path).pipe( - Effect.map((text): TraceFileReadResult => ({ _tag: "Loaded", path, text })), + onLine: (line: string) => void, +): Effect.Effect { + return fileSystem.stream(path).pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runForEachArray((lines) => Effect.sync(() => lines.forEach(onLine))), + Effect.as(true), Effect.catchTags({ PlatformError: (cause) => isNotFoundError(cause) - ? Effect.succeed({ _tag: "Missing", path }) + ? Effect.succeed(false) : Effect.fail( new TraceFileReadError({ traceFilePath: path, @@ -420,10 +424,11 @@ export const make = Effect.gen(function* () { const readAt = options.readAt ?? (yield* DateTime.now); const slowSpanThresholdMs = options.slowSpanThresholdMs ?? DEFAULT_SLOW_SPAN_THRESHOLD_MS; const paths = toRotatedTracePaths(options.traceFilePath, options.maxFiles); + const aggregator = makeTraceDiagnosticsAggregator(slowSpanThresholdMs); const results = yield* Effect.forEach( paths, (path) => - readTraceFile(fileSystem, path).pipe( + streamTraceFileLines(fileSystem, path, aggregator.addLine).pipe( Effect.tapError((cause) => Effect.logWarning("Failed to read local trace file.").pipe( Effect.annotateLogs({ @@ -435,15 +440,10 @@ export const make = Effect.gen(function* () { ), Effect.result, ), - { - concurrency: 1, - }, - ); - const files = results.flatMap((result) => - Result.isSuccess(result) && result.success._tag === "Loaded" - ? [{ path: result.success.path, text: result.success.text }] - : [], + // Every file feeds one aggregator, so read them one at a time, oldest first. + { concurrency: 1 }, ); + const foundFile = results.some((result) => Result.isSuccess(result) && result.success); const readFailure = results.find(Result.isFailure); const readFailureError = readFailure ? ({ @@ -452,7 +452,7 @@ export const make = Effect.gen(function* () { } satisfies TraceDiagnosticsErrorSummary) : undefined; - if (files.length === 0) { + if (!foundFile) { return makeEmptyDiagnostics({ traceFilePath: options.traceFilePath, scannedFilePaths: paths, @@ -467,12 +467,10 @@ export const make = Effect.gen(function* () { }); } - return aggregateTraceDiagnostics({ + return aggregator.finish({ traceFilePath: options.traceFilePath, - files, scannedFilePaths: paths, readAt, - slowSpanThresholdMs, ...(readFailureError ? { partialFailure: true, error: readFailureError } : {}), }); }, diff --git a/apps/server/src/http.ts b/apps/server/src/http.ts index 7533b8c1db19..30b368e62167 100644 --- a/apps/server/src/http.ts +++ b/apps/server/src/http.ts @@ -311,9 +311,12 @@ export const serverEnvironmentHttpApiLayer = HttpApiBuilder.group( class DecodeOtlpTraceRecordsError extends Data.TaggedError("DecodeOtlpTraceRecordsError")<{ readonly cause: unknown; - readonly bodyJson: OtlpTracer.TraceData; }> {} +// Renderers export up to once a second while they have spans buffered, so +// tracing this proxy would add more server spans than it forwards. +// withTracerEnabled(false) drops the handler's spans, including the forward. +// untracedRequestsLayer drops the HTTP server span. export const otlpTracesProxyRouteLayer = HttpRouter.add( "POST", OTLP_TRACES_PROXY_PATH, @@ -330,15 +333,10 @@ export const otlpTracesProxyRouteLayer = HttpRouter.add( yield* Effect.try({ try: () => decodeOtlpTraceRecords(bodyJson), - catch: (cause) => new DecodeOtlpTraceRecordsError({ cause, bodyJson }), + catch: (cause) => new DecodeOtlpTraceRecordsError({ cause }), }).pipe( Effect.flatMap((records) => browserTraceCollector.record(records)), - Effect.catch((cause) => - Effect.logWarning("Failed to decode browser OTLP traces", { - cause, - bodyJson, - }), - ), + Effect.catch((cause) => Effect.logWarning("Failed to decode browser OTLP traces", { cause })), ); if (otlpTracesUrl === undefined) { @@ -369,9 +367,25 @@ export const otlpTracesProxyRouteLayer = HttpRouter.add( EnvironmentInternalError: HttpServerRespondable.toResponse, EnvironmentScopeRequiredError: HttpServerRespondable.toResponse, }), + Effect.withTracerEnabled(false), ), ); +const UNTRACED_REQUEST_PATHS: ReadonlySet = new Set([OTLP_TRACES_PROXY_PATH]); + +// Skips the HTTP server span for UNTRACED_REQUEST_PATHS. That span starts +// before routing, so a route handler cannot skip it. TracerDisabledWhen is one +// predicate for the whole server and the last layer to provide it wins, so +// makeRoutesLayer provides this one last. Add paths here instead of providing +// TracerDisabledWhen again; server.test.ts fails if a later layer replaces it. +// The query string is ignored, as in routing. +export const untracedRequestsLayer = Layer.succeed(HttpMiddleware.TracerDisabledWhen)((request) => { + const queryIndex = request.url.indexOf("?"); + return UNTRACED_REQUEST_PATHS.has( + queryIndex === -1 ? request.url : request.url.slice(0, queryIndex), + ); +}); + export const assetRouteLayer = HttpRouter.add( "GET", `${ASSET_ROUTE_PREFIX}/*`, diff --git a/apps/server/src/observability/EventLoopMonitor.test.ts b/apps/server/src/observability/EventLoopMonitor.test.ts new file mode 100644 index 000000000000..fbe30157aec2 --- /dev/null +++ b/apps/server/src/observability/EventLoopMonitor.test.ts @@ -0,0 +1,77 @@ +import { assert, describe, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Tracer from "effect/Tracer"; +import * as TestClock from "effect/testing/TestClock"; + +import { type EventLoopReadings, layerWith, stallMs } from "./EventLoopMonitor.ts"; + +const ms = (value: number) => value * 1e6; + +// Node reports a stall of S as a gap of up to S + 1 s, the histogram resolution. +const stalled: EventLoopReadings = { + delayMaxNs: ms(5_950), + activeMs: 6_200, + utilization: 0.176, + usage: { + userCPUTime: 310_400, + systemCPUTime: 95_600, + majorPageFault: 8_412, + minorPageFault: 20_031, + involuntaryContextSwitches: 57, + }, + rssBytes: 1536 * 1024 * 1024, +}; +// Over the threshold as read, but not once the resolution is subtracted. +const quiet: EventLoopReadings = { ...stalled, delayMaxNs: ms(2_950) }; + +describe("EventLoopMonitor", () => { + it.effect("records a warning span only for samples that saw a stall", () => + Effect.gen(function* () { + const spans: Array = []; + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options); + spans.push(span); + return span; + }, + }); + // The first sample covers startup, so the monitor discards it. + const samples = [stalled, quiet, stalled]; + + yield* Effect.gen(function* () { + yield* Layer.build(layerWith(Effect.succeed(Effect.sync(() => samples.shift() ?? quiet)))); + yield* TestClock.adjust("60 seconds"); + assert.lengthOf(spans, 0); + yield* TestClock.adjust("30 seconds"); + }).pipe(Effect.scoped, Effect.withTracer(tracer)); + + assert.deepStrictEqual( + spans.map((span) => span.name), + ["server.eventLoop.stall"], + ); + const [span] = spans; + assert.deepStrictEqual(Object.fromEntries(span!.attributes), { + delayMaxMs: 4_950, + utilization: 0.18, + cpuUserMs: 310, + cpuSystemMs: 96, + majorPageFaults: 8_412, + minorPageFaults: 20_031, + involuntaryContextSwitches: 57, + rssMb: 1536, + }); + assert.deepStrictEqual( + span!.events.map(([name, , attributes]) => [name, attributes["effect.logLevel"]]), + [["event loop stalled for 4950 ms", "WARN"]], + ); + }), + ); + + it("ignores delay the loop spent idle, such as a system sleep", () => { + // Waking from sleep reads as a long gap, but the loop was idle in poll for it. + const asleep: EventLoopReadings = { ...stalled, delayMaxNs: ms(600_000), activeMs: 900 }; + assert.isUndefined(stallMs(asleep)); + assert.strictEqual(stallMs({ ...asleep, activeMs: 600_000 }), 599_000); + }); +}); diff --git a/apps/server/src/observability/EventLoopMonitor.ts b/apps/server/src/observability/EventLoopMonitor.ts new file mode 100644 index 000000000000..13b7b48c4cf7 --- /dev/null +++ b/apps/server/src/observability/EventLoopMonitor.ts @@ -0,0 +1,135 @@ +// @effect-diagnostics nodeBuiltinImport:off - only node:perf_hooks exposes the event loop delay histogram. +import * as NodePerfHooks from "node:perf_hooks"; + +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import type * as Scope from "effect/Scope"; + +// Node's delay histogram wakes a native timer every RESOLUTION_MS and records the +// gap between wakeups, so an idle loop reads about RESOLUTION_MS and a stall of S +// reads between S and S + RESOLUTION_MS. We subtract the resolution, so a delay can +// undercount a stall by up to RESOLUTION_MS. With these values every stall over 3 s +// is caught, at 1 wakeup per second that never enters JS. +const RESOLUTION_MS = 1000; +const STALL_THRESHOLD_MS = 2000; +const SAMPLE_INTERVAL = "30 seconds"; + +/** One sample interval as Node reports it. Delay in ns, active time in ms, CPU in µs. */ +export interface EventLoopReadings { + readonly delayMaxNs: number; + readonly activeMs: number; + readonly utilization: number; + readonly usage: Pick< + NodeJS.ResourceUsage, + | "userCPUTime" + | "systemCPUTime" + | "majorPageFault" + | "minorPageFault" + | "involuntaryContextSwitches" + >; + readonly rssBytes: number; +} + +// Enables the delay histogram for the layer's lifetime. Each read returns the +// readings since the previous read and resets the histogram. Node skips the first +// gap after a reset, so a stall right at a sample boundary can be missed. +const makeNodeSampler = Effect.gen(function* () { + const histogram = yield* Effect.acquireRelease( + Effect.sync(() => { + const histogram = NodePerfHooks.monitorEventLoopDelay({ resolution: RESOLUTION_MS }); + histogram.enable(); + return histogram; + }), + (histogram) => Effect.sync(() => histogram.disable()), + ); + let elu = NodePerfHooks.performance.eventLoopUtilization(); + let usage = process.resourceUsage(); + + // @effect-diagnostics-next-line returnEffectInGen:off - the read effect is the result. + return Effect.sync(() => { + const nextElu = NodePerfHooks.performance.eventLoopUtilization(); + const nextUsage = process.resourceUsage(); + const loop = NodePerfHooks.performance.eventLoopUtilization(nextElu, elu); + const readings: EventLoopReadings = { + delayMaxNs: histogram.max, + activeMs: loop.active, + utilization: loop.utilization, + usage: { + userCPUTime: nextUsage.userCPUTime - usage.userCPUTime, + systemCPUTime: nextUsage.systemCPUTime - usage.systemCPUTime, + majorPageFault: nextUsage.majorPageFault - usage.majorPageFault, + minorPageFault: nextUsage.minorPageFault - usage.minorPageFault, + involuntaryContextSwitches: + nextUsage.involuntaryContextSwitches - usage.involuntaryContextSwitches, + }, + rssBytes: process.memoryUsage.rss(), + }; + histogram.reset(); + elu = nextElu; + usage = nextUsage; + return readings; + }); +}); + +/** + * Returns the stall to report for one sample in ms, or undefined when there was none. + */ +export const stallMs = ({ delayMaxNs, activeMs }: EventLoopReadings) => { + const delayMs = Math.round(delayMaxNs / 1e6) - RESOLUTION_MS; + // A stall is time the loop spent running code, so it counts as active time. libuv's + // clock keeps running while the system sleeps on macOS and Windows, so a sleep also + // reads as delay, but the loop spent it idle in poll. + if (delayMs <= STALL_THRESHOLD_MS || activeMs < delayMs) return undefined; + return delayMs; +}; + +/** + * Samples event loop health every 30 s and records a `server.eventLoop.stall` span + * with a warning when the loop stalled for more than 2 s, so stalls land in + * the local trace file and Settings > Diagnostics without OTLP. Takes the sampler + * so tests can inject readings. + */ +export const layerWith = ( + makeSampler: Effect.Effect, never, Scope.Scope>, +) => + Layer.effectDiscard( + Effect.gen(function* () { + const sample = yield* makeSampler; + const tick = Effect.gen(function* () { + const readings = yield* sample; + const delayMaxMs = stallMs(readings); + if (delayMaxMs === undefined) return; + const { utilization, usage, rssBytes } = readings; + // Root, as the stall has no caller to attach to. Warn level keeps it when + // T3CODE_TRACE_MIN_LEVEL is raised to cut trace noise. + yield* Effect.logWarning(`event loop stalled for ${delayMaxMs} ms`).pipe( + Effect.withSpan("server.eventLoop.stall", { + root: true, + level: "Warn", + attributes: { + delayMaxMs, + utilization: Math.round(utilization * 100) / 100, + cpuUserMs: Math.round(usage.userCPUTime / 1000), + cpuSystemMs: Math.round(usage.systemCPUTime / 1000), + majorPageFaults: usage.majorPageFault, + minorPageFaults: usage.minorPageFault, + involuntaryContextSwitches: usage.involuntaryContextSwitches, + rssMb: Math.round(rssBytes / 1024 / 1024), + }, + }), + ); + }); + const wait = Effect.sleep(SAMPLE_INTERVAL); + // The layer builds before the rest of the server, so the first sample covers + // startup work such as migrations and projection bootstrap. That can block the + // loop for seconds on a large database, so skip it rather than warn at every + // launch. Layers build outside any span, so this fiber retains no parent span. + yield* wait.pipe( + Effect.andThen(sample), + Effect.andThen(wait.pipe(Effect.andThen(tick), Effect.forever)), + Effect.forkScoped, + ); + }), + ); + +export const layer = layerWith(makeNodeSampler); diff --git a/apps/server/src/observability/HeapSnapshot.test.ts b/apps/server/src/observability/HeapSnapshot.test.ts new file mode 100644 index 000000000000..0beb466e5019 --- /dev/null +++ b/apps/server/src/observability/HeapSnapshot.test.ts @@ -0,0 +1,36 @@ +// @effect-diagnostics nodeBuiltinImport:off - tests fake a failed write at the native v8 boundary. +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import * as NodeV8 from "node:v8"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import { vi } from "vite-plus/test"; + +import { writeHeapSnapshot } from "./HeapSnapshot.ts"; + +vi.mock("node:v8", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, writeHeapSnapshot: vi.fn(actual.writeHeapSnapshot) }; +}); + +it.layer(NodeServices.layer)("writeHeapSnapshot", (it) => { + it.effect("removes the partial file when the write fails", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const logsDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-heap-snapshot-test-" }); + let partialPath: string | undefined; + vi.mocked(NodeV8.writeHeapSnapshot).mockImplementationOnce((path) => { + partialPath = path; + if (path) NodeFS.writeFileSync(path, "partial"); + throw new Error("ENOSPC: no space left on device"); + }); + + yield* writeHeapSnapshot(logsDir); + + assert.strictEqual(NodePath.dirname(partialPath ?? ""), logsDir); + assert.deepEqual(yield* fs.readDirectory(logsDir), []); + }), + ); +}); diff --git a/apps/server/src/observability/HeapSnapshot.ts b/apps/server/src/observability/HeapSnapshot.ts new file mode 100644 index 000000000000..827dfab2c6d6 --- /dev/null +++ b/apps/server/src/observability/HeapSnapshot.ts @@ -0,0 +1,52 @@ +// @effect-diagnostics nodeBuiltinImport:off - v8.writeHeapSnapshot has no Effect equivalent. +import * as NodePath from "node:path"; +import * as NodeV8 from "node:v8"; + +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; + +import * as ServerConfig from "../config.ts"; + +/** + * Writes one V8 heap snapshot into `logsDir` and logs its path. A failed write + * logs a warning and removes any partial file, because that file can hold + * secrets and the failure is often a full disk. + */ +export const writeHeapSnapshot = Effect.fn("server.heapSnapshot", { root: true })( + function* (logsDir: string) { + const fs = yield* FileSystem.FileSystem; + const timestamp = DateTime.formatIso(yield* DateTime.now).replaceAll(":", "-"); + const path = NodePath.join(logsDir, `server-${process.pid}-${timestamp}.heapsnapshot`); + yield* Effect.annotateCurrentSpan({ path }); + yield* Effect.try(() => NodeV8.writeHeapSnapshot(path)).pipe( + Effect.tapError(() => fs.remove(path, { force: true }).pipe(Effect.ignore)), + ); + yield* Effect.logInfo("Wrote heap snapshot.", { path }); + }, + Effect.catch((cause) => Effect.logWarning("Failed to write heap snapshot.", { cause })), +); + +/** + * Writes a heap snapshot when the process gets SIGUSR2 (`kill -USR2 `), + * so a maintainer can see what a long-running server holds. See "Heap + * Snapshots" in docs/operations/observability.md. + * + * The write blocks the event loop, so two snapshots never overlap: a signal + * sent during a write waits until it finishes. Windows has no SIGUSR2, so the + * layer does nothing there. + */ +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + if ((yield* HostProcessPlatform) === "win32") return; + const { logsDir } = yield* ServerConfig.ServerConfig; + const runFork = Effect.runForkWith(yield* Effect.context()); + const onSignal = () => void runFork(writeHeapSnapshot(logsDir)); + yield* Effect.acquireRelease( + Effect.sync(() => process.on("SIGUSR2", onSignal)), + () => Effect.sync(() => process.off("SIGUSR2", onSignal)), + ); + }), +); diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts index f855771b8f01..9ccf78ca1744 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts @@ -435,6 +435,7 @@ describe("OrchestrationEngine", () => { updatedAt: projectionSnapshot.updatedAt, }), getDeletedWorktreeThreads: () => Effect.die("unused"), + listThreadsWithPullRequests: () => Effect.die("unused"), getArchivedShellSnapshot: () => Effect.succeed({ snapshotSequence: projectionSnapshot.snapshotSequence, diff --git a/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts b/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts index d25c442ac8b2..207ad9a20d34 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts @@ -108,6 +108,7 @@ describe("OrchestrationReactor", () => { Layer.provideMerge( Layer.succeed(AgentAwarenessRelay.AgentAwarenessRelay, { publishThread: () => Effect.void, + requestCatchUp: () => Effect.void, start: () => { started.push("agent-awareness-relay"); return Effect.void; diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts index 179d04843c7e..e8c7154f38e7 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts @@ -21,6 +21,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as Tracer from "effect/Tracer"; import * as SqlClient from "effect/unstable/sql/SqlClient"; import { makeSqlStatementCounter } from "../../../integration/SqlStatementCounter.integration.ts"; @@ -114,6 +115,66 @@ it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-curs }, ); +it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-cleanup-span-")))( + "OrchestrationProjectionPipeline attachment cleanup span", + (it) => { + it.effect("runs attachment cleanup only for events that remove attachments", () => + Effect.gen(function* () { + const projectionPipeline = yield* OrchestrationProjectionPipeline; + const eventStore = yield* OrchestrationEventStore; + let cleanupSpans = 0; + const tracer = Tracer.make({ + span: (options) => { + if (options.name === "applyAttachmentSideEffects") cleanupSpans += 1; + return new Tracer.NativeSpan(options); + }, + }); + const now = "2026-01-01T00:00:00.000Z"; + const projectId = ProjectId.make("project-cleanup-span"); + const threadId = ThreadId.make("thread-cleanup-span"); + + const projectCreated = yield* eventStore.append({ + type: "project.created", + eventId: EventId.make("evt-cleanup-span-project"), + aggregateKind: "project", + aggregateId: projectId, + occurredAt: now, + commandId: CommandId.make("cmd-cleanup-span-project"), + causationEventId: null, + correlationId: null, + metadata: {}, + payload: { + projectId, + title: "Cleanup span project", + workspaceRoot: "/tmp/project-cleanup-span", + defaultModelSelection: null, + scripts: [], + createdAt: now, + updatedAt: now, + }, + }); + yield* projectionPipeline.projectEvent(projectCreated).pipe(Effect.withTracer(tracer)); + assert.strictEqual(cleanupSpans, 0); + + const threadDeleted = yield* eventStore.append({ + type: "thread.deleted", + eventId: EventId.make("evt-cleanup-span-thread-delete"), + aggregateKind: "thread", + aggregateId: threadId, + occurredAt: now, + commandId: CommandId.make("cmd-cleanup-span-thread-delete"), + causationEventId: null, + correlationId: null, + metadata: {}, + payload: { threadId, deletedAt: now }, + }); + yield* projectionPipeline.projectEvent(threadDeleted).pipe(Effect.withTracer(tracer)); + assert.strictEqual(cleanupSpans, 1); + }), + ); + }, +); + it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-import-shell-")))( "imported thread shell projection", (it) => { diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts index 62991daadf0e..50e88e8cc867 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts @@ -1992,13 +1992,6 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti const applyAttachmentSideEffects = Effect.fn("applyAttachmentSideEffects")( function* (event: OrchestrationEvent, sideEffects: AttachmentSideEffects) { - if ( - sideEffects.deletedThreadIds.size === 0 && - sideEffects.prunedThreadRelativePaths.size === 0 - ) { - return; - } - const deletedThreadIds = new Set(); for (const threadId of sideEffects.deletedThreadIds) { const recreatedLater = yield* eventStore.hasEventAfter({ @@ -2114,9 +2107,15 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti ); }), ); + const hasCleanup = + attachmentSideEffects.deletedThreadIds.size > 0 || + attachmentSideEffects.prunedThreadRelativePaths.size > 0; // Return the cleanup effect so the caller runs it after the outer transaction commits. + // Most events have no cleanup, so they skip the call and write no cleanup span. // @effect-diagnostics-next-line returnEffectInGen:off - return applyAttachmentSideEffects(event, attachmentSideEffects).pipe(Effect.asVoid); + return hasCleanup + ? applyAttachmentSideEffects(event, attachmentSideEffects).pipe(Effect.asVoid) + : Effect.void; }, Effect.provideService(FileSystem.FileSystem, fileSystem), Effect.provideService(Path.Path, path), diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts index d6079d4bcff4..71af928a9175 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts @@ -30,6 +30,7 @@ import * as ThreadPlanProgress from "../ThreadPlanProgress.ts"; import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; import { encodeThreadDetailPageCursor } from "../threadDetailCursor.ts"; import { projectThreadDetailSnapshot } from "../ActivityPayloadProjection.ts"; +import { readSweepSnapshot } from "../ThreadPullRequestReactor.ts"; import { makeSqlStatementCounter } from "../../../integration/SqlStatementCounter.integration.ts"; const asProjectId = (value: string): ProjectId => ProjectId.make(value); @@ -3505,6 +3506,224 @@ it.effect("omits foreign-host PRs from legacy snapshots while preserving native }).pipe(Effect.provide(layer)); }); +it.effect( + "lists linked threads like the shell snapshot, in one query and without identities", + () => { + const resolved: string[] = []; + const layer = OrchestrationProjectionSnapshotQueryLive.pipe( + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide( + Layer.succeed(RepositoryIdentityResolver.RepositoryIdentityResolver, { + resolve: (root) => + Effect.sync(() => { + resolved.push(root); + return null; + }), + }), + ), + Layer.provideMerge(SqlitePersistenceMemory), + ); + return Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const query = yield* ProjectionSnapshotQuery; + yield* sql`INSERT INTO projection_projects (project_id, title, workspace_root, scripts_json, created_at, updated_at) + VALUES ('p1', 'One', '/one', '[]', '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z'), + ('p2', 'Two', '/two', '[]', '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z')`; + yield* sql`INSERT INTO projection_threads (thread_id, project_id, title, model_selection_json, runtime_mode, interaction_mode, created_at, updated_at, archived_at, deleted_at, settled_override, settled_at) + VALUES + ('t-late', 'p1', 'Late', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', '2026-09-03T00:00:00Z', '2026-09-03T00:00:00Z', NULL, NULL, 'settled', '2026-09-04T00:00:00Z'), + ('t-early', 'p2', 'Early', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z', NULL, NULL, NULL, NULL), + ('t-first', 'p1', 'First', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', '2026-09-02T00:00:00Z', '2026-09-02T00:00:00Z', NULL, NULL, NULL, NULL), + ('t-plain', 'p1', 'Plain', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', '2026-09-02T00:00:00Z', '2026-09-02T00:00:00Z', NULL, NULL, NULL, NULL), + ('t-archived', 'p1', 'Archived', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', '2026-09-02T00:00:00Z', '2026-09-02T00:00:00Z', '2026-09-05T00:00:00Z', NULL, NULL, NULL), + ('t-deleted', 'p1', 'Deleted', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', '2026-09-02T00:00:00Z', '2026-09-02T00:00:00Z', NULL, '2026-09-05T00:00:00Z', NULL, NULL)`; + yield* sql`INSERT INTO projection_thread_pull_requests (thread_id, host, repository, number, url, source, linked_at, snapshot_json) + VALUES + ('t-late', 'github.com', 'acme/web', 3, 'https://github.com/acme/web/pull/3', 'manual', '2026-09-03T00:00:00Z', NULL), + ('t-early', 'github.com', 'acme/api', 4, 'https://github.com/acme/api/pull/4', 'agent', '2026-09-01T00:00:00Z', NULL), + ('t-first', 'github.com', 'acme/web', 2, 'https://github.com/acme/web/pull/2', 'stack-dismissed', '2026-09-02T00:00:00Z', NULL), + ('t-first', 'github.com', 'acme/web', 1, 'https://github.com/acme/web/pull/1', 'created', '2026-09-02T00:00:00Z', + '{"state":"open","title":"One","headBranch":"one","baseBranch":"main","isDraft":false,"updatedAt":null,"syncedAt":"2026-09-02T00:00:00Z"}'), + ('t-archived', 'github.com', 'acme/web', 5, 'https://github.com/acme/web/pull/5', 'manual', '2026-09-02T00:00:00Z', NULL), + ('t-deleted', 'github.com', 'acme/web', 6, 'https://github.com/acme/web/pull/6', 'manual', '2026-09-02T00:00:00Z', NULL)`; + const expected = (yield* query.getShellSnapshot()).threads + .filter((thread) => thread.pullRequests.length > 0) + .map(({ id, projectId, settledOverride, settledAt, pullRequests }) => ({ + id, + projectId, + settledOverride, + settledAt, + pullRequests, + })); + resolved.length = 0; + + const counter = makeSqlStatementCounter(); + const threads = yield* query + .listThreadsWithPullRequests() + .pipe(Effect.withTracer(counter.tracer)); + assert.deepStrictEqual( + threads.map((thread) => [thread.id, thread.pullRequests.map((link) => link.number)]), + [ + ["t-first", [1, 2]], + ["t-late", [3]], + ["t-early", [4]], + ], + ); + assert.deepStrictEqual(threads, expected); + assert.strictEqual(counter.count(), 1); + assert.deepStrictEqual(resolved, []); + }).pipe(Effect.provide(layer)); + }, +); + +it.effect("reads one sweep thread and its projects like the shell snapshot", () => { + const layer = OrchestrationProjectionSnapshotQueryLive.pipe( + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide( + Layer.succeed(RepositoryIdentityResolver.RepositoryIdentityResolver, { + resolve: () => + Effect.succeed({ + canonicalKey: "github.com/acme/web", + provider: "github", + displayName: "acme/web", + locator: { + source: "git-remote" as const, + remoteName: "origin", + remoteUrl: "https://github.com/acme/web.git", + }, + }), + }), + ), + Layer.provideMerge(SqlitePersistenceMemory), + ); + return Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const query = yield* ProjectionSnapshotQuery; + yield* sql`INSERT INTO projection_projects (project_id, title, workspace_root, scripts_json, created_at, updated_at) + VALUES ('p1', 'One', '/one', '[]', '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z'), + ('p2', 'Two', '/two', '[]', '2026-09-02T00:00:00Z', '2026-09-02T00:00:00Z'), + ('p3', 'Three', '/three', '[]', '2026-09-03T00:00:00Z', '2026-09-03T00:00:00Z')`; + yield* sql`INSERT INTO projection_threads (thread_id, project_id, title, model_selection_json, runtime_mode, interaction_mode, branch, worktree_path, branch_pull_request_json, latest_turn_id, latest_user_message_at, pending_approval_count, snoozed_until, snoozed_at, created_at, updated_at, settled_override, settled_at) + VALUES + ('t-linked', 'p1', 'Linked', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'feature', '/one/wt', NULL, 'turn-1', '2026-09-02T00:00:00Z', 1, NULL, NULL, '2026-09-01T00:00:00Z', '2026-09-02T00:00:00Z', NULL, NULL), + ('t-branch', 'p1', 'Branch', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'other', NULL, + '{"projectId":"p2","repository":"acme/web","number":8,"url":"https://github.com/acme/web/pull/8"}', + NULL, NULL, 0, '2026-09-10T00:00:00Z', '2026-09-02T00:00:00Z', '2026-09-01T00:00:00Z', '2026-09-02T00:00:00Z', 'settled', '2026-09-03T00:00:00Z'), + ('t-other', 'p3', 'Other', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, NULL, NULL, NULL, 0, NULL, NULL, '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z', NULL, NULL)`; + yield* sql`INSERT INTO projection_thread_pull_requests (thread_id, host, repository, number, url, source, linked_at) + VALUES ('t-linked', 'github.com', 'acme/web', 7, 'https://github.com/acme/web/pull/7', 'agent', '2026-09-02T00:00:00Z')`; + yield* sql`INSERT INTO projection_turns (thread_id, turn_id, state, requested_at, started_at, completed_at, checkpoint_files_json) + VALUES ('t-linked', 'turn-1', 'completed', '2026-09-02T00:00:00Z', '2026-09-02T00:00:01Z', '2026-09-02T00:00:02Z', '[]')`; + yield* sql`INSERT INTO projection_thread_sessions (thread_id, status, provider_name, active_turn_id, last_error, updated_at) + VALUES ('t-linked', 'ready', 'codex', NULL, NULL, '2026-09-02T00:00:03Z')`; + for (const projector of Object.values(ORCHESTRATION_PROJECTOR_NAMES)) { + yield* sql`INSERT INTO projection_state (projector, last_applied_sequence, updated_at) + VALUES (${projector}, 9, '2026-09-02T00:00:03Z')`; + } + + const full = yield* query.getShellSnapshot(); + // The seeded fields must reach the snapshot, or the parity check is empty. + const linked = full.threads.find((thread) => thread.id === ThreadId.make("t-linked")); + assert.strictEqual(full.snapshotSequence, 9); + assert.strictEqual(linked?.linkedPullRequest?.number, 7); + assert.strictEqual(linked?.latestTurn?.turnId, asTurnId("turn-1")); + assert.strictEqual(linked?.session?.status, "ready"); + + for (const [threadId, projectIds] of [ + [ThreadId.make("t-linked"), [asProjectId("p1")]], + // Settlement also needs the project that the saved branch PR names. + [ThreadId.make("t-branch"), [asProjectId("p1"), asProjectId("p2")]], + ] as const) { + assert.deepStrictEqual(yield* readSweepSnapshot(query, threadId), { + snapshotSequence: full.snapshotSequence, + projects: full.projects.filter((project) => projectIds.includes(project.id)), + threads: full.threads.filter((thread) => thread.id === threadId), + }); + } + }).pipe(Effect.provide(layer)); +}); + +it.effect("reads a full sweep from unsettled threads and every project", () => { + const resolved: string[] = []; + const layer = OrchestrationProjectionSnapshotQueryLive.pipe( + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide( + Layer.succeed(RepositoryIdentityResolver.RepositoryIdentityResolver, { + resolve: (root) => + Effect.sync(() => { + resolved.push(root); + return null; + }), + }), + ), + Layer.provideMerge(SqlitePersistenceMemory), + ); + return Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const query = yield* ProjectionSnapshotQuery; + yield* sql`INSERT INTO projection_projects (project_id, title, workspace_root, scripts_json, created_at, updated_at) + VALUES ('p1', 'One', '/one', '[]', '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z'), + ('p2', 'Two', '/two', '[]', '2026-09-02T00:00:00Z', '2026-09-02T00:00:00Z'), + ('p3', 'Three', '/three', '[]', '2026-09-03T00:00:00Z', '2026-09-03T00:00:00Z'), + ('p4', 'Four', '/four', '[]', '2026-09-04T00:00:00Z', '2026-09-04T00:00:00Z')`; + yield* sql`INSERT INTO projection_threads (thread_id, project_id, title, model_selection_json, runtime_mode, interaction_mode, branch_pull_request_json, latest_turn_id, created_at, updated_at, archived_at, settled_override, settled_at) + VALUES + ('t-open', 'p1', 'Open', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, 'turn-open', '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z', NULL, NULL, NULL), + ('t-resumed', 'p1', 'Resumed', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z', NULL, 'active', NULL), + ('t-branch', 'p1', 'Branch', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', + '{"projectId":"p2","repository":"acme/web","number":8,"url":"https://github.com/acme/web/pull/8"}', + NULL, '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z', NULL, NULL, NULL), + ('t-settled', 'p3', 'Settled', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, 'turn-settled', '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z', NULL, 'settled', '2026-09-03T00:00:00Z'), + ('t-archived', 'p4', 'Archived', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z', '2026-09-04T00:00:00Z', NULL, NULL)`; + // The open and the settled thread both have a row in each joined table. The + // settled thread's turn and session are the newest rows, so updatedAt shows + // whether those two reads skip it. + yield* sql`INSERT INTO projection_thread_pull_requests (thread_id, host, repository, number, url, source, linked_at) + VALUES ('t-open', 'github.com', 'acme/web', 7, 'https://github.com/acme/web/pull/7', 'agent', '2026-09-02T00:00:00Z'), + ('t-settled', 'github.com', 'acme/web', 9, 'https://github.com/acme/web/pull/9', 'agent', '2026-09-02T00:00:00Z')`; + yield* sql`INSERT INTO projection_turns (thread_id, turn_id, state, requested_at, checkpoint_files_json) + VALUES ('t-open', 'turn-open', 'completed', '2026-09-02T00:00:00Z', '[]'), + ('t-settled', 'turn-settled', 'completed', '2026-09-09T00:00:00Z', '[]')`; + yield* sql`INSERT INTO projection_thread_sessions (thread_id, status, provider_name, active_turn_id, last_error, updated_at) + VALUES ('t-open', 'ready', 'codex', NULL, NULL, '2026-09-02T00:00:00Z'), + ('t-settled', 'stopped', 'codex', NULL, NULL, '2026-09-10T00:00:00Z')`; + + const full = yield* query.getShellSnapshot(); + // The settled thread's rows must reach the full read, or skipping them proves nothing. + const settled = full.threads.find((thread) => thread.id === ThreadId.make("t-settled")); + assert.strictEqual(settled?.pullRequests[0]?.number, 9); + assert.strictEqual(settled?.latestTurn?.turnId, asTurnId("turn-settled")); + assert.strictEqual(settled?.session?.status, "stopped"); + assert.strictEqual(full.updatedAt, "2026-09-10T00:00:00Z"); + resolved.length = 0; + + const sweep = yield* readSweepSnapshot(query, null); + assert.strictEqual(sweep.snapshotSequence, full.snapshotSequence); + assert.deepStrictEqual( + sweep.threads, + full.threads.filter((thread) => thread.id !== ThreadId.make("t-settled")), + ); + assert.deepStrictEqual( + sweep.threads.map((thread) => thread.id), + ["t-branch", "t-open", "t-resumed"], + ); + // Like the full read, the sweep resolves every project, so it keeps the + // repository identity cache warm for client connects. + assert.deepStrictEqual(sweep.projects, full.projects); + assert.deepStrictEqual(resolved.toSorted(), ["/four", "/one", "/three", "/two"]); + // A settled thread's link that no longer decodes breaks the full read, but + // not the sweep, which never reads it. + yield* sql`UPDATE projection_thread_pull_requests SET snapshot_json = 'invalid-json' WHERE thread_id = 't-settled'`; + assert.strictEqual((yield* Effect.exit(query.getShellSnapshot()))._tag, "Failure"); + const unsettled = yield* query.getShellSnapshot({ unsettledOnly: true }); + assert.deepStrictEqual(unsettled.threads, sweep.threads); + assert.strictEqual(unsettled.updatedAt, "2026-09-04T00:00:00Z"); + }).pipe(Effect.provide(layer)); +}); + projectionSnapshotLayer("ProjectionSnapshotQuery activities by kind", (it) => { it.effect("lists one kind across active threads only, without hydrating the threads", () => Effect.gen(function* () { diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index 236d031d7a62..51f9b6b4819f 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -12,7 +12,7 @@ import { OrchestrationProposedPlanId, OrchestrationReadModel, OrchestrationThreadSearchSource, - OrchestrationShellSnapshot, + type OrchestrationShellSnapshot, OrchestrationThread, OrchestrationThreadDetailSnapshot, ProjectScript, @@ -77,11 +77,11 @@ import { type ProjectionSnapshotCounts, type ProjectionThreadCheckpointContext, type ProjectionThreadDetailQuery, + type ProjectionThreadPullRequests, type ProjectionSnapshotQueryShape, } from "../Services/ProjectionSnapshotQuery.ts"; const decodeReadModel = Schema.decodeUnknownEffect(OrchestrationReadModel); -const decodeShellSnapshot = Schema.decodeUnknownEffect(OrchestrationShellSnapshot); const decodeThread = Schema.decodeUnknownEffect(OrchestrationThread); const decodeImportedTranscriptsPayload = Schema.decodeUnknownOption( Schema.fromJsonString( @@ -185,6 +185,7 @@ const EventReplayStatsRowSchema = Schema.Struct({ eventCount: Schema.Number, payloadBytes: Schema.Number, }); +const ActiveThreadRowsRequest = Schema.Struct({ unsettledOnly: Schema.Boolean }); const ProjectionThreadSearchRequest = Schema.Struct({ pattern: Schema.String, limit: Schema.Int, @@ -601,10 +602,16 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { `, }); + // Background sweeps skip settled threads, the same check PR discovery makes. + const unsettledThreadsFilter = (unsettledOnly: boolean) => + unsettledOnly + ? sql`AND threads.settled_at IS NULL AND threads.settled_override IS NOT 'settled'` + : sql``; + const listActiveThreadRows = SqlSchema.findAll({ - Request: Schema.Void, + Request: ActiveThreadRowsRequest, Result: ProjectionThreadDbRowSchema, - execute: () => + execute: (request) => sql` SELECT thread_id AS "threadId", @@ -639,9 +646,10 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { pending_user_input_count AS "pendingUserInputCount", has_actionable_proposed_plan AS "hasActionableProposedPlan", deleted_at AS "deletedAt" - FROM projection_threads + FROM projection_threads threads WHERE deleted_at IS NULL AND archived_at IS NULL + ${unsettledThreadsFilter(request.unsettledOnly)} ORDER BY project_id ASC, created_at ASC, thread_id ASC `, }); @@ -782,9 +790,9 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { }); const listActiveThreadPullRequestRows = SqlSchema.findAll({ - Request: Schema.Void, + Request: ActiveThreadRowsRequest, Result: ProjectionThreadPullRequestDbRowSchema, - execute: () => + execute: (request) => sql` SELECT links.thread_id AS "threadId", @@ -801,10 +809,46 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { ON threads.thread_id = links.thread_id WHERE threads.deleted_at IS NULL AND threads.archived_at IS NULL + ${unsettledThreadsFilter(request.unsettledOnly)} ORDER BY links.thread_id ASC, links.linked_at ASC, links.number ASC `, }); + // One row per link, in the shell snapshot's thread order and link order. + const listActiveThreadPullRequestSyncRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionThreadPullRequestDbRowSchema.mapFields( + Struct.assign({ + projectId: ProjectionThread.fields.projectId, + settledOverride: ProjectionThread.fields.settledOverride, + settledAt: ProjectionThread.fields.settledAt, + }), + ), + execute: () => + sql` + SELECT + links.thread_id AS "threadId", + threads.project_id AS "projectId", + threads.settled_override AS "settledOverride", + threads.settled_at AS "settledAt", + links.host, + links.repository, + links.number, + links.url, + links.source, + links.linked_at AS "linkedAt", + links.snapshot_json AS "snapshot", + links.stack_json AS "stack" + FROM projection_thread_pull_requests links + INNER JOIN projection_threads threads + ON threads.thread_id = links.thread_id + WHERE threads.deleted_at IS NULL + AND threads.archived_at IS NULL + ORDER BY threads.project_id ASC, threads.created_at ASC, threads.thread_id ASC, + links.linked_at ASC, links.number ASC + `, + }); + const listArchivedThreadPullRequestRows = SqlSchema.findAll({ Request: Schema.Void, Result: ProjectionThreadPullRequestDbRowSchema, @@ -875,9 +919,9 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { }); const listActiveThreadSessionRows = SqlSchema.findAll({ - Request: Schema.Void, + Request: ActiveThreadRowsRequest, Result: ProjectionThreadSessionDbRowSchema, - execute: () => + execute: (request) => sql` SELECT sessions.thread_id AS "threadId", @@ -895,6 +939,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { ON threads.thread_id = sessions.thread_id WHERE threads.deleted_at IS NULL AND threads.archived_at IS NULL + ${unsettledThreadsFilter(request.unsettledOnly)} ORDER BY sessions.thread_id ASC `, }); @@ -969,9 +1014,9 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { }); const listActiveLatestTurnRows = SqlSchema.findAll({ - Request: Schema.Void, + Request: ActiveThreadRowsRequest, Result: ProjectionLatestTurnDbRowSchema, - execute: () => + execute: (request) => sql` SELECT turns.thread_id AS "threadId", @@ -990,6 +1035,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { WHERE threads.deleted_at IS NULL AND threads.archived_at IS NULL AND threads.latest_turn_id IS NOT NULL + ${unsettledThreadsFilter(request.unsettledOnly)} ORDER BY turns.thread_id ASC `, }); @@ -2625,8 +2671,9 @@ pending_approval_requests AS ( }), ); - const getShellSnapshot: ProjectionSnapshotQueryShape["getShellSnapshot"] = () => - sql + const getShellSnapshot: ProjectionSnapshotQueryShape["getShellSnapshot"] = (options) => { + const unsettledOnly = options?.unsettledOnly === true; + return sql .withTransaction( Effect.all([ listProjectRows(undefined).pipe( @@ -2637,7 +2684,7 @@ pending_approval_requests AS ( ), ), ), - listActiveThreadRows(undefined).pipe( + listActiveThreadRows({ unsettledOnly }).pipe( Effect.mapError( toPersistenceSqlOrDecodeError( "ProjectionSnapshotQuery.getShellSnapshot:listThreads:query", @@ -2645,7 +2692,7 @@ pending_approval_requests AS ( ), ), ), - listActiveThreadSessionRows(undefined).pipe( + listActiveThreadSessionRows({ unsettledOnly }).pipe( Effect.mapError( toPersistenceSqlOrDecodeError( "ProjectionSnapshotQuery.getShellSnapshot:listThreadSessions:query", @@ -2653,7 +2700,7 @@ pending_approval_requests AS ( ), ), ), - listActiveThreadPullRequestRows(undefined).pipe( + listActiveThreadPullRequestRows({ unsettledOnly }).pipe( Effect.mapError( toPersistenceSqlOrDecodeError( "ProjectionSnapshotQuery.getShellSnapshot:listThreadPullRequests:query", @@ -2661,7 +2708,7 @@ pending_approval_requests AS ( ), ), ), - listActiveLatestTurnRows(undefined).pipe( + listActiveLatestTurnRows({ unsettledOnly }).pipe( Effect.mapError( toPersistenceSqlOrDecodeError( "ProjectionSnapshotQuery.getShellSnapshot:listLatestTurns:query", @@ -2716,7 +2763,10 @@ pending_approval_requests AS ( ); const pullRequestsByThread = groupPullRequestRowsByThread(pullRequestRows); - const snapshot = { + // Built from schema-decoded rows, so no second decode here. The HTTP + // and RPC layers encode it against OrchestrationShellSnapshot on the + // way out, like the per-item shells from getThreadShellById. + return { snapshotSequence: computeSnapshotSequence(stateRows), projects: Arr.filterMap(projectRows, (row) => row.deletedAt === null @@ -2771,15 +2821,7 @@ pending_approval_requests AS ( : Result.failVoid, ), updatedAt: updatedAt ?? "1970-01-01T00:00:00.000Z", - }; - - return yield* decodeShellSnapshot(snapshot).pipe( - Effect.mapError( - toPersistenceDecodeError( - "ProjectionSnapshotQuery.getShellSnapshot:decodeShellSnapshot", - ), - ), - ); + } satisfies OrchestrationShellSnapshot; }), ), Effect.mapError((error) => { @@ -2789,6 +2831,36 @@ pending_approval_requests AS ( return toPersistenceSqlError("ProjectionSnapshotQuery.getShellSnapshot:query")(error); }), ); + }; + + const listThreadsWithPullRequests: ProjectionSnapshotQueryShape["listThreadsWithPullRequests"] = + () => + listActiveThreadPullRequestSyncRows(undefined).pipe( + Effect.map((rows) => { + const threads = new Map< + ThreadId, + ProjectionThreadPullRequests & { readonly pullRequests: Array } + >(); + for (const row of rows) { + const thread = threads.get(row.threadId) ?? { + id: row.threadId, + projectId: row.projectId, + settledOverride: row.settledOverride, + settledAt: row.settledAt, + pullRequests: [], + }; + thread.pullRequests.push(mapPullRequestRow(row)); + threads.set(row.threadId, thread); + } + return [...threads.values()]; + }), + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.listThreadsWithPullRequests:query", + "ProjectionSnapshotQuery.listThreadsWithPullRequests:decodeRows", + ), + ), + ); const getArchivedShellSnapshot: ProjectionSnapshotQueryShape["getArchivedShellSnapshot"] = () => sql @@ -2883,7 +2955,7 @@ pending_approval_requests AS ( sessionRows.map((row) => [row.threadId, mapSessionRow(row)] as const), ); - const snapshot = { + return { snapshotSequence: computeSnapshotSequence(stateRows), projects: Arr.filterMap(projectRows, (row) => row.deletedAt === null && activeProjectIds.has(row.projectId) @@ -2934,15 +3006,7 @@ pending_approval_requests AS ( planProgress: threadPlanProgress.getThreadPlanProgress(row.threadId), })), updatedAt: updatedAt ?? "1970-01-01T00:00:00.000Z", - }; - - return yield* decodeShellSnapshot(snapshot).pipe( - Effect.mapError( - toPersistenceDecodeError( - "ProjectionSnapshotQuery.getArchivedShellSnapshot:decodeShellSnapshot", - ), - ), - ); + } satisfies OrchestrationShellSnapshot; }), ), Effect.mapError((error) => { @@ -3788,6 +3852,7 @@ pending_approval_requests AS ( listActivitiesByKind, getSnapshot, getShellSnapshot, + listThreadsWithPullRequests, getArchivedShellSnapshot, getDeletedWorktreeThreads, searchThreads, diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts index 254aec4bde7e..87f9bd03d46c 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts @@ -891,8 +891,92 @@ describe("ProviderCommandReactor", () => { expect(thread?.session?.threadId).toBe("thread-1"); expect(thread?.session?.status).toBe("starting"); expect(thread?.session?.runtimeMode).toBe("approval-required"); + expect(harness.startSession.mock.calls[0]?.[1]).not.toHaveProperty("title"); }); + effectIt.effect("forwards only a user-renamed title when starting a provider session", () => + Effect.gen(function* () { + const harness = yield* Effect.promise(() => + createHarness({ initialTitle: "Add a progressive blur as you scroll" }), + ); + const now = "2026-01-01T00:00:00.000Z"; + const modelSelection = { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }; + const startTurn = (threadId: string, text: string, titleSeed: string) => + harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make(`cmd-title-${threadId}`), + threadId: ThreadId.make(threadId), + message: { + messageId: asMessageId(`message-${threadId}`), + role: "user", + text, + attachments: [], + }, + titleSeed, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt: now, + }); + + yield* startTurn( + "thread-1", + "Add a progressive blur as you scroll", + "Add a progressive blur as you scroll", + ); + yield* Effect.promise(() => waitFor(() => harness.startSession.mock.calls.length === 1)); + expect(harness.startSession.mock.calls[0]?.[1]).not.toHaveProperty("title"); + + yield* harness.engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-thread-create-renamed"), + threadId: ThreadId.make("thread-renamed"), + projectId: asProjectId("project-1"), + title: "New thread", + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: null, + createdAt: now, + }); + yield* harness.engine.dispatch({ + type: "thread.meta.update", + commandId: CommandId.make("cmd-thread-rename"), + threadId: ThreadId.make("thread-renamed"), + title: "Keep this name", + }); + yield* startTurn("thread-renamed", "hello there", "hello there"); + yield* Effect.promise(() => waitFor(() => harness.startSession.mock.calls.length === 2)); + expect(harness.startSession.mock.calls[1]?.[1]).toMatchObject({ title: "Keep this name" }); + + yield* harness.engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-thread-create-seeded"), + threadId: ThreadId.make("thread-seeded"), + projectId: asProjectId("project-1"), + title: "New thread", + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: null, + createdAt: now, + }); + yield* harness.engine.dispatch({ + type: "thread.meta.update", + commandId: CommandId.make("cmd-thread-autotitle"), + threadId: ThreadId.make("thread-seeded"), + title: "hello there", + }); + yield* startTurn("thread-seeded", "hello there", "hello there"); + yield* Effect.promise(() => waitFor(() => harness.startSession.mock.calls.length === 3)); + expect(harness.startSession.mock.calls[2]?.[1]).not.toHaveProperty("title"); + }), + ); + effectIt.effect("projects inline context before sending the provider turn", () => Effect.gen(function* () { const harness = yield* Effect.promise(() => createHarness()); diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts index f6794135a925..77022a8518d7 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts @@ -575,6 +575,9 @@ const make = Effect.gen(function* () { options?: { readonly modelSelection?: ModelSelection; readonly pendingTurnStart?: boolean; + // First-turn prompt seed. A manual title that still equals this seed was + // written by the client's auto-title, not a user rename. + readonly titleSeed?: string; }, ) { const thread = yield* resolveThreadShell(threadId); @@ -712,6 +715,15 @@ const make = Effect.gen(function* () { .refreshWorkspaceSnapshot({ instanceId: desiredInstanceId, cwd: effectiveCwd }) .pipe(Effect.forkDetach) : Effect.void; + // OpenCode skips SessionPrompt.ensureTitle when session.create already has + // a title. Prompt seeds and "New thread" are not user titles, so omit them + // and let the provider generate one. A real rename is source "manual" and + // differs from the first-turn prompt seed (the web client writes that seed + // through thread.meta.update, which also marks the title manual). + const manualTitle = thread.titleState?.source === "manual" ? thread.title.trim() : ""; + const promptSeed = options?.titleSeed?.trim(); + const sessionTitle = + manualTitle.length > 0 && manualTitle !== promptSeed ? thread.title : undefined; const startProviderSession = (input?: { readonly resumeCursor?: unknown; @@ -723,7 +735,7 @@ const make = Effect.gen(function* () { ...(preferredProvider ? { provider: preferredProvider } : {}), providerInstanceId: desiredInstanceId, ...(effectiveCwd ? { cwd: effectiveCwd } : {}), - ...(thread.title ? { title: thread.title } : {}), + ...(sessionTitle ? { title: sessionTitle } : {}), modelSelection: desiredModelSelection, ...(input?.resumeCursor !== undefined ? { resumeCursor: input.resumeCursor } : {}), runtimeMode: desiredRuntimeMode, @@ -839,6 +851,7 @@ const make = Effect.gen(function* () { readonly modelSelection?: ModelSelection; readonly interactionMode?: "default" | "plan"; readonly createdAt: string; + readonly titleSeed?: string; }) { const thread = yield* resolveThreadShell(input.threadId); if (!thread) { @@ -848,6 +861,7 @@ const make = Effect.gen(function* () { } yield* ensureSessionForThread(input.threadId, input.createdAt, { ...(input.modelSelection !== undefined ? { modelSelection: input.modelSelection } : {}), + ...(input.titleSeed !== undefined ? { titleSeed: input.titleSeed } : {}), pendingTurnStart: true, }); if (input.modelSelection !== undefined) { @@ -1488,6 +1502,11 @@ const make = Effect.gen(function* () { : {}), interactionMode: event.payload.interactionMode, createdAt: event.payload.createdAt, + // Later turns must not reuse the current title as titleSeed. Only the + // first prompt seed should suppress a not-yet-renamed session title. + ...(!hasOtherUserMessages && event.payload.titleSeed !== undefined + ? { titleSeed: event.payload.titleSeed } + : {}), }).pipe( Effect.asSome, Effect.catchCause((cause) => handleTurnStartFailure(cause).pipe(Effect.as(Option.none()))), diff --git a/apps/server/src/orchestration/PullRequestSyncReactor.test.ts b/apps/server/src/orchestration/PullRequestSyncReactor.test.ts index 5502376e691e..e6a18c2856a1 100644 --- a/apps/server/src/orchestration/PullRequestSyncReactor.test.ts +++ b/apps/server/src/orchestration/PullRequestSyncReactor.test.ts @@ -172,6 +172,7 @@ const makeHarness = Effect.fn("makePullRequestSyncHarness")(function* (options: const snapshots = yield* Ref.make(options.snapshot); const events = yield* PubSub.unbounded(); const snapshotReads = yield* Queue.unbounded(); + const shellSnapshotReads = yield* Ref.make(0); const syncCommands = yield* Ref.make>([]); const linkCommands = yield* Ref.make>([]); const summaryCalls = yield* Ref.make>([]); @@ -209,8 +210,16 @@ const makeHarness = Effect.fn("makePullRequestSyncHarness")(function* (options: const dependencies = Layer.mergeAll( Layer.mock(ProjectionSnapshotQuery)({ + listThreadsWithPullRequests: () => + Queue.offer(snapshotReads, undefined).pipe( + Effect.andThen(Ref.get(snapshots)), + Effect.map((snapshot) => snapshot.threads), + ), getShellSnapshot: () => - Queue.offer(snapshotReads, undefined).pipe(Effect.andThen(Ref.get(snapshots))), + Ref.update(shellSnapshotReads, (count) => count + 1).pipe( + Effect.andThen(Queue.offer(snapshotReads, undefined)), + Effect.andThen(Ref.get(snapshots)), + ), }), Layer.mock(PullRequestService)({ summary, @@ -235,6 +244,7 @@ const makeHarness = Effect.fn("makePullRequestSyncHarness")(function* (options: activation, snapshots, snapshotReads, + shellSnapshotReads, syncCommands, linkCommands, summaryCalls, @@ -511,6 +521,8 @@ describe("PullRequestSyncReactor", () => { ], ); assert.strictEqual((yield* Ref.get(fixture.stackCalls)).length, 1); + // Reads only linked threads, never the full shell snapshot of every thread. + assert.strictEqual(yield* Ref.get(fixture.shellSnapshotReads), 0); }).pipe(Effect.provide(fixture.layer)); }), ), diff --git a/apps/server/src/orchestration/PullRequestSyncReactor.ts b/apps/server/src/orchestration/PullRequestSyncReactor.ts index 2bdf3ff9e71d..ffcc9fb3c26d 100644 --- a/apps/server/src/orchestration/PullRequestSyncReactor.ts +++ b/apps/server/src/orchestration/PullRequestSyncReactor.ts @@ -1,7 +1,6 @@ import { siblingPullRequestUrl } from "@t3tools/shared/changeRequestUrl"; import { CommandId, - type OrchestrationThreadShell, type PullRequestSummary, type ThreadPullRequestKey, type ThreadPullRequestLink, @@ -36,7 +35,7 @@ const SLOW_SYNC_INTERVAL_MS = 15 * 60 * 1_000; type SnapshotFields = Omit; interface LinkEntry { - readonly thread: OrchestrationThreadShell; + readonly thread: ProjectionSnapshotQuery.ProjectionThreadPullRequests; readonly link: ThreadPullRequestLink; } @@ -104,15 +103,15 @@ function stacksEqual( ); } -function isUnsettled(thread: OrchestrationThreadShell): boolean { +function isUnsettled(thread: ProjectionSnapshotQuery.ProjectionThreadPullRequests): boolean { return thread.settledOverride !== "settled" && thread.settledAt === null; } /** * Keeps every thread ↔ pull request link's host snapshot current. One sweep a minute reads - * the shell snapshot, groups visible links by pull request so the host is asked once per PR - * no matter how many threads share it, and writes back only what changed. Native stacks the - * host reports are auto-linked to the thread as `source: "stack"`. + * only the active threads that have links, groups visible links by pull request so the host + * is asked once per PR no matter how many threads share it, and writes back only what + * changed. Native stacks the host reports are auto-linked to the thread as `source: "stack"`. */ export class PullRequestSyncReactor extends Context.Service< PullRequestSyncReactor, @@ -153,14 +152,13 @@ export const make = Effect.gen(function* () { Cause.hasInterruptsOnly(cause) ? Effect.failCause(cause) : Effect.logWarning(message, fields); const sweep = Effect.fn("PullRequestSyncReactor.sweep")(function* (requestedKey?: string) { - const snapshot = yield* snapshots.getShellSnapshot(); + const threads = yield* snapshots.listThreadsWithPullRequests(); const now = yield* DateTime.now; const nowMs = DateTime.toEpochMillis(now); const nowIso = DateTime.formatIso(now); const groups = new Map>(); - for (const thread of snapshot.threads) { - if (thread.archivedAt !== null) continue; + for (const thread of threads) { for (const link of visibleThreadPullRequests(thread.pullRequests)) { const key = threadPullRequestKeyOf(link); const entries = groups.get(key) ?? []; diff --git a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts index eac3ede9c1ee..08da6e00940c 100644 --- a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts @@ -64,6 +64,12 @@ export interface ProjectionFullThreadDiffContext { readonly toCheckpointRef: CheckpointRef | null; } +/** The thread fields pull request sync reads, for a thread with at least one link. */ +export type ProjectionThreadPullRequests = Pick< + OrchestrationThreadShell, + "id" | "projectId" | "settledOverride" | "settledAt" | "pullRequests" +>; + export interface ProjectionThreadDetailQuery { /** * Limit activities before SQLite returns and decodes their payloads. @@ -114,11 +120,15 @@ export interface ProjectionSnapshotQueryShape { * * Returns only projects and thread shell summaries so clients can bootstrap * lightweight navigation state without hydrating every thread body. + * + * `unsettledOnly` is for background sweeps, not clients. It skips settled + * threads and their sessions, PR links, and turns, and its `updatedAt` + * ignores those rows. It still resolves every project, which keeps + * repository identities cached for client connects. */ - readonly getShellSnapshot: () => Effect.Effect< - OrchestrationShellSnapshot, - ProjectionRepositoryError - >; + readonly getShellSnapshot: (options?: { + readonly unsettledOnly?: boolean; + }) => Effect.Effect; /** * Read archived thread shell summaries for the archive page. @@ -131,6 +141,16 @@ export interface ProjectionSnapshotQueryShape { ProjectionRepositoryError >; + /** + * Read active (not deleted, not archived) threads that have at least one pull + * request link, in shell snapshot order. Skips repository identity, so no + * legacy `linkedPullRequest` is derived. + */ + readonly listThreadsWithPullRequests: () => Effect.Effect< + ReadonlyArray, + ProjectionRepositoryError + >; + /** Durable worktree ownership retained after thread deletion, including across restarts. */ readonly getDeletedWorktreeThreads: () => Effect.Effect< ReadonlyArray<{ diff --git a/apps/server/src/orchestration/ThreadPullRequestReactor.test.ts b/apps/server/src/orchestration/ThreadPullRequestReactor.test.ts index d6572083dad2..2a08dd03bc54 100644 --- a/apps/server/src/orchestration/ThreadPullRequestReactor.test.ts +++ b/apps/server/src/orchestration/ThreadPullRequestReactor.test.ts @@ -21,18 +21,27 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; import { TestClock } from "effect/testing"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; import { GitManager, type GitBranchPullRequest } from "../git/GitManager.ts"; +import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; import { PullRequestService } from "../pullRequest/PullRequestService.ts"; import { RepositoryIdentityResolver } from "../project/RepositoryIdentityResolver.ts"; import { ServerActivation } from "../serverActivation.ts"; +import { OrchestrationProjectionSnapshotQueryLive } from "./Layers/ProjectionSnapshotQuery.ts"; import { OrchestrationEngineService } from "./Services/OrchestrationEngine.ts"; -import { ProjectionSnapshotQuery } from "./Services/ProjectionSnapshotQuery.ts"; +import { + ProjectionSnapshotQuery, + type ProjectionSnapshotQueryShape, +} from "./Services/ProjectionSnapshotQuery.ts"; +import * as ThreadBackgroundLiveness from "./ThreadBackgroundLiveness.ts"; +import * as ThreadPlanProgress from "./ThreadPlanProgress.ts"; import * as ThreadPullRequestReactor from "./ThreadPullRequestReactor.ts"; const NOW = "2026-09-01T12:00:00.000Z"; @@ -134,6 +143,8 @@ const makeHarness = Effect.fn("makeThreadPullRequestHarness")(function* (options readonly existingWorktrees?: ReadonlyArray; readonly project?: OrchestrationProjectShell; readonly resolveRepositoryIdentity?: RepositoryIdentityResolver["Service"]["resolve"]; + /** Serve full sweep reads from this instead of `threads`. */ + readonly getShellSnapshot?: ProjectionSnapshotQueryShape["getShellSnapshot"]; }) { const activation = yield* Deferred.make(); const snapshots = yield* Ref.make({ @@ -142,7 +153,8 @@ const makeHarness = Effect.fn("makeThreadPullRequestHarness")(function* (options threads: options.threads, updatedAt: NOW, }); - const reads = yield* Queue.unbounded(); + // Each shell read: a thread id for a one-thread read, null for a full read. + const reads = yield* Queue.unbounded(); const events = yield* PubSub.unbounded(); const commands = yield* Ref.make>([]); const branchCalls = yield* Ref.make< @@ -152,8 +164,27 @@ const makeHarness = Effect.fn("makeThreadPullRequestHarness")(function* (options let uuid = 0; const dependencies = Layer.mergeAll( Layer.mock(ProjectionSnapshotQuery)({ - getShellSnapshot: () => - Ref.get(snapshots).pipe(Effect.tap(() => Queue.offer(reads, undefined))), + getShellSnapshot: (readOptions) => + (options.getShellSnapshot?.(readOptions) ?? Ref.get(snapshots)).pipe( + Effect.tap(() => Queue.offer(reads, null)), + ), + getSnapshotSequence: () => + Ref.get(snapshots).pipe(Effect.map(({ snapshotSequence }) => ({ snapshotSequence }))), + getThreadShellById: (threadId) => + Ref.get(snapshots).pipe( + Effect.map(({ threads }) => + Option.fromUndefinedOr( + threads.find((thread) => thread.id === threadId && thread.archivedAt === null), + ), + ), + Effect.tap(() => Queue.offer(reads, threadId)), + ), + getProjectShells: (projectIds) => + Ref.get(snapshots).pipe( + Effect.map(({ projects }) => + projects.filter((project) => projectIds?.includes(project.id) ?? true), + ), + ), }), Layer.mock(GitManager)({ branchPullRequest: (input, readOptions) => @@ -376,7 +407,7 @@ describe("ThreadPullRequestReactor", () => { : [checkpointEvent, sessionEvent]; for (const event of events) { yield* fixture.publish(event); - yield* Queue.take(fixture.reads); + expect(yield* Queue.take(fixture.reads)).toBe(current.id); yield* reactor.drain; } expect((yield* Ref.get(fixture.commands))[0]?.branchPullRequest).toEqual(reference(42)); @@ -388,6 +419,51 @@ describe("ThreadPullRequestReactor", () => { ), ); + it.effect("refreshes the project identity when a turn adds the remote", () => + Effect.scoped( + Effect.gen(function* () { + const current = thread("new-remote"); + const fixture = yield* makeHarness({ + threads: [current], + project: { ...project, repositoryIdentity: null }, + branchPullRequest: () => Effect.succeed(branchPullRequest()), + resolveRepositoryIdentity: (_cwd, options) => + Effect.succeed(options?.refresh ? project.repositoryIdentity : null), + }); + yield* Effect.gen(function* () { + const reactor = yield* fixture.start(); + expect(yield* Ref.get(fixture.commands)).toHaveLength(0); + + yield* fixture.publish({ + type: "thread.turn-diff-completed", + sequence: 2, + eventId: EventId.make("checkpoint-finished"), + aggregateKind: "thread", + aggregateId: current.id, + occurredAt: NOW, + commandId: null, + causationEventId: null, + correlationId: null, + metadata: {}, + payload: { + threadId: current.id, + turnId: TurnId.make("turn"), + checkpointTurnCount: 1, + checkpointRef: CheckpointRef.make("checkpoint"), + status: "ready", + files: [], + assistantMessageId: null, + completedAt: NOW, + }, + }); + yield* Queue.take(fixture.reads); + yield* reactor.drain; + expect((yield* Ref.get(fixture.commands))[0]?.branchPullRequest).toEqual(reference(42)); + }).pipe(Effect.provide(fixture.layer)); + }), + ), + ); + it.effect("uses live worktrees and falls back to the project for removed worktrees", () => Effect.scoped( Effect.gen(function* () { @@ -516,6 +592,23 @@ describe("ThreadPullRequestReactor", () => { yield* Effect.gen(function* () { const reactor = yield* fixture.start(); expect(yield* Ref.get(fixture.commands)).toHaveLength(0); + // A one-thread read cannot show that other pending threads are gone. + const gone = ThreadId.make("gone"); + yield* fixture.publish({ + type: "thread.unarchived", + sequence: 2, + eventId: EventId.make("gone-unarchived"), + aggregateKind: "thread", + aggregateId: gone, + occurredAt: NOW, + commandId: null, + causationEventId: null, + correlationId: null, + metadata: {}, + payload: { threadId: gone, updatedAt: NOW }, + }); + expect(yield* Queue.take(fixture.reads)).toBe(gone); + yield* reactor.drain; yield* Ref.set(online, true); yield* TestClock.adjust("1 minute"); yield* Queue.take(fixture.reads); @@ -564,6 +657,113 @@ describe("ThreadPullRequestReactor", () => { ), ); + it.effect("discovers the same PRs from the unsettled read as from the full read", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const query = yield* ProjectionSnapshotQuery; + yield* sql`INSERT INTO projection_projects (project_id, title, workspace_root, scripts_json, created_at, updated_at) + VALUES ('project', 'Project', '/workspace/project', '[]', ${NOW}, ${NOW}), + ('dormant', 'Dormant', '/workspace/dormant', '[]', ${NOW}, ${NOW})`; + yield* sql`INSERT INTO projection_threads (thread_id, project_id, title, model_selection_json, runtime_mode, interaction_mode, branch, branch_pull_request_json, created_at, updated_at, archived_at, settled_override, settled_at) + VALUES + ('open', 'project', 'Open', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'open', NULL, ${NOW}, ${NOW}, NULL, NULL, NULL), + ('resumed', 'project', 'Resumed', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'resumed', NULL, ${NOW}, ${NOW}, NULL, 'active', NULL), + ('linked', 'project', 'Linked', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'linked', '{"projectId":"project","repository":"owner/repository","number":3,"url":"https://github.com/owner/repository/pull/3"}', ${NOW}, ${NOW}, NULL, NULL, NULL), + ('settled', 'project', 'Settled', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'settled', '{"projectId":"project","repository":"owner/repository","number":4,"url":"https://github.com/owner/repository/pull/4"}', ${NOW}, ${NOW}, NULL, 'settled', ${NOW}), + ('backfill', 'project', 'Backfill', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'backfill', NULL, ${NOW}, ${NOW}, NULL, 'settled', ${NOW}), + ('imported', 'dormant', 'Imported', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, ${NOW}, ${NOW}, NULL, 'settled', ${NOW}), + ('archived', 'project', 'Archived', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'archived', NULL, ${NOW}, ${NOW}, ${NOW}, NULL, NULL)`; + const numbers = new Map([ + ["open", 1], + ["resumed", 2], + ["linked", 3], + ["settled", 4], + ["backfill", 5], + ["archived", 6], + ]); + + // Startup, then two periodic passes. The startup backfill lookup fails, + // so the first periodic pass retries it from the full read. + const discover = (read: ProjectionSnapshotQueryShape["getShellSnapshot"]) => + Effect.gen(function* () { + const online = yield* Ref.make(false); + const reads: Array> = []; + const fixture = yield* makeHarness({ + threads: [], + getShellSnapshot: (options) => + read(options).pipe( + Effect.tap((snapshot) => + Effect.sync(() => reads.push(snapshot.threads.map(({ id }) => id).toSorted())), + ), + ), + branchPullRequest: ({ cwd, branch }) => + Effect.gen(function* () { + if (branch === "backfill" && !(yield* Ref.get(online))) { + return yield* new GitManagerError({ + operation: "branchPullRequest", + cwd, + detail: "Offline", + }); + } + const number = numbers.get(branch); + return number === undefined ? null : branchPullRequest(number); + }), + }); + return yield* Effect.gen(function* () { + const reactor = yield* fixture.start(); + const startupCalls = (yield* Ref.get(fixture.branchCalls)).length; + const startupCommands = (yield* Ref.get(fixture.commands)).length; + yield* Ref.set(online, true); + for (let pass = 0; pass < 2; pass++) { + yield* TestClock.adjust("1 minute"); + yield* Queue.take(fixture.reads); + yield* reactor.drain; + } + return { + reads, + branchCalls: (yield* Ref.get(fixture.branchCalls)) + .slice(startupCalls) + .map(({ branch }) => branch) + .toSorted(), + commands: (yield* Ref.get(fixture.commands)) + .slice(startupCommands) + .map( + ({ threadId, branchPullRequest }) => `${threadId} ${branchPullRequest?.number}`, + ) + .toSorted(), + }; + }).pipe(Effect.provide(fixture.layer)); + }).pipe(Effect.scoped); + + const { reads: unsettledReads, ...unsettled } = yield* discover(query.getShellSnapshot); + const { reads: fullReads, ...full } = yield* discover(() => query.getShellSnapshot()); + expect(unsettled).toEqual(full); + expect(unsettled.commands).toEqual([ + "backfill 5", + "open 1", + "open 1", + "resumed 2", + "resumed 2", + ]); + // The last pass has no backfill left, so it reads no settled thread. + expect(fullReads.at(-1)).toContain("imported"); + expect(unsettledReads.at(-1)).toEqual(["linked", "open", "resumed"]); + }).pipe( + Effect.provide( + OrchestrationProjectionSnapshotQueryLive.pipe( + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide( + Layer.succeed(RepositoryIdentityResolver, { + resolve: () => Effect.succeed(project.repositoryIdentity), + }), + ), + Layer.provideMerge(SqlitePersistenceMemory), + ), + ), + ), + ); + it.effect("matches Azure SSH projects to HTTPS PRs with the provider repository selector", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration/ThreadPullRequestReactor.ts b/apps/server/src/orchestration/ThreadPullRequestReactor.ts index 17efc74e4f40..66ff81d5a9e7 100644 --- a/apps/server/src/orchestration/ThreadPullRequestReactor.ts +++ b/apps/server/src/orchestration/ThreadPullRequestReactor.ts @@ -6,6 +6,7 @@ import { CommandId, type OrchestrationEvent, type OrchestrationProjectShell, + type OrchestrationShellSnapshot, type ThreadId, type ThreadLinkedPullRequest, } from "@t3tools/contracts"; @@ -16,11 +17,13 @@ import * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Schedule from "effect/Schedule"; import type * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as GitManager from "../git/GitManager.ts"; +import type { ProjectionRepositoryError } from "../persistence/Errors.ts"; import * as PullRequestService from "../pullRequest/PullRequestService.ts"; import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; import { forkParked } from "../serverActivation.ts"; @@ -69,6 +72,37 @@ export function pullRequestMatchesProject( ); } +/** + * Read the shell state for a discovery or settlement sweep. A sweep for one + * thread reads that thread and the projects it names, not every thread. A + * sweep over all threads reads only unsettled threads, since both sweeps skip + * settled ones. Discovery's backfill does its own full read. + */ +export const readSweepSnapshot = ( + snapshots: ProjectionSnapshotQuery.ProjectionSnapshotQueryShape, + threadId: ThreadId | null, +): Effect.Effect< + Pick, + ProjectionRepositoryError +> => + threadId === null + ? snapshots.getShellSnapshot({ unsettledOnly: true }) + : Effect.gen(function* () { + // Read the sequence first. The thread is then at least this new, so a + // command guarded by the sequence is rejected rather than missing a change. + const { snapshotSequence } = yield* snapshots.getSnapshotSequence(); + const thread = yield* snapshots.getThreadShellById(threadId); + if (Option.isNone(thread)) return { snapshotSequence, projects: [], threads: [] }; + // Settlement also checks the project a saved pull request names. + const reference = thread.value.linkedPullRequest ?? thread.value.branchPullRequest; + const projects = yield* snapshots.getProjectShells( + reference == null + ? [thread.value.projectId] + : [thread.value.projectId, reference.projectId], + ); + return { snapshotSequence, projects, threads: [thread.value] }; + }); + /** @public Service construction is part of the canonical Effect module API. */ export const make = Effect.gen(function* () { const engine = yield* OrchestrationEngine.OrchestrationEngineService; @@ -97,7 +131,11 @@ export const make = Effect.gen(function* () { const synchronize = Effect.fn("ThreadPullRequestReactor.synchronize")(function* ( request: RefreshRequest, ) { - const snapshot = yield* snapshots.getShellSnapshot(); + // Backfill looks up settled threads, so its passes read every thread. + const snapshot = + request.threadId === null && (request.backfill || pendingBackfill.size > 0) + ? yield* snapshots.getShellSnapshot() + : yield* readSweepSnapshot(snapshots, request.threadId); const projects = new Map(snapshot.projects.map((project) => [project.id, project])); if (request.backfill) { for (const thread of snapshot.threads) { @@ -109,14 +147,19 @@ export const make = Effect.gen(function* () { } } } - const threadIds = new Set(snapshot.threads.map((thread) => thread.id)); - for (const threadId of pendingBackfill.keys()) { - if (!threadIds.has(threadId)) pendingBackfill.delete(threadId); + // A single-thread read only shows whether its own thread is gone. A thread + // with no branch has nothing to look up, and its entry would keep every + // periodic pass on the full read. + const branchThreadIds = new Set( + snapshot.threads.filter((thread) => thread.branch !== null).map((thread) => thread.id), + ); + const checkedIds = request.threadId === null ? pendingBackfill.keys() : [request.threadId]; + for (const threadId of checkedIds) { + if (!branchThreadIds.has(threadId)) pendingBackfill.delete(threadId); } const threads = snapshot.threads.filter( (thread) => thread.archivedAt === null && - (request.threadId === null || thread.id === request.threadId) && ((thread.settledOverride !== "settled" && thread.settledAt === null) || request.threadId !== null || pendingBackfill.has(thread.id)) && @@ -131,8 +174,19 @@ export const make = Effect.gen(function* () { (group) => Effect.gen(function* () { const first = group[0]!; - const project = projects.get(first.projectId); - if (project === undefined) return finishBackfill(group); + const snapshotProject = projects.get(first.projectId); + if (snapshotProject === undefined) return finishBackfill(group); + // A finished turn may have added the remote this PR lives on. A failed + // refresh resolves to null, so keep the snapshot's identity then. + const project = request.refresh + ? { + ...snapshotProject, + repositoryIdentity: + (yield* repositoryIdentities.resolve(snapshotProject.workspaceRoot, { + refresh: true, + })) ?? snapshotProject.repositoryIdentity, + } + : snapshotProject; const repository = sourceControlRepositorySelector(project.repositoryIdentity); if (first.branch !== null && repository === null) return finishBackfill(group); const worktreeExists = diff --git a/apps/server/src/orchestration/ThreadSettlementReactor.test.ts b/apps/server/src/orchestration/ThreadSettlementReactor.test.ts index 07afbaf05e6a..cad45007665c 100644 --- a/apps/server/src/orchestration/ThreadSettlementReactor.test.ts +++ b/apps/server/src/orchestration/ThreadSettlementReactor.test.ts @@ -24,13 +24,17 @@ import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; import { TestClock } from "effect/testing"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; import { GitManager, type GitBranchPullRequest } from "../git/GitManager.ts"; +import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; +import { RepositoryIdentityResolver } from "../project/RepositoryIdentityResolver.ts"; import { PullRequestService, type PullRequestMergeEvent, @@ -38,11 +42,17 @@ import { import { ServerActivation } from "../serverActivation.ts"; import { ServerSettingsService } from "../serverSettings.ts"; import { OrchestrationCommandInvariantError } from "./Errors.ts"; +import { OrchestrationProjectionSnapshotQueryLive } from "./Layers/ProjectionSnapshotQuery.ts"; import { OrchestrationEngineService, type OrchestrationEngineShape, } from "./Services/OrchestrationEngine.ts"; -import { ProjectionSnapshotQuery } from "./Services/ProjectionSnapshotQuery.ts"; +import { + ProjectionSnapshotQuery, + type ProjectionSnapshotQueryShape, +} from "./Services/ProjectionSnapshotQuery.ts"; +import * as ThreadBackgroundLiveness from "./ThreadBackgroundLiveness.ts"; +import * as ThreadPlanProgress from "./ThreadPlanProgress.ts"; import * as ThreadSettlementReactor from "./ThreadSettlementReactor.ts"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Path from "effect/Path"; @@ -168,6 +178,8 @@ function makeBranchPullRequest( interface HarnessOptions { readonly snapshot: OrchestrationShellSnapshot; + /** Serve full sweep reads from this instead of `snapshot`. */ + readonly getShellSnapshot?: ProjectionSnapshotQueryShape["getShellSnapshot"]; readonly settings?: ServerSettings; readonly branchPullRequest?: GitManager["Service"]["branchPullRequest"]; readonly pullRequestSummary?: PullRequestService["Service"]["summary"]; @@ -181,7 +193,8 @@ const makeHarness = Effect.fn("makeThreadSettlementHarness")(function* (options: const activation = yield* Deferred.make(); const snapshots = yield* Ref.make(options.snapshot); const snapshotReadCount = yield* Ref.make(0); - const snapshotReads = yield* Queue.unbounded(); + // Each shell read: a thread id for a one-thread read, null for a full read. + const snapshotReads = yield* Queue.unbounded(); const settings = yield* Ref.make(options.settings ?? DEFAULT_SERVER_SETTINGS); const settingsReads = yield* Queue.unbounded(); const settingsChanges = yield* PubSub.unbounded(); @@ -254,10 +267,27 @@ const makeHarness = Effect.fn("makeThreadSettlementHarness")(function* (options: const dependencies = Layer.mergeAll( Layer.mock(ProjectionSnapshotQuery)({ - getShellSnapshot: () => - Ref.updateAndGet(snapshotReadCount, (count) => count + 1).pipe( - Effect.tap((count) => Queue.offer(snapshotReads, count)), - Effect.andThen(Ref.get(snapshots)), + getShellSnapshot: (readOptions) => + Ref.update(snapshotReadCount, (count) => count + 1).pipe( + Effect.andThen(Queue.offer(snapshotReads, null)), + Effect.andThen(options.getShellSnapshot?.(readOptions) ?? Ref.get(snapshots)), + ), + getSnapshotSequence: () => + Ref.get(snapshots).pipe(Effect.map(({ snapshotSequence }) => ({ snapshotSequence }))), + getThreadShellById: (threadId) => + Ref.get(snapshots).pipe( + Effect.map(({ threads }) => + Option.fromUndefinedOr( + threads.find((thread) => thread.id === threadId && thread.archivedAt === null), + ), + ), + Effect.tap(() => Queue.offer(snapshotReads, threadId)), + ), + getProjectShells: (projectIds) => + Ref.get(snapshots).pipe( + Effect.map(({ projects }) => + projects.filter((project) => projectIds?.includes(project.id) ?? true), + ), ), }), Layer.mock(GitManager)({ @@ -313,7 +343,7 @@ const makeHarness = Effect.fn("makeThreadSettlementHarness")(function* (options: const startHarness = Effect.fn("startThreadSettlementHarness")(function* ( reactor: ThreadSettlementReactor.ThreadSettlementReactor["Service"], activation: Deferred.Deferred, - snapshotReads: Queue.Queue, + snapshotReads: Queue.Queue, ) { yield* reactor.start(); yield* Deferred.succeed(activation, undefined); @@ -443,7 +473,7 @@ describe("ThreadSettlementReactor", () => { updatedAt: NOW, }, }); - yield* Queue.take(fixture.snapshotReads); + assert.strictEqual(yield* Queue.take(fixture.snapshotReads), thread.id); yield* reactor.drain; } assert.deepStrictEqual( @@ -463,7 +493,7 @@ describe("ThreadSettlementReactor", () => { aggregateId: readySession.threadId, payload: { threadId: readySession.threadId, session: readySession }, }); - yield* Queue.take(fixture.snapshotReads); + assert.strictEqual(yield* Queue.take(fixture.snapshotReads), readySession.threadId); yield* reactor.drain; assert.deepStrictEqual( (yield* Ref.get(fixture.commands)).map(({ threadId }) => threadId), @@ -1418,6 +1448,88 @@ describe("ThreadSettlementReactor", () => { }), ), ); + + it.effect("settles the same threads from the unsettled read as from the full read", () => + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse(NOW)); + const sql = yield* SqlClient.SqlClient; + const query = yield* ProjectionSnapshotQuery; + yield* sql`INSERT INTO projection_projects (project_id, title, workspace_root, scripts_json, created_at, updated_at) + VALUES ('settlement-project', 'Project', '/workspace/project', '[]', ${NOW}, ${NOW}), + ('linked-settlement-project', 'Linked', '/workspace/linked', '[]', ${NOW}, ${NOW}), + ('dormant-project', 'Dormant', '/workspace/dormant', '[]', ${NOW}, ${NOW})`; + yield* sql`INSERT INTO projection_threads (thread_id, project_id, title, model_selection_json, runtime_mode, interaction_mode, branch, branch_pull_request_json, latest_user_message_at, created_at, updated_at, archived_at, settled_override, settled_at) + VALUES + ('idle', 'settlement-project', 'Idle', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, '2026-08-20T00:00:00.000Z', '2026-08-01T00:00:00.000Z', ${NOW}, NULL, NULL, NULL), + ('merged', 'settlement-project', 'Merged', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'feature', + '{"projectId":"linked-settlement-project","repository":"owner/repository","number":42,"url":"https://example.test/owner/repository/pull/42"}', + '2026-08-27T00:00:00.000Z', '2026-08-01T00:00:00.000Z', ${NOW}, NULL, NULL, NULL), + ('linked', 'settlement-project', 'Linked', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, '2026-08-27T00:00:00.000Z', '2026-08-01T00:00:00.000Z', ${NOW}, NULL, NULL, NULL), + ('open', 'settlement-project', 'Open', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'open-feature', NULL, '2026-08-27T00:00:00.000Z', '2026-08-01T00:00:00.000Z', ${NOW}, NULL, NULL, NULL), + ('resumed', 'settlement-project', 'Resumed', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, '2026-08-20T00:00:00.000Z', '2026-08-01T00:00:00.000Z', ${NOW}, NULL, 'active', NULL), + ('settled', 'dormant-project', 'Settled', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'done', NULL, '2026-08-20T00:00:00.000Z', '2026-08-01T00:00:00.000Z', ${NOW}, NULL, 'settled', '2026-08-21T00:00:00.000Z'), + ('archived', 'settlement-project', 'Archived', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, '2026-08-20T00:00:00.000Z', '2026-08-01T00:00:00.000Z', ${NOW}, ${NOW}, NULL, NULL)`; + yield* sql`INSERT INTO projection_thread_pull_requests (thread_id, host, repository, number, url, source, linked_at, snapshot_json) + VALUES ('linked', 'example.test', 'owner/repository', 7, 'https://example.test/owner/repository/pull/7', 'manual', ${NOW}, + '{"state":"merged","title":"Review","headBranch":"linked","baseBranch":"main","isDraft":false,"updatedAt":"2026-08-28T12:00:00.000Z","syncedAt":"2026-08-28T12:00:00.000Z","mergedAt":"2026-08-28T12:00:00.000Z","closedAt":null}')`; + + const sweep = (read: ProjectionSnapshotQueryShape["getShellSnapshot"]) => + Effect.gen(function* () { + const readThreadIds: Array = []; + const fixture = yield* makeHarness({ + snapshot: makeSnapshot([]), + getShellSnapshot: (options) => + read(options).pipe( + Effect.tap((snapshot) => + Effect.sync(() => readThreadIds.push(...snapshot.threads.map(({ id }) => id))), + ), + ), + branchPullRequest: ({ branch }) => + Effect.succeed(branch === "open-feature" ? makeBranchPullRequest("open") : null), + pullRequestSummary: (input) => + Effect.succeed(makePullRequestSummary({ ...input, state: "merged" })), + }); + return yield* Effect.gen(function* () { + const reactor = yield* ThreadSettlementReactor.ThreadSettlementReactor; + yield* startHarness(reactor, fixture.activation, fixture.snapshotReads); + return { + readThreadIds: readThreadIds.toSorted(), + commands: (yield* Ref.get(fixture.commands)) + .map(({ threadId, settledAt }) => `${threadId} ${settledAt}`) + .toSorted(), + branchCalls: (yield* Ref.get(fixture.branchCalls)) + .map(({ branch }) => branch) + .toSorted(), + summaryCalls: yield* Ref.get(fixture.summaryCalls), + }; + }).pipe(Effect.provide(fixture.layer)); + }).pipe(Effect.scoped); + + const { readThreadIds: unsettledReads, ...unsettled } = yield* sweep(query.getShellSnapshot); + const { readThreadIds: fullReads, ...full } = yield* sweep(() => query.getShellSnapshot()); + assert.deepStrictEqual(unsettled, full); + // A settle for inactivity, for a synced merged link, and for a saved + // branch PR whose project only that PR names. + assert.deepStrictEqual(unsettled.commands, [ + "idle 2026-08-20T00:00:00.000Z", + "linked 2026-08-27T00:00:00.000Z", + "merged 2026-08-27T00:00:00.000Z", + ]); + assert.deepStrictEqual(fullReads, [...unsettledReads, "settled"].toSorted()); + assert.deepStrictEqual(unsettledReads, ["idle", "linked", "merged", "open", "resumed"]); + }).pipe( + Effect.provide( + OrchestrationProjectionSnapshotQueryLive.pipe( + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide( + Layer.succeed(RepositoryIdentityResolver, { resolve: () => Effect.succeed(null) }), + ), + Layer.provideMerge(SqlitePersistenceMemory), + ), + ), + ), + ); }); describe("storage cleanup", () => { diff --git a/apps/server/src/orchestration/ThreadSettlementReactor.ts b/apps/server/src/orchestration/ThreadSettlementReactor.ts index 4192896efed5..22d21ff27996 100644 --- a/apps/server/src/orchestration/ThreadSettlementReactor.ts +++ b/apps/server/src/orchestration/ThreadSettlementReactor.ts @@ -23,7 +23,7 @@ import * as ServerSettings from "../serverSettings.ts"; import { forkParked } from "../serverActivation.ts"; import * as OrchestrationEngine from "./Services/OrchestrationEngine.ts"; import * as ProjectionSnapshotQuery from "./Services/ProjectionSnapshotQuery.ts"; -import { pullRequestMatchesProject } from "./ThreadPullRequestReactor.ts"; +import { pullRequestMatchesProject, readSweepSnapshot } from "./ThreadPullRequestReactor.ts"; import { isAutoSettlementCandidate, resolveAutoSettlementAt, @@ -95,20 +95,16 @@ export const make = Effect.gen(function* () { if (!autoSettlementConfigured(settings)) { return; } - const snapshot = yield* snapshots.getShellSnapshot(); + const snapshot = yield* readSweepSnapshot(snapshots, threadId ?? null); const now = DateTime.formatIso(yield* DateTime.now); const projects = new Map(snapshot.projects.map((project) => [project.id, project])); // A merge rechecks all candidates, including branches that discovery has // not linked yet. Those lookups can still have cached the PR as open. - const candidates = snapshot.threads.filter( - (thread) => - (threadId === undefined || thread.id === threadId) && - isAutoSettlementCandidate(thread, now), - ); + const candidates = snapshot.threads.filter((thread) => isAutoSettlementCandidate(thread, now)); // Return the thread when it still needs a pull request decision. A rejected // dispatch skips it for this snapshot instead of retrying through a lookup. - const settleThread = Effect.fn("ThreadSettlementReactor.settleThread")( + const settleThread = Effect.fnUntraced( function* (thread: (typeof candidates)[number], pullRequest: SettlementPullRequest | null) { const settings = resolveProjectSettings( yield* settingsService.getSettings, diff --git a/apps/server/src/orchestration/projector.ts b/apps/server/src/orchestration/projector.ts index 54166e417192..96b4a3dd26c4 100644 --- a/apps/server/src/orchestration/projector.ts +++ b/apps/server/src/orchestration/projector.ts @@ -117,12 +117,26 @@ function settledTurnStateForSessionStatus( } } +// Runs for every thread event (including streaming deltas) against every +// thread the server has ever seen, so copy the array rather than map it. function updateThread( threads: ReadonlyArray, threadId: ThreadId, patch: ThreadPatch, -): OrchestrationThread[] { - return threads.map((thread) => (thread.id === threadId ? { ...thread, ...patch } : thread)); +): ReadonlyArray { + const index = threads.findIndex((thread) => thread.id === threadId); + return index === -1 ? threads : patchThreadAt(threads, index, patch); +} + +/** For callers that already located the thread and must not scan again. */ +function patchThreadAt( + threads: ReadonlyArray, + index: number, + patch: ThreadPatch, +): ReadonlyArray { + const next = threads.slice(); + next[index] = { ...threads[index]!, ...patch }; + return next; } /** Patch that swaps a thread's links and re-derives the legacy single-PR field from them. */ @@ -768,7 +782,8 @@ export function projectEvent( event.type, "payload", ); - const thread = nextBase.threads.find((entry) => entry.id === payload.threadId); + const threadIndex = nextBase.threads.findIndex((entry) => entry.id === payload.threadId); + const thread = nextBase.threads[threadIndex]; if (!thread) { return nextBase; } @@ -816,7 +831,7 @@ export function projectEvent( return { ...nextBase, - threads: updateThread(nextBase.threads, payload.threadId, { + threads: patchThreadAt(nextBase.threads, threadIndex, { messages: cappedMessages, updatedAt: event.occurredAt, }), @@ -1056,7 +1071,8 @@ export function projectEvent( "payload", ).pipe( Effect.map((payload) => { - const thread = nextBase.threads.find((entry) => entry.id === payload.threadId); + const threadIndex = nextBase.threads.findIndex((entry) => entry.id === payload.threadId); + const thread = nextBase.threads[threadIndex]; if (!thread) { return nextBase; } @@ -1070,7 +1086,7 @@ export function projectEvent( return { ...nextBase, - threads: updateThread(nextBase.threads, payload.threadId, { + threads: patchThreadAt(nextBase.threads, threadIndex, { activities, updatedAt: event.occurredAt, }), diff --git a/apps/server/src/persistence/Layers/Sqlite.test.ts b/apps/server/src/persistence/Layers/Sqlite.test.ts index 0b64e4f7fdcb..5bcbd35e918c 100644 --- a/apps/server/src/persistence/Layers/Sqlite.test.ts +++ b/apps/server/src/persistence/Layers/Sqlite.test.ts @@ -10,7 +10,11 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as SqlClient from "effect/unstable/sql/SqlClient"; -import { SqlitePersistenceMemory, makeSqlitePersistenceLive } from "./Sqlite.ts"; +import { + SqlitePersistenceMemory, + WAL_SIZE_LIMIT_BYTES, + makeSqlitePersistenceLive, +} from "./Sqlite.ts"; const lockHolderSource = ` const { DatabaseSync } = require("node:sqlite"); @@ -57,6 +61,32 @@ it.effect("waits out a concurrent writer instead of failing with SQLITE_BUSY", ( ); }); +it.effect("shrinks the WAL file back to the size limit after a large write", () => { + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-sqlite-wal-")); + const dbPath = NodePath.join(tempDir, "state.sqlite"); + const walFileSize = () => NodeFS.statSync(`${dbPath}-wal`).size; + // About 25% more 4 KB rows than the limit holds, in one transaction. + const rowCount = Math.ceil((WAL_SIZE_LIMIT_BYTES * 1.25) / 4000); + + return Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE wal_probe(payload BLOB)`; + yield* sql` + WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ${rowCount}) + INSERT INTO wal_probe(payload) SELECT randomblob(4000) FROM n + `; + assert.isAbove(walFileSize(), WAL_SIZE_LIMIT_BYTES); + + // The auto-checkpoint after the large commit copied every frame into the + // database, so the next commit restarts the WAL and cuts the file back. + yield* sql`INSERT INTO wal_probe(payload) VALUES (x'00')`; + assert.isAtMost(walFileSize(), WAL_SIZE_LIMIT_BYTES); + }).pipe( + Effect.provide(makeSqlitePersistenceLive(dbPath).pipe(Layer.provide(NodeServices.layer))), + Effect.ensuring(Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true }))), + ); +}); + it.effect("applies busy_timeout in the shared persistence setup", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; diff --git a/apps/server/src/persistence/Layers/Sqlite.ts b/apps/server/src/persistence/Layers/Sqlite.ts index 1a2854c3d9cf..802f8bad3130 100644 --- a/apps/server/src/persistence/Layers/Sqlite.ts +++ b/apps/server/src/persistence/Layers/Sqlite.ts @@ -13,6 +13,9 @@ import { } from "../ForkMigrations.ts"; import { ServerConfig } from "../../config.ts"; +// Size the -wal file is cut back to on the first commit after a WAL reset. +export const WAL_SIZE_LIMIT_BYTES = 32 * 1024 * 1024; + const setup = Layer.effectDiscard( Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; @@ -20,6 +23,9 @@ const setup = Layer.effectDiscard( yield* sql`PRAGMA busy_timeout = 5000;`; yield* sql`PRAGMA foreign_keys = ON;`; yield* sql`PRAGMA journal_mode = WAL;`; + // PASSIVE checkpoints never shrink the -wal file, so it otherwise keeps its + // largest size until the last connection closes. + yield* sql.unsafe(`PRAGMA journal_size_limit = ${WAL_SIZE_LIMIT_BYTES};`); yield* realignSharedMigrationLedger(); yield* verifySharedMigrationLedger(); yield* runMigrations(); diff --git a/apps/server/src/persistence/ProviderSessionRuntime.ts b/apps/server/src/persistence/ProviderSessionRuntime.ts index 80588f58858f..1ec7adcd623b 100644 --- a/apps/server/src/persistence/ProviderSessionRuntime.ts +++ b/apps/server/src/persistence/ProviderSessionRuntime.ts @@ -102,11 +102,14 @@ export class ProviderSessionRuntimeRepository extends Context.Service< >; /** - * List all provider runtime rows. + * List provider runtime rows. * - * Returned in ascending last-seen order. + * Returned in ascending last-seen order. `excludeStopped` filters stopped + * rows in SQL. Long-lived installs keep thousands for their resume cursors. */ - readonly list: () => Effect.Effect< + readonly list: (options?: { + readonly excludeStopped?: boolean; + }) => Effect.Effect< ReadonlyArray, ProviderSessionRuntimeRepositoryError >; @@ -336,9 +339,9 @@ export const make = Effect.gen(function* () { }); const listRuntimeRows = SqlSchema.findAll({ - Request: Schema.Void, + Request: Schema.Struct({ excludeStopped: Schema.Boolean }), Result: ProviderSessionRuntimeRawDbRowSchema, - execute: () => + execute: ({ excludeStopped }) => sql` SELECT thread_id AS "threadId", @@ -351,6 +354,7 @@ export const make = Effect.gen(function* () { resume_cursor_json AS "resumeCursor", runtime_payload_json AS "runtimePayload" FROM provider_session_runtime + ${excludeStopped ? sql`WHERE status != 'stopped'` : sql``} ORDER BY last_seen_at ASC, thread_id ASC `, }); @@ -414,8 +418,8 @@ export const make = Effect.gen(function* () { ), ); - const list: ProviderSessionRuntimeRepository["Service"]["list"] = () => - listRuntimeRows(undefined).pipe( + const list: ProviderSessionRuntimeRepository["Service"]["list"] = (options) => + listRuntimeRows({ excludeStopped: options?.excludeStopped === true }).pipe( Effect.mapError( toPersistenceSqlOrDecodeError( "ProviderSessionRuntimeRepository.list:query", diff --git a/apps/server/src/preview/PortScanner.test.ts b/apps/server/src/preview/PortScanner.test.ts index 7fa15defeca9..d790002f95f1 100644 --- a/apps/server/src/preview/PortScanner.test.ts +++ b/apps/server/src/preview/PortScanner.test.ts @@ -18,6 +18,7 @@ import * as Layer from "effect/Layer"; import * as PlatformError from "effect/PlatformError"; import * as Scope from "effect/Scope"; import * as TestClock from "effect/testing/TestClock"; +import * as Tracer from "effect/Tracer"; import { expect } from "vite-plus/test"; import { FetchHttpClient } from "effect/unstable/http"; @@ -441,6 +442,26 @@ effectIt.effect("stops probing a subscriber's configured paths after its scope c }).pipe(Effect.scoped, Effect.provide(layer)); }); +effectIt.effect("writes no poll span while no client retains the scanner", () => { + let pollSpans = 0; + const tracer = Tracer.make({ + span: (options) => { + if (options.name === "PortDiscovery.pollTick") pollSpans += 1; + return new Tracer.NativeSpan(options); + }, + }); + const layer = makeProbeFailureLayer(processProbeFailure); + + return Effect.gen(function* () { + const scanner = yield* PortScanner.PortDiscovery; + yield* TestClock.adjust(Duration.seconds(15)); + expect(pollSpans).toBe(0); + + yield* scanner.retain; + expect(pollSpans).toBe(1); + }).pipe(Effect.scoped, Effect.provide(layer), Effect.withTracer(tracer)); +}); + effectIt.effect("uses the current configured fragment when readiness comes from cache", () => { const requests: string[] = []; const fetchFn = ((input: Parameters[0]) => { diff --git a/apps/server/src/preview/PortScanner.ts b/apps/server/src/preview/PortScanner.ts index f4d73d62320d..9eee1a3e215e 100644 --- a/apps/server/src/preview/PortScanner.ts +++ b/apps/server/src/preview/PortScanner.ts @@ -550,7 +550,6 @@ export const make = Effect.gen(function* PortDiscoveryMake() { const pollTick = Effect.fn("PortDiscovery.pollTick")( function* () { - if ((yield* Ref.get(stateRef)).retainCount <= 0) return; const configuredUrls = [ ...new Set( [...(yield* Ref.get(stateRef)).listeners.values()].flatMap( @@ -579,9 +578,12 @@ export const make = Effect.gen(function* PortDiscoveryMake() { ), ); - // Single layer-scoped polling fiber. Ticks are no-ops when no client is - // currently retained, so the cost is one Ref.get every POLL_INTERVAL. - yield* Effect.forkScoped(pollTick().pipe(Effect.repeat(Schedule.spaced(POLL_INTERVAL)))); + // Single layer-scoped polling fiber. Ticks skip the scan and its span when no + // client is currently retained, so the cost is one Ref.get every POLL_INTERVAL. + const pollIfRetained = Ref.get(stateRef).pipe( + Effect.flatMap((state) => (state.retainCount > 0 ? pollTick() : Effect.void)), + ); + yield* Effect.forkScoped(pollIfRetained.pipe(Effect.repeat(Schedule.spaced(POLL_INTERVAL)))); const acquireRetention = Effect.fn("PortDiscovery.retain")(function* () { const wasIdle = yield* Ref.modify(stateRef, (state) => [ diff --git a/apps/server/src/process/externalLauncher.test.ts b/apps/server/src/process/externalLauncher.test.ts index f714a70f783d..aab4d78f4d71 100644 --- a/apps/server/src/process/externalLauncher.test.ts +++ b/apps/server/src/process/externalLauncher.test.ts @@ -923,6 +923,18 @@ it.effect("discovers editors through the service API", () => ); for (const { platform, installPath, editor, args } of [ + { + platform: "darwin", + installPath: "Applications/Antigravity IDE.app/Contents/Resources/app/bin/antigravity-ide", + editor: "antigravity", + args: ["--goto", "/workspace with spaces/file.ts:12:4"], + }, + { + platform: "linux", + installPath: ".local/bin/antigravity-ide", + editor: "antigravity", + args: ["--goto", "/workspace with spaces/file.ts:12:4"], + }, { platform: "darwin", installPath: "Applications/Cursor.app/Contents/Resources/app/bin/code", @@ -1028,6 +1040,45 @@ for (const { platform, installPath, editor, args } of [ ); } +// `agy` is the standalone Antigravity CLI, which installs to ~/.local/bin on +// macOS and Linux and to its own bin folder on Windows. It is not the IDE. +for (const { platform, installPath, onPath } of [ + { platform: "darwin", installPath: ".local/bin/agy", onPath: true }, + { platform: "linux", installPath: ".local/bin/agy", onPath: false }, + { platform: "win32", installPath: "agy/bin/agy.cmd", onPath: true }, +] as const) { + it.effect.skipIf(windowsHost && platform !== "win32")( + `does not report the agy CLI as the Antigravity IDE on ${platform}`, + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-agy-cli-" }); + const executable = path.join(home, installPath); + yield* fs.makeDirectory(path.dirname(executable), { recursive: true }); + yield* fs.writeFileString(executable, "#!/bin/sh\n"); + yield* fs.chmod(executable, 0o755); + const editors = yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + return yield* launcher.resolveAvailableEditors(); + }).pipe( + Effect.provide( + testLayer({ + platform, + env: { + HOME: home, + LOCALAPPDATA: home, + PATH: onPath ? path.dirname(executable) : path.join(home, "empty"), + PATHEXT: ".COM;.EXE;.BAT;.CMD", + }, + }), + ), + ); + assert.notInclude(editors, "antigravity"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +} + it.effect.skipIf(windowsHost)("ignores unusable app bundles and keeps PATH launchers first", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/process/externalLauncher.ts b/apps/server/src/process/externalLauncher.ts index 29c25e790c61..ab287e4d78ee 100644 --- a/apps/server/src/process/externalLauncher.ts +++ b/apps/server/src/process/externalLauncher.ts @@ -20,7 +20,11 @@ import { } from "@t3tools/contracts"; import { resolveEditorCommand } from "@t3tools/shared/editor"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; -import { isCommandAvailable, resolveSpawnCommand } from "@t3tools/shared/shell"; +import { + isCommandAvailable, + resolveSpawnCommand, + withPathDirectoryListings, +} from "@t3tools/shared/shell"; import * as Clock from "effect/Clock"; import * as Config from "effect/Config"; import * as Context from "effect/Context"; @@ -442,7 +446,7 @@ const resolveBrowserLaunch = Effect.fn("externalLauncher.resolveBrowserLaunch")( const resolveAvailableEditors = Effect.fn("externalLauncher.resolveAvailableEditors")(function* () { const platform = yield* HostProcessPlatform; const env = { ...(yield* readBrowserLaunchEnv), ...(yield* readCommandLookupEnv) }; - return yield* buildAvailableEditors(platform, env); + return yield* buildAvailableEditors(platform, env).pipe(withPathDirectoryListings); }); const resolveFileManagerRevealKind = Effect.fn("externalLauncher.resolveFileManagerRevealKind")( diff --git a/apps/server/src/processRunner.test.ts b/apps/server/src/processRunner.test.ts index e264ba7849da..d7f3799e103b 100644 --- a/apps/server/src/processRunner.test.ts +++ b/apps/server/src/processRunner.test.ts @@ -412,3 +412,11 @@ describe("isWindowsCommandNotFound", () => { }), ); }); + +describe("commandName", () => { + it("drops the directory from POSIX and Windows paths", () => { + expect(ProcessRunner.commandName("/Users/me/.local/bin/claude")).toBe("claude"); + expect(ProcessRunner.commandName("C:\\Program Files\\nodejs\\npx.cmd")).toBe("npx.cmd"); + expect(ProcessRunner.commandName("git")).toBe("git"); + }); +}); diff --git a/apps/server/src/processRunner.ts b/apps/server/src/processRunner.ts index 0a9bb9b04a43..049125de3fbf 100644 --- a/apps/server/src/processRunner.ts +++ b/apps/server/src/processRunner.ts @@ -171,7 +171,8 @@ export const isWindowsCommandNotFound = Effect.fn("processRunner.isWindowsComman }, ); -const collectText = Effect.fn("processRunner.collectText")(function* (input: { +// Untraced: no attributes, and its time is the runProcessCore span. Errors fail that span. +const collectText = Effect.fnUntraced(function* (input: { readonly command: string; readonly args: ReadonlyArray; readonly cwd?: string | undefined; @@ -285,10 +286,14 @@ function finalizeRunProcess( ); } +/** The executable name without its directory, recorded as `process.command` on process spans. */ +export const commandName = (command: string) => command.replace(/^.*[\\/]/, ""); + const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* ( spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], input: ProcessRunInput, ): Effect.fn.Return { + yield* Effect.annotateCurrentSpan("process.command", commandName(input.command)); const maxOutputBytes = input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES; const outputMode = input.outputMode ?? "error"; const truncatedMarker = input.truncatedMarker ?? ""; diff --git a/apps/server/src/project/AgentSessionScanner.test.ts b/apps/server/src/project/AgentSessionScanner.test.ts index 792ef92310ff..2dde0643ed60 100644 --- a/apps/server/src/project/AgentSessionScanner.test.ts +++ b/apps/server/src/project/AgentSessionScanner.test.ts @@ -48,6 +48,7 @@ const makeProjectionSnapshotQueryLayer = (importedWorkspaceRoots: ReadonlyArray< updatedAt: "2026-01-01T00:00:00.000Z", }), getDeletedWorktreeThreads: () => Effect.die("unused"), + listThreadsWithPullRequests: () => Effect.die("unused"), getArchivedShellSnapshot: () => Effect.die("unused"), getSnapshotSequence: () => Effect.die("unused"), getCounts: () => Effect.die("unused"), diff --git a/apps/server/src/project/ProjectSetupScriptRunner.test.ts b/apps/server/src/project/ProjectSetupScriptRunner.test.ts index 650f304e8481..69c553a4deaf 100644 --- a/apps/server/src/project/ProjectSetupScriptRunner.test.ts +++ b/apps/server/src/project/ProjectSetupScriptRunner.test.ts @@ -34,6 +34,7 @@ const makeProjectionSnapshotQueryLayer = (project: OrchestrationProject) => getSnapshot: () => Effect.die("unused"), getShellSnapshot: () => Effect.die("unused"), getDeletedWorktreeThreads: () => Effect.die("unused"), + listThreadsWithPullRequests: () => Effect.die("unused"), getArchivedShellSnapshot: () => Effect.die("unused"), getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 1 }), getCounts: () => Effect.die("unused"), diff --git a/apps/server/src/project/RepositoryIdentityResolver.test.ts b/apps/server/src/project/RepositoryIdentityResolver.test.ts index d6ddb0b9263f..58f199b834e2 100644 --- a/apps/server/src/project/RepositoryIdentityResolver.test.ts +++ b/apps/server/src/project/RepositoryIdentityResolver.test.ts @@ -94,6 +94,8 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; const first = yield* resolver.resolve("/repo/packages/web"); rootPath = "/repo/packages/web"; + // Longer than the one-minute cadence of the background sweeps. + yield* TestClock.adjust(Duration.minutes(10)); const second = yield* resolver.resolve("/repo/packages/web"); expect(first?.canonicalKey).toBe("github.com/t3tools/t3code"); @@ -123,10 +125,10 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { const unavailable = yield* resolver.resolve(rootPath, { refresh: true }); expect(unavailable?.webUrl).toBeUndefined(); expect(unavailable?.canonicalKey).toBe("ssh.forge.test/team/repo"); - }).pipe(Effect.provide(resolverLayer)); + }).pipe(Effect.provide(Layer.merge(TestClock.layer(), resolverLayer))); }); - it.effect("retries Git root discovery after a failed lookup", () => { + it.effect("retries Git root discovery after the negative TTL", () => { const calls: Array> = []; let rootAttempts = 0; const processRunner = Layer.succeed(ProcessRunner.ProcessRunner, { @@ -159,7 +161,9 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { return Effect.gen(function* () { const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; expect(yield* resolver.resolve("/repo/packages/web")).toBeNull(); + expect(yield* resolver.resolve("/repo/packages/web")).toBeNull(); + yield* TestClock.adjust(Duration.minutes(1)); const recovered = yield* resolver.resolve("/repo/packages/web"); expect(recovered?.rootPath).toBe("/repo"); expect(calls).toEqual([ @@ -167,7 +171,7 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { ["-C", "/repo/packages/web", "rev-parse", "--show-toplevel"], ["-C", "/repo", "remote", "-v"], ]); - }).pipe(Effect.provide(resolverLayer)); + }).pipe(Effect.provide(Layer.merge(TestClock.layer(), resolverLayer))); }); it.effect("normalizes equivalent GitHub remotes into a stable repository identity", () => diff --git a/apps/server/src/project/RepositoryIdentityResolver.ts b/apps/server/src/project/RepositoryIdentityResolver.ts index 2d7f5d02d02e..5acafa47e2e2 100644 --- a/apps/server/src/project/RepositoryIdentityResolver.ts +++ b/apps/server/src/project/RepositoryIdentityResolver.ts @@ -13,7 +13,11 @@ import * as Layer from "effect/Layer"; import * as ProcessRunner from "../processRunner.ts"; const DEFAULT_REPOSITORY_IDENTITY_CACHE_CAPACITY = 512; -const DEFAULT_POSITIVE_CACHE_TTL = Duration.minutes(1); +// Background sweeps resolve every project each minute. A long TTL keeps them +// from spawning git each time. Clone, publish, and PR discovery (after a turn +// and before it saves links) resolve with `refresh: true`. +const DEFAULT_POSITIVE_CACHE_TTL = Duration.minutes(15); +// Short, so a folder that gains a repository or a remote shows up quickly. const DEFAULT_NEGATIVE_CACHE_TTL = Duration.minutes(1); export interface RepositoryIdentityResolverOptions { @@ -142,20 +146,23 @@ export const make = Effect.fn("RepositoryIdentityResolver.make")(function* ( const processRunner = yield* ProcessRunner.ProcessRunner; const cacheCapacity = options.cacheCapacity ?? DEFAULT_REPOSITORY_IDENTITY_CACHE_CAPACITY; const refine = options.refine ?? Effect.succeed; + // Git errors and timeouts resolve to null, so they use the negative TTL like + // "no repository" or "no remote". Only interrupts and defects skip the cache. + const timeToLive = (exit: Exit.Exit) => + Exit.match(exit, { + onSuccess: (value) => + value === null + ? (options.negativeCacheTtl ?? DEFAULT_NEGATIVE_CACHE_TTL) + : (options.positiveCacheTtl ?? DEFAULT_POSITIVE_CACHE_TTL), + onFailure: () => Duration.zero, + }); const repositoryRootCache = yield* Cache.makeWith( (cwd) => resolveRepositoryIdentityCacheKey(cwd).pipe( Effect.provideService(ProcessRunner.ProcessRunner, processRunner), ), - { - capacity: cacheCapacity, - timeToLive: Exit.match({ - onSuccess: (value) => - value === null ? Duration.zero : (options.positiveCacheTtl ?? DEFAULT_POSITIVE_CACHE_TTL), - onFailure: () => Duration.zero, - }), - }, + { capacity: cacheCapacity, timeToLive }, ); const repositoryIdentityCache = yield* Cache.makeWith( @@ -167,27 +174,20 @@ export const make = Effect.fn("RepositoryIdentityResolver.make")(function* ( (identity) => refine(identity).pipe(Effect.orElseSucceed(() => identity)), ), ), - { - capacity: cacheCapacity, - timeToLive: Exit.match({ - onSuccess: (value) => - value === null - ? (options.negativeCacheTtl ?? DEFAULT_NEGATIVE_CACHE_TTL) - : (options.positiveCacheTtl ?? DEFAULT_POSITIVE_CACHE_TTL), - onFailure: () => Duration.zero, - }), - }, + { capacity: cacheCapacity, timeToLive }, ); - const resolve: RepositoryIdentityResolver["Service"]["resolve"] = Effect.fn( - "RepositoryIdentityResolver.resolve", - )(function* (cwd, options) { - if (options?.refresh) yield* Cache.invalidate(repositoryRootCache, cwd); - const cacheKey = yield* Cache.get(repositoryRootCache, cwd); - if (cacheKey === null) return null; - if (options?.refresh) yield* Cache.invalidate(repositoryIdentityCache, cacheKey); - return yield* Cache.get(repositoryIdentityCache, cacheKey); - }); + // Untraced because almost every call is a cache hit. The lookups that spawn + // git keep their own spans. + const resolve: RepositoryIdentityResolver["Service"]["resolve"] = Effect.fnUntraced( + function* (cwd, options) { + if (options?.refresh) yield* Cache.invalidate(repositoryRootCache, cwd); + const cacheKey = yield* Cache.get(repositoryRootCache, cwd); + if (cacheKey === null) return null; + if (options?.refresh) yield* Cache.invalidate(repositoryIdentityCache, cacheKey); + return yield* Cache.get(repositoryIdentityCache, cacheKey); + }, + ); return RepositoryIdentityResolver.of({ resolve }); }); diff --git a/apps/server/src/provider/Layers/AntigravityAdapter.ts b/apps/server/src/provider/Layers/AntigravityAdapter.ts index e12a320c4d48..c5e6400d11b4 100644 --- a/apps/server/src/provider/Layers/AntigravityAdapter.ts +++ b/apps/server/src/provider/Layers/AntigravityAdapter.ts @@ -786,9 +786,9 @@ export const makeAntigravityAdapter = Effect.fn("makeAntigravityAdapter")(functi stopOwned, Effect.gen(function* () { const mcp = McpProviderSession.readMcpProviderSession(input.threadId); - // The attachments dir grant lets the agent read pasted files at - // the paths ProviderService injects into the turn text. It is a - // leaf directory holding only uploads. + // The attachments dir grant lets the agent read path-only uploads + // at the paths ProviderService injects into the turn text. It is + // a leaf directory holding only uploads. const runtime = yield* options.makeRuntime({ cwd, clientInfo: { name: "t3-code", version: "0.0.0" }, diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts index 6609e728816e..350d5a518d7c 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts @@ -6811,6 +6811,56 @@ describe("ClaudeAdapterLive", () => { ); }); + it.effect("completed turns keep their ids but not the SDK messages", () => { + const harness = makeHarness(); + return Effect.gen(function* () { + const adapter = yield* ClaudeAdapter; + const session = yield* adapter.startSession({ + threadId: THREAD_ID, + provider: ProviderDriverKind.make("claudeAgent"), + runtimeMode: "full-access", + }); + const turn = yield* adapter.sendTurn({ + threadId: session.threadId, + input: "hello", + attachments: [], + }); + const completedFiber = yield* Stream.filter( + adapter.streamEvents, + (event) => event.type === "turn.completed", + ).pipe(Stream.runHead, Effect.forkChild); + + harness.query.emit({ + type: "assistant", + session_id: "sdk-session-1", + uuid: "assistant-1", + parent_tool_use_id: null, + message: { + id: "assistant-message-1", + content: [{ type: "text", text: "Hi" }], + }, + } as unknown as SDKMessage); + harness.query.emit({ + type: "result", + subtype: "success", + is_error: false, + errors: [], + session_id: "sdk-session-1", + uuid: "result-1", + } as unknown as SDKMessage); + yield* Fiber.join(completedFiber); + + const snapshot = yield* adapter.readThread(session.threadId); + assert.deepEqual( + snapshot.turns.map((entry) => ({ id: String(entry.id), items: entry.items })), + [{ id: String(turn.turnId), items: [] }], + ); + }).pipe( + Effect.provideService(Random.Random, makeDeterministicRandomService()), + Effect.provide(harness.layer), + ); + }); + it.effect("rewinds Claude history when the fork omits retained system messages", () => { const forkCalls: Array>> = []; let firstTurnId = ""; diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index 13275a56aae5..f577418375bf 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -267,7 +267,6 @@ interface ClaudeTurnState { * steered instead (the queued message continues the same turn). */ readonly synthetic?: boolean; - readonly items: Array; readonly assistantTextBlocks: Map; readonly assistantTextBlockOrder: Array; readonly capturedProposedPlanKeys: Set; @@ -422,10 +421,11 @@ interface ClaudeSessionContext { resumeSessionId: string | undefined; readonly pendingApprovals: Map; readonly pendingUserInputs: Map; - readonly turns: Array<{ - id: TurnId; - items: Array; - }>; + /** Completed turn ids, reported by readThread and trimmed on rollback. + * SDK messages are not kept: rollback reads Claude's own history through + * turnStartMessageIds, and a long-lived session would otherwise hold every + * message it ever produced. */ + readonly turns: Array<{ readonly id: TurnId }>; readonly inFlightTools: Map; readonly claudeTasks: Map; readonly taskAgents: Map; @@ -2181,10 +2181,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( } return { threadId, - turns: context.turns.map((turn) => ({ - id: turn.id, - items: [...turn.items], - })), + turns: context.turns.map((turn) => ({ id: turn.id, items: [] })), }; }); @@ -2813,10 +2810,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( }); } - context.turns.push({ - id: turnState.turnId, - items: [...turnState.items], - }); + context.turns.push({ id: turnState.turnId }); yield* emitThreadTokenUsage(context, usageSnapshot, { rawMethod: "claude/result", @@ -3180,10 +3174,6 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( return; } - if (context.turnState) { - context.turnState.items.push(message.message); - } - for (const toolResult of toolResultBlocksFromUserMessage(message)) { const toolEntry = Array.from(context.inFlightTools.entries()).find( ([, tool]) => tool.itemId === toolResult.toolUseId, @@ -3383,7 +3373,6 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( turnId, startedAt, synthetic: true, - items: [], assistantTextBlocks: new Map(), assistantTextBlockOrder: [], capturedProposedPlanKeys: new Set(), @@ -3466,7 +3455,6 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( cwd: path.resolve(context.session.cwd ?? "."), }); } - context.turnState.items.push(message.message); if ( normalizeClaudeActiveTokenUsage( message.message.usage, @@ -5227,7 +5215,6 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( const turnState: ClaudeTurnState = { turnId, startedAt: yield* nowIso, - items: [], assistantTextBlocks: new Map(), assistantTextBlockOrder: [], capturedProposedPlanKeys: new Set(), diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index 36d884d8ac98..7f0465b9c401 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -648,6 +648,78 @@ it.effect("ProviderServiceLive catches stopAll failures during shutdown", () => }), ); +it.effect("ProviderServiceLive shutdown leaves settled session rows untouched", () => + Effect.gen(function* () { + const recordedAnalytics = makeRecordingAnalytics(); + const codex = makeFakeCodexAdapter(); + const persistence = yield* Layer.build( + ProviderSessionDirectoryLive.pipe( + Layer.provide(ProviderSessionRuntime.layer.pipe(Layer.provide(SqlitePersistenceMemory))), + ), + ); + const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory.pipe( + Effect.provide(persistence), + ); + const seed = (threadId: ThreadId, status: "running" | "stopped", activeTurnId: TurnId | null) => + directory.upsert({ + threadId, + provider: CODEX_DRIVER, + providerInstanceId: codexInstanceId, + status, + runtimePayload: { cwd: "/repo", activeTurnId }, + }); + const readBindings = directory + .listBindings() + .pipe( + Effect.map((bindings) => new Map(bindings.map((binding) => [binding.threadId, binding]))), + ); + const settledId = asThreadId("shutdown-settled"); + const runningId = asThreadId("shutdown-running"); + const stoppedWithTurnId = asThreadId("shutdown-stopped-with-turn"); + yield* seed(settledId, "stopped", null); + yield* seed(runningId, "running", asTurnId("running-turn")); + yield* seed(stoppedWithTurnId, "stopped", asTurnId("stale-turn")); + const settledBefore = (yield* readBindings).get(settledId); + assert(settledBefore !== undefined); + + const scope = yield* Scope.make(); + yield* Layer.build( + makeProviderServiceLive().pipe( + Layer.provide(NodeServices.layer), + Layer.provide(Layer.succeed(ProviderSessionDirectory.ProviderSessionDirectory, directory)), + Layer.provide( + Layer.succeed( + ProviderAdapterRegistry.ProviderAdapterRegistry, + makeStaticInstanceRegistry([[codexInstanceId, codex.adapter]]), + ), + ), + Layer.provide(defaultServerSettingsLayer), + Layer.provide(serverConfigTestLayer), + Layer.provide(recordedAnalytics.layer), + Layer.provide( + Layer.succeed( + ProviderEventLoggers.ProviderEventLoggers, + ProviderEventLoggers.NoOpProviderEventLoggers, + ), + ), + ), + ).pipe(Scope.provide(scope)); + yield* TestClock.adjust("1 minute"); + yield* Scope.close(scope, Exit.void); + + const byThread = yield* readBindings; + assert.deepStrictEqual(byThread.get(settledId), settledBefore); + for (const threadId of [runningId, stoppedWithTurnId]) { + const binding = byThread.get(threadId); + assert.equal(binding?.status, "stopped"); + assert.propertyVal(binding?.runtimePayload, "activeTurnId", null); + assert.propertyVal(binding?.runtimePayload, "lastRuntimeEvent", "provider.stopAll"); + } + const [stoppedAll] = recordedAnalytics.eventsByName("provider.sessions.stopped_all"); + assert.equal(stoppedAll?.properties?.stoppedSessionCount, 2); + }).pipe(Effect.provide(NodeServices.layer)), +); + it.effect("ProviderServiceLive flushes deferred completions during shutdown", () => Effect.gen(function* () { const recordedAnalytics = makeRecordingAnalytics(); @@ -4729,6 +4801,57 @@ validation.layer("ProviderServiceLive validation", (it) => { }), ); + it.effect("rejects a file when its path cannot fit in the prompt", () => + Effect.gen(function* () { + const provider = yield* ProviderService.ProviderService; + validation.codex.sendTurn.mockClear(); + const failure = yield* provider + .sendTurn({ + threadId: asThreadId("thread-file-path-context-limit"), + input: "x".repeat(PROVIDER_SEND_TURN_MAX_INPUT_CHARS), + attachments: [ + { + type: "file", + id: "thread-attach-12345678-1234-1234-1234-123456789abc-zip", + name: "archive.zip", + mimeType: "application/zip", + sizeBytes: 1024, + }, + ], + }) + .pipe(Effect.flip); + + assert.instanceOf(failure, ProviderValidationError); + assert.include(failure.issue, String(PROVIDER_SEND_TURN_MAX_INPUT_CHARS)); + assert.equal(validation.codex.sendTurn.mock.calls.length, 0); + }), + ); + + it.effect("sends a native image when its path cannot fit in the prompt", () => + Effect.gen(function* () { + const provider = yield* ProviderService.ProviderService; + const threadId = asThreadId("thread-image-path-context-limit"); + yield* provider.startSession(threadId, { + provider: ProviderDriverKind.make("codex"), + providerInstanceId: codexInstanceId, + threadId, + runtimeMode: "full-access", + }); + validation.codex.sendTurn.mockClear(); + const attachment = { + type: "image" as const, + id: "thread-attach-12345678-1234-1234-1234-123456789abc-png", + name: "screen.png", + mimeType: "image/png", + sizeBytes: 1024, + }; + const input = "x".repeat(PROVIDER_SEND_TURN_MAX_INPUT_CHARS); + yield* provider.sendTurn({ threadId, input, attachments: [attachment] }); + assert.equal(validation.codex.sendTurn.mock.calls[0]?.[0].input, input); + assert.deepEqual(validation.codex.sendTurn.mock.calls[0]?.[0].attachments, [attachment]); + }), + ); + it.effect("rejects citation-expanded input over the provider character limit", () => Effect.gen(function* () { const provider = yield* ProviderService.ProviderService; @@ -4969,6 +5092,7 @@ describe("agent browser access", () => { getSnapshot: () => Effect.die("unused"), getShellSnapshot: () => Effect.die("unused"), getDeletedWorktreeThreads: () => Effect.die("unused"), + listThreadsWithPullRequests: () => Effect.die("unused"), getArchivedShellSnapshot: () => Effect.die("unused"), getSnapshotSequence: () => Effect.die("unused"), getCounts: () => Effect.die("unused"), diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index f6a918cb20f3..64f9eedb689b 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -427,6 +427,14 @@ function readPersistedCwd( return trimmed.length > 0 ? trimmed : undefined; } +/** Stopped rows with no active turn are settled; shutdown leaves them untouched. */ +function isSettledBinding(binding: ProviderSessionDirectory.ProviderRuntimeBinding): boolean { + if (binding.status !== "stopped") return false; + const payload = binding.runtimePayload; + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return true; + return !("activeTurnId" in payload) || payload.activeTurnId == null; +} + const dieOnMissingBindingInstanceId = ( operation: string, payload: { @@ -1635,10 +1643,12 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ? `[Pasted text "${attachment.name}" is saved at: ${attachmentPath}. Inspect it as needed.]` : `[Attached ${attachment.type} "${attachment.name}" is saved at: ${attachmentPath}]`, ); - if (isPastedText && !appended) { + // Most adapters see generic files only through this path line, so a file + // without one would be silently dropped. Images still go natively. + if (!appended && attachment.type === "file") { return yield* toValidationError( "ProviderService.sendTurn", - `Input plus pasted-text attachment context exceeds the ${PROVIDER_SEND_TURN_MAX_INPUT_CHARS} character limit`, + `Input plus attachment context exceeds the ${PROVIDER_SEND_TURN_MAX_INPUT_CHARS} character limit`, ); } } @@ -2340,7 +2350,6 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( return [completed, state] as const; }); yield* recordCompletedTurnProperties(properties); - const threadIds = yield* directory.listThreadIds(); const currentAdapters = yield* getAdapterEntries; const activeSessions = yield* Effect.forEach(currentAdapters, ([instanceId, adapter]) => adapter.listSessions().pipe( @@ -2371,7 +2380,12 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( yield* Effect.forEach(currentAdapters, ([, adapter]) => adapter.stopAll()).pipe(Effect.asVoid); yield* McpSessionRegistry.revokeAllActiveMcpCredentials(); McpProviderSession.clearAllMcpProviderSessions(); - const bindings = yield* directory.listBindings().pipe(Effect.orElseSucceed(() => [])); + // Stopped rows stay for their resume cursors, so long-lived installs hold + // thousands. Only rewrite the ones this shutdown actually stops. + const bindings = yield* directory.listBindings().pipe( + Effect.map((all) => all.filter((binding) => !isSettledBinding(binding))), + Effect.orElseSucceed(() => []), + ); yield* Effect.forEach(bindings, (binding) => Effect.gen(function* () { const providerInstanceId = dieOnMissingBindingInstanceId( @@ -2391,8 +2405,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( }); }), ).pipe(Effect.asVoid); + // Not `sessionCount`: that older property counted every row, so a new name + // keeps the two meanings in separate series. yield* analytics.record("provider.sessions.stopped_all", { - sessionCount: threadIds.length, + stoppedSessionCount: bindings.length, }); yield* analytics.flush; }); diff --git a/apps/server/src/provider/Layers/ProviderSessionDirectory.test.ts b/apps/server/src/provider/Layers/ProviderSessionDirectory.test.ts index 8b41bd3e518c..45615fe7b440 100644 --- a/apps/server/src/provider/Layers/ProviderSessionDirectory.test.ts +++ b/apps/server/src/provider/Layers/ProviderSessionDirectory.test.ts @@ -372,6 +372,41 @@ it.layer(makeDirectoryLayer(SqlitePersistenceMemory))("ProviderSessionDirectoryL }), ); + it.effect("lists only bindings that are not stopped when asked", () => + Effect.gen(function* () { + const directory = yield* ProviderSessionDirectory; + const runtimeRepository = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + const statuses = ["running", "starting", "error", "stopped"] as const; + const threadIds = new Set(); + + for (const status of statuses) { + const threadId = ThreadId.make(`thread-exclude-stopped-${status}`); + threadIds.add(threadId); + yield* runtimeRepository.upsert({ + threadId, + providerName: "codex", + providerInstanceId: ProviderInstanceId.make("codex"), + adapterKey: "codex", + runtimeMode: "full-access", + status, + lastSeenAt: "2026-04-14T12:00:00.000Z", + resumeCursor: null, + runtimePayload: null, + }); + } + + const liveStatuses = (yield* directory.listBindings({ excludeStopped: true })) + .filter((binding) => threadIds.has(binding.threadId)) + .map((binding) => binding.status); + const allStatuses = (yield* directory.listBindings()) + .filter((binding) => threadIds.has(binding.threadId)) + .map((binding) => binding.status); + + assert.deepEqual(liveStatuses.toSorted(), ["error", "running", "starting"]); + assert.deepEqual(allStatuses.toSorted(), ["error", "running", "starting", "stopped"]); + }), + ); + it.effect( "resets adapterKey to the new provider when provider changes without an explicit adapter key", () => diff --git a/apps/server/src/provider/Layers/ProviderSessionDirectory.ts b/apps/server/src/provider/Layers/ProviderSessionDirectory.ts index 03f1ece0b5e0..0e9de8fb868f 100644 --- a/apps/server/src/provider/Layers/ProviderSessionDirectory.ts +++ b/apps/server/src/provider/Layers/ProviderSessionDirectory.ts @@ -182,8 +182,8 @@ const makeProviderSessionDirectory = Effect.gen(function* () { Effect.map((rows) => rows.map((row) => row.threadId)), ); - const listBindings: ProviderSessionDirectoryShape["listBindings"] = () => - repository.list().pipe( + const listBindings: ProviderSessionDirectoryShape["listBindings"] = (options) => + repository.list(options).pipe( Effect.mapError(toPersistenceError("ProviderSessionDirectory.listBindings:list")), Effect.flatMap((rows) => Effect.forEach( diff --git a/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts b/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts index d8226648e9f3..7b1fec90f867 100644 --- a/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts +++ b/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts @@ -239,6 +239,7 @@ describe("ProviderSessionReaper", () => { getSnapshot: () => Effect.die("unused"), getShellSnapshot: () => Effect.die("unused"), getDeletedWorktreeThreads: () => Effect.die("unused"), + listThreadsWithPullRequests: () => Effect.die("unused"), getArchivedShellSnapshot: () => Effect.die("unused"), getSnapshotSequence: () => Effect.succeed({ snapshotSequence: input.readModel.snapshotSequence }), diff --git a/apps/server/src/provider/Layers/ProviderSessionReaper.ts b/apps/server/src/provider/Layers/ProviderSessionReaper.ts index 8f04bd1821fd..bf8199f80eac 100644 --- a/apps/server/src/provider/Layers/ProviderSessionReaper.ts +++ b/apps/server/src/provider/Layers/ProviderSessionReaper.ts @@ -35,15 +35,13 @@ const makeProviderSessionReaper = (options?: ProviderSessionReaperLiveOptions) = const sweepIntervalMs = Math.max(1, options?.sweepIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS); const sweep = Effect.gen(function* () { - const bindings = yield* directory.listBindings(); + // Stopped rows stay for their resume cursors and far outnumber live + // ones, so the query skips them. + const bindings = yield* directory.listBindings({ excludeStopped: true }); const now = yield* Clock.currentTimeMillis; let reapedCount = 0; for (const binding of bindings) { - if (binding.status === "stopped") { - continue; - } - const lastSeenMs = Date.parse(binding.lastSeenAt); if (Number.isNaN(lastSeenMs)) { yield* Effect.logWarning("provider.session.reaper.invalid-last-seen", { @@ -122,7 +120,7 @@ const makeProviderSessionReaper = (options?: ProviderSessionReaperLiveOptions) = if (reapedCount > 0) { yield* Effect.logInfo("provider.session.reaper.sweep-complete", { reapedCount, - totalBindings: bindings.length, + liveBindings: bindings.length, }); } }); diff --git a/apps/server/src/provider/Services/ProviderSessionDirectory.ts b/apps/server/src/provider/Services/ProviderSessionDirectory.ts index 9dbafd3e804e..1b5c47ec1d62 100644 --- a/apps/server/src/provider/Services/ProviderSessionDirectory.ts +++ b/apps/server/src/provider/Services/ProviderSessionDirectory.ts @@ -70,7 +70,10 @@ export interface ProviderSessionDirectoryShape { ProviderSessionDirectoryPersistenceError >; - readonly listBindings: () => Effect.Effect< + /** `excludeStopped` skips stopped rows in the query, not after decoding. */ + readonly listBindings: (options?: { + readonly excludeStopped?: boolean; + }) => Effect.Effect< ReadonlyArray, ProviderSessionDirectoryPersistenceError >; diff --git a/apps/server/src/provider/acp/AntigravityAcpSupport.test.ts b/apps/server/src/provider/acp/AntigravityAcpSupport.test.ts index aee10782a613..7d6cfb1e2620 100644 --- a/apps/server/src/provider/acp/AntigravityAcpSupport.test.ts +++ b/apps/server/src/provider/acp/AntigravityAcpSupport.test.ts @@ -2,7 +2,6 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; import { ANTIGRAVITY_DEFAULT_MODEL, - PROVIDER_SEND_TURN_MAX_FILE_BYTES, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, type ChatAttachment, type RuntimeMode, @@ -15,7 +14,6 @@ import type * as EffectAcpSchema from "effect-acp/schema"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { - ANTIGRAVITY_MAX_TEXT_ATTACHMENT_BYTES, antigravityPermissionMode, applyAntigravityAcpModelSelection, buildAntigravityPrompt, @@ -324,6 +322,41 @@ it.layer(NodeServices.layer)("buildAntigravityPrompt", (it) => { }), ); + it.effect.each([ + { name: "archive.zip", mimeType: "application/zip" }, + { name: "clip.mp4", mimeType: "video/mp4" }, + { name: "recording.aiff", mimeType: "audio/aiff" }, + { name: "large.pdf", mimeType: "application/pdf", sizeBytes: 75_000_000 }, + { name: "large.txt", mimeType: "text/plain", sizeBytes: 1024 * 1024 + 1 }, + { name: "large.wav", mimeType: "audio/wav", sizeBytes: 20 * 1024 * 1024 + 1 }, + ])("keeps $name as a file path without reading or spending the native media budget", (file) => + Effect.gen(function* () { + const fixture = yield* makeAttachmentFixture(); + const attachment = { ...textAttachment, sizeBytes: 50 * 1024 * 1024, ...file }; + const upload = yield* fixture.write(attachment, ""); + yield* fixture.fs.truncate(upload.filePath, attachment.sizeBytes); + const pdf = yield* fixture.write(pdfAttachment, ""); + yield* fixture.fs.truncate(pdf.filePath, 50 * 1024 * 1024); + const input = `Inspect the file at ${upload.filePath}`; + const prompt = yield* buildAntigravityPrompt({ + input, + attachments: [attachment, pdfAttachment], + attachmentsDir: fixture.attachmentsDir, + }).pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fixture.fs, + stream: () => { + throw new Error("Path attachments must not be read into the prompt"); + }, + }), + ); + expect(prompt).toEqual([ + { type: "text", text: input }, + { type: "resource_link", uri: pdf.uri, name: "report.pdf", mimeType: "application/pdf" }, + ]); + }), + ); + it.effect("sends supported audio files as native audio content", () => Effect.gen(function* () { const fixture = yield* makeAttachmentFixture(); @@ -378,8 +411,6 @@ it.layer(NodeServices.layer)("buildAntigravityPrompt", (it) => { it.effect.each([ { ...imageAttachment, name: "animation.gif", mimeType: "image/gif" }, - { ...textAttachment, name: "archive.zip", mimeType: "application/zip" }, - { ...textAttachment, name: "recording.aiff", mimeType: "audio/aiff" }, ] satisfies ReadonlyArray)( "rejects $name instead of silently dropping it from a valid prompt", (attachment) => @@ -400,32 +431,25 @@ it.layer(NodeServices.layer)("buildAntigravityPrompt", (it) => { }), ); - it.effect.each([ - { attachment: textAttachment, bytes: ANTIGRAVITY_MAX_TEXT_ATTACHMENT_BYTES + 1 }, - { attachment: imageAttachment, bytes: PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1 }, - { attachment: pdfAttachment, bytes: PROVIDER_SEND_TURN_MAX_FILE_BYTES + 1 }, - ])( - "rejects oversized $attachment.name using file size instead of upload metadata", - ({ attachment, bytes }) => - Effect.gen(function* () { - const fixture = yield* makeAttachmentFixture(); - const upload = yield* fixture.write(attachment, ""); - yield* fixture.fs.truncate(upload.filePath, bytes); - const error = yield* buildAntigravityPrompt({ - input: "Read this attachment.", - attachments: [attachment], - attachmentsDir: fixture.attachmentsDir, - }).pipe(Effect.flip); - - expect(error).toMatchObject({ - _tag: "AcpRequestError", - code: -32602, - errorMessage: expect.stringContaining(`'${attachment.name}' is too large`), - }); - }), + it.effect("rejects oversized images using file size instead of upload metadata", () => + Effect.gen(function* () { + const fixture = yield* makeAttachmentFixture(); + const upload = yield* fixture.write(imageAttachment, ""); + yield* fixture.fs.truncate(upload.filePath, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1); + const error = yield* buildAntigravityPrompt({ + input: "Read this attachment.", + attachments: [imageAttachment], + attachmentsDir: fixture.attachmentsDir, + }).pipe(Effect.flip); + expect(error).toMatchObject({ + _tag: "AcpRequestError", + code: -32602, + errorMessage: expect.stringContaining("'screen.png' is too large"), + }); + }), ); - it.effect("accepts 50 MiB in total but rejects one byte more across files", () => + it.effect("keeps PDF overflow on the file-path route", () => Effect.gen(function* () { const fixture = yield* makeAttachmentFixture(); const secondAttachment = { @@ -435,25 +459,57 @@ it.layer(NodeServices.layer)("buildAntigravityPrompt", (it) => { }; const first = yield* fixture.write(pdfAttachment, ""); const second = yield* fixture.write(secondAttachment, ""); - yield* fixture.fs.truncate(first.filePath, PROVIDER_SEND_TURN_MAX_FILE_BYTES / 2); - yield* fixture.fs.truncate(second.filePath, PROVIDER_SEND_TURN_MAX_FILE_BYTES / 2); + yield* fixture.fs.truncate(first.filePath, (50 * 1024 * 1024) / 2); + yield* fixture.fs.truncate(second.filePath, (50 * 1024 * 1024) / 2); const input = { - input: undefined, + input: `Read ${first.filePath} and ${second.filePath}`, attachments: [pdfAttachment, secondAttachment], attachmentsDir: fixture.attachmentsDir, }; const prompt = yield* buildAntigravityPrompt(input); expect(prompt).toEqual([ - { type: "resource_link", uri: first.uri, name: "report.pdf", mimeType: "application/pdf" }, - { type: "resource_link", uri: second.uri, name: "second.pdf", mimeType: "application/pdf" }, + { type: "text", text: input.input }, + { + type: "resource_link", + uri: first.uri, + name: "report.pdf", + mimeType: "application/pdf", + }, + { + type: "resource_link", + uri: second.uri, + name: "second.pdf", + mimeType: "application/pdf", + }, ]); - yield* fixture.fs.truncate(second.filePath, PROVIDER_SEND_TURN_MAX_FILE_BYTES / 2 + 1); - const error = yield* buildAntigravityPrompt(input).pipe(Effect.flip); + yield* fixture.fs.truncate(second.filePath, (50 * 1024 * 1024) / 2 + 1); + expect(yield* buildAntigravityPrompt(input)).toEqual([ + { type: "text", text: input.input }, + { + type: "resource_link", + uri: first.uri, + name: "report.pdf", + mimeType: "application/pdf", + }, + ]); + }), + ); + + it.effect("still rejects images when the native budget is full", () => + Effect.gen(function* () { + const fixture = yield* makeAttachmentFixture(); + const pdf = yield* fixture.write(pdfAttachment, ""); + yield* fixture.fs.truncate(pdf.filePath, 50 * 1024 * 1024); + yield* fixture.write(imageAttachment, new Uint8Array([1])); + const error = yield* buildAntigravityPrompt({ + input: "Inspect both attachments.", + attachments: [pdfAttachment, imageAttachment], + attachmentsDir: fixture.attachmentsDir, + }).pipe(Effect.flip); expect(error).toMatchObject({ _tag: "AcpRequestError", - code: -32602, - errorMessage: expect.stringContaining("'second.pdf' is too large"), + errorMessage: expect.stringContaining("'screen.png' is too large"), }); }), ); @@ -463,7 +519,7 @@ it.layer(NodeServices.layer)("buildAntigravityPrompt", (it) => { const fixture = yield* makeAttachmentFixture(); const pdf = yield* fixture.write(pdfAttachment, ""); const text = yield* fixture.write(textAttachment, "a"); - yield* fixture.fs.truncate(pdf.filePath, PROVIDER_SEND_TURN_MAX_FILE_BYTES - 1); + yield* fixture.fs.truncate(pdf.filePath, 50 * 1024 * 1024 - 1); const error = yield* buildAntigravityPrompt({ input: undefined, attachments: [pdfAttachment, textAttachment], diff --git a/apps/server/src/provider/acp/AntigravityAcpSupport.ts b/apps/server/src/provider/acp/AntigravityAcpSupport.ts index f2f370068181..8f2877330bbe 100644 --- a/apps/server/src/provider/acp/AntigravityAcpSupport.ts +++ b/apps/server/src/provider/acp/AntigravityAcpSupport.ts @@ -243,10 +243,14 @@ const TEXT_FILE_EXTENSIONS = new Set([ ".ini", ".conf", ]); -export const ANTIGRAVITY_MAX_TEXT_ATTACHMENT_BYTES = 1024 * 1024; +const ANTIGRAVITY_MAX_TEXT_ATTACHMENT_BYTES = 1024 * 1024; const MAX_TOTAL_ATTACHMENT_BYTES = PROVIDER_SEND_TURN_MAX_FILE_BYTES; -/** Sends uploads as native ACP content instead of workspace path hints. */ +/** + * Sends supported uploads as native ACP content. Other files, and native + * candidates over their limits, reach the agent through the saved path + * ProviderService puts in the text block. + */ export const buildAntigravityPrompt = Effect.fn("buildAntigravityPrompt")(function* (input: { readonly input: ProviderSendTurnInput["input"]; readonly attachments: ProviderSendTurnInput["attachments"]; @@ -280,7 +284,9 @@ export const buildAntigravityPrompt = Effect.fn("buildAntigravityPrompt")(functi (mimeType.startsWith("text/") || TEXT_MIME_TYPES.has(mimeType) || TEXT_FILE_EXTENSIONS.has(path.extname(attachment.name).toLowerCase())); - if (!image && !audio && !pdf && !textFile) { + const isPathOnly = + attachment.type === "file" && (isPastedText || (!audio && !pdf && !textFile)); + if (attachment.type === "image" && !image) { return yield* EffectAcpErrors.AcpRequestError.invalidParams( `Antigravity does not support '${attachment.name}' (${attachment.mimeType}). Attach a BMP, JPEG, PNG, WebP, PDF, audio, or text file.`, ); @@ -303,26 +309,30 @@ export const buildAntigravityPrompt = Effect.fn("buildAntigravityPrompt")(functi ), ), ); - if (isPastedText) { - if (info.type !== "File") { - return yield* EffectAcpErrors.AcpRequestError.invalidParams( - `Could not read attachment '${attachment.name}'.`, - ); - } - continue; + if (info.type !== "File") { + return yield* EffectAcpErrors.AcpRequestError.invalidParams( + `Could not read attachment '${attachment.name}'.`, + ); } + if (isPathOnly) continue; const size = Number(info.size); const limit = image ? PROVIDER_SEND_TURN_MAX_IMAGE_BYTES : audio ? ANTIGRAVITY_MAX_AUDIO_ATTACHMENT_BYTES : pdf - ? PROVIDER_SEND_TURN_MAX_FILE_BYTES + ? MAX_TOTAL_ATTACHMENT_BYTES : ANTIGRAVITY_MAX_TEXT_ATTACHMENT_BYTES; + if ( + attachment.type === "file" && + (size > limit || totalBytes + size > MAX_TOTAL_ATTACHMENT_BYTES) + ) { + continue; + } totalBytes += size; - if (info.type !== "File" || size > limit || totalBytes > MAX_TOTAL_ATTACHMENT_BYTES) { + if (size > limit || totalBytes > MAX_TOTAL_ATTACHMENT_BYTES) { return yield* EffectAcpErrors.AcpRequestError.invalidParams( - `Attachment '${attachment.name}' is too large. Antigravity accepts text files up to 1 MiB, images up to 10 MiB, audio up to 20 MiB, and 50 MiB total attachments.`, + `Image '${attachment.name}' is too large. Antigravity accepts images up to 10 MiB and 50 MiB of native attachments per message.`, ); } const uri = yield* path.toFileUrl(attachmentPath).pipe( diff --git a/apps/server/src/provider/cursorCredentialStore.test.ts b/apps/server/src/provider/cursorCredentialStore.test.ts index 051cb5425c69..f212ce110ea6 100644 --- a/apps/server/src/provider/cursorCredentialStore.test.ts +++ b/apps/server/src/provider/cursorCredentialStore.test.ts @@ -1,6 +1,9 @@ import { assert, describe, it } from "@effect/vitest"; -import { makeCachedCursorAccessTokenReader } from "./cursorCredentialStore.ts"; +import { + CursorKeychainTimeoutError, + makeCachedCursorAccessTokenReader, +} from "./cursorCredentialStore.ts"; describe("Cursor Keychain reader", () => { it("shares concurrent reads and rechecks after the cache expires", async () => { @@ -19,4 +22,25 @@ describe("Cursor Keychain reader", () => { time = 5 * 60_000; assert.strictEqual(await read(), "token-2"); }); + + it("gives up on an unanswered prompt and reuses it on the next read", async () => { + let reads = 0; + let allow: (token: string) => void = () => {}; + const read = makeCachedCursorAccessTokenReader( + () => { + reads++; + return new Promise((resolve) => { + allow = resolve; + }); + }, + () => 0, + 1, + ); + const error = await read().catch((cause: unknown) => cause); + assert.instanceOf(error, CursorKeychainTimeoutError); + const retry = read(); + allow("token"); + assert.strictEqual(await retry, "token"); + assert.strictEqual(reads, 1); + }); }); diff --git a/apps/server/src/provider/cursorCredentialStore.ts b/apps/server/src/provider/cursorCredentialStore.ts index 9d7d1c3bccff..2857683e4821 100644 --- a/apps/server/src/provider/cursorCredentialStore.ts +++ b/apps/server/src/provider/cursorCredentialStore.ts @@ -4,17 +4,31 @@ const CACHE_MS = 5 * 60_000; const requireForKeyring = NodeModule.createRequire(import.meta.url); -/** Share one Keychain request across usage history and limits in this server process. */ +/** Rejected when nobody answers the macOS Keychain prompt in time. */ +export class CursorKeychainTimeoutError extends Error { + constructor() { + super("Timed out waiting for Keychain access."); + } +} + +/** + * Share one Keychain request across usage history and limits in this server process. + * + * macOS shows the access prompt on the server's own screen, which a remote + * client cannot answer, so callers give up after `timeoutMs`. The read stays in + * flight: the next call reuses it instead of stacking a second prompt, and picks + * up the token once someone allows access. + */ export function makeCachedCursorAccessTokenReader( read: () => Promise, now: () => number = Date.now, + timeoutMs = 30_000, ): () => Promise { let cached: { token: string; until: number } | null = null; let pending: Promise | null = null; return () => { if (cached && cached.until > now()) return Promise.resolve(cached.token); - if (pending) return pending; - pending = read() + pending ??= read() .then((token) => { cached = token ? { token, until: now() + CACHE_MS } : null; return token; @@ -22,7 +36,15 @@ export function makeCachedCursorAccessTokenReader( .finally(() => { pending = null; }); - return pending; + const deadline = AbortSignal.timeout(timeoutMs); + return Promise.race([ + pending, + new Promise((_, reject) => { + deadline.addEventListener("abort", () => reject(new CursorKeychainTimeoutError()), { + once: true, + }); + }), + ]); }; } diff --git a/apps/server/src/provider/opencodeRuntime.ts b/apps/server/src/provider/opencodeRuntime.ts index e87f758e0e3d..d8319bc44a71 100644 --- a/apps/server/src/provider/opencodeRuntime.ts +++ b/apps/server/src/provider/opencodeRuntime.ts @@ -78,7 +78,6 @@ export function resolveOpenCodeServerPassword( : input.environment.OPENCODE_SERVER_PASSWORD; } -const OPENCODE_SERVER_READY_PREFIX = "opencode server listening"; const DEFAULT_OPENCODE_SERVER_TIMEOUT_MS = 30_000; const DEFAULT_HOSTNAME = "127.0.0.1"; const OPENCODE_SERVER_STARTUP_MAX_OUTPUT_CHARS = 64 * 1024; @@ -289,11 +288,8 @@ export interface OpenCodeRuntimeShape { function parseServerUrlFromOutput(output: string): string | null { for (const line of output.split("\n")) { - if (!line.startsWith(OPENCODE_SERVER_READY_PREFIX)) { - continue; - } - const match = line.match(/on\s+(https?:\/\/[^\s]+)/); - return match?.[1] ?? null; + const match = line.match(/server listening on\s+(https?:\/\/[^\s]+)/i); + if (match?.[1]) return match[1]; } return null; } diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index db4f1f53b710..71ec0719325d 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -28,6 +28,7 @@ import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import * as Tracer from "effect/Tracer"; +import * as TestClock from "effect/testing/TestClock"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; @@ -975,3 +976,114 @@ describe.sequential("signRelayAgentActivityPublishProof", () => { ), ); }); + +describe.sequential("startup catch-up", () => { + // An unlinked relay with publishing off. `link` writes the link secrets and + // `enablePublishing` the opt-in. Counts link checks (relay URL reads) and + // catch-up publishes (shell snapshot reads). + function makeUnlinkedRelay() { + const secrets = makeMemorySecretStore(); + const counts = { linkChecks: 0, catchUpPublishes: 0 }; + const countingStore = { + ...secrets.store, + get: (name: string) => + Effect.suspend(() => { + if (name === RELAY_URL_SECRET) counts.linkChecks += 1; + return secrets.store.get(name); + }), + } satisfies ServerSecretStore.ServerSecretStore["Service"]; + + const layer = AgentAwarenessRelay.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed(ServerSecretStore.ServerSecretStore, countingStore), + Layer.succeed(ServerEnvironment.ServerEnvironment, { + getEnvironmentId: Effect.succeed("env-1" as EnvironmentId), + getDescriptor: Effect.die("unused descriptor"), + }), + Layer.succeed(OrchestrationEngineService, { + streamDomainEvents: Stream.never, + } as unknown as OrchestrationEngineShape), + Layer.succeed(ProjectionSnapshotQuery, { + getShellSnapshot: () => + Effect.sync(() => { + counts.catchUpPublishes += 1; + return { + snapshotSequence: 1, + projects: [], + threads: [], + updatedAt: "2026-05-25T00:00:00.000Z", + } satisfies OrchestrationShellSnapshot; + }), + } as unknown as ProjectionSnapshotQueryShape), + ), + ), + Layer.provideMerge(NodeServices.layer), + ); + const link = Effect.all( + [ + secrets.setString(RELAY_URL_SECRET, "https://relay.example.test"), + secrets.setString(RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "relay-credential"), + ], + { discard: true }, + ); + const enablePublishing = secrets.setString(PUBLISH_AGENT_ACTIVITY_SECRET, "true"); + return { counts, layer, link, enablePublishing }; + } + + it.effect("checks an unlinked environment once a minute and still catches up once linked", () => { + const { counts, layer, link, enablePublishing } = makeUnlinkedRelay(); + return Effect.gen(function* () { + const relay = yield* AgentAwarenessRelay.AgentAwarenessRelay; + yield* enablePublishing; + yield* relay.start(); + + // Get past the backoff ramp, then count checks in a steady window. + yield* TestClock.adjust("10 minutes"); + const checksBeforeWindow = counts.linkChecks; + yield* TestClock.adjust("10 minutes"); + expect(counts.linkChecks - checksBeforeWindow).toBe(10); + expect(counts.catchUpPublishes).toBe(0); + + yield* link; + yield* TestClock.adjust("1 minute"); + expect(counts.catchUpPublishes).toBe(1); + }).pipe(Effect.provide(layer), Effect.scoped); + }); + + it.effect("publishes at once when this process links while the check is backed off", () => { + const { counts, layer, link, enablePublishing } = makeUnlinkedRelay(); + return Effect.gen(function* () { + const relay = yield* AgentAwarenessRelay.AgentAwarenessRelay; + yield* enablePublishing; + yield* relay.start(); + + // Backed off to 60 s: the next check is still seconds away. + yield* TestClock.adjust("10 minutes"); + yield* link; + yield* TestClock.adjust("1 second"); + expect(counts.catchUpPublishes).toBe(0); + + yield* relay.requestCatchUp(); + yield* TestClock.adjust("1 second"); + expect(counts.catchUpPublishes).toBe(1); + }).pipe(Effect.provide(layer), Effect.scoped); + }); + + it.effect("catches up within 5 s when another process enables publishing on a link", () => { + const { counts, layer, link, enablePublishing } = makeUnlinkedRelay(); + return Effect.gen(function* () { + const relay = yield* AgentAwarenessRelay.AgentAwarenessRelay; + yield* link; + yield* relay.start(); + + yield* TestClock.adjust("10 minutes"); + expect(counts.catchUpPublishes).toBe(0); + + // `t3 connect publish` writes the opt-in without waking this process. + yield* enablePublishing; + yield* TestClock.adjust("5 seconds"); + expect(counts.catchUpPublishes).toBe(1); + }).pipe(Effect.provide(layer), Effect.scoped); + }); +}); diff --git a/apps/server/src/relay/AgentAwarenessRelay.ts b/apps/server/src/relay/AgentAwarenessRelay.ts index 29d2abecebb7..c9acf6b510fc 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.ts @@ -25,6 +25,7 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import type * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; @@ -51,6 +52,8 @@ export class AgentAwarenessRelay extends Context.Service< AgentAwarenessRelay, { readonly publishThread: (threadId: ThreadId) => Effect.Effect; + /** Retries a pending catch-up publish now. Call after this process links or enables publishing. */ + readonly requestCatchUp: () => Effect.Effect; readonly start: () => Effect.Effect; } >()("t3/relay/AgentAwarenessRelay") {} @@ -309,6 +312,8 @@ export const make = Effect.gen(function* () { const cloudLinkKeyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(secrets); const startedAt = (yield* DateTime.now).epochMilliseconds; const activeSnapshotPublishedRef = yield* Ref.make(false); + // Holds at most one pending wake, so a burst of requests costs one retry. + const catchUpRequests = yield* Queue.dropping(1); const publishedStateByThreadRef = yield* Ref.make(new Map()); const readSecretString = (name: string) => @@ -534,18 +539,29 @@ export const make = Effect.gen(function* () { withRelayClientTracing, ); + // Publishes the active threads once. Returns why it did not, so the retry + // knows whether it is waiting on a link or on the publish setting. const publishActiveThreadsUnsafe = Effect.gen(function* () { + // One secret read settles the common never-linked case; the full link + // config is read only once publishing is on. + const relayUrl = yield* readSecretString(RELAY_URL_SECRET).pipe( + Effect.orElseSucceed(() => null), + ); + if (!relayUrl) { + yield* Effect.logDebug("agent activity snapshot skipped; relay link credentials unavailable"); + return "unlinked" as const; + } const publishAgentActivity = yield* readPublishAgentActivityEnabled.pipe( Effect.orElseSucceed(() => false), ); if (!publishAgentActivity) { yield* Effect.logDebug("agent activity snapshot skipped; publication disabled"); - return false; + return "disabled" as const; } const relayConfig = yield* readRelayConfig.pipe(Effect.orElseSucceed(() => null)); if (!relayConfig) { yield* Effect.logDebug("agent activity snapshot skipped; relay link credentials unavailable"); - return false; + return "unlinked" as const; } const environmentId = yield* serverEnvironment.getEnvironmentId; const snapshot = yield* snapshotQuery.getShellSnapshot(); @@ -557,20 +573,29 @@ export const make = Effect.gen(function* () { }); if (activeThreadIds.length === 0) { yield* Effect.logDebug("agent activity snapshot has no publishable threads"); - return true; + return "published" as const; } yield* Effect.logInfo("publishing active agent activity snapshot", { count: activeThreadIds.length, }); yield* Effect.forEach(activeThreadIds, publishThread, { concurrency: 4, discard: true }); - return true; + return "published" as const; }); + // Publishes the catch-up snapshot of active threads once the environment is + // linked and publishing is enabled. Many environments never link, so while + // unlinked the retry backs off from 5 s to 60 s. Only this process writes + // the link, and it calls `requestCatchUp`, which ends the wait early. A + // linked environment keeps the 5 s retry, because `t3 connect publish` can + // turn publishing on from another process. const publishActiveThreadsOnceWhenConfigured = (logEnabledWhenReady: boolean) => Effect.gen(function* () { + let unlinkedRetryDelayMs = 5_000; while (!(yield* Ref.get(activeSnapshotPublishedRef))) { - const published = yield* publishActiveThreadsUnsafe.pipe(Effect.orElseSucceed(() => false)); - if (published) { + const result = yield* publishActiveThreadsUnsafe.pipe( + Effect.orElseSucceed(() => "failed" as const), + ); + if (result === "published") { yield* Ref.set(activeSnapshotPublishedRef, true); if (logEnabledWhenReady) { const relayConfig = yield* readRelayConfig.pipe(Effect.orElseSucceed(() => null)); @@ -580,7 +605,11 @@ export const make = Effect.gen(function* () { } return; } - yield* Effect.sleep("5 seconds"); + const retryDelayMs = result === "unlinked" ? unlinkedRetryDelayMs : 5_000; + yield* Effect.race(Effect.sleep(retryDelayMs), Queue.take(catchUpRequests)); + if (result === "unlinked") { + unlinkedRetryDelayMs = Math.min(unlinkedRetryDelayMs * 2, 60_000); + } } }); @@ -657,6 +686,7 @@ export const make = Effect.gen(function* () { return AgentAwarenessRelay.of({ publishThread, + requestCatchUp: () => Queue.offer(catchUpRequests, undefined).pipe(Effect.asVoid), start, }); }); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index aa01591e9302..dc9e63d8010e 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -179,10 +179,12 @@ import * as VcsProcess from "./vcs/VcsProcess.ts"; import * as GitWorkflowService from "./git/GitWorkflowService.ts"; import * as ReviewService from "./review/ReviewService.ts"; import * as SourceControlRepositoryService from "./sourceControl/SourceControlRepositoryService.ts"; +import { REPLAY_MARKER_MAX_AGE } from "./auth/replayMarkers.ts"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; import * as PairingGrantStore from "./auth/PairingGrantStore.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; +import * as AgentAwarenessRelay from "./relay/AgentAwarenessRelay.ts"; import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts"; import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts"; import * as HostResources from "./resourceTelemetry/HostResources.ts"; @@ -565,6 +567,7 @@ const buildAppUnderTest = (options?: { CloudManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"] >; relayClient?: Partial; + agentAwarenessRelay?: Partial; cloudCliTokenManager?: Partial; httpClient?: HttpClient.HttpClient; nativeTelemetryClient?: Partial; @@ -1209,14 +1212,20 @@ const buildAppUnderTest = (options?: { }), ), Layer.provide( - Layer.succeed( - CloudManagedEndpointRuntime.CloudManagedEndpointRuntime, - CloudManagedEndpointRuntime.CloudManagedEndpointRuntime.of({ - applyConfig: () => Effect.succeed({ status: "disabled" }), - recoveryRequests: Stream.empty, - requestRecovery: () => Effect.void, - withLinkStateLock: (effect) => effect, - ...options?.layers?.cloudManagedEndpointRuntime, + Layer.mergeAll( + Layer.succeed( + CloudManagedEndpointRuntime.CloudManagedEndpointRuntime, + CloudManagedEndpointRuntime.CloudManagedEndpointRuntime.of({ + applyConfig: () => Effect.succeed({ status: "disabled" }), + recoveryRequests: Stream.empty, + requestRecovery: () => Effect.void, + withLinkStateLock: (effect) => effect, + ...options?.layers?.cloudManagedEndpointRuntime, + }), + ), + Layer.mock(AgentAwarenessRelay.AgentAwarenessRelay)({ + requestCatchUp: () => Effect.void, + ...options?.layers?.agentAwarenessRelay, }), ), ), @@ -2718,6 +2727,40 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("rejects a DPoP replay by time alone once its marker can be pruned", () => + Effect.gen(function* () { + yield* buildAppUnderTest(); + + const ownerCookie = yield* getAuthenticatedSessionCookieHeader(); + const credentialResponse = yield* HttpClient.post("/api/auth/pairing-token", { + headers: { cookie: ownerCookie }, + body: yield* HttpBody.json({}), + }); + const credential = (yield* credentialResponse.json) as { readonly credential: string }; + const tokenUrl = yield* getHttpServerUrl("/oauth/token"); + const acceptedAt = yield* DateTime.now; + // The longest-lived proof: `iat` at the 5 s future skew the verifier allows. + const dpop = makeDpopProof({ + method: "POST", + url: tokenUrl, + iat: Math.floor(acceptedAt.epochMilliseconds / 1_000) + 5, + }); + const exchange = exchangeAccessToken(credential.credential, { + headers: { dpop: dpop.proof }, + scope: "orchestration:read orchestration:operate terminal:operate review:write", + }); + + assert.equal((yield* exchange).response.status, 200); + // While the proof is fresh, only the replay marker rejects it. + assert.equal((yield* exchange).body.dpopFailureReason, "replay"); + // Once the marker can be pruned, the time check rejects the proof by itself. + yield* TestClock.setTime( + acceptedAt.epochMilliseconds + Duration.toMillis(REPLAY_MARKER_MAX_AGE), + ); + assert.equal((yield* exchange).body.dpopFailureReason, "time_window"); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("ignores forwarded host headers when validating token exchange DPoP URLs", () => Effect.gen(function* () { yield* buildAppUnderTest(); @@ -3154,6 +3197,55 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("wakes the agent awareness relay when this server links or changes publishing", () => + Effect.gen(function* () { + let catchUpRequests = 0; + yield* buildAppUnderTest({ + layers: { + agentAwarenessRelay: { + requestCatchUp: () => + Effect.sync(() => { + catchUpRequests += 1; + }), + }, + }, + }); + + const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const ownerCookie = yield* getAuthenticatedSessionCookieHeader(); + const relayConfigResponse = yield* fetchEffect( + yield* getHttpServerUrl("/api/connect/relay-config"), + { + method: "POST", + headers: { cookie: ownerCookie, "content-type": "application/json" }, + body: jsonRequestBody({ + relayUrl: "https://relay.example.test", + cloudUserId: "user_123", + environmentCredential: "t3env_test_credential", + cloudMintPublicKey: cloudKeyPair.publicKey, + endpointRuntime: null, + }), + }, + ); + assert.equal(relayConfigResponse.status, 200); + assert.equal(catchUpRequests, 1); + + const preferencesResponse = yield* fetchEffect( + yield* getHttpServerUrl("/api/connect/preferences"), + { + method: "POST", + headers: { cookie: ownerCookie, "content-type": "application/json" }, + body: jsonRequestBody({ publishAgentActivity: true }), + }, + ); + assert.equal(preferencesResponse.status, 200); + assert.equal(catchUpRequests, 2); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("rejects relay config with an invalid cloud mint public key", () => Effect.gen(function* () { yield* buildAppUnderTest(); @@ -3785,6 +3877,82 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("rejects cloud replays by time alone once their markers can be pruned", () => + Effect.gen(function* () { + yield* buildAppUnderTest(); + + const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const ownerCookie = yield* getAuthenticatedSessionCookieHeader(); + const relayConfigResponse = yield* fetchEffect( + yield* getHttpServerUrl("/api/connect/relay-config"), + { + method: "POST", + headers: { cookie: ownerCookie, "content-type": "application/json" }, + body: jsonRequestBody({ + relayUrl: "https://relay.example.test", + cloudUserId: "user_123", + environmentCredential: "t3env_test_credential", + cloudMintPublicKey: cloudKeyPair.publicKey, + endpointRuntime: null, + }), + }, + ); + assert.equal(relayConfigResponse.status, 200); + + const acceptedAt = yield* DateTime.now; + // The longest-lived proofs: `iat` at the 60 s future skew the handlers + // allow, and the 5 minute maximum lifetime. + const issuedAt = DateTime.add(acceptedAt, { minutes: 1 }); + const proofTimes = { + issuedAt: DateTime.formatIso(issuedAt), + expiresAt: DateTime.formatIso(DateTime.add(issuedAt, { minutes: 5 })), + }; + const requests = [ + [ + "/api/t3-connect/health", + makeCloudEnvironmentHealthRequest({ + privateKey: cloudKeyPair.privateKey, + environmentId: testEnvironmentDescriptor.environmentId, + nonce: "cloud-health-nonce-pruned", + ...proofTimes, + }), + ], + [ + "/api/t3-connect/mint-credential", + makeCloudMintCredentialRequest({ + privateKey: cloudKeyPair.privateKey, + environmentId: testEnvironmentDescriptor.environmentId, + clientProofKeyThumbprint: "client-proof-key-thumbprint", + nonce: "cloud-mint-nonce-pruned", + ...proofTimes, + }), + ], + ] as const; + const postAll = Effect.forEach(requests, ([pathname, request]) => + Effect.gen(function* () { + const response = yield* fetchEffect(yield* getHttpServerUrl(pathname), { + method: "POST", + headers: { "content-type": "application/json" }, + body: jsonRequestBody(request), + }); + return response.status; + }), + ); + + assert.deepStrictEqual(yield* postAll, [200, 200]); + // While the proofs are fresh, only the replay markers reject them (409). + assert.deepStrictEqual(yield* postAll, [409, 409]); + // Once the markers can be pruned, the time checks reject the proofs by themselves (401). + yield* TestClock.setTime( + acceptedAt.epochMilliseconds + Duration.toMillis(REPLAY_MARKER_MAX_AGE), + ); + assert.deepStrictEqual(yield* postAll, [401, 401]); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect( "validates cloud proofs against the configured relay issuer, not the transport URL", () => @@ -5736,6 +5904,56 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("does not trace browser OTLP trace exports on the server", () => + Effect.gen(function* () { + const spanNames: Array = []; + const forwardedUrls: Array = []; + yield* buildAppUnderTest({ + config: { otlpTracesUrl: "http://collector.test/v1/traces" }, + layers: { + httpClient: HttpClient.make((request) => + Effect.sync(() => { + forwardedUrls.push(request.url); + return HttpClientResponse.fromWeb(request, new Response(null, { status: 204 })); + }), + ), + }, + }).pipe( + Effect.provideService( + Tracer.Tracer, + Tracer.make({ + span: (options) => { + spanNames.push(options.name); + return new Tracer.NativeSpan(options); + }, + }), + ), + ); + const cookie = yield* getAuthenticatedSessionCookieHeader(); + spanNames.length = 0; + + // The query string must not bring back the HTTP server span. + for (const url of ["/api/observability/v1/traces", "/api/observability/v1/traces?x=1"]) { + const response = yield* HttpClient.post(url, { + headers: { cookie, "content-type": "application/json" }, + body: yield* HttpBody.json({ resourceSpans: [] }), + }); + assert.equal(response.status, 204); + } + + assert.deepEqual(forwardedUrls, [ + "http://collector.test/v1/traces", + "http://collector.test/v1/traces", + ]); + assert.deepEqual(spanNames, []); + + // Other routes keep their HTTP server span. + const session = yield* HttpClient.get("/api/auth/session", { headers: { cookie } }); + assert.equal(session.status, 200); + assert.include(spanNames, "http.server GET"); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("routes websocket rpc server.upsertKeybinding", () => Effect.gen(function* () { const rule: KeybindingRule = { diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 0d211fcdeb0c..160e864cd174 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -33,6 +33,7 @@ import { staticAndDevRouteLayer, browserApiCorsLayer, httpCompressionLayer, + untracedRequestsLayer, } from "./http.ts"; import { guardHttpResponseWriteErrors } from "./httpResponseErrorGuard.ts"; import { fixPath } from "./os-jank.ts"; @@ -127,9 +128,12 @@ import * as SourceControlRepositoryService from "./sourceControl/SourceControlRe import * as ProjectSetupScriptRunner from "./project/ProjectSetupScriptRunner.ts"; import * as WorktreeSetupTracker from "./project/WorktreeSetupTracker.ts"; import { ObservabilityLive } from "./observability/Layers/Observability.ts"; +import * as HeapSnapshot from "./observability/HeapSnapshot.ts"; +import * as EventLoopMonitor from "./observability/EventLoopMonitor.ts"; import * as ServerEnvironment from "./environment/ServerEnvironment.ts"; import * as RemoteOpenTargets from "./environment/RemoteOpenTargets.ts"; import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http.ts"; +import * as ReplayMarkers from "./auth/replayMarkers.ts"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; import { @@ -190,7 +194,8 @@ export const HTTP_ROUTER_CONFIG = { // those finalizers get a chance to run. const HTTP_PREEMPTIVE_SHUTDOWN_GRACE_MS = 0; const ResourceAttributionLayerLive = ResourceAttribution.layer; -const ApplicationObservabilityLive = ObservabilityLive.pipe( +const ApplicationObservabilityLive = EventLoopMonitor.layer.pipe( + Layer.provideMerge(ObservabilityLive), Layer.provideMerge(ResourceAttributionLayerLive), ); @@ -507,6 +512,7 @@ const AntigravityInstallationRefreshLive = Layer.effectDiscard( const RuntimeCoreDependenciesWithoutThreadBootstrapLive = ReactorLayerLive.pipe( Layer.provideMerge(AntigravityInstallationRefreshLive), + Layer.provideMerge(ReplayMarkers.layer), Layer.provideMerge(ProviderAuthServiceLive), // Core Services Layer.provideMerge(ServerSettingsLayerLive), @@ -627,6 +633,8 @@ export const makeRoutesLayerWith = (mcpToolkitDependencies: Layer.La Layer.provide(McpSessionRegistry.layer), Layer.provide(mcpToolkitDependencies), ), + // Last, so no route layer can replace the server's one TracerDisabledWhen. + untracedRequestsLayer, ).pipe( // Both transports consume the same service instance, so caches single-flight across clients // and mutations observed on WebSocket invalidate patches subsequently read over HTTP. @@ -1008,6 +1016,7 @@ const makeServerLayer = Layer.unwrap( runtimeStateLayer.pipe(Layer.provide(launcherLayer)), tailscaleServeLayer, cloudDesiredLinkReconcileLayer, + HeapSnapshot.layer, ); return serverApplicationLayer.pipe( diff --git a/apps/server/src/serverRuntimeStartup.test.ts b/apps/server/src/serverRuntimeStartup.test.ts index df879a2cf307..4bbdb2e9c8b5 100644 --- a/apps/server/src/serverRuntimeStartup.test.ts +++ b/apps/server/src/serverRuntimeStartup.test.ts @@ -169,6 +169,7 @@ it.effect("resolveAutoBootstrapWelcomeTargets returns existing project and threa getSnapshot: () => Effect.die("unused"), getShellSnapshot: () => Effect.die("unused"), getDeletedWorktreeThreads: () => Effect.die("unused"), + listThreadsWithPullRequests: () => Effect.die("unused"), getArchivedShellSnapshot: () => Effect.die("unused"), getSnapshotSequence: () => Effect.die("unused"), getCounts: () => Effect.die("unused"), @@ -298,6 +299,7 @@ it.effect.each([ getSnapshot: () => Effect.die("unused"), getShellSnapshot: () => Effect.die("unused"), getDeletedWorktreeThreads: () => Effect.die("unused"), + listThreadsWithPullRequests: () => Effect.die("unused"), getArchivedShellSnapshot: () => Effect.die("unused"), getSnapshotSequence: () => Effect.die("unused"), getCounts: () => Effect.die("unused"), @@ -385,6 +387,7 @@ it.effect( getSnapshot: () => Effect.die("unused"), getShellSnapshot: () => Effect.die("unused"), getDeletedWorktreeThreads: () => Effect.die("unused"), + listThreadsWithPullRequests: () => Effect.die("unused"), getArchivedShellSnapshot: () => Effect.die("unused"), getSnapshotSequence: () => Effect.die("unused"), getCounts: () => Effect.die("unused"), @@ -450,6 +453,7 @@ it.effect("resolveAutoBootstrapWelcomeTargets preserves typed UUID generation fa getSnapshot: () => Effect.die("unused"), getShellSnapshot: () => Effect.die("unused"), getDeletedWorktreeThreads: () => Effect.die("unused"), + listThreadsWithPullRequests: () => Effect.die("unused"), getArchivedShellSnapshot: () => Effect.die("unused"), getSnapshotSequence: () => Effect.die("unused"), getCounts: () => Effect.die("unused"), diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index a80a31f48ae6..15ea4b673f22 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -121,6 +121,46 @@ function totalOutputTokens(summary: { buckets: readonly { totals: { outputTokens } describe("UsageService", () => { + it.live("omits Cursor account usage when no file login is saved", () => + Effect.gen(function* () { + const { settings, home } = yield* setup; + for (const platform of ["linux", "win32", "darwin"] as const) { + const service = yield* UsageService.make.pipe( + Effect.provide( + serviceLayers({ + prefix: `usage-service-cursor-no-login-${platform}`, + home, + settings, + platform, + environment: { AGENT_CLI_CREDENTIAL_STORE: "file" }, + }), + ), + ); + const summary = yield* service.readSummary(WINDOW); + assert.isFalse(summary.sources.some((source) => source.fingerprint.provider === "cursor")); + } + }).pipe(Effect.scoped), + ); + + it.live("keeps Cursor credential errors visible when a saved login cannot be read", () => + Effect.gen(function* () { + const { settings, home } = yield* setup; + const authPath = NodePath.join(home, "config", "cursor", "auth.json"); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(NodePath.dirname(authPath), { recursive: true }); + await NodeFSP.writeFile(authPath, "invalid json"); + }); + const service = yield* UsageService.make.pipe( + Effect.provide( + serviceLayers({ prefix: "usage-service-cursor-invalid-login", home, settings }), + ), + ); + const summary = yield* service.readSummary(WINDOW); + const cursor = summary.sources.find((source) => source.fingerprint.provider === "cursor"); + assert.strictEqual(cursor?.message, "Cursor credentials could not be read."); + }).pipe(Effect.scoped), + ); + it.live("does not read the macOS Cursor Keychain before account usage is enabled", () => Effect.gen(function* () { const { settings, home } = yield* setup; @@ -233,10 +273,7 @@ describe("UsageService", () => { const summary = yield* service.readSummary(WINDOW); assert.strictEqual(summary.buckets[0]?.provider, "opencode"); assert.isFalse(summary.buckets.some((bucket) => bucket.provider === "cursor")); - assert.strictEqual( - summary.sources.find((source) => source.fingerprint.provider === "cursor")?.status, - "missing", - ); + assert.isFalse(summary.sources.some((source) => source.fingerprint.provider === "cursor")); assert.strictEqual( summary.buckets[0]?.sourcePath, yield* Effect.promise(() => NodeFSP.realpath(root)), @@ -247,10 +284,6 @@ describe("UsageService", () => { ?.distinctSessions, 1, ); - assert.include( - summary.sources.find((source) => source.fingerprint.provider === "cursor")?.message ?? "", - "Cursor account history needs a Cursor CLI login", - ); }).pipe(Effect.scoped), ); diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index d4d7e0786ec6..59ce5f8aabde 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -641,6 +641,8 @@ export const make = Effect.gen(function* () { cursorUntilMs, ), ); + // No saved login means there is no account source to report, not a setup error. + if (account.missing && account.error === null) return scanned; if (account.accountKey !== null && account.error === null && !account.missing) { // The same account includes CLI and desktop history from every machine. // A stable remote fingerprint prevents connected environments counting it twice. diff --git a/apps/server/src/usage/cursorUsageReader.test.ts b/apps/server/src/usage/cursorUsageReader.test.ts new file mode 100644 index 000000000000..4bfa647c0aff --- /dev/null +++ b/apps/server/src/usage/cursorUsageReader.test.ts @@ -0,0 +1,22 @@ +import { assert, describe, it } from "@effect/vitest"; + +import { CursorKeychainTimeoutError } from "../provider/cursorCredentialStore.ts"; +import { readCursorAccountUsage } from "./cursorUsageReader.ts"; + +describe("readCursorAccountUsage", () => { + it("asks for Keychain approval when the prompt goes unanswered", async () => { + const result = await readCursorAccountUsage( + { kind: "keychain" }, + 0, + 1, + () => Promise.reject(new Error("no network expected")), + () => Promise.reject(new CursorKeychainTimeoutError()), + ); + assert.deepStrictEqual(result, { + accountKey: null, + records: [], + missing: false, + error: "Allow Keychain access on the Mac running T3 Code, then refresh.", + }); + }); +}); diff --git a/apps/server/src/usage/cursorUsageReader.ts b/apps/server/src/usage/cursorUsageReader.ts index 1e07fcc7a825..573506d818a1 100644 --- a/apps/server/src/usage/cursorUsageReader.ts +++ b/apps/server/src/usage/cursorUsageReader.ts @@ -5,7 +5,10 @@ import * as NodeCrypto from "node:crypto"; import * as NodeTimersPromises from "node:timers/promises"; import type { UsageRecord } from "./usageTranscripts.ts"; -import { readMacCursorAccessToken } from "../provider/cursorCredentialStore.ts"; +import { + CursorKeychainTimeoutError, + readMacCursorAccessToken, +} from "../provider/cursorCredentialStore.ts"; function object(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) @@ -86,7 +89,9 @@ export async function readCursorAccountUsage( ? null : typeof credentialSource === "string" ? "Cursor credentials could not be read." - : "Cursor Keychain credentials could not be read.", + : cause instanceof CursorKeychainTimeoutError + ? "Allow Keychain access on the Mac running T3 Code, then refresh." + : "Cursor Keychain credentials could not be read.", }; } if (typeof accessToken !== "string" || !accessToken) { diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index 47ca4e178700..85c4d1a59d1d 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -1918,6 +1918,35 @@ it.layer(TestLayer)("GitVcsDriver core integration", (it) => { }), ); + it.effect("does not start Git auto-maintenance from background upstream fetches", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-vcs-driver-remote-"); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "-u", "origin", initialBranch]); + yield* git(cwd, ["repack", "-d"]); + yield* writeTextFile(cwd, "second.txt", "second\n"); + yield* git(cwd, ["add", "second.txt"]); + yield* git(cwd, ["commit", "-m", "second commit"]); + yield* git(cwd, ["push"]); + yield* git(cwd, ["repack", "-d"]); + // Two packs make `git gc --auto` due, and without detaching it would run inside the fetch. + yield* git(cwd, ["config", "gc.autoPackLimit", "1"]); + yield* git(cwd, ["config", "gc.autoDetach", "false"]); + yield* git(cwd, ["config", "maintenance.autoDetach", "false"]); + const packCount = git(cwd, ["count-objects", "-v"]).pipe( + Effect.map((stdout) => stdout.match(/^packs: (\d+)$/m)?.[1]), + ); + assert.equal(yield* packCount, "2"); + + yield* (yield* GitVcsDriver.GitVcsDriver).statusDetailsRemote(cwd); + + assert.equal(yield* packCount, "2"); + }), + ); + it.effect("uses origin HEAD for default-branch detection with a non-origin upstream", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 8ec274a46611..0dd73af687f9 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -549,7 +549,9 @@ function trace2ChildKey(record: Record): string | null { const Trace2Record = Schema.Record(Schema.String, Schema.Unknown); const decodeTrace2Record = decodeJsonResult(Trace2Record); -const createTrace2Monitor = Effect.fn("createTrace2Monitor")(function* ( +// Untraced because it runs on every git spawn and returns at once without hook +// callbacks. Its errors fail the runGitCommand span. +const createTrace2Monitor = Effect.fnUntraced(function* ( input: Pick, progress: GitVcsDriver.ExecuteGitProgress | undefined, ): Effect.fn.Return< @@ -1125,10 +1127,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ): Effect.Effect => { const fetchCwd = path.basename(gitCommonDir) === ".git" ? path.dirname(gitCommonDir) : gitCommonDir; + // `--no-auto-gc` (a synonym of `--no-auto-maintenance` that older Git also knows) keeps + // this poll from starting `git gc --auto`. When that gc fails, for example on a repository + // with missing objects, Git retries it on every fetch and leaves a full-size `tmp_pack_*` + // behind each time, so a background poll could fill the disk. return executeGit( "GitVcsDriver.fetchRemoteForStatus", fetchCwd, - ["--git-dir", gitCommonDir, "fetch", "--quiet", "--no-tags", remoteName], + ["--git-dir", gitCommonDir, "fetch", "--quiet", "--no-tags", "--no-auto-gc", remoteName], { env: STATUS_UPSTREAM_REFRESH_ENV, fallbackErrorDetail: "Background Git fetch exited with a non-zero status.", diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 1e4d56a36a93..ae09cd38ac45 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -2207,9 +2207,13 @@ const makeWsRpcLayer = ( [WS_METHODS.sourceControlPublishRepository]: (input) => observeRpcEffect( WS_METHODS.sourceControlPublishRepository, - sourceControlRepositories - .publishRepository(input) - .pipe(Effect.tap(() => refreshGitStatus(input.cwd))), + sourceControlRepositories.publishRepository(input).pipe( + // A new remote can change the cached identity. Only the `cwd` entry + // refreshes, so after a publish from a linked worktree the project + // root entry waits for its TTL. + Effect.tap(() => repositoryIdentityResolver.resolve(input.cwd, { refresh: true })), + Effect.tap(() => refreshGitStatus(input.cwd)), + ), { "rpc.aggregate": "source-control", }, diff --git a/apps/web/package.json b/apps/web/package.json index ec7334e1fd86..665cdbf71736 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -75,6 +75,7 @@ "@vitejs/plugin-react": "^6.0.0", "babel-plugin-react-compiler": "1.0.0", "compression": "^1.8.1", + "jsdom": "^30.0.1", "react-test-renderer": "19.2.6", "tailwindcss": "^4.0.0", "unified": "^11.0.5", diff --git a/apps/web/src/components/AppSidebarLayout.tsx b/apps/web/src/components/AppSidebarLayout.tsx index 0cf965c4bd22..05d965d95d06 100644 --- a/apps/web/src/components/AppSidebarLayout.tsx +++ b/apps/web/src/components/AppSidebarLayout.tsx @@ -79,6 +79,7 @@ function readInitialThreadSidebarWidth(): number { } function SidebarControl() { + const usagePageOpen = useLocation({ select: (location) => location.pathname === "/usage" }); const keybindings = useAtomValue(primaryServerKeybindingsAtom); const { toggleSidebar } = useSidebar(); const isSidebarVisible = useSidebarVisibility(); @@ -86,7 +87,9 @@ function SidebarControl() { const stageBackdropVariant = useSidebarStageBackdropVariant( environmentIdentificationMode === "artwork", ); - const shortcutLabel = shortcutLabelForCommand(keybindings, "sidebar.toggle"); + const shortcutLabel = shortcutLabelForCommand(keybindings, "sidebar.toggle", { + context: { usagePageOpen }, + }); useEffect(() => { const onKeyDown = (event: KeyboardEvent) => { @@ -106,7 +109,11 @@ function SidebarControl() { // available everywhere else, including the plain-text composer. return; } - if (resolveShortcutCommand(event, keybindings) !== "sidebar.toggle") return; + if ( + resolveShortcutCommand(event, keybindings, { context: { usagePageOpen } }) !== + "sidebar.toggle" + ) + return; event.preventDefault(); event.stopPropagation(); @@ -116,7 +123,7 @@ function SidebarControl() { // Capture before focused editors consume commands such as Mod+B for rich-text formatting. window.addEventListener("keydown", onKeyDown, true); return () => window.removeEventListener("keydown", onKeyDown, true); - }, [keybindings, toggleSidebar]); + }, [keybindings, toggleSidebar, usagePageOpen]); return ( // The right-side layout controls carry mr-px (border compensation inside @@ -297,6 +304,8 @@ export function AppSidebarLayout({ children }: { children: ReactNode }) { side="left" collapsible="offcanvas" data-app-sidebar="" + role="navigation" + aria-label={isOnSettings ? "Settings" : "Threads"} resizable={{ maxWidth: sidebarMaximumWidth, minWidth: THREAD_SIDEBAR_MIN_WIDTH, diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 43b908feca52..06a79868785c 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -328,12 +328,13 @@ import { terminalContextReference, } from "../lib/composerContextRecords"; import { - isQueuedMessageDue, latestCompletedToolActivityId, type QueuedComposerMessage, + type QueuedMessageSendSettings, useQueuedMessages, useQueuedMessageStore, } from "../queuedMessageStore"; +import { sendQueuedMessage } from "./chat/sendQueuedMessage"; import { type ReviewCommentContext } from "../reviewCommentContext"; import { environmentCatalog } from "../connection/catalog"; import { isDesktopLocalConnectionTarget } from "../connection/desktopLocal"; @@ -924,7 +925,14 @@ const PersistentThreadTerminalDrawer = memo(function PersistentThreadTerminalDra const writeTerminal = useAtomCommand(terminalEnvironment.write, "terminal write"); const closeTerminalMutation = useAtomCommand(terminalEnvironment.close, "terminal close"); const draftThread = useComposerDraftStore((store) => store.getDraftThreadByRef(threadRef)); - const serverThread = useThread(threadRef, { waitForShell: draftThread !== null }); + // Hidden drawers stay mounted (see MAX_HIDDEN_MOUNTED_TERMINAL_THREADS), so they read only + // the shell: a detail subscription would keep each hidden thread's history in memory. The + // active drawer shares ChatView's detail, which also covers archived threads (no shell). + const activeServerThread = useThread(active ? threadRef : null, { + waitForShell: draftThread !== null, + }); + const serverThreadShell = useThreadShell(threadRef); + const serverThread = activeServerThread ?? serverThreadShell; const projectRef = serverThread ? scopeProjectRef(serverThread.environmentId, serverThread.projectId) : draftThread @@ -2704,68 +2712,40 @@ export default function ChatView(props: ChatViewProps) { unavailableConnection !== null && (unavailableConnection.phase === "connecting" || unavailableConnection.phase === "reconnecting"); - // Reconnecting to a version-skewed server with no update in flight - // usually means the server is restarting mid-update and a refresh wiped - // the in-memory update state. Fold the reconnect and version banners - // into one calm line instead of stacking "Failed to connect" on - // "versions differ". A failed update never folds: its error and retry - // action must stay visible. - const reconnectingThroughVersionSkew = - serverUpdateState.status === "idle" && environmentReconnecting && versionMismatch !== null; // While an update runs, transient connect blips are expected (the server // restarts) and the update banner already shows progress. Hard failure // phases still surface so the Reconnect action stays reachable. const suppressUnavailableBanner = - environmentReconnecting && - (updateRunning || (!reconnectingThroughVersionSkew && !reconnectWarningGraceElapsed)); + environmentReconnecting && (updateRunning || !reconnectWarningGraceElapsed); if (activeEnvironmentUnavailableState && unavailableConnection && !suppressUnavailableBanner) { - if (reconnectingThroughVersionSkew) { - items.push({ - id: `environment-unavailable:${activeEnvironmentUnavailableState.environmentId}`, - variant: "default", - // Prioritize live connection progress among the notices. - priority: "urgent", - icon: ( -
{isDraftHeroState ? (
diff --git a/apps/web/src/components/NoProjectsHero.tsx b/apps/web/src/components/NoProjectsHero.tsx index de125d384eb1..2da1c0aadb8c 100644 --- a/apps/web/src/components/NoProjectsHero.tsx +++ b/apps/web/src/components/NoProjectsHero.tsx @@ -2,9 +2,11 @@ import { PlusIcon } from "lucide-react"; import { useCallback } from "react"; import { openCommandPalette } from "../commandPaletteBus"; +import { isElectron } from "../env"; import { Button } from "./ui/button"; import { Empty, EmptyDescription, EmptyHeader, EmptyTitle } from "./ui/empty"; import { SidebarInset } from "./ui/sidebar"; +import { WorkspacePageHeader } from "./WorkspacePageHeader"; export function NoProjectsHero() { const openAddProject = useCallback(() => openCommandPalette({ open: "add-project" }), []); @@ -12,6 +14,8 @@ export function NoProjectsHero() { return (
+ {/* The desktop window only moves where CSS opts in, so keep a titlebar strip. */} + {isElectron ? : null}
diff --git a/apps/web/src/components/QueuedMessageSender.test.tsx b/apps/web/src/components/QueuedMessageSender.test.tsx new file mode 100644 index 000000000000..dc31694f8b96 --- /dev/null +++ b/apps/web/src/components/QueuedMessageSender.test.tsx @@ -0,0 +1,231 @@ +import { scopeThreadRef, scopedThreadKey } from "@t3tools/client-runtime/environment"; +import { EnvironmentId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import { act, createElement } from "react"; +import { create, type ReactTestRenderer } from "react-test-renderer"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +import { useQueuedMessageStore, type QueuedComposerMessage } from "../queuedMessageStore"; +import { sendQueuedMessage } from "./chat/sendQueuedMessage"; +import { QueuedMessageSender } from "./QueuedMessageSender"; + +const io = vi.hoisted(() => ({ + run: vi.fn(), + upload: vi.fn(), + toast: vi.fn(), + thread: null as unknown, + shell: { runtimeMode: "full-access", interactionMode: "default" } as Record, +})); +const config = { + environment: { capabilities: { attachmentUploads: true, inlineMessageContext: true } }, +}; +vi.mock("@t3tools/client-runtime/state/runtime", async (load) => ({ + ...(await load()), + runAtomCommand: (...args: unknown[]) => io.run(...args), +})); +vi.mock("../rpc/atomRegistry", () => ({ + appAtomRegistry: { get: () => new Map([["env-a", config]]) }, +})); +vi.mock("../state/server", () => ({ environmentServerConfigsAtom: {} })); +vi.mock("../state/threads", () => ({ + threadEnvironment: { + updateMetadata: "metadata", + setRuntimeMode: "runtime", + setInteractionMode: "interaction", + startTurn: "start", + }, +})); +vi.mock("../state/environments", () => ({ + useEnvironment: () => ({ connection: { phase: "connected" } }), +})); +vi.mock("../state/entities", () => ({ + useThread: () => io.thread, + useThreadStatus: () => "live", + useServerConfigs: () => new Map([["env-a", config]]), + readThreadShell: () => io.shell, + readThread: () => io.thread, +})); +vi.mock("./ui/toast", () => ({ toastManager: { add: (...args: unknown[]) => io.toast(...args) } })); +vi.mock("../lib/attachmentUploadQueue", () => ({ + startAttachmentUpload: vi.fn(), + awaitAttachmentUploads: (...args: unknown[]) => io.upload(...args), + getUploadedAttachments: () => [ + { type: "image", id: "uploaded", name: "a.png", mimeType: "image/png", sizeBytes: 4 }, + ], + releaseDraftAttachments: vi.fn(), +})); + +const threadRef = scopeThreadRef(EnvironmentId.make("env-a"), ThreadId.make("thread-a")); +const threadKey = scopedThreadKey(threadRef); +const modelSelection = { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5" }; + +function enqueue(overrides: Partial = {}) { + return useQueuedMessageStore.getState().enqueue(threadKey, { + prompt: "follow up", + images: [], + files: [], + terminalContexts: [], + previewAnnotations: [], + reviewComments: [], + sendSettings: { + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + promptEffort: null, + }, + queuedAfterToolActivityId: null, + createdAt: "2026-09-25T00:00:00Z", + ...overrides, + }); +} + +const commandsRun = () => io.run.mock.calls.map((call) => call[1]); +const queue = () => useQueuedMessageStore.getState().queuesByThreadKey[threadKey]; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + useQueuedMessageStore.setState({ queuesByThreadKey: {}, lastDispatchByThreadKey: {} }); + io.thread = null; + io.run.mockReset().mockResolvedValue({ _tag: "Success", value: undefined }); + io.upload.mockReset().mockResolvedValue(undefined); + io.toast.mockReset(); + io.shell = { + modelSelection, + branch: null, + runtimeMode: "full-access", + interactionMode: "default", + }; +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("QueuedMessageSender", () => { + const thread = ( + status: string, + { toolActivityIds = [] as string[], userMessageIds = [] as string[] } = {}, + ) => ({ + session: { status, activeTurnId: null, updatedAt: status }, + activities: toolActivityIds.map((id, index) => ({ + id, + kind: "tool.completed", + sequence: index, + createdAt: "2026-09-25T00:00:01Z", + })), + messages: userMessageIds.map((id) => ({ id, role: "user" })), + latestTurn: null, + }); + let root: ReactTestRenderer | null = null; + const render = () => + act(() => { + if (root) root.update(createElement(QueuedMessageSender)); + else root = create(createElement(QueuedMessageSender)); + }); + afterEach(async () => { + await act(() => root?.unmount()); + root = null; + }); + + it("sends a queued message when the turn ends, with no chat view open", async () => { + enqueue(); + io.thread = thread("running"); + await render(); + expect(commandsRun()).toEqual([]); + + io.thread = thread("ready"); + await render(); + + expect(commandsRun()).toEqual(["start"]); + expect(io.run.mock.calls[0]?.[2]).toMatchObject({ + environmentId: "env-a", + input: { threadId: "thread-a", message: { text: "follow up" }, modelSelection }, + }); + expect(queue()).toBeUndefined(); + }); + + it("holds the next message until the server picks up the one before it", async () => { + enqueue({ prompt: "first" }); + enqueue({ prompt: "second" }); + io.thread = thread("ready"); + await render(); + await render(); + expect(commandsRun()).toEqual(["start"]); + + // The first message started a turn; the second waits for its next tool call. + io.thread = thread("running", { userMessageIds: ["first"] }); + await render(); + expect(commandsRun()).toEqual(["start"]); + io.thread = thread("running", { userMessageIds: ["first"], toolActivityIds: ["tool-1"] }); + await render(); + expect(commandsRun()).toEqual(["start", "start"]); + }); + + it("moves on to the next message after a failed one is cancelled", async () => { + io.run.mockResolvedValueOnce({ _tag: "Failure", cause: Cause.fail(new Error("offline")) }); + const first = enqueue({ prompt: "first" }); + enqueue({ prompt: "second" }); + io.thread = thread("ready"); + await render(); + expect(queue()?.[0]).toMatchObject({ prompt: "first", holdUntilUserAction: true }); + + await act(() => { + useQueuedMessageStore.getState().remove(threadKey, first.id); + }); + await render(); + + expect(commandsRun()).toEqual(["start", "start"]); + expect(io.run.mock.calls[1]?.[2]).toMatchObject({ input: { message: { text: "second" } } }); + }); +}); + +describe("sendQueuedMessage", () => { + it("saves a mode changed before queueing, then starts the turn", async () => { + io.shell = { ...io.shell, runtimeMode: "approval-required" }; + const message = enqueue(); + + await sendQueuedMessage(threadRef, message.id); + + expect(commandsRun()).toEqual(["runtime", "start"]); + expect(io.run.mock.calls[1]?.[2]).toMatchObject({ input: { runtimeMode: "full-access" } }); + expect(queue()).toBeUndefined(); + }); + + it("gives a message back to Stop while its upload runs, without starting a turn", async () => { + let finishUpload!: () => void; + io.upload.mockReturnValue(new Promise((resolve) => (finishUpload = resolve))); + const image = { + type: "image" as const, + id: "image-1", + name: "a.png", + mimeType: "image/png", + sizeBytes: 4, + previewUrl: "data:image/png;base64,AAAA", + file: new File(["AAAA"], "a.png", { type: "image/png" }), + }; + const message = enqueue({ images: [image] }); + + const sending = sendQueuedMessage(threadRef, message.id); + expect(useQueuedMessageStore.getState().drain(threadKey)).toHaveLength(1); + finishUpload(); + await sending; + + expect(commandsRun()).toEqual([]); + expect(io.toast).not.toHaveBeenCalled(); + expect(queue()).toBeUndefined(); + }); + + it("holds a message at the head when the turn start fails", async () => { + io.run.mockResolvedValue({ _tag: "Failure", cause: Cause.fail(new Error("offline")) }); + enqueue({ prompt: "first" }); + const second = enqueue({ prompt: "second" }); + + await sendQueuedMessage(threadRef, second.id); + + expect(queue()?.map((entry) => [entry.prompt, entry.holdUntilUserAction])).toEqual([ + ["second", true], + ["first", undefined], + ]); + expect(io.toast).toHaveBeenCalledWith(expect.objectContaining({ description: "offline" })); + }); +}); diff --git a/apps/web/src/components/QueuedMessageSender.tsx b/apps/web/src/components/QueuedMessageSender.tsx new file mode 100644 index 000000000000..86dfa8a265fe --- /dev/null +++ b/apps/web/src/components/QueuedMessageSender.tsx @@ -0,0 +1,100 @@ +import { parseScopedThreadKey } from "@t3tools/client-runtime/environment"; +import { derivePendingRequests } from "@t3tools/client-runtime/pending-requests"; +import { useEffect, useMemo } from "react"; +import { useShallow } from "zustand/react/shallow"; + +import { useComposerDraftStore } from "../composerDraftStore"; +import { + isQueuedMessageDue, + latestCompletedToolActivityId, + useQueuedMessageStore, + useQueuedMessages, +} from "../queuedMessageStore"; +import { derivePhase } from "../session-logic"; +import { useServerConfigs, useThread, useThreadStatus } from "../state/entities"; +import { useEnvironment } from "../state/environments"; +import { hasServerAcknowledgedLocalDispatch, latestTurnStartFailureId } from "./ChatView.logic"; +import { sendQueuedMessage } from "./chat/sendQueuedMessage"; + +/** + * Sends queued messages when they are due, for every thread with a queue, + * whether or not the thread is on screen. Mounted once at the root. + */ +export function QueuedMessageSender() { + const threadKeys = useQueuedMessageStore( + useShallow((state) => Object.keys(state.queuesByThreadKey)), + ); + return threadKeys.map((threadKey) => ); +} + +/** + * Watches one thread while it has queued messages. Reading the thread keeps + * its detail subscribed, so tool boundaries and the end of the turn are + * visible while the user is elsewhere. + */ +function ThreadQueueSender({ threadKey }: { threadKey: string }) { + const threadRef = useMemo(() => parseScopedThreadKey(threadKey), [threadKey]); + const thread = useThread(threadRef); + const threadStatus = useThreadStatus(threadRef); + const environmentId = threadRef?.environmentId ?? null; + const environment = useEnvironment(environmentId); + const serverConfigs = useServerConfigs(); + const serverConfigLoaded = environmentId !== null && serverConfigs.has(environmentId); + const rewinding = useComposerDraftStore((store) => store.rewindingThreadKeys.has(threadKey)); + const queue = useQueuedMessages(threadKey); + const next = queue[0]; + const sending = queue.some((message) => message.sending); + const activities = thread?.activities; + const latestToolActivityId = useMemo( + () => latestCompletedToolActivityId(activities ?? []), + [activities], + ); + const pendingRequests = useMemo(() => derivePendingRequests(activities ?? []), [activities]); + const phase = derivePhase(thread?.session ?? null); + + // A send that starts a new turn leaves the thread idle until the server + // picks it up. Hold the next message until then, as the composer does for + // its own sends. + const lastDispatch = useQueuedMessageStore( + (state) => state.lastDispatchByThreadKey[threadKey]?.thread ?? null, + ); + const latestUserMessageId = thread?.messages.findLast((m) => m.role === "user")?.id ?? null; + const waitingForServer = + lastDispatch !== null && + !hasServerAcknowledgedLocalDispatch({ + localDispatch: lastDispatch, + phase, + latestTurn: thread?.latestTurn ?? null, + latestUserMessageId, + session: thread?.session ?? null, + hasPendingApproval: pendingRequests.approvals.length > 0, + hasPendingUserInput: pendingRequests.userInputs.length > 0, + latestTurnStartFailureId: latestTurnStartFailureId(thread ?? undefined, latestUserMessageId), + threadError: null, + }); + + // Approvals and questions block the agent; a steer landing on top of them + // would answer nothing and confuse the turn, so the queue holds until the + // user resolves them. + const blocked = + threadRef === null || + thread === null || + threadStatus !== "live" || + (environment !== null && environment.connection.phase !== "connected") || + !serverConfigLoaded || + rewinding || + sending || + waitingForServer || + pendingRequests.approvals.length > 0 || + pendingRequests.userInputs.length > 0; + const due = + next !== undefined && + !blocked && + isQueuedMessageDue({ message: next, phase, latestToolActivityId }); + const nextId = next?.id; + useEffect(() => { + if (!due || !threadRef || nextId === undefined) return; + void sendQueuedMessage(threadRef, nextId); + }, [due, nextId, threadRef]); + return null; +} diff --git a/apps/web/src/components/Sidebar.logic.test.ts b/apps/web/src/components/Sidebar.logic.test.ts index 9525f1b5eda2..e183e129ad31 100644 --- a/apps/web/src/components/Sidebar.logic.test.ts +++ b/apps/web/src/components/Sidebar.logic.test.ts @@ -23,6 +23,7 @@ import { isTrailingDoubleClick, orderItemsByPreferredIds, resolveProjectStatusIndicator, + resolveSidebarRowAccessibility, resolveSidebarThreadStatus, resolveThreadStatusPill, resolveWorkingStartedAt, @@ -31,7 +32,6 @@ import { shouldClearThreadSelectionOnMouseDown, shouldRecedeSidebarThread, sortLogicalProjectsForSidebar, - sortSettledThreadsForSidebar, resolveSidebarDropTarget, pinOrderKeyBetween, planPinnedReorder, @@ -52,6 +52,7 @@ import { resolveSidebarDropVerb, } from "./Sidebar.logic"; import { threadSearchMatchKey } from "@t3tools/client-runtime/state/thread-search"; +import { sortSettledThreads } from "@t3tools/client-runtime/state/thread-sort"; import { EnvironmentId, OrchestrationLatestTurn, @@ -70,6 +71,35 @@ import { const localEnvironmentId = EnvironmentId.make("environment-local"); +describe("resolveSidebarRowAccessibility", () => { + it.each([ + { + title: "Can you audit the UI?", + statusLabel: "Working", + projectDisplayName: "T3 Code", + isActive: true, + expected: { label: "Can you audit the UI?, Working, T3 Code", current: "page" }, + }, + { + title: "The audit is done", + statusLabel: null, + projectDisplayName: "T3 Code", + isActive: false, + expected: { label: "The audit is done, T3 Code", current: undefined }, + }, + { + title: "Untitled task", + statusLabel: null, + projectDisplayName: null, + isActive: false, + expected: { label: "Untitled task", current: undefined }, + }, + ])("leads with the title without folding row actions into its name: %j", (input) => { + const { expected, ...state } = input; + expect(resolveSidebarRowAccessibility(state)).toEqual(expected); + }); +}); + describe("animateSidebarLayoutChanges", () => { const baseArgs: Parameters[0] = { active: null, @@ -1691,12 +1721,12 @@ describe("applySidebarThreadDrop", () => { }; const existing = { ...newer, settledOverride: "settled" as const, settledAt: newer.createdAt }; expect(preview).toEqual({ ...final, settledAt: now }); - expect(sortSettledThreadsForSidebar([existing, preview]).map((row) => row.id)).toEqual([ + expect(sortSettledThreads([existing, preview]).map((row) => row.id)).toEqual([ "dragged", "newer", ]); - expect(sortSettledThreadsForSidebar([existing, preview]).map((row) => row.id)).toEqual( - sortSettledThreadsForSidebar([existing, final]).map((row) => row.id), + expect(sortSettledThreads([existing, preview]).map((row) => row.id)).toEqual( + sortSettledThreads([existing, final]).map((row) => row.id), ); }); @@ -1711,7 +1741,7 @@ describe("applySidebarThreadDrop", () => { const final = { ...source, snoozedAt: null, snoozedUntil: null }; const existing = { ...newer, settledOverride: "settled" as const, settledAt: newer.createdAt }; expect(preview).toEqual(final); - expect(sortSettledThreadsForSidebar([existing, preview]).map((row) => row.id)).toEqual([ + expect(sortSettledThreads([existing, preview]).map((row) => row.id)).toEqual([ "newer", "dragged", ]); @@ -1821,74 +1851,6 @@ describe("sortPinnedThreadsForSidebar", () => { }); }); -describe("sortSettledThreadsForSidebar", () => { - const settled = (input: { - id: string; - settledAt?: string | null; - latestUserMessageAt?: string | null; - latestTurn?: OrchestrationLatestTurn | null; - updatedAt?: string; - }) => ({ - id: input.id, - settledAt: input.settledAt ?? null, - latestUserMessageAt: input.latestUserMessageAt ?? null, - latestTurn: input.latestTurn ?? null, - updatedAt: input.updatedAt ?? "2026-03-09T09:00:00.000Z", - }); - - it("orders by settle time, most recently settled first", () => { - const sorted = sortSettledThreadsForSidebar([ - settled({ - id: "settled-first", - settledAt: "2026-03-09T10:00:00.000Z", - // Created/active later than the other thread: settle time must win. - latestUserMessageAt: "2026-03-09T09:59:00.000Z", - }), - settled({ - id: "settled-last", - settledAt: "2026-03-09T12:00:00.000Z", - latestUserMessageAt: "2026-03-09T08:00:00.000Z", - }), - ]); - - expect(sorted.map((thread) => thread.id)).toEqual(["settled-last", "settled-first"]); - }); - - it("falls back to last activity for auto-settled threads without a settledAt stamp", () => { - const sorted = sortSettledThreadsForSidebar([ - settled({ id: "auto-old", latestUserMessageAt: "2026-03-09T08:00:00.000Z" }), - settled({ id: "explicit", settledAt: "2026-03-09T10:00:00.000Z" }), - settled({ id: "auto-recent", latestUserMessageAt: "2026-03-09T11:00:00.000Z" }), - ]); - - expect(sorted.map((thread) => thread.id)).toEqual(["auto-recent", "explicit", "auto-old"]); - }); - - it("counts a turn completion as activity for auto-settled threads", () => { - // The message came in before the other thread's, but its turn finished - // after: completion time is the real "work ended" moment. - const sorted = sortSettledThreadsForSidebar([ - settled({ id: "message-only", latestUserMessageAt: "2026-03-09T10:04:00.000Z" }), - settled({ - id: "completed-later", - latestUserMessageAt: "2026-03-09T10:00:00.000Z", - latestTurn: makeLatestTurn({ completedAt: "2026-03-09T10:30:00.000Z" }), - }), - ]); - - expect(sorted.map((thread) => thread.id)).toEqual(["completed-later", "message-only"]); - }); - - it("breaks timestamp ties by id so the order is stable", () => { - const sorted = sortSettledThreadsForSidebar([ - settled({ id: "b", settledAt: "2026-03-09T10:00:00.000Z" }), - settled({ id: "a", settledAt: "2026-03-09T10:00:00.000Z" }), - ]); - - expect(sorted.map((thread) => thread.id)).toEqual(["a", "b"]); - }); -}); - describe("resolveWorkingStartedAt", () => { const session = { threadId: ThreadId.make("thread-1"), @@ -2376,6 +2338,49 @@ describe("sortProjectsForSidebar", () => { ]); }); + it.each(["updated_at", "created_at"] as const)( + "matches the per-comparison %s order on a shuffled list with ties", + (sortOrder) => { + const minute = (value: number) => `2026-03-09T10:0${value}:00.000Z`; + // (index * 7) % 24 scrambles the input order. Titles repeat, and + // projects 16-23 have no threads, so they use their own stamps. + const projects = Array.from({ length: 24 }, (_, index) => { + const n = (index * 7) % 24; + return makeProject({ + id: ProjectId.make(`project-${n}`), + title: n % 2 === 0 ? "Alpha" : "Beta", + createdAt: minute(n % 3), + updatedAt: n % 5 === 0 ? "invalid" : minute(n % 2), + }); + }); + const threads = Array.from({ length: 48 }, (_, n) => ({ + projectId: ProjectId.make(`project-${n % 16}`), + createdAt: minute(n % 6), + updatedAt: minute(n % 3), + latestUserMessageAt: n % 4 === 0 ? null : minute(n % 5), + })); + // The comparator this sort replaced: it walked each project's threads + // on every call. + const timestamp = (project: Project) => + getProjectSortTimestamp( + project, + threads.filter((thread) => thread.projectId === project.id), + sortOrder, + ); + const expected = projects.toSorted((left, right) => { + const rightTimestamp = timestamp(right); + const leftTimestamp = timestamp(left); + const byTimestamp = + rightTimestamp === leftTimestamp ? 0 : rightTimestamp > leftTimestamp ? 1 : -1; + return ( + byTimestamp || left.title.localeCompare(right.title) || left.id.localeCompare(right.id) + ); + }); + + expect(sortProjectsForSidebar(projects, threads, sortOrder)).toEqual(expected); + }, + ); + it("returns the project timestamp when no threads are present", () => { const timestamp = getProjectSortTimestamp( makeProject({ updatedAt: "2026-03-09T10:10:00.000Z" }), diff --git a/apps/web/src/components/Sidebar.logic.ts b/apps/web/src/components/Sidebar.logic.ts index a895dad3c429..ac59800a8d3a 100644 --- a/apps/web/src/components/Sidebar.logic.ts +++ b/apps/web/src/components/Sidebar.logic.ts @@ -16,10 +16,8 @@ import { } from "@t3tools/client-runtime/state/thread-settled"; import { getThreadSortTimestamp, - resolveSettledThreadTimestamp, sortThreads, toSortableTimestamp, - type SettledThreadTimestampInput, type ThreadSortInput, } from "../lib/threadSort"; import type { SidebarThreadSummary, Thread } from "../types"; @@ -44,6 +42,21 @@ export function shouldNavigateAfterThreadPark(input: { const THREAD_SELECTION_SAFE_SELECTOR = "[data-thread-item], [data-thread-selection-safe]"; export const THREAD_JUMP_HINT_SHOW_DELAY_MS = 200; + +export function resolveSidebarRowAccessibility(input: { + readonly title: string; + readonly statusLabel: string | null; + readonly projectDisplayName: string | null; + readonly isActive: boolean; +}): { readonly label: string; readonly current: "page" | undefined } { + return { + // The title is the row's identity and must lead when users scan tasks. + // Only static context belongs here; nested action labels remain separate controls. + label: [input.title, input.statusLabel, input.projectDisplayName].filter(Boolean).join(", "), + current: input.isActive ? "page" : undefined, + }; +} + // Visible sidebar rows are prewarmed into the thread-detail cache so opening a // nearby thread usually reuses an already-hot subscription. Each prewarmed // thread holds a live, fully hydrated detail subscription (all messages and @@ -937,20 +950,6 @@ export function reduceSidebarProjectScopeMenuState( } } -// Settled rows are history, so they order by when the work ENDED, not when -// the thread was created or last touched. -export function sortSettledThreadsForSidebar< - T extends SettledThreadTimestampInput & { readonly id: string }, ->(threads: readonly T[]): T[] { - const timestampMs = (thread: T) => { - const timestamp = resolveSettledThreadTimestamp(thread); - return timestamp === null ? 0 : Date.parse(timestamp); - }; - return [...threads].toSorted( - (left, right) => timestampMs(right) - timestampMs(left) || left.id.localeCompare(right.id), - ); -} - /** The timestamp a working thread's elapsed label counts from: the running turn's start (request time until adoption), falling back to the session's last transition when the turn projection lags behind. Malformed @@ -1136,13 +1135,19 @@ function sortProjectsByActivity( return [...projects]; } - return [...projects].toSorted((left, right) => { - const rightTimestamp = getProjectSortTimestamp(right, getProjectThreads(right), sortOrder); - const leftTimestamp = getProjectSortTimestamp(left, getProjectThreads(left), sortOrder); - const byTimestamp = - rightTimestamp === leftTimestamp ? 0 : rightTimestamp > leftTimestamp ? 1 : -1; - return byTimestamp || compareTies(left, right); - }); + // Each project's timestamp walks all of its threads, so compute it once + // per project instead of once per comparison. + return projects + .map((project) => ({ + project, + timestamp: getProjectSortTimestamp(project, getProjectThreads(project), sortOrder), + })) + .sort((left, right) => { + const byTimestamp = + right.timestamp === left.timestamp ? 0 : right.timestamp > left.timestamp ? 1 : -1; + return byTimestamp || compareTies(left.project, right.project); + }) + .map(({ project }) => project); } export function sortProjectsForSidebar< diff --git a/apps/web/src/components/Sidebar.tsx b/apps/web/src/components/Sidebar.tsx index 0bea385ccce0..cf591597f405 100644 --- a/apps/web/src/components/Sidebar.tsx +++ b/apps/web/src/components/Sidebar.tsx @@ -21,7 +21,10 @@ import { effectiveSnoozed, threadWokeAt, } from "@t3tools/client-runtime/state/thread-settled"; -import { resolveSettledThreadTimestamp } from "@t3tools/client-runtime/state/thread-sort"; +import { + resolveSettledThreadTimestamp, + sortSettledThreads, +} from "@t3tools/client-runtime/state/thread-sort"; import { threadSearchMatchKey, type EnvironmentThreadSearchMatch, @@ -172,6 +175,7 @@ import { resolveAdjacentThreadId, resolveSidebarDropTarget, resolveSidebarDropVerb, + resolveSidebarRowAccessibility, type SidebarDropVerb, resolveSidebarThreadStatus, searchSidebarThreads, @@ -183,7 +187,6 @@ import { sidebarMarkerId, sortLogicalProjectsForSidebar, sortPinnedThreadsForSidebar, - sortSettledThreadsForSidebar, sortThreadsForSidebar, useRetainedValue, useSidebarRowSubscriptionLease, @@ -729,6 +732,12 @@ const SidebarDraftRow = memo(function SidebarDraftRow(props: { promptPreview.length > 0 ? promptPreview : `${attachmentCount} attachment${attachmentCount === 1 ? "" : "s"}`; + const accessibility = resolveSidebarRowAccessibility({ + title: preview, + statusLabel: "Unsent draft", + projectDisplayName: props.projectDisplayName, + isActive: props.isActive, + }); const handleActivate = useCallback(() => onNavigate(draftId), [draftId, onNavigate]); const handleKeyDown = useCallback( (event: ReactKeyboardEvent) => { @@ -756,14 +765,17 @@ const SidebarDraftRow = memo(function SidebarDraftRow(props: {
+ {preview}
@@ -791,7 +803,9 @@ const SidebarDraftRow = memo(function SidebarDraftRow(props: {
-
{preview}
+
+ {preview} +
@@ -1397,7 +1411,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { // a useful hierarchy nor a reliable hover cue. Status now lives in the row // content; surface is reserved for interaction (hover, multi-select, route). const rowSurfaceClassName = cn( - "group/sidebar-row relative w-full cursor-pointer overflow-hidden rounded-md text-left outline-none select-none", + "group/sidebar-row relative w-full cursor-pointer overflow-hidden rounded-md text-left outline-none select-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-ring", variantAction === "unsettle" && "[&:not(:hover):not(:focus-within)_*]:text-secondary-label/70", props.isActive ? "bg-sidebar-row-active text-sidebar-foreground" @@ -1452,6 +1466,13 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { ) : null; + const accessibility = resolveSidebarRowAccessibility({ + title: thread.title, + statusLabel: topStatus?.label ?? null, + projectDisplayName: props.projectDisplayName, + isActive: props.isActive, + }); + const title = isRenaming ? ( ) : ( ); + const accessibleTitle = isRenaming ? null : {thread.title}; // Stacks show their layer count; multiple unrelated links show their total count. // Plain clicks open T3; individual PR links also support opening the host in a new tab. @@ -1598,6 +1621,8 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { ref={rowRef} role="button" tabIndex={0} + aria-label={accessibility.label} + aria-current={accessibility.current} data-testid="sidebar-row-slim" aria-busy={isRegeneratingTitle || undefined} className={cn(rowSurfaceClassName, "flex h-9 items-center gap-2.5 px-2.5")} @@ -1608,6 +1633,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { /> } > + {accessibleTitle} {/* Settled history recedes: dimmed favicon at rest, restored on hover so the tail stays scannable when you're hunting. */} } > + {accessibleTitle}
{subagentDepth > 0 && ( @@ -2023,6 +2052,12 @@ const SidebarSearchResultRow = memo(function SidebarSearchResultRow(props: { onFileDropThreads: (threadRef: ScopedThreadRef, files: File[]) => void; }) { const { thread } = props; + const accessibility = resolveSidebarRowAccessibility({ + title: thread.title, + statusLabel: null, + projectDisplayName: props.projectDisplayName, + isActive: props.isRouteActive, + }); const threadRef = useMemo( () => scopeThreadRef(thread.environmentId, thread.id), [thread.environmentId, thread.id], @@ -2099,12 +2134,8 @@ const SidebarSearchResultRow = memo(function SidebarSearchResultRow(props: { // which owns all keyboard interaction for the listbox. tabIndex={-1} aria-selected={props.isHighlighted} - aria-current={props.isRouteActive ? "page" : undefined} - aria-label={ - props.projectDisplayName - ? `${thread.title}, ${props.projectDisplayName}` - : thread.title - } + aria-current={accessibility.current} + aria-label={accessibility.label} onMouseMove={props.onHighlight} onClick={props.onSelect} className={cn( @@ -2687,7 +2718,7 @@ export default function Sidebar() { firstValidTimestampMs(left.snoozedUntil ?? null) - firstValidTimestampMs(right.snoozedUntil ?? null), ), - settledThreads: sortSettledThreadsForSidebar(settled), + settledThreads: sortSettledThreads(settled), snoozeNow: preciseNow, }; }, [nowMinute, optimisticDrop, scopedProjectKeys, serverConfigs, snoozeWakeTick, threads]); @@ -3513,7 +3544,7 @@ export default function Sidebar() { if (dragState === null || thread === undefined) return []; const key = (candidate: EnvironmentThreadShell) => scopedThreadKey(scopeThreadRef(candidate.environmentId, candidate.id)); - return sortSettledThreadsForSidebar([ + return sortSettledThreads([ ...settledThreads.filter((candidate) => key(candidate) !== dragState.activeKey), applySidebarThreadDrop(thread, "settled", dragState.occurredAt), ]).map(key); @@ -4635,7 +4666,7 @@ export default function Sidebar() { } > - + {isSearchingThreads ? ( threadSearchResults.length > 0 ? (