From 5fb334a68114c647366d19f90edb9042aed38298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Wed, 28 Oct 2026 01:13:01 +0800 Subject: [PATCH] feat(webui): probe the event channel for half-open links and degrade a never-accepted watcher The channel's health rested on close edges alone: a half-open socket (the network dropped the mapping, neither end was told) delivers no close event, so a silently dead link kept reading connected until the next send; and the startup grace let a watcher that was never accepted stay unknown for its full window even when the channel had already gone fatal on it. - Liveness probe: while a socket is open, the channel timestamps every inbound frame (any frame, decoded or not, is proof of life) and asks for one every 10 s - the version operation, the lightest read the surface offers: a synchronous in-memory answer whose validation refuses a body, carried on a request id derived from this connection's, so the answer needs no handler of its own and drops through the existing request-id filter. No new frame kind, no server change. 25 s of silence is the verdict: the channel forces a reconnect (resetting the backoff deliberately, so a server that accepts sockets but never acknowledges watchEvents gets a steady retry rhythm rather than an exponentially spacing one) and the existing close/down rules degrade it while it dials. The numbers are anchored to deadlines the repo already keeps - the server's 15 s socket heartbeat, the 30 s request deadline in wire.ts, the 30 s retry ceiling. A hidden page's throttled timers suspend both probe and verdict; the visibilitychange handler already forces a reconnect on return. - Grace expiry: a fatal rejection on an unaccepted watcher ends the startup grace immediately instead of stacking silent seconds on a settled fact; already-accepted watchers are a pure no-op. The remaining gap is deliberate and documented in ConnectionStatus: a fatal channel still borrows the reconnecting copy and offers no retry button, because wiring health.reconnect() into the merge would change the component's semantics - left out rather than half-added. --- .../src/client/platform/connection-health.ts | 27 ++- .../src/client/platform/event-channel.ts | 140 ++++++++++++ .../src/client/shell/ConnectionStatus.tsx | 18 +- .../webui/test/unit/connection-health.test.ts | 46 ++++ .../test/unit/webui-channel-health.test.ts | 213 +++++++++++++++++- 5 files changed, 435 insertions(+), 9 deletions(-) diff --git a/packages/webui/src/client/platform/connection-health.ts b/packages/webui/src/client/platform/connection-health.ts index c21becfc1..96ce5698d 100644 --- a/packages/webui/src/client/platform/connection-health.ts +++ b/packages/webui/src/client/platform/connection-health.ts @@ -82,8 +82,9 @@ function notifyIfChanged(before: boolean): void { /** Node's timer exposes `unref`; the DOM lib types `setTimeout` as returning * a plain number, so the capability is read through a narrow cast rather - * than assumed on the type. */ -function unrefTimer(timer: ReturnType): void { + * than assumed on the type. Exported because the event channel's liveness + * probe schedules the same kind of don't-hold-the-process timers. */ +export function unrefTimer(timer: ReturnType): void { (timer as { unref?: () => void }).unref?.(); } @@ -113,6 +114,28 @@ export function registerWebuiEventWatcher(): number { return nextToken; } +/** Ends the startup grace for one watcher now, ahead of the timer. + * + * Called by the channel's fatal path: a fatal rejection that arrives while + * the subscription is still unaccepted has already decided the link will + * never come up on its own, so waiting out the grace would only extend the + * silent "connected" window on top of a settled fact. The body mirrors the + * timer's exactly — clear the timer, keep an accepted watcher untouched + * (this is a pure no-op for it, listeners included), flip `startupExpired` + * on the unaccepted one — so the two paths cannot drift apart. */ +export function expireWebuiEventWatcherGrace(token: number): void { + const watcher = watchers.get(token); + if (!watcher) return; + if (watcher.graceTimer !== undefined) { + clearTimeout(watcher.graceTimer); + watcher.graceTimer = undefined; + } + if (watcher.healthy) return; + const before = degraded(); + watcher.startupExpired = true; + notifyIfChanged(before); +} + /** The watcher's `watchEvents` request was accepted — its socket is live. */ export function markWebuiEventWatcherHealthy(token: number): void { const watcher = watchers.get(token); diff --git a/packages/webui/src/client/platform/event-channel.ts b/packages/webui/src/client/platform/event-channel.ts index d680b01fa..ed25c5dd5 100644 --- a/packages/webui/src/client/platform/event-channel.ts +++ b/packages/webui/src/client/platform/event-channel.ts @@ -15,17 +15,31 @@ // backs off with capped exponential delay plus jitter, so a fleet of clients that // dropped together does not return in lockstep. // +// A close event is not the only way a link dies, either: a half-open socket (the +// network dropped the mapping, neither end was told) delivers no close at all, so +// the channel probes the live socket periodically and a probe timeout forces a +// reconnect rather than waiting for an event that will never come. That forced +// reconnect goes through `reconnectNow`, which resets the backoff attempt count — +// deliberately. A server that accepts sockets but never acknowledges +// `watchEvents` thereby gets a steady ~25-30 s retry rhythm (the startup grace +// still degrades the channel during it, and any acknowledgement restores it) +// instead of an exponential curve that would space the same hopeless attempt +// further and further apart. +// // Split out of `transport.ts` so the channel's lifecycle is one module: the // transport keeps request/response IO and the independent streams (session and // terminal), and this file owns the one channel that outlives them. import { + expireWebuiEventWatcherGrace, markWebuiEventWatcherDown, markWebuiEventWatcherHealthy, registerWebuiEventWatcher, unregisterWebuiEventWatcher, + unrefTimer, } from "./connection-health.js"; import type { WebuiRuntimeEvent } from "../../shared/contracts/stream.js"; +import { VERSION_OPERATION_NAME } from "../../shared/operation-names.js"; import { decodeWebuiServerFrame, WebuiErrorCode, @@ -139,6 +153,29 @@ export interface WebuiEventChannelOptions { export const WEBUI_CHANNEL_MAX_RETRY_MS = 30_000; const DEFAULT_BASE_RETRY_MS = 250; +/** + * How often the liveness probe asks the live `watchEvents` socket for an + * answer, and how long a silence on that socket may last before the channel + * treats the link as dead and forces a reconnect. + * + * Every number is anchored to a deadline this repo already keeps: + * + * - The server pings its sockets every 15 s and terminates after one missed + * pong (`WEBUI_WEBSOCKET_HEARTBEAT_INTERVAL_MS`, `websocket-server.ts`), so + * a 10 s client-side probe discovers a dead link in step with — or ahead of + * — the server's own eviction, rather than after it. + * - The 25 s timeout is ~2.5 intervals: one lost probe or lost answer is + * tolerated, and the verdict still lands inside the window the repo treats + * as its anti-half-open budget elsewhere — the 30 s request deadline in + * `wire.ts` (which exists precisely because "a request pending forever on a + * half-open socket" is the failure this probe catches for the channel) and + * the 30 s retry ceiling above. A false kill costs one recoverable + * reconnect; a missed kill costs a permanently lying "connected", so the + * bias is toward the shorter timeout. + */ +export const WEBUI_CHANNEL_LIVENESS_PING_INTERVAL_MS = 10_000; +export const WEBUI_CHANNEL_LIVENESS_TIMEOUT_MS = 25_000; + /** * Whether a service refusal means "this will keep failing". * @@ -233,6 +270,81 @@ export function createWebuiEventChannel( if (reconnectTimer !== undefined) clearTimeout(reconnectTimer); reconnectTimer = undefined; }; + // --------------------------------------------------------------------- + // Liveness watchdog. A half-open link delivers no close event, so health + // cannot rest on one: the channel timestamps every inbound frame — any + // frame, decoded or not, is proof the peer is alive — and, while a socket + // is open, periodically asks the server to produce one. `version` is the + // probe payload: the lightest read the surface offers (a synchronous + // in-memory answer; its validation refuses a body at all, so the frame + // omits `body` entirely), carried on a request id derived from this + // connection's. The answer needs no handling of its own — arriving is the + // whole point: it refreshes the timestamp, then falls through the + // `frame.requestId !== requestId` drop in the message handler exactly like + // a frame from anyone else's request. No fork of that filter, no new + // frame kind, no server change. + // --------------------------------------------------------------------- + /** When a frame last arrived on the live socket. Refreshed before + * decoding: even a frame this client cannot read is evidence of life. */ + let lastInboundAt = 0; + /** The request id of the connection the probe currently watches. */ + let probedRequestId = ""; + let probeTimer: ReturnType | undefined; + const clearProbe = (): void => { + if (probeTimer !== undefined) clearTimeout(probeTimer); + probeTimer = undefined; + }; + const scheduleProbe = (): void => { + // Terminal shapes do not schedule: the probe's next start is the next + // `open` event, not a timer that outlives the socket it watches. + if (stopped || fatal || socket === undefined) return; + probeTimer = setTimeout(probeTick, WEBUI_CHANNEL_LIVENESS_PING_INTERVAL_MS); + unrefTimer(probeTimer); + }; + const probeTick = (): void => { + probeTimer = undefined; + if (stopped || fatal || socket === undefined) return; + // A hidden page's timers are throttled hard (Chromium aligns them to + // roughly one wake-up per minute after five minutes hidden), so a tick + // here can be arbitrarily late; judging death on a throttled clock + // would kill links that were merely quiet while nobody was reading. + // Suspend both the probe and the verdict for the hidden window: the + // visibilitychange handler forces a reconnect the moment the page is + // seen again, and inbound frames keep refreshing the timestamp + // meanwhile (message events are not throttled). + if ( + typeof document !== "undefined" && + document.visibilityState === "hidden" + ) { + scheduleProbe(); + return; + } + const age = Date.now() - lastInboundAt; + if (age >= WEBUI_CHANNEL_LIVENESS_TIMEOUT_MS) { + // The verdict: nothing has arrived for longer than the timeout, so + // the link is dead even though no close event said so. reconnectNow() + // marks the watcher down — the channel degrades through the existing + // health rules — and dials a fresh socket, which is why this tick + // must return rather than fall through to `send`: `socket` already + // points at the CONNECTING replacement, and writing to a connecting + // socket throws in a real browser. The new socket's `open` event + // restarts the probe. + reconnectNow(); + return; + } + if (age >= WEBUI_CHANNEL_LIVENESS_PING_INTERVAL_MS) { + // Quiet, not yet dead: ask for traffic. + socket.send( + JSON.stringify({ + protocolVersion: 1, + kind: "request", + requestId: `${probedRequestId}:liveness`, + operation: VERSION_OPERATION_NAME, + }), + ); + } + scheduleProbe(); + }; const connect = () => { if (stopped) return; clearTimer(); @@ -240,6 +352,7 @@ export function createWebuiEventChannel( const ws = new webSocket(websocketUrl()); socket = ws; const requestId = crypto.randomUUID(); + probedRequestId = requestId; // Fires when the server *accepts* the `watchEvents` request, not when // the socket is created and not when the request is written. It is an // acceptance signal, **not** proof that the runtime's event bus has @@ -250,6 +363,11 @@ export function createWebuiEventChannel( let acknowledged = false; ws.addEventListener("open", () => { if (stopped) return; + // The probe watches exactly this socket, from its first instant of + // life to the close that ends it. + lastInboundAt = Date.now(); + clearProbe(); + scheduleProbe(); ws.send( JSON.stringify({ protocolVersion: 1, @@ -261,6 +379,11 @@ export function createWebuiEventChannel( ); }); ws.addEventListener("message", (event) => { + // First, before any judging: a frame arrived, so the link is alive. + // This is the liveness evidence the probe runs on — it counts even a + // frame the decoder below will refuse, and it counts the probe's own + // answer, which the request-id filter further down then drops. + lastInboundAt = Date.now(); if (stopped) return; // Judged by the same decoder the request sockets use. The event channel // used to parse and trust whatever arrived, so a frame from a different @@ -304,6 +427,9 @@ export function createWebuiEventChannel( ws.addEventListener("close", () => { if (stopped || socket !== ws) return; socket = undefined; + // The socket the probe watches is gone; the retry loop's next `open` + // restarts it. + clearProbe(); markWebuiEventWatcherDown(watcherToken); if (fatal) { // The fatal verdict already decided not to come back; the socket @@ -324,7 +450,16 @@ export function createWebuiEventChannel( const enterFatal = (code: WebuiWireErrorCode, message: string): void => { fatal = true; clearTimer(); + // A fatal verdict is exempt from the probe: it is a decision that + // retrying cannot change, and the socket below is about to be closed + // anyway. + clearProbe(); markWebuiEventWatcherDown(watcherToken); + // The verdict has already answered "will this link ever come up", so an + // unaccepted watcher must not sit out the startup grace on top of it — + // the grace ends here and the channel degrades immediately instead of + // after another silent 30 s. + expireWebuiEventWatcherGrace(watcherToken); setState({ kind: "fatal", code, message }); report( "[webui] event channel stopped: the service rejected the connection in a way that will not change by retrying", @@ -338,6 +473,10 @@ export function createWebuiEventChannel( if (stopped) return; fatal = false; attempt = 0; + // Stop the probe on the socket being replaced — same reason as the + // explicit markDown below: the close handler guards on `socket !== ws`, + // so this path cannot rely on the handler's own teardown. + clearProbe(); const previous = socket; // The close handler above guards on `socket !== ws`, and this function // has already cleared `socket`, so that guard would skip @@ -375,6 +514,7 @@ export function createWebuiEventChannel( return () => { stopped = true; clearTimer(); + clearProbe(); if (typeof document !== "undefined") document.removeEventListener("visibilitychange", reconnectWhenAvailable); if (typeof window !== "undefined") diff --git a/packages/webui/src/client/shell/ConnectionStatus.tsx b/packages/webui/src/client/shell/ConnectionStatus.tsx index 2479e4f63..653dbc23f 100644 --- a/packages/webui/src/client/shell/ConnectionStatus.tsx +++ b/packages/webui/src/client/shell/ConnectionStatus.tsx @@ -17,12 +17,18 @@ import type { WebuiStreamState } from "../features/turn/projection/stream-state. * on `close`, and the component merges the two: failed beats reconnecting * beats connected. * - * What the channel signal does **not** cover, so nothing downstream over-trusts - * it: the client sends no heartbeat on that socket, so detection is - * close-edge only and a half-open link is invisible; and a watcher that was - * never accepted stays "unknown" rather than degraded, so a host that is - * already unreachable at load time reads as connected until something else - * moves. + * What the channel signal covers, and what it still does not. The watcher + * socket sends a periodic liveness probe, so a silently dead (half-open) link + * — one that would never deliver a close event — is forced to reconnect once + * the probe timeout runs out, and reads as reconnecting while it does. A + * watcher that was never accepted is degraded by the 30 s startup grace + * instead of staying unknown forever, and immediately when the channel goes + * fatal. The remaining gap is deliberate: a fatal channel still borrows the + * reconnecting copy — 「正在自动恢复」 is not literally true there, since fatal + * never retries on its own — and offers no retry button, because that state's + * only exits are an explicit `health.reconnect()` (no UI consumer yet) or a + * fresh subscription; wiring either into this component would change the + * merge semantics below, so it is left out rather than half-added. */ export type WebuiConnectionState = "connected" | "reconnecting" | "failed"; diff --git a/packages/webui/test/unit/connection-health.test.ts b/packages/webui/test/unit/connection-health.test.ts index 3ef41fbca..9bf953c30 100644 --- a/packages/webui/test/unit/connection-health.test.ts +++ b/packages/webui/test/unit/connection-health.test.ts @@ -24,10 +24,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { + expireWebuiEventWatcherGrace, isWebuiEventChannelDegraded, markWebuiEventWatcherDown, markWebuiEventWatcherHealthy, registerWebuiEventWatcher, + subscribeWebuiConnectionHealth, unregisterWebuiEventWatcher, } from "../../src/client/platform/connection-health.js"; import { createWebuiTransport } from "../../src/client/platform/transport.js"; @@ -229,6 +231,50 @@ describe("the startup grace on a never-accepted watcher", () => { }); }); +describe("ending the grace ahead of the timer", () => { + // `expireWebuiEventWatcherGrace` is the channel's fatal path: a fatal + // rejection that arrives before the first acceptance has already decided + // the link will never come up, so the 30 s of silence the grace would add + // on top is not caution, it is a lie of omission. The three cases pin the + // exact, unchanged aggregation semantics: immediate for the unaccepted + // lone watcher, silent while any live link exists, and a true no-op for a + // watcher that was already accepted. + + it("degrades an unaccepted watcher at once, without waiting out the timer", () => { + const token = register(); + expect(isWebuiEventChannelDegraded()).toBe(false); + expireWebuiEventWatcherGrace(token); + expect(isWebuiEventChannelDegraded()).toBe(true); + }); + + it("does not degrade a channel that still holds a live link", () => { + const shell = register(); + const panel = register(); + markWebuiEventWatcherHealthy(shell); + expireWebuiEventWatcherGrace(panel); + // Rule 2 only speaks when no watcher is healthy: a healthy shell with a + // panel that will never come up is a channel with a live link. + expect(isWebuiEventChannelDegraded()).toBe(false); + }); + + it("is a pure no-op for a watcher that was already accepted", () => { + const token = register(); + markWebuiEventWatcherHealthy(token); + const listener = vi.fn(); + const detach = subscribeWebuiConnectionHealth(listener); + try { + expireWebuiEventWatcherGrace(token); + expect(isWebuiEventChannelDegraded()).toBe(false); + // No-op means no-op: not one listener fires. + expect(listener).not.toHaveBeenCalled(); + } finally { + // The listener set is module-level like the store; leave nothing + // behind for later suites in this worker. + detach(); + } + }); +}); + describe("the transport's watcher reporting", () => { it("marks healthy on the server's acceptance and down on socket close", () => { const { transport, sockets } = watchEventsHarness(); diff --git a/packages/webui/test/unit/webui-channel-health.test.ts b/packages/webui/test/unit/webui-channel-health.test.ts index 412c6b57d..8ab285b9e 100644 --- a/packages/webui/test/unit/webui-channel-health.test.ts +++ b/packages/webui/test/unit/webui-channel-health.test.ts @@ -7,7 +7,7 @@ // frame; and a code this build has never seen is transient, because an unfamiliar // code is a newer server rather than a malformed one. -import { describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createWebuiEventChannel, @@ -15,6 +15,7 @@ import { WEBUI_CHANNEL_MAX_RETRY_MS, webuiChannelRetryDelayMs, } from "../../src/client/platform/event-channel.js"; +import { isWebuiEventChannelDegraded } from "../../src/client/platform/connection-health.js"; import { WebuiErrorCode, WEBUI_PROTOCOL_VERSION } from "../../src/shared/envelope.js"; /** A socket that records what the channel wrote and lets a test drive it. */ @@ -72,6 +73,216 @@ function channel(options: Partial[0]> return { channel: created, onEvent, onReconnect, unsubscribe }; } +describe("the liveness probe on an open socket", () => { + // This suite must stay FIRST in the file. The health store behind + // `connection-health` is module-level, and the older suites below never + // call the `unsubscribe` they get — their watchers (some permanently + // healthy) stay registered for the rest of this worker's life. Vitest runs + // describes in definition order, so running here means the store is clean + // and these cases own the only watchers in it; the cases' own afterEach + // keeps it that way. The one case that needs a degraded channel to clear + // (the fatal one) would read false against a correct implementation if a + // leftover healthy watcher from an earlier suite were present. + const unsubscribers: Array<() => void> = []; + + beforeEach(() => { + vi.useFakeTimers(); + // The socket double's ledger is file-level; each case counts from zero. + FakeSocket.instances.length = 0; + }); + + afterEach(() => { + for (const unsubscribe of unsubscribers.splice(0)) unsubscribe(); + expect(isWebuiEventChannelDegraded()).toBe(false); + vi.useRealTimers(); + }); + + const subscribe = () => { + const created = channel(); + unsubscribers.push(created.unsubscribe); + return created; + }; + + /** Drives the newest socket through open and the `watchEvents` acceptance. */ + const accept = (socket: FakeSocket): void => { + socket.emit("open"); + socket.emit( + "message", + JSON.stringify({ + protocolVersion: WEBUI_PROTOCOL_VERSION, + kind: "response", + requestId: socket.requestId, + body: { ok: true, source: [] }, + }), + ); + }; + + const sentFrames = (socket: FakeSocket): Array> => + socket.sent.map((text) => JSON.parse(text) as Record); + const livenessFrames = (socket: FakeSocket): Array> => + sentFrames(socket).filter((frame) => frame.operation === "version"); + + it("N1 · forces a reconnect when an accepted socket goes silent, and recovers on the next acceptance", () => { + subscribe(); + const socket = lastSocket(); + // The acceptance clears the startup grace, so whatever degrades or dials + // from here on can only be the probe's doing. + accept(socket); + + vi.advanceTimersByTime(60_000); // and never emits another frame + + // The verdict dialled a replacement and closed the silent one — the test + // emitted no "close", so the probe is the only possible cause. + expect(FakeSocket.instances).toHaveLength(2); + expect(FakeSocket.instances[0]!.closed).toBe(true); + // Probes were really sent before the verdict; this is a probe kill, not a + // stray close. + expect(livenessFrames(FakeSocket.instances[0]!)).not.toHaveLength(0); + // reconnectNow marks the watcher down before dialling, so the channel + // reads degraded at the moment of the kill. + expect(isWebuiEventChannelDegraded()).toBe(true); + + // The round trip starts with the replacement opening — and its first + // written frame being its own `watchEvents` request, which is also the + // guard that the verdict did not fall through to `send`: a liveness frame + // written to the CONNECTING replacement would sit in front of it. + const replacement = FakeSocket.instances[1]!; + replacement.emit("open"); + const firstOnReplacement = JSON.parse(replacement.sent[0] ?? "{}") as Record< + string, + unknown + >; + expect(firstOnReplacement.operation).toBe("watchEvents"); + // …and the acceptance restores the channel. + replacement.emit( + "message", + JSON.stringify({ + protocolVersion: WEBUI_PROTOCOL_VERSION, + kind: "response", + requestId: replacement.requestId, + body: { ok: true, source: [] }, + }), + ); + expect(isWebuiEventChannelDegraded()).toBe(false); + }); + + it("N2 · never probes — and never kills — a socket that keeps delivering events", () => { + subscribe(); + const socket = lastSocket(); + accept(socket); + // An event every 8 s: quieter than the 10 s probe interval, but never + // silent. Live traffic is its own liveness evidence. + for (let tick = 0; tick < 8; tick += 1) { + vi.advanceTimersByTime(8_000); + socket.emit( + "message", + JSON.stringify({ + protocolVersion: WEBUI_PROTOCOL_VERSION, + kind: "event", + requestId: socket.requestId, + body: { type: "session.updated", payload: {}, timestamp: 0, source: "runtime" }, + }), + ); + } + expect(sentFrames(socket)).toHaveLength(1); // only the watchEvents request + expect(FakeSocket.instances).toHaveLength(1); + expect(socket.closed).toBe(false); + expect(isWebuiEventChannelDegraded()).toBe(false); + }); + + it("N3 · a probe that is answered keeps the channel alive", () => { + subscribe(); + const socket = lastSocket(); + accept(socket); + for (let round = 0; round < 3; round += 1) { + vi.advanceTimersByTime(10_000); // the quiet gap fires one probe + const probe = JSON.parse(socket.sent.at(-1) ?? "{}") as Record; + expect(probe.operation).toBe("version"); + // The answer arrives within the tick: like any inbound frame it is the + // liveness evidence, and the request-id filter drops it right after the + // refresh — the probe's answer is never mistaken for an acceptance. + socket.emit( + "message", + JSON.stringify({ + protocolVersion: WEBUI_PROTOCOL_VERSION, + kind: "response", + requestId: String(probe.requestId), + body: { version: "probe-answer", protocolVersion: 1 }, + }), + ); + } + expect(FakeSocket.instances).toHaveLength(1); + expect(isWebuiEventChannelDegraded()).toBe(false); + }); + + it("N4 · a hidden page suspends both the probe and the verdict until it is seen again", () => { + // This node environment has no `document`; the stub supplies the + // visibility surface the channel reads, installed before `channel()` so + // the subscribe wiring sees it (the precedent is the document stub in + // `connection-health.test.ts`). + const globals = globalThis as unknown as Record; + const saved = globals.document; + const document = { + visibilityState: "hidden", + addEventListener: () => undefined, + removeEventListener: () => undefined, + }; + globals.document = document; + try { + subscribe(); + const socket = lastSocket(); + // The acceptance clears the startup grace; without it the 30 s grace + // would degrade during the 120 s advance below and the case would test + // the grace, not the probe. + accept(socket); + vi.advanceTimersByTime(120_000); // silent AND hidden the whole time + // Suspended, not dead: no probe was sent, no verdict fired, the socket + // was neither closed nor replaced. + expect(sentFrames(socket)).toHaveLength(1); + expect(FakeSocket.instances).toHaveLength(1); + expect(socket.closed).toBe(false); + expect(isWebuiEventChannelDegraded()).toBe(false); + // The control: the same silence, once the page is visible again, gets + // the verdict it deserved all along. + document.visibilityState = "visible"; + vi.advanceTimersByTime(10_000); + expect(FakeSocket.instances).toHaveLength(2); + expect(socket.closed).toBe(true); + expect(isWebuiEventChannelDegraded()).toBe(true); + } finally { + globals.document = saved; + } + }); + + it("N5 · a fatal rejection degrades immediately, and neither probes nor retries", () => { + subscribe(); + const socket = lastSocket(); + socket.emit("open"); + socket.emit( + "message", + JSON.stringify({ + protocolVersion: WEBUI_PROTOCOL_VERSION, + kind: "error", + requestId: socket.requestId, + code: WebuiErrorCode.capabilityNotReady, + message: "terminal bridge is not ready", + }), + ); + // Immediately — no advancing the 30 s startup grace. The fatal verdict + // ended the grace on the spot, so the never-accepted watcher degrades + // now; before that behaviour this read false for the first 30 s of a + // fatal outage, which the banner projected as connected. + expect(isWebuiEventChannelDegraded()).toBe(true); + + vi.advanceTimersByTime(60_000); + // Fatal exempts itself from the probe and from retrying: nothing was + // written beyond the watchEvents request, and no second socket exists. + expect(sentFrames(socket)).toHaveLength(1); + expect(FakeSocket.instances).toHaveLength(1); + expect(isWebuiEventChannelDegraded()).toBe(true); + }); +}); + describe("V26 · the delay before a retry is bounded and spread", () => { it("grows exponentially, jitters, and stops at the ceiling", () => { const at = (attempt: number, jitter: number) =>