From 9787996f75c17308ea20f91b374a7567963b70c3 Mon Sep 17 00:00:00 2001 From: ufna <271578+ufna@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:41:02 +0300 Subject: [PATCH 1/7] perf(tray): parse hgs output off the GUI thread MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner direction 2026-10-27: data parsing and similar processing run outside the GUI thread. First milestone of the event data plane (zerus-604d.1). - ProcessRunner passes every result through one serial preparation thread, hgs-prepare, keeping completion order. run(request, owner, prepare, apply) and post(owner, prepare, apply) shape values there; the owner's thread only applies them. - Every HgsClient reply, the connector panels, the speech client, the update feed and startup fleet discovery are parsed in prepare steps. The task parse pool and the 8 MiB worktree snapshot re-parse per lookup are gone. - JsonParse is the tray's only JSON parser and reports a parse on the GUI thread; every Qt suite runs with ZERUS_GUI_THREAD_CHECKS=fatal. check-source.py rejects QJsonDocument::fromJson elsewhere and limits JsonParse::local() to a listed, shrink-only set of small local values (owner: "согласен, оставляем исключения до M2"). - SettingsJson parses QSettings JSON values once per change, warms the large keys at startup and records what writers stored, so per-poll read-mark reads no longer re-parse settings. - IdentityHash remembers per-row SHA-256 identities, 3.9 % of the Mac GUI thread in a 2026-10-27 sample. Design: docs/superpowers/specs/2026-10-27-event-data-plane-design.md. Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 12 + .../2026-10-27-event-data-plane-design.md | 289 ++++++++ scripts/ci/check-source.py | 45 +- scripts/ci/gui.sh | 2 + tests/test_source_json_parse.py | 53 ++ tray/CMakeLists.txt | 6 +- tray/src/AccountCatalog.h | 4 +- tray/src/AccountFilter.cpp | 3 +- tray/src/AnalyticsClient.cpp | 5 +- tray/src/AnalyticsData.cpp | 7 +- tray/src/AttentionTracker.cpp | 3 +- tray/src/ComposerDraftStore.cpp | 35 +- tray/src/DownloadCache.cpp | 8 +- tray/src/HgsClient.cpp | 630 ++++++++++-------- tray/src/HgsClient.h | 6 + tray/src/IdentityHash.h | 26 + tray/src/JsonParse.cpp | 93 +++ tray/src/JsonParse.h | 51 ++ tray/src/LinuxAttentionNotifier.cpp | 3 +- tray/src/NewSessionDialog.cpp | 9 +- tray/src/ProcessRunner.cpp | 74 +- tray/src/ProcessRunner.h | 48 +- tray/src/ProjectsDialog.cpp | 8 +- tray/src/QuestionReply.h | 7 +- tray/src/ReadMarks.h | 5 +- tray/src/RecoverySettings.h | 3 +- tray/src/RelayConfiguration.h | 5 +- tray/src/RepositoryClone.cpp | 3 +- tray/src/SessionCreationController.cpp | 8 +- tray/src/SessionList.cpp | 7 +- tray/src/SessionRowPreferences.h | 4 +- tray/src/SessionsWindow.cpp | 11 +- tray/src/SettingsJson.cpp | 61 ++ tray/src/SettingsJson.h | 27 + tray/src/SwarmController.cpp | 7 +- tray/src/TaskDrag.h | 3 +- tray/src/TaskInspector.cpp | 3 +- tray/src/TaskMessageOutbox.h | 3 +- tray/src/TrayAgent.cpp | 31 +- tray/src/UpdateController.cpp | 23 +- tray/src/WindowsMobileSettings.h | 15 +- tray/src/WorkspaceSettings.h | 14 +- tray/src/WorktreeSettings.h | 8 +- tray/src/WorktreesDialog.cpp | 3 +- tray/src/speech/SpeechClient.cpp | 22 +- tray/src/speech/SpeechSettings.cpp | 3 +- tray/tests/CMakeLists.txt | 28 +- tray/tests/support/OffGuiThread.h | 15 + tray/tests/test_analytics.cpp | 43 +- tray/tests/test_dashboard.cpp | 16 +- tray/tests/test_externalfile.cpp | 3 +- tray/tests/test_jsonparse.cpp | 96 +++ tray/tests/test_processrunner.cpp | 54 ++ tray/tests/test_sessionswindow.cpp | 6 + 54 files changed, 1541 insertions(+), 416 deletions(-) create mode 100644 docs/superpowers/specs/2026-10-27-event-data-plane-design.md create mode 100644 tests/test_source_json_parse.py create mode 100644 tray/src/IdentityHash.h create mode 100644 tray/src/JsonParse.cpp create mode 100644 tray/src/JsonParse.h create mode 100644 tray/src/SettingsJson.cpp create mode 100644 tray/src/SettingsJson.h create mode 100644 tray/tests/support/OffGuiThread.h create mode 100644 tray/tests/test_jsonparse.cpp diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 641b97955..179d1b3d5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,6 +70,8 @@ stable release, and raising them is routine maintenance. Run The replica crate has its own lockfile: run `cargo +1.99.1 test --locked --manifest-path crates/replica/Cargo.toml`; its golden vectors must keep decoding, and `ZERUS_BLESS=1` only adds new ones. +The event stream's frame crate has one too: run +`cargo +1.99.1 test --locked --manifest-path crates/watch-protocol/Cargo.toml --target-dir target/watch-protocol`. On Linux, the three worktree modules run through `scripts/ci/worktree-tests.py` in a fresh PID/mount namespace, including their live process blockers and private @@ -110,6 +112,16 @@ class) and before it starts. `CHILD_PROCESS_EXEMPT` in the script names the files that may start children otherwise, each with its reason. +The desktop parses JSON only through [JsonParse.h](tray/src/JsonParse.h), off +the GUI thread: `hgs` output in a `ProcessRunner` prepare step, other data +through `ProcessRunner::post` or an existing worker, and QSettings values +through `SettingsJson`. `check-source.py` rejects `QJsonDocument::fromJson` in +`tray/src` outside `JsonParse.cpp`, and lists the files that may still call +`JsonParse::local()` on the GUI thread and how often; that list only shrinks. +Every Qt suite runs with `ZERUS_GUI_THREAD_CHECKS=fatal`, so a parse on the GUI +thread aborts the test with its call site. A GUI suite that checks a parser +directly calls it through `offGuiThread()` (`tray/tests/support/OffGuiThread.h`). + A change to tmux or `ps` call sites must keep Unix behaviour unchanged: compare a [tmux census](docs/ci-and-aur.md#unix-tmux-census) of the branch with one of main. `scripts/ci/check-windows-target.sh` type-checks the Windows target from Linux or diff --git a/docs/superpowers/specs/2026-10-27-event-data-plane-design.md b/docs/superpowers/specs/2026-10-27-event-data-plane-design.md new file mode 100644 index 000000000..c0e17c66c --- /dev/null +++ b/docs/superpowers/specs/2026-10-27-event-data-plane-design.md @@ -0,0 +1,289 @@ +# Event data plane + +Epic: `zerus-604d`. Owner direction, 2026-10-27: an event-driven data channel +is mandatory, and parsing and similar processing must run off the GUI (main) +thread. This design replaces per-request `hgs` process polling with one +long-lived event stream per computer and moves all data preparation in the +desktop off the GUI thread. The stream contract is shared by the Qt desktop, +the planned Tauri client (`2026-10-27-tauri-web-ui-design.md`), the mobile +connector and a future plugin host's `events.subscribe`. + +## Baseline (2026-10-27) + +Measured on the owner's Mac (M1 Pro, all sessions on `arch`) and on `arch` +itself, both running Dev 0.47.1-dev.1398 with `hgs 1.56.0`. Raw logs, scripts +and hashes are kept with the review report, outside the repository. + +| Measure | arch GUI | Mac GUI | +|---|---|---| +| `hgs` children started per second (60 s idle window) | 3.63 | 3.60 | +| of which `git-status` per minute | 123 | 114 (each one SSH to arch) | +| of which local `ls` / peer `ls` / `inspect` per minute | 42 / 13 / 20 | 29 / 10 / 16 | +| GUI process CPU (self) | 2.4 % | 17.0 % | +| CPU of reaped children | 35.5 % of a core | n/a | +| RSS | 166 MiB | 139 MiB | + +Command costs (median of 3–5 runs): + +| Command | on arch | from the Mac to arch | +|---|---|---| +| `ls --json --local --no-archives` (35 live rows) | 280 ms, 294 KB | 599 ms, 294 KB | +| `inspect NAME --after CURSOR` | 117 ms, 171 KB | 524 ms, 171 KB | +| same with `--skip-processes` | 50 ms, 51 KB | — | +| `git-status --path P --json` | 21–223 ms, 0.4 KB | 500 ms | +| `ssh arch true` | — | 240 ms | + +Of the 171 KB incremental inspection, 120 KB is the `processes` section. The +desktop includes it every 2.5 s while the Processes tab is shown and every 30 s +otherwise; other polls pass `--skip-processes` (51 KB). `--after` trims only +`events`. Every other section is rebuilt and resent each time. + +JSON parsing itself is cheap. Qt `fromJson` + `boxFromObject` of the 294 KB +listing takes 1.75 ms on arch, and 6.9 ms for the 1.1 MB listing with archives. +The GUI-thread cost lies after the parse. A 15 s `sample` of the Mac GUI +(5,834 main-thread samples, about 2.6 ms each) attributes: + +- 11.4 % of the main thread to `ProcessRunner` result callbacks; +- 7.2 % to `TrayAgent::onLocalReady`, about **150 ms per local poll**: three + QSettings JSON re-parses, several full unread passes, `SessionsWindow::rebuild` + and `AccountsPage::render`; +- 3.9 % to SHA-256 through OpenSSL (`QCryptographicHash`) inside `rebuild`, for + account identity keys and row preference keys recomputed for every row on + every poll; +- about 52 ms of GUI time per applied inspection, and 87 ms per peer listing; +- 19 % to painting, which is outside this design. + +Each local listing row changes on every poll even when nothing happened: +`goal_observed_at` and `process_work.sampled_at` are observation timestamps. +Any delta protocol must mask them, or every delta becomes a full snapshot. + +Session open from the Mac is therefore at least one SSH inspection (0.4–0.5 s) +plus about 50 ms of GUI work before the first Activity render. A reply appears +on the next 2.5 s poll plus one inspection: on average about 1.75 s and at worst +about 3 s after the agent wrote it. These two values are derived from component +timings. Milestone 1 adds instrumentation that measures them end to end. + +## Rule: no data preparation on the GUI thread + +The GUI thread posts requests and applies prepared, immutable values. It never +parses JSON, merges, diffs, sorts, filters, hashes or builds models from data +that came from `hgs`, the stream or disk. + +### ProcessRunner preparation stage + +`ProcessRunner` already starts children on its worker thread. Every result now +also passes a single serial preparation thread, `hgs-prepare`, before it reaches +the owner: + +```cpp +ProcessRunner::run(request, owner, + [](ProcessRunner::Result result) { return prepareListing(result); }, // hgs-prepare + [this](PreparedListing listing) { apply(std::move(listing)); }); // owner thread +``` + +The prepare step sees only the result and values it captured. It must not touch +the owner or any other application-thread object. Results keep the order in +which their children finished, whether or not they have a prepare step, because +the legacy `run(request, owner, done)` passes the same serial thread unchanged. +A destroyed owner still drops a prepared value. + +`HgsClient` converts each `runHgs` call to a prepared variant. In the first +step, signals keep their existing types (`BoxState`, `QJsonObject`, parsed +lists). Those are built off-thread and handed over by implicit sharing, so +receivers stay unchanged. Later steps replace them with typed prepared models. + +### Enforcement + +- `tray/src/JsonParse.h` is the tray's only JSON parser. In a GUI application it + records a violation when it runs on the application thread. With + `ZERUS_GUI_THREAD_CHECKS=fatal`, which the Qt test suites set, a violation + aborts the test with the call site. +- `scripts/ci/check-source.py` rejects `QJsonDocument::fromJson` and + `QCborValue::fromCbor` in `tray/src` outside `JsonParse.cpp`. +- During migration, small local preference values that are still read on the + GUI thread go through `JsonParse::localSetting()`. It is capped at 64 KiB, and + `check-source.py` lists the files and call counts allowed to use it. The list + may only shrink, and each entry has a Beads task. Owner decision, + 2026-10-27: «согласен, оставляем исключения до M2» — the list must be empty + at the end of M2. All `hgs` and stream output is parsed off the GUI thread + from M1 on, without exception. +- JSON values kept in QSettings go through `SettingsJson`. It parses a value + only when its stored bytes changed since the last read. The tray warms the + large keys (organization, swarm observations, read marks, worktree + snapshots) on the preparation thread at startup, and writers record what + they stored, so the GUI thread does not parse them again. On arch the saved + organization is 66 KB and the swarm versions 102 KB, both above the + `local()` limit, so a cold read of either on the GUI thread counts as a + violation. +- Tests: preparation runs on `hgs-prepare` and application happens on the owner + thread; global result order is preserved; a destroyed owner drops prepared + values; a fixture listing and inspection through `HgsClient` parse off the GUI + thread; and a GUI-thread parse fails under the fatal check. + +### Model preparation off the GUI thread + +After the parse moves, the dominant cost is the work that follows each listing: +unread and attention computation, fleet merges, account catalogs, sorting, +grouping and per-row roles. Milestone 2 introduces a `FleetModel` worker. It +owns the merged fleet, read marks and organization inputs, and publishes an +immutable `PreparedFleet`: rows in display order with precomputed roles, group +headers, counts, unread and attention results, and a changed-row set. The GUI +applies only the changed rows to `SessionList` and the pages that are visible. +Read marks and attention marks are held in memory, written through on change, +and never re-parsed from QSettings per poll. Account and row-preference +identity hashes are computed once per identity, on the worker. + +## Event stream: `hgs [@host] watch --json` + +### Transport + +- One long-lived `hgs watch --json` process per computer. Local: a child of the + desktop. Peer: `hgs @host watch --json`, one SSH connection with + `ServerAliveInterval=10` and `ServerAliveCountMax=3`. The remote command is + guarded by the `watch-v1` marker, the same way other newer commands are, so + an old peer never receives an unknown command as an agent launch. +- Server to client: NDJSON on stdout, UTF-8, one frame per line. A frame is at + most 1 MiB. Larger content is paged or truncated with a reference that the + client fetches with an existing bounded command. +- Client to server: NDJSON control operations on stdin. EOF on stdin ends the + stream, so a closed desktop or a dropped SSH connection never leaves an + orphan watcher behind. +- `hgs -h` prints the line `Event stream ABI: watch-v1`. Clients that do not + find it keep today's polling for that computer. + +### Frames + +Every frame has `v` (1), `t` (type) and `seq`. `seq` is monotonic per stream, +and each stream has a random `stream` ID that appears in its `hello`. + +| `t` | Fields | Meaning | +|---|---|---| +| `hello` | `machine_id`, `host`, `hgs_version`, `stream`, `heartbeat_ms`, `max_frame_bytes`, `topics`, `limits` | first frame | +| `snapshot` | `topic`, `key?`, `items` or `value`, `page`, `last`, `cursor` | bounded replacement of one topic or key. Retain the previous display until `last` | +| `delta` | `topic`, `upsert[]`, `remove[]` with `key` and `reason`, `cursor` | keyed collection change | +| `events` | `topic: "activity"`, `key`, `events[]`, `cursor` | append-only Activity events after `cursor` | +| `event` | `topic: "activity"`, `key`, `id`, `rev`, `event` | replace one open event by stable ID; a higher `rev` wins | +| `state` | `topic`, `key`, `value`, `rev` | replace one keyed object (git status, process set, session summary) | +| `heartbeat` | `at` | sent after 10 s without other frames | +| `gap` | `topic`, `key?`, `reason` | the client must discard that topic's state; a snapshot follows | +| `error` | `code`, `message`, `topic?`, `fatal` | a fatal error ends the stream | + +Control operations: `subscribe` and `unsubscribe` with `topic` and its key +(`session` as `{name, run_id, conversation_id}`, `path` or `project`), an +optional `after` cursor and `tail` count for Activity, and `ping`. The catalog +topic is always on. + +### Topics + +| Topic | Key | Content | Resume | +|---|---|---|---| +| `catalog` | `run:`, or `name:` for rows without a run | exactly today's `ls --json --local --no-archives` row objects. A rename is an upsert on the same key. A resume or restart that starts a new run removes the old key with `reason: "replaced"`. Removals carry `ended`, `archived`, `forgotten` or `replaced` when known | a fresh bounded snapshot. Clients diff it against their retained rows | +| `catalog.meta` | — | the non-row fields of the listing (`projects`, `peers`, `preferences`, `desktop_presence`, `archive_revision`, `hgs_version`, `metrics`) | snapshot | +| `activity` | session key | `events`: the same event objects as `inspect`, cursor = journal `seq`. `state`: the inspection summary without `events` and `processes`. `event`: open-event updates such as provider messages between tools | `after` = the last `seq`. Below retention the server sends `gap` and a tail snapshot | +| `processes` | session key | `state`: the `processes` section of `inspect` | snapshot | +| `git` | checkout path | `state`: the `git-status --json` object | snapshot | +| `tasks` | project | `delta` of task rows, as `task list` | snapshot | + +Archives stay outside the stream. They remain a keyset-paged read (`archives +--json`), so the snapshot is bounded by live sessions, not by history. + +Status semantics do not change. Rows carry the same fields; Working, +Background, Scheduled, Needs attention and unread reply identity are derived +by the clients exactly as today. + +Change detection masks the volatile observation fields (`goal_observed_at`, +`process_work.sampled_at`). A row is sent when anything else in it changes. +`catalog.meta` with fresh `metrics` and presence is sent at most every 10 s. + +### Event sources + +No file watcher or async runtime exists in `hgs` today, and v1 adds no +dependency. One watcher process uses `std::thread` and cheap change detection +on a 250 ms pulse: + +- **Activity**: a held read-only connection to `events.sqlite3`. When + `PRAGMA data_version` changes, read `seq > cursor` for subscribed + conversations (at most 200 rows per read). `seq` is global and monotonic and + already serves as the `inspect --after` cursor. Hooks write it on every + native hook event, and retention keeps 50,000 rows. Text between tools exists + only in native transcripts. For subscribed sessions the watcher stats the + transcript and, when it grows, re-derives provider messages from an offset + cursor keyed on device, inode, prefix hash and offset (the existing + tail-reader pattern). DeepSeek and OpenCode keep their inspect RPCs, polled + only while subscribed. +- **Catalog**: a stamp vector over the binding directories, `renames/`, + `name_routes/`, `read-marks.json`, `notifications.json`, `waits.sqlite3`, + `launches/`, the archive revision and the provider binding directories. When + the stamp changes (150 ms debounce, at most every 500 ms), or after a 5 s + safety tick, the watcher re-runs the in-process listing builder that + `ls --json --local --no-archives` uses and diffs rows by key. The safety tick + covers what hooks never report: process exit, tmux kill or rename, ended + background jobs and terminal choosers. Bindings are replaced by rename; + temporary `.hgs-*` files are ignored. +- **Processes**: hook-recorded shell jobs (the binding stamp) plus one `ps` + work sample every 2 s per subscribed session. The full snapshot, which + re-reads the transcript tail, runs at most every 10 s. +- **Git**: stats of `HEAD`, `index`, the current branch ref, `packed-refs` and + `FETCH_HEAD` in the git directory and common directory of each subscribed + path. Activity events from file-changing tools in sessions under that path + also trigger a check. There is a 30 s safety tick, and the existing + `git-status` command (with its 60 s remote cache) runs on change. +- **Tasks**: stats of `replica//project.loro` and its `tasks/` + directory, then the existing task listing. Loro stays inside `crates/replica`. + +A later version may replace stat pulses with inotify/kqueue/FSEvents behind +the same frames. The protocol does not depend on how changes are detected. + +### Bounds and backpressure + +- Subscriptions: at most 16 Activity sessions, 8 process sets, 64 git paths + and 8 task projects per stream. Excess subscriptions fail with an `error` + frame. +- A writer thread owns stdout with a bounded queue (4 MiB). When the client + stops reading, pending `state` and `delta` frames coalesce per topic and key + (latest wins). If the queue still overflows, the affected topic receives + `gap` followed by a fresh snapshot once the client drains. +- Snapshot pages are at most 512 KiB. Activity tails are at most 100 events, + with older history read through the existing paged commands. +- Listing work is bounded exactly as today. The watcher never adds per-session + caches beyond subscribed sessions. + +### Consumers + +- **Qt desktop**: one `EventStream` per computer. Its reader, line framing, + JSON parsing and per-computer store run on a worker thread. The store + publishes immutable prepared deltas to the GUI, coalesced to at most one + application per 50 ms. While a computer's stream is healthy, the desktop + stops its `ls`, inspection, `git-status` and `processes` polls for it. A + missing marker, or three stream failures within 60 s, falls back to today's + polling, and the desktop retries the stream every 5 minutes. +- **Tauri client**: its Rust backend spawns the same subprocess per computer, + parses frames in Rust (never in the webview) and pushes view deltas to the + webview, coalesced per animation frame. A small serde crate + `crates/watch-protocol` defines the frame types. It is shared by `hgs` and + the Tauri crate and contains neither Loro nor Qt. +- **Mobile connector**: it keeps polling until it opts in, then consumes the + same frames. API v1 and its journals stay unchanged. +- **Plugin host**: `events.subscribe` maps onto these topics, filtered by the + manifest's permissions. + +## Compatibility + +- `hgs` CLI commands, `HGS_*` variables, state paths, API v1 and the connector + journal stay unchanged. The stream is additive. +- Older peers lack the `watch-v1` marker and keep today's polling. +- An older desktop never starts `watch`. The command runs only while a client + holds its stdin open. +- Session counts may reach tens of thousands. The live catalog is bounded by + live sessions, archives stay paged, Activity is cursor-based, and the + watcher holds no unbounded per-session state. + +## Milestones + +| | Scope | Measure afterwards | +|---|---|---| +| M1 | Preparation stage, every parse of `hgs`, connector and network output off-thread, enforcement, settings values parsed once per change, cached identity hashes | Mac `sample` main-thread share, per-poll GUI time | +| M2 | `FleetModel` worker: unread, attention, sort, group and row roles prepared off-thread; the local-value list empty; open/reply latency instrumentation | per-poll GUI time below 5 ms | +| M3 | `hgs watch` server: catalog and Activity topics, `crates/watch-protocol`, `docs/reference.md` | spawns per second, idle CPU on arch | +| M4 | Qt `EventStream` consumer with fallback, local and peers | session open and reply latency from the Mac | +| M5 | git, processes and tasks topics; connector and plugin adoption | spawns per second close to zero while idle | diff --git a/scripts/ci/check-source.py b/scripts/ci/check-source.py index 1d67f8226..068fd9e27 100644 --- a/scripts/ci/check-source.py +++ b/scripts/ci/check-source.py @@ -21,6 +21,27 @@ "tray/src/WindowsTerminalBackend.cpp": "the app links it only on Windows, where inheritStdioOnly() does nothing", } CHILD_PROCESS_SOURCES = re.compile(r"tray/src/.+\.(?:cpp|h|mm)") +# The desktop parses JSON only through tray/src/JsonParse.h, which reports a +# parse on the GUI thread. JsonParse::local() is the exception for small values +# the tray wrote itself; these files may call it this many times. Owner +# decision, 2026-10-27: the list only shrinks and is empty at the end of M2 +# (zerus-604d.2). +JSON_PARSER = "tray/src/JsonParse.cpp" +LOCAL_JSON_CALLS = { + "tray/src/AccountFilter.cpp": 1, # binding keys the filter built + "tray/src/AttentionTracker.cpp": 1, # notification tokens the tray encoded + "tray/src/ComposerDraftStore.cpp": 1, # question draft identities + "tray/src/QuestionReply.h": 1, # the user's own answers in a prompt; M2 prepares them with the row + "tray/src/RelayConfiguration.h": 1, # private connector configuration, at most 64 KiB + "tray/src/SessionList.cpp": 3, # drag data the list wrote + "tray/src/SessionsWindow.cpp": 1, # pending task launches, at most 64 KiB + "tray/src/SettingsJson.cpp": 1, # a changed QSettings value before warm() parsed it + "tray/src/TaskDrag.h": 1, # drag data the board wrote + "tray/src/UpdateController.cpp": 3, # the saved update catalog and installer record + "tray/src/WorktreesDialog.cpp": 1, # review keys the dialog built +} +JSON_DECODER = re.compile(r"\b(?:QJsonDocument\s*::\s*fromJson|QCborValue\s*::\s*fromCbor)\b") +LOCAL_JSON = re.compile(r"\bJsonParse\s*::\s*local(?:Object|Array)?\s*\(") SEE_CHILD_PROCESS = "(tray/src/ChildProcess.h)" LEXEME = re.compile(r"//|/\*|[\"']") LITERAL = {'"': re.compile(r'"(?:\\.|[^"\\\n])*"?'), "'": re.compile(r"'(?:\\.|[^'\\\n])*'?")} @@ -249,6 +270,24 @@ def child_process_findings(files): return [f"{name}:{line}: {reason} {SEE_CHILD_PROCESS}." for name, line, reason in sorted(problems)] +def json_parse_findings(files): + """"file:line: reason" for JSON parsed outside JsonParse.h or local() calls beyond LOCAL_JSON_CALLS.""" + problems = [] + for name, text in sorted(files.items()): + code = code_only(text) + if name != JSON_PARSER: + for match in JSON_DECODER.finditer(code): + line = code.count("\n", 0, match.start()) + 1 + problems.append(f"{name}:{line}: parse JSON with JsonParse (tray/src/JsonParse.h), off the GUI thread") + calls = len(LOCAL_JSON.findall(code)) + allowed = LOCAL_JSON_CALLS.get(name, 0) + if calls > allowed: + problems.append(f"{name}: {calls} JsonParse::local() calls, {allowed} allowed; parse off the GUI thread instead") + elif calls < allowed: + problems.append(f"{name}: lower LOCAL_JSON_CALLS to {calls} in scripts/ci/check-source.py") + return problems + + def main(): paths = subprocess.check_output(["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"], cwd=ROOT).decode().split("\0") # First, so that a desktop child the unit tests would also find shows only its findings. @@ -258,6 +297,10 @@ def main(): raise SystemExit(f"Remove the child process exemption of missing {', '.join(stale)}.") if problems := child_process_findings(sources): raise SystemExit("\n".join(problems)) + if stale := sorted(set(LOCAL_JSON_CALLS) - set(sources)): + raise SystemExit(f"Remove the local JSON allowance of missing {', '.join(stale)}.") + if problems := json_parse_findings(sources): + raise SystemExit("\n".join(problems)) subprocess.run([sys.executable, "-m", "unittest", "discover", "-q", "-s", "tests", "-p", "test_source_*.py"], cwd=ROOT, check=True) subprocess.run([sys.executable, str(ROOT / "scripts/check-source-privacy.py")], check=True) subprocess.run(["git", "diff", "--check"], cwd=ROOT, check=True) @@ -316,7 +359,7 @@ def main(): ast.parse(path.read_text(), filename=name) elif path.read_bytes().split(b"\n", 1)[0].startswith((b"#!/bin/sh", b"#!/usr/bin/env bash")) or path.name in {"PKGBUILD", "PKGBUILD.in"}: subprocess.run(["bash", "-n", str(path)], check=True) - print("Source syntax, privacy, child process and release version checks passed.") + print("Source syntax, privacy, child process, JSON parsing and release version checks passed.") if __name__ == "__main__": diff --git a/scripts/ci/gui.sh b/scripts/ci/gui.sh index fbbd53ce7..840806551 100644 --- a/scripts/ci/gui.sh +++ b/scripts/ci/gui.sh @@ -63,6 +63,8 @@ if [ "$phase" = build ]; then exit 0 fi export QT_QPA_PLATFORM=offscreen QT_QUICK_BACKEND=software +# JSON parsed on the GUI thread fails the suite (tray/src/JsonParse.h). +export ZERUS_GUI_THREAD_CHECKS=fatal export QTWEBENGINE_CHROMIUM_FLAGS=--disable-gpu rm -rf artifacts/test-results/qt-functions mkdir -p artifacts/test-results/qt-functions diff --git a/tests/test_source_json_parse.py b/tests/test_source_json_parse.py new file mode 100644 index 000000000..84fa6c8aa --- /dev/null +++ b/tests/test_source_json_parse.py @@ -0,0 +1,53 @@ +"""Exercise the desktop JSON parsing rule of scripts/ci/check-source.py.""" +import importlib.util +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("check_source", ROOT / "scripts/ci/check-source.py") +check = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(check) + + +class CurrentTreeTest(unittest.TestCase): + def test_desktop_parses_json_only_through_json_parse(self): + files = {path.relative_to(ROOT).as_posix(): path.read_text(encoding="utf-8") + for path in (ROOT / "tray/src").rglob("*") if path.suffix in {".cpp", ".h", ".mm"}} + self.assertEqual(check.json_parse_findings(files), []) + self.assertLessEqual(set(check.LOCAL_JSON_CALLS), set(files), "remove allowances of deleted files") + + +class RuleTest(unittest.TestCase): + def test_direct_decoders_are_rejected_outside_the_parser(self): + problems = check.json_parse_findings({ + "tray/src/View.cpp": "void f(const QByteArray &b) {\n auto d = QJsonDocument::fromJson(b);\n}\n", + "tray/src/Cbor.cpp": "auto v = QCborValue :: fromCbor(bytes);\n", + check.JSON_PARSER: "QJsonDocument document = QJsonDocument::fromJson(bytes, &error);\n", + }) + self.assertEqual(problems, [ + "tray/src/Cbor.cpp:1: parse JSON with JsonParse (tray/src/JsonParse.h), off the GUI thread", + "tray/src/View.cpp:2: parse JSON with JsonParse (tray/src/JsonParse.h), off the GUI thread", + ]) + + def test_comments_and_strings_do_not_count(self): + problems = check.json_parse_findings({ + "tray/src/Notes.h": "// QJsonDocument::fromJson is rejected here\n" + "const char *text = \"JsonParse::local(bytes)\";\n", + }) + self.assertEqual(problems, []) + + def test_local_calls_must_match_the_allowance_exactly(self): + listed = next(iter(check.LOCAL_JSON_CALLS)) + allowed = check.LOCAL_JSON_CALLS[listed] + extra = "".join("JsonParse::localObject(a);\n" for _ in range(allowed + 1)) + fewer = "".join("JsonParse::local(a);\n" for _ in range(allowed - 1)) + self.assertEqual(check.json_parse_findings({"tray/src/New.cpp": "JsonParse::localArray(key);\n"}), + ["tray/src/New.cpp: 1 JsonParse::local() calls, 0 allowed; parse off the GUI thread instead"]) + self.assertEqual(check.json_parse_findings({listed: extra}), + [f"{listed}: {allowed + 1} JsonParse::local() calls, {allowed} allowed; parse off the GUI thread instead"]) + self.assertEqual(check.json_parse_findings({listed: fewer}), + [f"{listed}: lower LOCAL_JSON_CALLS to {allowed - 1} in scripts/ci/check-source.py"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tray/CMakeLists.txt b/tray/CMakeLists.txt index 443606095..384f12ce2 100644 --- a/tray/CMakeLists.txt +++ b/tray/CMakeLists.txt @@ -99,7 +99,7 @@ endif() add_library(hgs-updates STATIC src/UpdateController.cpp src/UpdateController.h src/WindowsUpdateTrust.cpp src/WindowsUpdateTrust.h) target_include_directories(hgs-updates PUBLIC src) -target_link_libraries(hgs-updates PUBLIC Qt6::Widgets Qt6::Network) +target_link_libraries(hgs-updates PUBLIC Qt6::Widgets Qt6::Network hgs-tray-core) if(WIN32) # WindowsUpdateTrust: CNG for the update manifest, WinVerifyTrust for installers. target_link_libraries(hgs-updates PRIVATE bcrypt wintrust crypt32) @@ -143,7 +143,7 @@ include(cmake/BuildVersion.cmake) # link; tests/CMakeLists.txt fails when a suite lists a source that one of them # builds. Each layer adds Qt modules, so a suite that links the lowest layer # covering its code keeps its QTEST_MAIN application type. -add_library(hgs-tray-core STATIC src/ProcessRunner.cpp src/HgsClient.cpp src/FleetState.cpp +add_library(hgs-tray-core STATIC src/ProcessRunner.cpp src/JsonParse.cpp src/SettingsJson.cpp src/HgsClient.cpp src/FleetState.cpp src/SessionCreationController.cpp src/AnalyticsClient.cpp src/AnalyticsData.cpp src/ReadMarkSync.cpp src/AppConfig.cpp src/AccountUsageStore.cpp src/ComposerDraftStore.cpp @@ -192,7 +192,7 @@ target_link_libraries(hgs-tray-ui PUBLIC hgs-tray-core Qt6::Widgets add_library(hgs-tray-activity STATIC src/ActivityView.cpp) target_include_directories(hgs-tray-activity PUBLIC src) -target_link_libraries(hgs-tray-activity PUBLIC Qt6::Widgets Qt6::Multimedia PRIVATE hgs-markdown hgs-round-video) +target_link_libraries(hgs-tray-activity PUBLIC Qt6::Widgets Qt6::Multimedia PRIVATE hgs-tray-core hgs-markdown hgs-round-video) target_link_libraries(hgs-tray-ui PRIVATE hgs-tray-activity) add_library(hgs-tray-views STATIC src/ArchiveDatePicker.cpp src/ArchiveDatePicker.h src/ArchiveBrowser.cpp src/ArchiveBrowser.h src/SessionsWindow.cpp src/SessionsWindowShortcuts.cpp src/SessionsWindowGoTo.cpp src/SessionSelection.cpp src/NativeUiView.cpp diff --git a/tray/src/AccountCatalog.h b/tray/src/AccountCatalog.h index de01c6a2c..4013e1efa 100644 --- a/tray/src/AccountCatalog.h +++ b/tray/src/AccountCatalog.h @@ -1,6 +1,6 @@ #pragma once +#include "IdentityHash.h" #include "MachinePlatform.h" -#include #include #include #include @@ -22,7 +22,7 @@ inline QString key(const QJsonObject &p) { return "profile:" + profileKey(p); const auto scope = identity["organization"].toString().trimmed().toCaseFolded(); const auto identityKey = p["provider"].toString() + '\n' + (account.isEmpty() ? "email:" + email : "id:" + account) + '\n' + scope; - return "account:" + QString::fromLatin1(QCryptographicHash::hash(identityKey.toUtf8(), QCryptographicHash::Sha256).toHex()); + return "account:" + QString::fromLatin1(IdentityHash::sha256(identityKey.toUtf8()).toHex()); } // A quota belongs to the account: select a snapshot, never add quotas from // several machines or let a failed probe replace a good snapshot. diff --git a/tray/src/AccountFilter.cpp b/tray/src/AccountFilter.cpp index 056f1c29f..65b9121fe 100644 --- a/tray/src/AccountFilter.cpp +++ b/tray/src/AccountFilter.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "AccountFilter.h" #include "ScopeFilterWidgets.h" #include "WorkspaceIcons.h" @@ -27,7 +28,7 @@ QString AccountFilter::caption(const SessionAccounts::Account &account) const { })) { QStringList profiles; for (const auto &binding : account.bindings) { - const auto fields = QJsonDocument::fromJson(binding.toUtf8()).array(); + const auto fields = JsonParse::localArray(binding.toUtf8()); if (fields.size() != 4) continue; const auto host = fields[0].toString(), profile = fields[3].toString(); profiles.append((host.isEmpty() ? tr("This computer") : host) + " / " + (profile.isEmpty() ? tr("Not recorded") : profile)); diff --git a/tray/src/AnalyticsClient.cpp b/tray/src/AnalyticsClient.cpp index bf9613168..9eb3ea709 100644 --- a/tray/src/AnalyticsClient.cpp +++ b/tray/src/AnalyticsClient.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "AnalyticsClient.h" #include "AnalyticsData.h" #include "ProcessRunner.h" @@ -58,8 +59,8 @@ void AnalyticsClient::request(const QString &host, const QString &verb, const QJ if (mutation) { // Mutations return metadata with the default range. Only a // fresh explicit summary repopulates the controls/view. - QJsonParseError error; const auto json = QJsonDocument::fromJson(bytes,&error); - if (error.error == QJsonParseError::NoError && json.isObject() && json.object().value("schema") == 1 + QString error; const auto json = JsonParse::parse(bytes,&error); + if (error.isEmpty() && json.isObject() && json.object().value("schema") == 1 && json.object().value("source") == "agent" && !json.object().contains("error")) safe = QJsonObject{{"schema",1}}; } else safe = verb == "speech" ? AnalyticsData::speech(bytes,from,through) : AnalyticsData::summary(bytes,from,through); if (guard) QMetaObject::invokeMethod(guard,[guard,safe,done,mutation] { diff --git a/tray/src/AnalyticsData.cpp b/tray/src/AnalyticsData.cpp index b61887abe..3fd7a4e65 100644 --- a/tray/src/AnalyticsData.cpp +++ b/tray/src/AnalyticsData.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "AnalyticsData.h" #include #include @@ -24,9 +25,9 @@ bool uuid(const QJsonValue &value, bool allowNil = false) { } std::optional object(const QByteArray &bytes) { if (bytes.size() > MaximumBytes) return {}; - QJsonParseError error; - const auto document = QJsonDocument::fromJson(bytes, &error); - if (error.error != QJsonParseError::NoError || !document.isObject() || document.object().value("schema") != 1) return {}; + QString error; + const auto document = JsonParse::parse(bytes, &error); + if (!error.isEmpty() || !document.isObject() || document.object().value("schema") != 1) return {}; return document.object(); } std::optional codes(const QJsonValue &value, const QStringList &allowed) { diff --git a/tray/src/AttentionTracker.cpp b/tray/src/AttentionTracker.cpp index a28c0e849..903035c7b 100644 --- a/tray/src/AttentionTracker.cpp +++ b/tray/src/AttentionTracker.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "AttentionTracker.h" #include "SessionPresentation.h" #include "SessionNotifications.h" @@ -269,7 +270,7 @@ void AttentionTracker::deliveryFailed(const QString &token, qint64 now) QJsonObject AttentionTracker::target(const QString &token) { if (token.size() > 8192) return {}; - const auto result = QJsonDocument::fromJson(QByteArray::fromBase64(token.toLatin1(), QByteArray::Base64UrlEncoding)).object(); + const auto result = JsonParse::localObject(QByteArray::fromBase64(token.toLatin1(), QByteArray::Base64UrlEncoding)); if (!result.value("host").isString() || result.value("name").toString().isEmpty() || result.value("notice").toString().isEmpty()) return {}; return result; diff --git a/tray/src/ComposerDraftStore.cpp b/tray/src/ComposerDraftStore.cpp index 3a31980ed..7dd04e7a8 100644 --- a/tray/src/ComposerDraftStore.cpp +++ b/tray/src/ComposerDraftStore.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "ComposerDraftStore.h" #include @@ -138,9 +139,13 @@ void ComposerDraftStore::saveAsync(const QString &key, const ComposerDraft &draf owner, std::move(done)); } +// The blocking fixture API reads on the writer thread too, after every queued +// write, so no draft file is parsed on the GUI thread. ComposerDraft ComposerDraftStore::load(const QString &key) const { - flushWrites(); return loadDirect(key); + const auto store = *this; ComposerDraft draft; + DraftWrites::instance()->sync([&] { draft = store.loadDirect(key); return WriteResult{true, {}}; }); + return draft; } ComposerDraft ComposerDraftStore::loadDirect(const QString &key) const @@ -152,9 +157,9 @@ ComposerDraft ComposerDraftStore::loadDirect(const QString &key) const draft.storageError = QObject::tr("Could not restore this draft completely. Saved files have been kept locally."); }; if (!file.open(QIODevice::ReadOnly)) { invalid(); return draft; } - QJsonParseError error; - const auto object = QJsonDocument::fromJson(file.readAll(), &error).object(); - if (error.error != QJsonParseError::NoError || object.value("version").toInt() != 1 || object.value("key").toString() != key) { + QString error; + const auto object = JsonParse::parse(file.readAll(), &error).object(); + if (!error.isEmpty() || object.value("version").toInt() != 1 || object.value("key").toString() != key) { invalid(); return draft; } draft.text = object.value("text").toString(); @@ -218,7 +223,9 @@ bool ComposerDraftStore::saveDirect(const QString &key, ComposerDraft &draft) co QStringList ComposerDraftStore::keys() const { - flushWrites(); auto result = indexDirect().keys(); result.sort(); return result; + const auto store = *this; QStringList result; + DraftWrites::instance()->sync([&] { result = store.indexDirect().keys(); return WriteResult{true, {}}; }); + result.sort(); return result; } QHash ComposerDraftStore::indexDirect() const @@ -227,7 +234,7 @@ QHash ComposerDraftStore::indexDirect() const for (const auto &name : QDir(root()).entryList(QDir::Dirs | QDir::NoDotAndDotDot)) { QFile file(root() + '/' + name + "/draft.json"); if (!file.open(QIODevice::ReadOnly)) continue; - const auto object = QJsonDocument::fromJson(file.readAll()).object(); + const auto object = JsonParse::parse(file.readAll()).object(); const auto key = object.value("key").toString(); const auto text = object.value("text").toString(); const auto attachments = object.value("attachments").toArray(); @@ -260,7 +267,7 @@ ComposerDraftOwner ComposerDraftStore::owner(const QString &key) } if (key.startsWith(Question)) { // A question identity is a compact JSON array, so it holds no line break. - const auto identity = QJsonDocument::fromJson(key.mid(Question.size()).section('\n', 0, 0).toUtf8()).array(); + const auto identity = JsonParse::localArray(key.mid(Question.size()).section('\n', 0, 0).toUtf8()); auto result = owner(identity.at(0).toString()); if (result.kind != ComposerDraftOwner::Kind::Unknown) { result.kind = ComposerDraftOwner::Kind::Question; return result; } } @@ -328,11 +335,11 @@ void ComposerDraftStore::renameAsync(const QString &oldKey, const QString &newKe QString ComposerDraftStore::label(const QString &key) const { - flushWrites(); - QFile file(folder(key) + "/draft.json"); - if (file.open(QIODevice::ReadOnly)) { - const auto label = QJsonDocument::fromJson(file.readAll()).object().value("label").toString(); - if (!label.isEmpty()) return label; - } - return QString(key).replace('\n', " / "); + const auto path = folder(key) + "/draft.json"; QString label; + DraftWrites::instance()->sync([&] { + QFile file(path); + if (file.open(QIODevice::ReadOnly)) label = JsonParse::parse(file.readAll()).object().value("label").toString(); + return WriteResult{true, {}}; + }); + return label.isEmpty() ? QString(key).replace('\n', " / ") : label; } diff --git a/tray/src/DownloadCache.cpp b/tray/src/DownloadCache.cpp index 54e4b211d..db473f2a5 100644 --- a/tray/src/DownloadCache.cpp +++ b/tray/src/DownloadCache.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "DownloadCache.h" #include "LocalFile.h" #include @@ -138,10 +139,9 @@ struct Ledger { QFile file(QDir(root).filePath(Index)); if (!occupied(file.fileName())) return; if (!ordinary(QFileInfo(file)) || file.size() > 32 * 1024 * 1024 || !file.open(QIODevice::ReadOnly)) { error = text("Cannot read the downloaded files index."); return; } - QJsonParseError parse; - const auto document = QJsonDocument::fromJson(file.readAll(), &parse); - const auto object = document.object(); - if (parse.error != QJsonParseError::NoError || object.value("version") != 1 || !object.value("files").isArray()) { error = text("The downloaded files index is invalid. Existing files were preserved."); return; } + QString parse; + const auto object = JsonParse::parse(file.readAll(), &parse).object(); + if (!parse.isEmpty() || object.value("version") != 1 || !object.value("files").isArray()) { error = text("The downloaded files index is invalid. Existing files were preserved."); return; } entries = object.value("files").toArray(); QSet names; for (const auto &value : entries) { diff --git a/tray/src/HgsClient.cpp b/tray/src/HgsClient.cpp index 5e6e22b2c..e1187f7f8 100644 --- a/tray/src/HgsClient.cpp +++ b/tray/src/HgsClient.cpp @@ -2,10 +2,12 @@ #include #include "DownloadCache.h" #include "ChildProcess.h" +#include "JsonParse.h" #include "ProcessRunner.h" #include "LocalFile.h" #include "MachinePlatform.h" #include "SessionNotifications.h" +#include "SettingsJson.h" #include #include @@ -26,10 +28,6 @@ #include #include #include -#include -#include -#include -#include #include #include @@ -40,6 +38,35 @@ namespace { +// A JSON object reply parsed on the preparation thread. error is empty when the +// reply was a JSON object within limit bytes (0: any size). +struct ObjectReply { + QJsonObject object; + QString error; + bool ok() const { return error.isEmpty(); } +}; +ObjectReply objectReply(const QByteArray &bytes, qsizetype limit = 0) +{ + ObjectReply reply; + if (limit > 0 && bytes.size() > limit) { reply.error = QStringLiteral("reply exceeds %1 bytes").arg(limit); return reply; } + reply.object = JsonParse::object(bytes, &reply.error); + return reply; +} +const auto parseObject = [](const QByteArray &bytes) { return objectReply(bytes); }; +// The reply as an object, or an empty object for anything else. +const auto parseJsonObject = [](const QByteArray &bytes) { return JsonParse::object(bytes); }; +// A finished child's result with its standard output parsed as an object. +struct ParsedResult { + ProcessRunner::Result result; + ObjectReply reply; +}; +ParsedResult parsedResult(ProcessRunner::Result result, qsizetype limit = 0) +{ + ParsedResult parsed; parsed.reply = objectReply(result.standardOutput, limit); parsed.result = std::move(result); + return parsed; +} + + // The executable a start would use: a path as given, otherwise a PATH lookup // (which honors PATHEXT on Windows, where C:\Zerus\hgs.exe is a path as well). QString resolvedProgram(const QString &program) @@ -273,7 +300,9 @@ quint64 HgsClient::requestRecovery(const QString &host, const QString &operation QStringList args{"recovery", operation}; if (!host.isEmpty()) args.prepend('@' + host); auto process = processRequest(m_hgs, args); process.timeoutMs = host.isEmpty() ? 10000 : 25000; process.input = operation != "get" ? QJsonDocument(data).toJson(QJsonDocument::Compact) : QByteArray(); - ProcessRunner::run(std::move(process), this, [this, request, host, operation](const ProcessRunner::Result &result) { + ProcessRunner::run(std::move(process), this, [](ProcessRunner::Result result) { return parsedResult(std::move(result)); }, + [this, request, host, operation](const ParsedResult &parsed) { + const auto &result = parsed.result; const auto fail = [this, request](const QString &error) { emit recoveryFinished(request, false, {}, error); }; if (result.outcome == ProcessRunner::Result::TimedOut) return fail(tr("Recovery request timed out. Reload its state before trying again.")); if (result.outcome == ProcessRunner::Result::FailedToStart) return fail(result.errorString); @@ -281,9 +310,8 @@ quint64 HgsClient::requestRecovery(const QString &host, const QString &operation const auto error = QString::fromUtf8(result.standardError).trimmed().left(2000); return fail(error.isEmpty() ? tr("Could not update recovery. Reload its state.") : error); } - QJsonParseError error; const auto doc = QJsonDocument::fromJson(result.standardOutput, &error); - if (error.error != QJsonParseError::NoError || !doc.isObject()) return fail(tr("Invalid recovery response")); - emit recoveryFinished(request, true, doc.object(), {}); if (operation != "get") emit sessionWriteDone(host); + if (!parsed.reply.ok()) return fail(tr("Invalid recovery response")); + emit recoveryFinished(request, true, parsed.reply.object, {}); if (operation != "get") emit sessionWriteDone(host); }); return request; } @@ -301,6 +329,8 @@ HgsClient::~HgsClient() } } +BoxState HgsClient::box(const QJsonObject &object) { return boxFromObject(object); } + QList HgsClient::parseSessions(const QJsonArray &rows) { QJsonObject box{{"host","archive"},{"ok",true},{"sessions",rows}}; return boxFromObject(box).sessions; @@ -311,18 +341,14 @@ BoxState HgsClient::parseBox(const QByteArray &json, QString *error) if (error) error->clear(); - QJsonParseError perr; - const QJsonDocument doc = QJsonDocument::fromJson(json, &perr); - if (perr.error != QJsonParseError::NoError || !doc.isObject()) { - if (error) { - *error = perr.error != QJsonParseError::NoError - ? perr.errorString() - : QStringLiteral("expected a JSON object"); - } + QString parseError; + const QJsonObject object = JsonParse::object(json, &parseError); + if (!parseError.isEmpty()) { + if (error) *error = parseError; return BoxState{}; } - return boxFromObject(doc.object()); + return boxFromObject(object); } QList HgsClient::parseFleet(const QByteArray &json, QString *error) @@ -330,19 +356,14 @@ QList HgsClient::parseFleet(const QByteArray &json, QString *error) if (error) error->clear(); - QJsonParseError perr; - const QJsonDocument doc = QJsonDocument::fromJson(json, &perr); - if (perr.error != QJsonParseError::NoError || !doc.isArray()) { - if (error) { - *error = perr.error != QJsonParseError::NoError - ? perr.errorString() - : QStringLiteral("expected a JSON array"); - } + QString parseError; + const QJsonArray arr = JsonParse::array(json, &parseError); + if (!parseError.isEmpty()) { + if (error) *error = parseError; return {}; } QList boxes; - const QJsonArray arr = doc.array(); boxes.reserve(arr.size()); for (const QJsonValue &v : arr) boxes << boxFromObject(v.toObject()); @@ -354,19 +375,14 @@ QList HgsClient::parseProjects(const QByteArray &json, QString *err if (error) error->clear(); - QJsonParseError perr; - const QJsonDocument doc = QJsonDocument::fromJson(json, &perr); - if (perr.error != QJsonParseError::NoError || !doc.isArray()) { - if (error) { - *error = perr.error != QJsonParseError::NoError - ? perr.errorString() - : QStringLiteral("expected a JSON array"); - } + QString parseError; + const QJsonArray arr = JsonParse::array(json, &parseError); + if (!parseError.isEmpty()) { + if (error) *error = parseError; return {}; } QList projects; - const QJsonArray arr = doc.array(); projects.reserve(arr.size()); for (const QJsonValue &v : arr) projects << projectFromJson(v.toObject()); @@ -382,6 +398,22 @@ namespace { // instead of the GUI thread. ProcessRunner also reports exactly once: the finished() // that follows a timeout kill() must not reach onFailed() again (--selftest leaves // main() on the first failure). +// Why a call failed, or empty when it succeeded. +QString failureDetail(const QString &hgs, const ProcessRunner::Result &result) +{ + if (result.outcome == ProcessRunner::Result::TimedOut) + return QStringLiteral("timed out"); + if (result.outcome == ProcessRunner::Result::FailedToStart) + return QStringLiteral("cannot start '%1': %2").arg(hgs, result.errorString); + // hgs всегда возвращает 0 для "спящего пира" — ненулевой код или CrashExit + // означает, что сломан сам вызов, а не то, что застали пира офлайн. + if (result.exitStatus != QProcess::NormalExit || result.exitCode != 0) { + const QString detail = QString::fromLocal8Bit(result.standardError).trimmed().left(4000); + return detail.isEmpty() ? QStringLiteral("exit code %1").arg(result.exitCode) : detail; + } + return {}; +} + void runHgsDirect(const QString &hgs, const QStringList &args, int timeoutMs, QObject *parent, const std::function &onFinished, const std::function &onFailed, @@ -390,21 +422,34 @@ void runHgsDirect(const QString &hgs, const QStringList &args, int timeoutMs, QO auto request = processRequest(hgs, args); request.timeoutMs = timeoutMs; if (!input.isEmpty()) request.input = input; ProcessRunner::run(std::move(request), parent, [hgs, args, onFinished, onFailed](const ProcessRunner::Result &result) { - const QString what = hgs + QLatin1Char(' ') + args.join(QLatin1Char(' ')); - if (result.outcome == ProcessRunner::Result::TimedOut) - return onFailed(what, QStringLiteral("timed out")); - if (result.outcome == ProcessRunner::Result::FailedToStart) - return onFailed(what, QStringLiteral("cannot start '%1': %2").arg(hgs, result.errorString)); - // hgs всегда возвращает 0 для "спящего пира" — ненулевой код или CrashExit - // означает, что сломан сам вызов, а не то, что застали пира офлайн. - if (result.exitStatus != QProcess::NormalExit || result.exitCode != 0) { - const QString detail = QString::fromLocal8Bit(result.standardError).trimmed().left(4000); - return onFailed(what, detail.isEmpty() ? QStringLiteral("exit code %1").arg(result.exitCode) : detail); - } + if (const auto detail = failureDetail(hgs, result); !detail.isEmpty()) + return onFailed(hgs + QLatin1Char(' ') + args.join(QLatin1Char(' ')), detail); onFinished(result.standardOutput); }); } +// A successful call's output becomes parse(output) on the preparation thread +// (ProcessRunner.h); done(value) or onFailed then runs on parent's thread. +template +void runHgsParsedDirect(const QString &hgs, const QStringList &args, int timeoutMs, QObject *parent, Parse parse, Done done, + const std::function &onFailed, const QByteArray &input = {}) +{ + using Value = std::decay_t>; + struct Reply { QString detail; Value value{}; }; + auto request = processRequest(hgs, args); request.timeoutMs = timeoutMs; + if (!input.isEmpty()) request.input = input; + ProcessRunner::run(std::move(request), parent, + [hgs, parse = std::move(parse)](ProcessRunner::Result result) mutable { + Reply reply; reply.detail = failureDetail(hgs, result); + if (reply.detail.isEmpty()) reply.value = parse(result.standardOutput); + return reply; + }, + [hgs, args, done = std::move(done), onFailed](Reply reply) mutable { + if (!reply.detail.isEmpty()) return onFailed(hgs + QLatin1Char(' ') + args.join(QLatin1Char(' ')), reply.detail); + done(std::move(reply.value)); + }); +} + void guardMachine(QStringList &args,const QString &expected) { if(expected.isEmpty())return; @@ -424,6 +469,22 @@ void runHgs(const QString &hgs,const QStringList &args,int timeoutMs,QObject *pa },failed); } +// runHgs() for a JSON reply: parse runs on the preparation thread, done on +// parent's thread. Every hgs reply the client parses goes through here. +template +void runHgsParsed(const QString &hgs, const QStringList &args, int timeoutMs, QObject *parent, Parse parse, Done done, + const std::function &failed, const QByteArray &input = {}) +{ + const int at = !args.isEmpty() && args.first().startsWith('@') ? 1 : 0; + if (args.value(at) != "--expect-machine") { runHgsParsedDirect(hgs, args, timeoutMs, parent, std::move(parse), std::move(done), failed, input); return; } + QStringList help{"--help"}; if (at) help.prepend(args.first()); + runHgsDirect(hgs, help, 15000, parent, [=](const QByteArray &out) { + if (!out.contains("guarded-create-v1")) { failed({}, QObject::tr("The selected CLI cannot verify this creation's computer. Update hgs before continuing.")); return; } + runHgsParsedDirect(hgs, args, timeoutMs, parent, parse, done, failed, input); + }, failed); +} + + } // namespace void HgsClient::cancelArchivePages(){const auto jobs=std::exchange(m_archiveJobs,{});qDeleteAll(jobs);} @@ -433,18 +494,19 @@ quint64 HgsClient::requestArchivePage(const QString &host,const QJsonObject &que const quint64 request=++m_archiveRequest; QStringList args;if(!host.isEmpty())args<<'@'+host;args<<"archives"<<"--json"; const auto finish=[this,owner,request,host](const QJsonObject &data,const QString &error){m_archiveJobs.remove(host);owner->deleteLater();emit archivePageReady(request,host,data,error);}; - runHgs(m_hgs,args,30000,owner,[query,finish](const QByteArray &out){ - const auto doc=QJsonDocument::fromJson(out);const auto data=doc.object();QString error; + struct Page { QJsonObject data; QString error; }; + runHgsParsed(m_hgs,args,30000,owner,[query](const QByteArray &out){ + QString parseError;const auto data=JsonParse::object(out,&parseError);QString error; const auto rows=data["sessions"].toArray(); - if(!doc.isObject()||data["version"].toInt()!=1||!data["sessions"].isArray()||rows.size()>50||!data["next_cursor"].isString()||data["next_cursor"].toString().size()>1024||!data["total"].isDouble()||data["total"].toDouble()50||!data["next_cursor"].isString()||data["next_cursor"].toString().size()>1024||!data["total"].isDouble()||data["total"].toDouble()1024||!data["position"].isDouble()||position<0||position!=qFloor(position)||position+rows.size()>data["total"].toDouble())error=tr("Invalid archive window. Refresh the archive."); + if(!previous.isString()||previous.toString().size()>1024||!data["position"].isDouble()||position<0||position!=qFloor(position)||position+rows.size()>data["total"].toDouble())error=HgsClient::tr("Invalid archive window. Refresh the archive."); } - for(const auto &value:rows){const auto row=value.toObject();const auto id=row["archive_id"].toString();if(row["state"]!="archived"||row["name"].toString().isEmpty()||QUuid(id).toString(QUuid::WithoutBraces)!=id||!row["archived_at"].isDouble()||row["archived_at"].toDouble()=query["until"].toDouble())error=tr("Invalid archived session identity. Refresh the archive.");} - finish(data,error); - },[finish](const QString &,const QString &error){finish({},error);},QJsonDocument(query).toJson(QJsonDocument::Compact)); + for(const auto &value:rows){const auto row=value.toObject();const auto id=row["archive_id"].toString();if(row["state"]!="archived"||row["name"].toString().isEmpty()||QUuid(id).toString(QUuid::WithoutBraces)!=id||!row["archived_at"].isDouble()||row["archived_at"].toDouble()=query["until"].toDouble())error=HgsClient::tr("Invalid archived session identity. Refresh the archive.");} + return Page{data,error}; + },[finish](const Page &page){finish(page.data,page.error);},[finish](const QString &,const QString &error){finish({},error);},QJsonDocument(query).toJson(QJsonDocument::Compact)); return request; } @@ -468,10 +530,9 @@ void HgsClient::requestSessionReference(const QString &host, const QString &name if (!host.isEmpty()) args.prepend('@' + host); if (!archive.isEmpty()) args << "--archive" << archive; if (!agent.isEmpty()) args << "--agent" << agent; - runHgs(m_hgs,args,host.isEmpty()?15000:25000,this,[this](const QByteArray &out){ - QJsonParseError error;const auto data=QJsonDocument::fromJson(out,&error); - const auto uri=data.object().value("uri").toString(); - if(error.error!=QJsonParseError::NoError || !data.isObject() || !uri.startsWith("zerus://session?") || uri.size()>8192) + runHgsParsed(m_hgs,args,host.isEmpty()?15000:25000,this,parseObject,[this](const ObjectReply &data){ + const auto uri=data.object.value("uri").toString(); + if(!data.ok() || !uri.startsWith("zerus://session?") || uri.size()>8192) emit sessionReferenceFailed(tr("Could not read a session reference. Update hgs on this computer.")); else emit sessionReferenceReady(uri); },[this](const QString &,const QString &error){emit sessionReferenceFailed(error);}); @@ -484,11 +545,10 @@ quint64 HgsClient::requestAccounts(const QString &host, const QStringList &argum if (!host.isEmpty()) args << QLatin1Char('@') + host; args << QStringLiteral("account"); args.append(arguments); // Start waits up to 20 seconds for OpenCode's registration on that computer. - runHgs(m_hgs, args, arguments.value(0) == "set-key" ? 90000 : arguments.value(0) == "copy" || arguments.value(0) == "start" ? 60000 : 25000, this, - [this, request, host](const QByteArray &bytes) { - QJsonParseError error; const auto data = QJsonDocument::fromJson(bytes, &error); - if (error.error != QJsonParseError::NoError || !data.isObject()) emit accountsFailed(request, host, tr("Invalid account catalog response")); - else emit accountsReady(request, host, data.object()); + runHgsParsed(m_hgs, args, arguments.value(0) == "set-key" ? 90000 : arguments.value(0) == "copy" || arguments.value(0) == "start" ? 60000 : 25000, this, + parseObject, [this, request, host](const ObjectReply &data) { + if (!data.ok()) emit accountsFailed(request, host, tr("Invalid account catalog response")); + else emit accountsReady(request, host, data.object); }, [this, request, host](const QString &, const QString &error) { emit accountsFailed(request, host, error); }, input); return request; } @@ -500,16 +560,21 @@ quint64 HgsClient::requestOpenCodeModels(const QString &host, const QString &loc if (!host.isEmpty()) args << QLatin1Char('@') + host; args << QStringLiteral("account") << QStringLiteral("models") << QStringLiteral("native-opencode") << QStringLiteral("--location") << location << QStringLiteral("--json"); - runHgs(m_hgs, args, 30000, this, - [this, request, host, location](const QByteArray &bytes) { + struct Catalog { bool tooLarge = false; ObjectReply reply; }; + runHgsParsed(m_hgs, args, 30000, this, + [](const QByteArray &bytes) { // hgs keeps the catalog within 4 MiB; a larger answer is not one. - if (bytes.size() > 5 * 1024 * 1024) { + Catalog catalog; catalog.tooLarge = bytes.size() > 5 * 1024 * 1024; + if (!catalog.tooLarge) catalog.reply = objectReply(bytes); + return catalog; + }, + [this, request, host, location](const Catalog &catalog) { + if (catalog.tooLarge) { emit openCodeModelsFailed(request, host, location, tr("The OpenCode model catalog is too large")); return; } - QJsonParseError error; const auto data = QJsonDocument::fromJson(bytes, &error); - if (error.error != QJsonParseError::NoError || !data.isObject()) emit openCodeModelsFailed(request, host, location, tr("Invalid OpenCode model catalog response")); - else emit openCodeModelsReady(request, host, location, data.object()); + if (!catalog.reply.ok()) emit openCodeModelsFailed(request, host, location, tr("Invalid OpenCode model catalog response")); + else emit openCodeModelsReady(request, host, location, catalog.reply.object); }, [this, request, host, location](const QString &, const QString &error) { emit openCodeModelsFailed(request, host, location, error); }); return request; } @@ -524,11 +589,11 @@ quint64 HgsClient::requestOpenCodeFavorite(const QString &host, const QString &v args << QStringLiteral("account") << QStringLiteral("favorite") << QStringLiteral("native-opencode") << QStringLiteral("--model") << value << QStringLiteral("--favorite") << (favorite ? QStringLiteral("on") : QStringLiteral("off")) << QStringLiteral("--request-id") << identity << QStringLiteral("--json"); - runHgs(m_hgs, args, 25000, this, - [this, request, host, value, favorite, identity](const QByteArray &bytes) { - QJsonParseError error; const auto data = QJsonDocument::fromJson(bytes, &error); const auto receipt = data.object(); + runHgsParsed(m_hgs, args, 25000, this, parseObject, + [this, request, host, value, favorite, identity](const ObjectReply &data) { + const auto &receipt = data.object; // The receipt must name this exact change. - if (error.error != QJsonParseError::NoError || !data.isObject() || receipt["status"] != "applied" + if (!data.ok() || receipt["status"] != "applied" || receipt["request_id"] != identity || receipt["model"] != value || receipt["favorite"] != favorite) emit openCodeFavoriteFailed(request, host, tr("Invalid favorite response")); else emit openCodeFavoriteReady(request, host, receipt); @@ -552,10 +617,9 @@ quint64 HgsClient::requestMobile(const QString &host, const QStringList &argumen emit mobileFailed(request, host, tr("Update Zerus on %1 to manage phones from here.").arg(host.isEmpty() ? tr("this computer") : host)); return; } - runHgs(m_hgs, args, 40000, this, [this, request, host](const QByteArray &bytes) { - QJsonParseError error; const auto data = QJsonDocument::fromJson(bytes, &error); - if (error.error != QJsonParseError::NoError || !data.isObject()) emit mobileFailed(request, host, tr("Invalid phone management response")); - else emit mobileReady(request, host, data.object()); + runHgsParsed(m_hgs, args, 40000, this, parseObject, [this, request, host](const ObjectReply &data) { + if (!data.ok()) emit mobileFailed(request, host, tr("Invalid phone management response")); + else emit mobileReady(request, host, data.object); }, failed); }, failed); return request; @@ -565,11 +629,10 @@ quint64 HgsClient::requestMachines(const QStringList &arguments) { const quint64 request = ++m_machineRequest; QStringList args{QStringLiteral("machine")}; args.append(arguments); - runHgs(m_hgs, args, arguments.value(0) == "check" ? 25000 : 8000, this, - [this, request](const QByteArray &bytes) { - QJsonParseError error; const auto data = QJsonDocument::fromJson(bytes, &error); - if (error.error != QJsonParseError::NoError || !data.isObject()) emit machinesFailed(request, tr("Invalid machine settings response")); - else emit machinesReady(request, data.object()); + runHgsParsed(m_hgs, args, arguments.value(0) == "check" ? 25000 : 8000, this, parseObject, + [this, request](const ObjectReply &data) { + if (!data.ok()) emit machinesFailed(request, tr("Invalid machine settings response")); + else emit machinesReady(request, data.object); }, [this, request](const QString &, const QString &error) { emit machinesFailed(request, error); }); return request; } @@ -579,11 +642,10 @@ quint64 HgsClient::requestSwarm(const QStringList &arguments, const QJsonObject const quint64 request = ++m_swarmRequest; QStringList args{QStringLiteral("swarm")}; args.append(arguments); const bool network = QStringList{"preview", "join", "sync", "bind"}.contains(arguments.value(0)); - runHgs(m_hgs, args, network ? 180000 : 10000, this, - [this, request](const QByteArray &bytes) { - QJsonParseError error; const auto data = QJsonDocument::fromJson(bytes, &error); - if (error.error != QJsonParseError::NoError || !data.isObject()) emit swarmFailed(request, tr("Invalid project catalog response")); - else emit swarmReady(request, data.object()); + runHgsParsed(m_hgs, args, network ? 180000 : 10000, this, parseObject, + [this, request](const ObjectReply &data) { + if (!data.ok()) emit swarmFailed(request, tr("Invalid project catalog response")); + else emit swarmReady(request, data.object); }, [this, request](const QString &, const QString &error) { emit swarmFailed(request, error); }, input.isEmpty() ? QByteArray() : QJsonDocument(input).toJson(QJsonDocument::Compact)); return request; @@ -602,7 +664,14 @@ quint64 HgsClient::requestTask(const QStringList &arguments, const QString &inpu call.maximumOutputBytes = 36 * 1024 * 1024; // stdin always closes, even empty: a command that reads a text never waits on the GUI. call.input = input.toUtf8(); - ProcessRunner::run(std::move(call), this, [this, request](const ProcessRunner::Result &result) { + ProcessRunner::run(std::move(call), this, [](ProcessRunner::Result result) { + ParsedResult parsed; + if (!result.outputLimitExceeded && result.outcome == ProcessRunner::Result::Finished && result.exitStatus == QProcess::NormalExit + && result.exitCode == 0) parsed = parsedResult(std::move(result)); + else parsed.result = std::move(result); + return parsed; + }, [this, request](const ParsedResult &parsed) { + const auto &result = parsed.result; if (result.outputLimitExceeded) return emit taskFailed(request, tr("Task response exceeds 36 MiB")); if (result.outcome == ProcessRunner::Result::TimedOut) return emit taskFailed(request, tr("hgs task timed out")); if (result.outcome == ProcessRunner::Result::FailedToStart) @@ -611,19 +680,8 @@ quint64 HgsClient::requestTask(const QStringList &arguments, const QString &inpu const QString detail = QString::fromLocal8Bit(result.standardError).trimmed().left(4000); return emit taskFailed(request, detail.isEmpty() ? tr("exit code %1").arg(result.exitCode) : detail); } - auto promise = std::make_shared>(); promise->start(); - auto *watcher = new QFutureWatcher(this); - connect(watcher, &QFutureWatcher::finished, this, [this, request, watcher] { - const auto data = watcher->result(); watcher->deleteLater(); - if (!data.isObject()) emit taskFailed(request, tr("Invalid task response")); - else emit taskReady(request, data.object()); - }); - watcher->setFuture(promise->future()); - static QThreadPool parser; - static const bool configured = [] { parser.setMaxThreadCount(2); return true; }(); Q_UNUSED(configured); - parser.start(QRunnable::create([promise, bytes = result.standardOutput] { - promise->addResult(QJsonDocument::fromJson(bytes)); promise->finish(); - })); + if (!parsed.reply.ok()) emit taskFailed(request, tr("Invalid task response")); + else emit taskReady(request, parsed.reply.object); }); return request; } @@ -634,11 +692,10 @@ quint64 HgsClient::requestSessionHost(const QString &action) // install copies hgs.exe and a host image into the profile; status asks a // running host, which answers within five seconds or counts as unavailable. const int timeoutMs = action == QLatin1String("install") ? 60000 : action == QLatin1String("status") ? 15000 : 25000; - runHgs(m_hgs, {QStringLiteral("session-host"), action, QStringLiteral("--json")}, timeoutMs, this, - [this, request](const QByteArray &bytes) { - QJsonParseError error; const auto data = QJsonDocument::fromJson(bytes, &error); - if (error.error != QJsonParseError::NoError || !data.isObject()) emit sessionHostFailed(request, tr("Invalid session host response")); - else emit sessionHostReady(request, data.object()); + runHgsParsed(m_hgs, {QStringLiteral("session-host"), action, QStringLiteral("--json")}, timeoutMs, this, parseObject, + [this, request](const ObjectReply &data) { + if (!data.ok()) emit sessionHostFailed(request, tr("Invalid session host response")); + else emit sessionHostReady(request, data.object); }, [this, request](const QString &, const QString &error) { emit sessionHostFailed(request, error); }); return request; } @@ -689,12 +746,14 @@ void HgsClient::requestLocal() emit stateReadStarted({}); // Large native histories can take longer than five seconds. Keep one poll // in flight and allow it to finish instead of continually killing/restarting it. - runHgs(m_hgs, {QStringLiteral("ls"), QStringLiteral("--json"), QStringLiteral("--local"), QStringLiteral("--no-archives")}, + struct Listing { BoxState box; QString error; }; + runHgsParsed(m_hgs, {QStringLiteral("ls"), QStringLiteral("--json"), QStringLiteral("--local"), QStringLiteral("--no-archives")}, 15000, this, - [this](const QByteArray &out) { + [](const QByteArray &out) { Listing listing; listing.box = parseBox(out, &listing.error); return listing; }, + [this](const Listing &listing) { m_localInFlight = false; - QString err; - const BoxState box = parseBox(out, &err); + const QString &err = listing.error; + const BoxState &box = listing.box; if (!err.isEmpty()) { emit stateReadFailed(QString(), err); emit failed(m_hgs + QStringLiteral(" ls --json --local"), err); @@ -718,27 +777,32 @@ void HgsClient::requestPeer(const QString &peer) // ssh-рукопожатие может занять больше времени, чем локальный вызов — у самого hgs // уже есть ConnectTimeout=3, но обвязка (auth, motd) укладывается не всегда в 5с. - runHgs(m_hgs, {QLatin1Char('@') + peer, QStringLiteral("ls"), QStringLiteral("--json"), QStringLiteral("--no-archives")}, + struct Listing { BoxState box; QString error; }; + runHgsParsed(m_hgs, {QLatin1Char('@') + peer, QStringLiteral("ls"), QStringLiteral("--json"), QStringLiteral("--no-archives")}, 15000, this, - [this, peer](const QByteArray &out) { + [peer](const QByteArray &out) { + Listing listing; listing.box = parseBox(out, &listing.error); + // The CLI forwards the peer's own listing, whose host is the name + // that machine gives itself (HGS_SELF or its host name). A machine + // saved under another alias must still be found, polled and + // addressed by that alias. + if (listing.error.isEmpty()) listing.box.reportedHost = std::exchange(listing.box.host, peer); + return listing; + }, + [this, peer](const Listing &listing) { // Снятие метки здесь и в onFailed ниже -- ровно те же два колбэка, которыми // runHgs() гарантированно завершает вызов РОВНО один раз (см. reported в // runHgs). Без этого упавший или зависший по таймауту пир навсегда // застревал бы в m_peerInFlight, и его строка в меню замерла бы на // "polling…" до перезапуска трея. m_peerInFlight.remove(peer); - QString err; - BoxState box = parseBox(out, &err); + const QString &err = listing.error; + const BoxState &box = listing.box; if (!err.isEmpty()) { emit stateReadFailed(peer, err); emit failed(m_hgs + QStringLiteral(" ls --json"), err); return; } - // The CLI forwards the peer's own listing, whose host is the name - // that machine gives itself (HGS_SELF or its host name). A machine - // saved under another alias must still be found, polled and - // addressed by that alias. - box.reportedHost = std::exchange(box.host, peer); emit peerReady(box); }, [this, peer](const QString &what, const QString &detail) { @@ -760,12 +824,11 @@ quint64 HgsClient::requestSearch(const QString &host, const QString &query) const auto id = ++m_searchRequest; QStringList args{"search", "--query", query, "--limit", "100"}; if (!host.isEmpty()) args.prepend('@' + host); - runHgs(m_hgs, args, host.isEmpty() ? 12000 : 20000, this, - [this, id, host](const QByteArray &out) { - QJsonParseError error; const auto doc = QJsonDocument::fromJson(out, &error); - if (error.error != QJsonParseError::NoError || !doc.isObject() || !doc.object().value("results").isArray()) + runHgsParsed(m_hgs, args, host.isEmpty() ? 12000 : 20000, this, parseObject, + [this, id, host](const ObjectReply &doc) { + if (!doc.ok() || !doc.object.value("results").isArray()) emit searchFailed(id, host, tr("Invalid search response; update hgs on this machine.")); - else emit searchReady(id, host, doc.object()); + else emit searchReady(id, host, doc.object); }, [this, id, host](const QString &, const QString &error) { emit searchFailed(id, host, error); }); return id; } @@ -779,10 +842,9 @@ quint64 HgsClient::requestProcess(const QString &host, const QString &name, cons if(!archive.isEmpty())args<<"--archive"< 256 * 1024 || result.isEmpty()) error = tr("Invalid automatic approval journal response."); @@ -924,16 +986,20 @@ void HgsClient::transferFile(quint64 id, const QString &host, const QString &pat if (cacheKey.isEmpty()) emit fileDownloadFailed(id, error); else finishCachedDownload(cacheKey, {}, error); }; - runHgs(m_hgs, {'@' + host, "download", "--path", path, "--output", destination}, 135000, this, - [this, id, host, path, destination, pending, cacheKey, failed](const QByteArray &out) { - if (out.size() > 65536) { failed(tr("Invalid file transfer response.")); return; } - const auto object = QJsonDocument::fromJson(out).object(); + enum class Transfer { Saved, InvalidResponse, Unconfirmed }; + runHgsParsed(m_hgs, {'@' + host, "download", "--path", path, "--output", destination}, 135000, this, + [host, path, destination](const QByteArray &out) { + if (out.size() > 65536) return Transfer::InvalidResponse; + const auto object = JsonParse::object(out); const QFileInfo saved(destination); if (object.value("host").toString() != host || object.value("path").toString() != path || object.value("local_path").toString() != destination || !object.value("bytes").isDouble() - || !LocalFile::readable(saved) || object.value("bytes").toDouble() != LocalFile::size(saved)) { - failed(tr("The file transfer did not confirm the saved local file.")); return; - } + || !LocalFile::readable(saved) || object.value("bytes").toDouble() != LocalFile::size(saved)) return Transfer::Unconfirmed; + return Transfer::Saved; + }, + [this, id, host, path, destination, pending, cacheKey, failed](Transfer transfer) { + if (transfer == Transfer::InvalidResponse) { failed(tr("Invalid file transfer response.")); return; } + if (transfer == Transfer::Unconfirmed) { failed(tr("The file transfer did not confirm the saved local file.")); return; } if (!pending) { emit fileDownloaded(id, destination); return; } DownloadCache::async(this, [pending, host, path] { return DownloadCache::publish(pending, path, -1, {host, path}); }, @@ -976,16 +1042,22 @@ quint64 HgsClient::requestAttachment(const QString &host, const QString &name, c if(!archive.isEmpty())args<<"--archive"<MaximumAttachmentBytes*4/3+8192){emit attachmentFailed(id,tr("Attachment response is too large."));return;} - const auto object=QJsonDocument::fromJson(out).object(); + struct Attachment { bool tooLarge=false, valid=false; QJsonObject metadata; QByteArray bytes; }; + runHgsParsed(m_hgs,args,host.isEmpty()?10000:45000,this,[file](const QByteArray &out){ + Attachment attachment; + if(out.size()>MaximumAttachmentBytes*4/3+8192){attachment.tooLarge=true;return attachment;} + const auto object=JsonParse::object(out); const auto encoded=object.value("data_base64").toString().toLatin1(); const auto decoded=QByteArray::fromBase64Encoding(encoded,QByteArray::AbortOnBase64DecodingErrors); - if(out.size()>MaximumAttachmentBytes*4/3+8192 || object.value("request_id")!=file.value("request_id") - || object.value("index")!=file.value("index") || !decoded || decoded.decoded.isEmpty() || decoded.decoded.size()>MaximumAttachmentBytes) { - emit attachmentFailed(id,tr("Invalid attachment response. Try again."));return; - } - auto metadata=object;metadata.remove("data_base64");emit attachmentReady(id,metadata,decoded.decoded); + if(object.value("request_id")!=file.value("request_id") + || object.value("index")!=file.value("index") || !decoded || decoded.decoded.isEmpty() || decoded.decoded.size()>MaximumAttachmentBytes) + return attachment; + attachment.valid=true;attachment.metadata=object;attachment.metadata.remove("data_base64");attachment.bytes=decoded.decoded; + return attachment; + },[this,id](const Attachment &attachment){ + if(attachment.tooLarge){emit attachmentFailed(id,tr("Attachment response is too large."));return;} + if(!attachment.valid){emit attachmentFailed(id,tr("Invalid attachment response. Try again."));return;} + emit attachmentReady(id,attachment.metadata,attachment.bytes); },[this,id](const QString &,const QString &error){emit attachmentFailed(id,error);}); return id; } @@ -1019,14 +1091,13 @@ void HgsClient::requestSubagentInspection(const QString &host, const QString &na QStringList args{"inspect", name, "--agent", agentId, "--skip-processes"}; if (!archiveId.isEmpty()) args << "--archive" << archiveId; if (!host.isEmpty()) args.prepend('@' + host); - runHgs(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, - [this,host,name,agentId,archiveId,key](const QByteArray &out) { + runHgsParsed(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, parseObject, + [this,host,name,agentId,archiveId,key](const ObjectReply &doc) { m_inspectionsInFlight.remove(key); emit inspectionLoadingChanged(host, name, archiveId, agentId, false); - QJsonParseError error; const auto doc = QJsonDocument::fromJson(out,&error); - if (error.error != QJsonParseError::NoError || !doc.isObject()) + if (!doc.ok()) emit subagentInspectionFailed(host,name,agentId,archiveId,tr("Invalid subagent activity response")); - else emit subagentInspectionReady(host,name,agentId,archiveId,doc.object()); + else emit subagentInspectionReady(host,name,agentId,archiveId,doc.object); }, [this,host,name,agentId,archiveId,key](const QString &,const QString &error) { m_inspectionsInFlight.remove(key); emit inspectionLoadingChanged(host, name, archiveId, agentId, false); @@ -1038,7 +1109,16 @@ void HgsClient::runWrite(const QString &op, const QStringList &args, int timeout bool sessionWrite, const QString &host) { auto request = processRequest(m_hgs, args); request.timeoutMs = timeoutMs; - ProcessRunner::run(std::move(request), this, [this, op, sessionWrite, host](const ProcessRunner::Result &result) { + struct Write { ProcessRunner::Result result; QJsonObject cleanup; int cleanups = 0; }; + ProcessRunner::run(std::move(request), this, [](ProcessRunner::Result result) { + Write write; + for (const auto &line : result.standardError.split('\n')) if (line.startsWith("ZERUS_WORKTREE_CLEANUP ")) { + write.cleanup = JsonParse::object(line.mid(23)); ++write.cleanups; + } + write.result = std::move(result); + return write; + }, [this, op, sessionWrite, host](const Write &write) { + const auto &result = write.result; // Any QProcess error (failed start, crash) reports its own text, as the // errorOccurred() that precedes finished() did before. const bool timedOut = result.outcome == ProcessRunner::Result::TimedOut; @@ -1047,10 +1127,7 @@ void HgsClient::runWrite(const QString &op, const QStringList &args, int timeout emit writeDone(op, ok, timedOut ? QStringLiteral("timed out") : error ? result.errorString : QString::fromLocal8Bit(result.standardError).trimmed()); if (ok && (op.startsWith("archive ") || op.startsWith("terminate "))) { - QJsonObject cleanup; int count = 0; - for (const auto &line : result.standardError.split('\n')) if (line.startsWith("ZERUS_WORKTREE_CLEANUP ")) { - cleanup = QJsonDocument::fromJson(line.mid(23)).object(); ++count; - } + const auto &cleanup = write.cleanup; const int count = write.cleanups; if (count == 1 && !cleanup["path"].toString().isEmpty() && !cleanup["common_dir"].toString().isEmpty()) { if (cleanup["status"] == "removed") invalidateWorktreeCache(host, cleanup["common_dir"].toString()); emit worktreeCleanupFinished(host, op, cleanup); @@ -1106,13 +1183,12 @@ quint64 HgsClient::requestNativeUi(const QString &host, const QString &name) { const auto request = ++m_nativeUiRequest; QStringList args{"native-ui", name, "--json"}; if (!host.isEmpty()) args.prepend('@' + host); - runHgs(m_hgs, args, host.isEmpty() ? 45000 : 60000, this, - [this, request](const QByteArray &out) { - const auto doc = QJsonDocument::fromJson(out); - if (!doc.isObject() || !doc.object().value("url").isString()) { + runHgsParsed(m_hgs, args, host.isEmpty() ? 45000 : 60000, this, parseObject, + [this, request](const ObjectReply &doc) { + if (!doc.ok() || !doc.object.value("url").isString()) { emit nativeUiFailed(request, tr("Invalid native UI connection response")); return; } - emit nativeUiReady(request, doc.object()); + emit nativeUiReady(request, doc.object); }, [this, request](const QString &, const QString &error) { emit nativeUiFailed(request, error); }); return request; } @@ -1243,10 +1319,12 @@ void HgsClient::requestProjects(const QString &host) { QStringList args{QStringLiteral("project"), QStringLiteral("ls"), QStringLiteral("--json")}; if (!host.isEmpty()) args.prepend(QLatin1Char('@') + host); - runHgs(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, - [this, host](const QByteArray &out) { - QString err; - const auto projects = parseProjects(out, &err); + struct Projects { QList projects; QString error; }; + runHgsParsed(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, + [](const QByteArray &out) { Projects parsed; parsed.projects = parseProjects(out, &parsed.error); return parsed; }, + [this, host](const Projects &parsed) { + const QString &err = parsed.error; + const auto &projects = parsed.projects; if (!err.isEmpty()) { emit projectsFailed(host, err); return; } if (host.isEmpty()) emit projectsReady(projects); emit projectsForHostReady(host, projects); @@ -1254,23 +1332,36 @@ void HgsClient::requestProjects(const QString &host) } namespace { -void trimWorktreeCache(QHash &cache) { - qsizetype bytes=0;for(const auto &entry:cache)bytes+=QJsonDocument(entry).toJson(QJsonDocument::Compact).size(); +// sizes holds each entry's reply size, measured when its reply was parsed, +// so trimming never serializes the cache on the GUI thread. +void trimWorktreeCache(QHash &cache,QHash &sizes) { + qsizetype bytes=0;for(auto i=cache.cbegin();i!=cache.cend();++i)bytes+=sizes.value(i.key()); while(cache.size()>64||bytes>16*1024*1024) { auto oldest=cache.begin(); for(auto i=cache.begin();i!=cache.end();++i)if(i.value()["received_at"].toDouble()64||QJsonDocument(snapshots).toJson(QJsonDocument::Compact).size()>WorktreeSnapshotsLimit){QString oldest;double time=std::numeric_limits::max();for(auto i=snapshots.begin();i!=snapshots.end();++i)if(i.value().toObject()["sampled_at"].toDouble()8*1024*1024)return {}; - return QJsonDocument::fromJson(bytes).object().value(worktreeCacheKey(m_hgs,host,path)).toObject(); + return SettingsJson::object(WorktreeSnapshotsKey,WorktreeSnapshotsLimit).value(worktreeCacheKey(m_hgs,host,path)).toObject(); } QJsonObject HgsClient::gitStatusSnapshot(const QString &host, const QString &path) const { @@ -1311,8 +1402,9 @@ void HgsClient::pumpGitStatus() }; QStringList args{"git-status", "--path", path, "--json"}; if (!host.isEmpty()) args.prepend('@' + host); - runHgs(m_hgs, args, host.isEmpty() ? 5000 : 10000, this, - [finish](const QByteArray &out) { finish(out.size() <= 32768 ? QJsonDocument::fromJson(out).object() : QJsonObject()); }, + runHgsParsed(m_hgs, args, host.isEmpty() ? 5000 : 10000, this, + [](const QByteArray &out) { return out.size() <= 32768 ? JsonParse::object(out) : QJsonObject(); }, + [finish](const QJsonObject &data) { finish(data); }, [finish](const QString &, const QString &) { finish({}); }); } } @@ -1322,8 +1414,7 @@ quint64 HgsClient::requestWorktrees(const QString &host,const QString &path,bool if(!expectedMachine.isEmpty()){ const auto request=++m_worktreeRequest;QStringList args{"worktrees","--creation","--path",path,"--refresh","--json"}; if(!host.isEmpty())args.prepend('@'+host);guardMachine(args,expectedMachine); - runHgs(m_hgs,args,15000,this,[this,request,host,path](const QByteArray &out){ - const auto data=QJsonDocument::fromJson(out).object(); + runHgsParsed(m_hgs,args,15000,this,[](const QByteArray &out){return JsonParse::object(out);},[this,request,host,path](const QJsonObject &data){ if(!data["worktrees"].isArray()||!data["state"].isString())emit worktreesFailed(request,host,path,tr("Invalid verified worktree catalog.")); else emit worktreesReady(request,host,path,data); },[this,request,host,path](const QString &,const QString &error){emit worktreesFailed(request,host,path,error);}); @@ -1346,24 +1437,23 @@ quint64 HgsClient::requestWorktrees(const QString &host,const QString &path,bool if(m_worktreesInFlight.contains(key)){m_worktreesInFlight[key].append({request,path});return request;} m_worktreesInFlight[key].append({request,path}); QStringList args{"worktrees","--path",path,"--json"};if(refresh)args<<"--refresh";if(!host.isEmpty())args.prepend('@'+host); - runHgs(m_hgs,args,host.isEmpty()?6000:15000,this,[this,host,key,repository,generation,hostKey,hostGeneration](const QByteArray &out){ + runHgsParsed(m_hgs,args,host.isEmpty()?6000:15000,this,[](const QByteArray &out){return out.size()<=2*1024*1024?JsonParse::object(out):QJsonObject();}, + [this,host,key,repository,generation,hostKey,hostGeneration](QJsonObject data){ const auto requests=m_worktreesInFlight.take(key); if(m_worktreeCacheGenerations.value(repository)!=generation||m_worktreeCacheGenerations.value(hostKey)!=hostGeneration) { for(const auto &r:requests)emit worktreesFailed(r.first,host,r.second,tr("Folder changed. Refresh its catalog."));return; } - auto data=out.size()<=2*1024*1024?QJsonDocument::fromJson(out).object():QJsonObject(); data["received_at"] = QDateTime::currentSecsSinceEpoch(); if(!data.value("worktrees").isArray()||!data.value("state").isString()){ for(const auto &r:requests)emit worktreesFailed(r.first,host,r.second,tr("Invalid worktree catalog response"));return; } - QSettings settings;auto snapshots=QJsonDocument::fromJson(settings.value("workspace/worktreeSnapshots").toByteArray()).object(); + auto snapshots=SettingsJson::object(WorktreeSnapshotsKey); auto store=[&](const QString &path,QJsonObject value){snapshots[worktreeCacheKey(m_hgs,host,path)]=value;}; for(const auto &r:requests)store(r.second,data); if(data["state"]=="ok")for(const auto &v:data["worktrees"].toArray()){ const auto root=v.toObject()["path"].toString();auto sibling=data;sibling["path"]=root;sibling["selected_root"]=root;store(root,sibling); } - while(snapshots.size()>64||QJsonDocument(snapshots).toJson(QJsonDocument::Compact).size()>8*1024*1024){QString oldest;double time=std::numeric_limits::max();for(auto i=snapshots.begin();i!=snapshots.end();++i)if(i.value().toObject()["sampled_at"].toDouble()0)args<<"--offset"<0&&data["file"].toObject()["offset"].toInt(-1)!=offset)) { - finish({},tr("Worktree inspection could not be verified. Refresh and try again."));return; - } - finish(data,{}); - },[finish](const QString &,const QString &error){finish({},error.contains("usage: hgs worktrees")?tr("Update Zerus on this computer to inspect worktree changes."):error);}); + ||(offset>0&&data["file"].toObject()["offset"].toInt(-1)!=offset)); + return inspection; + },[finish](const Inspection &inspection){ + if(!inspection.verified){finish({},tr("Worktree inspection could not be verified. Refresh and try again."),0);return;} + finish(inspection.data,{},inspection.bytes); + },[finish](const QString &,const QString &error){finish({},error.contains("usage: hgs worktrees")?tr("Update Zerus on this computer to inspect worktree changes."):error,0);}); return request; } @@ -1549,9 +1647,11 @@ quint64 HgsClient::removeWorktree(const QString &host,const QString &path,const if(preserveArchiveRestore)args<<"--preserve-archive-restore"; const auto forget=[this,host,commonDir]{invalidateWorktreeCache(host,commonDir);}; auto process=processRequest(m_hgs,args); process.timeoutMs=330000; process.maximumOutputBytes=65536; - ProcessRunner::run(std::move(process),this,[this,request,token,path,commonDir,host,forget](const ProcessRunner::Result &reply){ + ProcessRunner::run(std::move(process),this,[](ProcessRunner::Result result){return parsedResult(std::move(result));}, + [this,request,token,path,commonDir,host,forget](const ParsedResult &parsed){ + const auto &reply=parsed.result; forget(); - const auto data=QJsonDocument::fromJson(reply.standardOutput).object(); + const auto &data=parsed.reply.object; const bool finished=reply.outcome==ProcessRunner::Result::Finished && reply.exitStatus==QProcess::NormalExit; const bool identity=data["request_id"]==token && data["path"]==path; if(finished && reply.exitCode==0 && identity && (data["status"]=="removed" || data["status"]=="forgotten")) { @@ -1574,8 +1674,7 @@ quint64 HgsClient::requestCreationDirectory(const QString &host,const QString &p if(!verified&&!expectedMachine.isEmpty()){emit directoriesFailed(request,host,tr("The original computer identity cannot be verified by this CLI."));return;} QStringList args{"dirs"};if(verified)args<<"--creation";args<<(path.isEmpty()?QStringLiteral("~"):path); if(!host.isEmpty())args.prepend('@'+host);guardMachine(args,expectedMachine); - runHgs(m_hgs,args,host.isEmpty()?10000:20000,this,[this,request,host,verified](const QByteArray &out){ - const auto data=QJsonDocument::fromJson(out).object(); + runHgsParsed(m_hgs,args,host.isEmpty()?10000:20000,this,parseJsonObject,[this,request,host,verified](const QJsonObject &data){ if(!data["path"].isString()||!data["directories"].isArray()||(verified&&(!data["creation_identity_v1"].toBool()||QUuid(data["computer_id"].toString()).isNull()))) emit directoriesFailed(request,host,tr("Invalid creation directory identity.")); else emit directoriesReady(request,host,data); @@ -1592,15 +1691,13 @@ quint64 HgsClient::requestDirectories(const QString &host, const QString &path, if (hidden) args << QStringLiteral("--hidden"); args << (path.isEmpty() ? QStringLiteral("~") : path); if (!host.isEmpty()) args.prepend(QLatin1Char('@') + host); - runHgs(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, - [this, host, request](const QByteArray &out) { - QJsonParseError error; - const auto doc = QJsonDocument::fromJson(out, &error); - if (!doc.isObject() || !doc.object().value("path").isString() - || !doc.object().value("directories").isArray()) { + runHgsParsed(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, parseObject, + [this, host, request](const ObjectReply &doc) { + if (!doc.ok() || !doc.object.value("path").isString() + || !doc.object.value("directories").isArray()) { emit directoriesFailed(request, host, tr("Invalid directory response")); return; } - emit directoriesReady(request, host, doc.object()); + emit directoriesReady(request, host, doc.object); }, [this, host, request](const QString &, const QString &detail) { emit directoriesFailed(request, host, detail); }); @@ -1621,9 +1718,8 @@ quint64 HgsClient::createDirectory(const QString &host, const QString &path) QStringList args{"dirs", "--create", path, "--request-id", token}; if (!host.isEmpty()) args.prepend('@' + host); const auto expected = MachinePlatform::clean(parent + '/' + leaf, style); - runHgs(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, - [this, host, path, request, token, style, expected, parent](const QByteArray &bytes) { - const auto data = QJsonDocument::fromJson(bytes).object(); + runHgsParsed(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, + parseJsonObject,[this, host, path, request, token, style, expected, parent](const QJsonObject &data){ const auto confirmed = data.value("path").toString(); if (data["status"] != "created" || data["request_id"] != token || data["requested_path"] != path || !MachinePlatform::absolute(confirmed, style) || !MachinePlatform::same(confirmed, expected, style) @@ -1652,9 +1748,8 @@ quint64 HgsClient::createDirectory(const QString &host, const QString &path) QStringList args{"dirs", parentPath}; if (!host.isEmpty()) args.prepend('@' + host); // Pin the canonical parent before mkdir, retaining symlink aliases. - runHgs(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, - [this, host, request, style, mutate](const QByteArray &bytes) { - const auto data = QJsonDocument::fromJson(bytes).object(); + runHgsParsed(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, + parseJsonObject,[this, host, request, style, mutate](const QJsonObject &data){ const auto parent = data.value("path").toString(); if (!MachinePlatform::absolute(parent, style) || !data.value("directories").isArray()) { emit directoryCreationFailed(request, host, @@ -1731,9 +1826,10 @@ quint64 HgsClient::requestAnswerQuestion(const QString &host, const QString &nam if (!host.isEmpty()) args.prepend(QLatin1Char('@') + host); m_answersInFlight.insert(key); auto job = processRequest(program, args); job.input = payload; job.timeoutMs = host.isEmpty() ? 30000 : 45000; - ProcessRunner::run(std::move(job), this, + ProcessRunner::run(std::move(job), this, [](ProcessRunner::Result result) { return parsedResult(std::move(result), 65536); }, [this, request, host, name, key, requestId, questionId, questionHash, runId, conversationId, asyncQuestion, canSkip] - (const ProcessRunner::Result &result) { + (const ParsedResult &parsed) { + const auto &result = parsed.result; m_answersInFlight.remove(key); const auto fail = [this, request, host, name](const QString &detail, bool uncertain) { emit questionAnswerFailed(request, host, name, detail, uncertain); }; if (result.outcome == ProcessRunner::Result::FailedToStart) { fail(tr("Cannot start hgs: %1").arg(result.errorString), false); return; } @@ -1745,8 +1841,8 @@ quint64 HgsClient::requestAnswerQuestion(const QString &host, const QString &nam fail(errorText.isEmpty() ? tr("Answer transfer failed (exit %1).").arg(result.exitCode) : errorText, uncertain); return; } if (result.standardOutput.size() > 65536) { fail(tr("The answer acknowledgement is too large. Check Terminal before retrying."), true); return; } - QJsonParseError error; const auto doc = QJsonDocument::fromJson(result.standardOutput, &error); const auto receipt = doc.object(); - if (error.error != QJsonParseError::NoError || !doc.isObject() + const auto &receipt = parsed.reply.object; + if (!parsed.reply.ok() || (receipt.value("status") != "answered" && !(asyncQuestion && !canSkip && receipt.value("status") == "submitted" && !receipt.value("skipped").toBool())) || receipt.value("request_id").toString() != requestId || receipt.value("name").toString() != name || receipt.value("question_id").toString() != questionId || receipt.value("question_hash").toString() != questionHash @@ -1875,11 +1971,10 @@ quint64 HgsClient::requestSendMessage(const QString &host, const QString &name, const bool uncertain = status != QProcess::NormalExit || code != 1 || detail.contains(QStringLiteral("delivery uncertain"), Qt::CaseInsensitive); fail(detail.isEmpty() ? tr("Message transfer failed (exit %1).").arg(code) : detail, uncertain); - } else { - QJsonParseError error; - const auto document = QJsonDocument::fromJson(process->readAllStandardOutput(), &error); - const QJsonObject receipt = document.object(); - if (error.error != QJsonParseError::NoError || !document.isObject() + } else ProcessRunner::post(this, [bytes = process->readAllStandardOutput()] { return objectReply(bytes); }, + [this, fail, reported, request, host, name, key, messageId, expectedRunId, expectedConversationId, agentId](const ObjectReply &document) { + const QJsonObject &receipt = document.object; + if (!document.ok() || receipt.value("status").toString() != QLatin1String("submitted") || receipt.value("request_id").toString() != messageId || receipt.value("agent_id").toString() != agentId @@ -1895,7 +1990,7 @@ quint64 HgsClient::requestSendMessage(const QString &host, const QString &name, emit messageSent(request, host, name, receipt); emit sessionWriteDone(host); } - } + }); process->deleteLater(); }); m_messagesInFlight.insert(key); @@ -2010,8 +2105,8 @@ quint64 HgsClient::runSessionAction(const QString &host, const QString &name, co const auto error = QString::fromUtf8(process->readAllStandardError()).trimmed().left(2000); if (code || status != QProcess::NormalExit) fail(!error.isEmpty() ? error : scoped ? tr("Could not complete the request. Check the session before retrying.") : tr("Could not complete the request. Check Terminal.")); - else { - const auto result = QJsonDocument::fromJson(process->readAllStandardOutput()).object(); + else ProcessRunner::post(this, [bytes = process->readAllStandardOutput()] { return JsonParse::object(bytes); }, + [this, fail, reported, request, data, operation, name, host, scoped](const QJsonObject &result) { const auto state = result.value("status").toString(); const bool identity = result.value("request_id") == data.value("request_id") && result.value("name") == name && result.value("run_id") == data.value("expected_run_id") && result.value("conversation_id") == data.value("expected_conversation_id"); @@ -2027,7 +2122,7 @@ quint64 HgsClient::runSessionAction(const QString &host, const QString &name, co || (operation == "clear-context" && state == "confirmed"))) fail(scoped ? tr("Invalid acknowledgement. Check the session before retrying.") : tr("Invalid acknowledgement. Check Terminal before retrying.")); else { *reported = true; emit sessionActionFinished(request,true,result,{}); if (operation != "attachment") emit sessionWriteDone(host); } - } + }); } process->deleteLater(); }); @@ -2070,13 +2165,14 @@ quint64 HgsClient::requestQueueSendNow(const QString &host, const QString &name, if (code || status != QProcess::NormalExit) { const QString error = QString::fromUtf8(process->readAllStandardError()).trimmed().left(2000); fail(error.isEmpty() ? tr("Could not confirm sending the queue. Check Terminal before retrying.") : error); - } else { - QJsonParseError error; const auto doc = QJsonDocument::fromJson(process->readAllStandardOutput(), &error); const auto receipt = doc.object(); - if (error.error != QJsonParseError::NoError || receipt.value("status") != "submitted" || receipt.value("request_id") != requestId + } else ProcessRunner::post(this, [bytes = process->readAllStandardOutput()] { return objectReply(bytes); }, + [this, fail, reported, request, requestId, name, run, conversation, queue, host](const ObjectReply &doc) { + const auto &receipt = doc.object; + if (!doc.ok() || receipt.value("status") != "submitted" || receipt.value("request_id") != requestId || receipt.value("name") != name || receipt.value("run_id") != run || receipt.value("conversation_id") != conversation || receipt.value("queue_id") != queue) fail(tr("Invalid queue acknowledgement. Check Terminal before retrying.")); else { *reported = true; emit queueSendFinished(request, true, receipt, {}); emit sessionWriteDone(host); } - } + }); } process->deleteLater(); }); @@ -2119,13 +2215,14 @@ quint64 HgsClient::requestEffort(const QString &host, const QString &name, const if (code || status != QProcess::NormalExit) { const QString error = QString::fromUtf8(process->readAllStandardError()).trimmed().left(2000); fail(error.isEmpty() ? tr("Could not confirm effort. Check Terminal before retrying.") : error); - } else { - QJsonParseError error; const auto doc = QJsonDocument::fromJson(process->readAllStandardOutput(), &error); const auto receipt = doc.object(); - if (error.error != QJsonParseError::NoError || receipt.value("status") != "confirmed" || receipt.value("request_id") != requestId + } else ProcessRunner::post(this, [bytes = process->readAllStandardOutput()] { return objectReply(bytes); }, + [this, fail, reported, request, requestId, name, run, conversation, effort, host](const ObjectReply &doc) { + const auto &receipt = doc.object; + if (!doc.ok() || receipt.value("status") != "confirmed" || receipt.value("request_id") != requestId || receipt.value("name") != name || receipt.value("run_id") != run || receipt.value("conversation_id") != conversation || receipt.value("effort") != effort) fail(tr("Invalid effort acknowledgement. Check Terminal before retrying.")); else { *reported = true; emit effortFinished(request, true, receipt, {}); emit sessionWriteDone(host); } - } + }); } process->deleteLater(); }); @@ -2176,14 +2273,15 @@ quint64 HgsClient::requestSettings(const QString &host, const QString &name, con if (code || status != QProcess::NormalExit) { const QString error = QString::fromUtf8(process->readAllStandardError()).trimmed().left(2000); fail(error.isEmpty() ? unconfirmed : error); - } else { - QJsonParseError error; const auto doc = QJsonDocument::fromJson(process->readAllStandardOutput(), &error); const auto receipt = doc.object(); - if (error.error != QJsonParseError::NoError || (receipt.value("status") != "applied" && receipt.value("status") != "scheduled") || receipt.value("request_id") != requestId + } else ProcessRunner::post(this, [bytes = process->readAllStandardOutput()] { return objectReply(bytes); }, + [this, fail, reported, request, requestId, name, run, conversation, model, effort, host, invalid](const ObjectReply &doc) { + const auto &receipt = doc.object; + if (!doc.ok() || (receipt.value("status") != "applied" && receipt.value("status") != "scheduled") || receipt.value("request_id") != requestId || receipt.value("name") != name || receipt.value("run_id") != run || receipt.value("conversation_id") != conversation || (!model.isEmpty() && receipt.value("model") != model) || (!effort.isEmpty() && receipt.value("effort") != effort)) fail(invalid); else { *reported = true; emit settingsFinished(request, true, receipt, {}); emit sessionWriteDone(host); } - } + }); } process->deleteLater(); }); @@ -2232,14 +2330,14 @@ quint64 HgsClient::requestNotifications(const QString &host, const QString &name job.input = QJsonDocument(payload).toJson(QJsonDocument::Compact); job.timeoutMs = host.isEmpty() ? 15000 : 35000; job.maximumOutputBytes = 512 * 1024; // Includes the bounded 256-child suppression baseline. - ProcessRunner::run(std::move(job), this, [this, request, payload, expected](const ProcessRunner::Result &reply) { - QJsonParseError parse; - const auto document = QJsonDocument::fromJson(reply.standardOutput, &parse); - const auto result = document.object(), preferences = result.value("preferences").toObject(); + ProcessRunner::run(std::move(job), this, [](ProcessRunner::Result result) { return parsedResult(std::move(result)); }, + [this, request, payload, expected](const ParsedResult &parsed) { + const auto &reply = parsed.result; + const auto result = parsed.reply.object, preferences = result.value("preferences").toObject(); const auto revision = SessionNotifications::revision(preferences.value("revision")); const bool applied = result.value("applied").toBool(); const bool confirmed = reply.outcome == ProcessRunner::Result::Finished && reply.exitStatus == QProcess::NormalExit - && reply.exitCode == 0 && parse.error == QJsonParseError::NoError && document.isObject() + && reply.exitCode == 0 && parsed.reply.ok() && SessionNotifications::keys(result, {"ok", "applied", "run_id", "conversation_id", "preferences"}) && result.value("ok").isBool() && result.value("ok").toBool() && result.value("applied").isBool() && result.value("run_id") == payload.value("run_id") && result.value("conversation_id") == payload.value("conversation_id") @@ -2278,21 +2376,25 @@ quint64 HgsClient::requestRead(const QString &host, const QString &name, const Q }); connect(process, &QProcess::finished, process, [this, process, timer, fail, reported, request, payload](int code, QProcess::ExitStatus status) { timer->stop(); - if (!*reported) { - QJsonParseError parse; const auto result = QJsonDocument::fromJson(process->readAllStandardOutput(), &parse).object(); + if (!*reported) ProcessRunner::post(this, + [bytes = process->readAllStandardOutput(), errorText = QString::fromUtf8(process->readAllStandardError()).trimmed().left(2000)] { + ObjectReply reply = objectReply(bytes); return std::make_pair(reply, errorText); + }, + [fail, reported, request, payload, code, status, this](const std::pair &prepared) { + const auto &result = prepared.first.object; const auto applied = result.value("applied").toObject(); const bool notFound = code == 1 && result == QJsonObject{{"ok", false}, {"error", "not_found"}}; const bool answered = code == 0 && result.value("ok") == true && applied.value("reply_id").isBool() && applied.value("attention_signature").isBool() && (result.value("read").isNull() || result.value("read").toObject().value("conversation_id") == payload.value("conversation_id")); - if (status == QProcess::NormalExit && parse.error == QJsonParseError::NoError && (notFound || answered)) { + if (status == QProcess::NormalExit && prepared.first.ok() && (notFound || answered)) { *reported = true; emit readFinished(request, true, result, {}); } else { - const auto error = QString::fromUtf8(process->readAllStandardError()).trimmed().left(2000); + const auto &error = prepared.second; fail(error.isEmpty() ? tr("Could not save read state on that machine.") : error); } - } + }); process->deleteLater(); }); timer->start(host.isEmpty() ? 15000 : 35000); process->start(); return request; @@ -2315,13 +2417,15 @@ quint64 HgsClient::repositoryOperation(const QString &host,const QString &action guardMachine(args,expectedMachine); auto process=processRequest(m_hgs,args);process.input=QByteArray();process.maximumOutputBytes=65536; process.timeoutMs=action=="clone"?650000:30000; - ProcessRunner::run(std::move(process),this,[this,request,action,name,token,host](const ProcessRunner::Result &result){ + ProcessRunner::run(std::move(process),this,[](ProcessRunner::Result result){return parsedResult(std::move(result));}, + [this,request,action,name,token,host](const ParsedResult &parsed){ + const auto &result=parsed.result; if(result.outcome!=ProcessRunner::Result::Finished||result.exitStatus!=QProcess::NormalExit||result.exitCode!=0){ auto error=QString::fromUtf8(result.standardError).trimmed().left(4000); if(error.isEmpty())error=tr("The computer did not confirm the repository operation. Check its result before trying again."); emit repositoryFinished(request,false,{},error);return; } - const auto data=QJsonDocument::fromJson(result.standardOutput).object(); + const auto &data=parsed.reply.object; const auto style=host.isEmpty()&&MachinePlatform::windowsShaped(data["parent"].toString())?MachinePlatform::Style::Windows:MachinePlatform::Style::Posix; const auto status=data["status"].toString(); const auto expected=MachinePlatform::clean(data["parent"].toString()+'/'+name,style); diff --git a/tray/src/HgsClient.h b/tray/src/HgsClient.h index 9ae92a1d5..46b2e9bd3 100644 --- a/tray/src/HgsClient.h +++ b/tray/src/HgsClient.h @@ -298,6 +298,8 @@ class HgsClient : public QObject { quint64 requestArchivePage(const QString &host, const QJsonObject &query); static QList parseSessions(const QJsonArray &rows); static BoxState parseBox(const QByteArray &json, QString *error); + // A listing that is already an object, as tests and prepared values hold it. + static BoxState box(const QJsonObject &object); static QList parseFleet(const QByteArray &json, QString *error); static QList parseProjects(const QByteArray &json, QString *error); @@ -450,6 +452,10 @@ class HgsClient : public QObject { QHash> m_reviewsInFlight; QHash m_worktreeReviews; QHash m_worktreeInspections; + // Reply sizes of cached reviews and inspections, and each inspection's + // {host, path, common dir} scope, so trimming and invalidation never parse. + QHash m_worktreeReviewBytes, m_worktreeInspectionBytes; + QHash m_worktreeInspectionScopes; QHash> m_worktreeInspectionsInFlight; QHash m_worktreeCacheGenerations; quint64 m_nativeUiRequest = 0; diff --git a/tray/src/IdentityHash.h b/tray/src/IdentityHash.h new file mode 100644 index 000000000..93d2d615a --- /dev/null +++ b/tray/src/IdentityHash.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include +#include +#include + +// SHA-256 of a short identity, remembered: rows hash the same identities on +// every listing, and on macOS each QCryptographicHash call set up an OpenSSL +// digest (3.9 % of the GUI thread in a 2026-10-27 sample). Bounded; any thread. +namespace IdentityHash { +inline QByteArray sha256(const QByteArray &input) +{ + static QMutex mutex; + static QHash cache; + { + QMutexLocker lock(&mutex); + if (const auto found = cache.constFind(input); found != cache.cend()) return *found; + } + const QByteArray digest = QCryptographicHash::hash(input, QCryptographicHash::Sha256); + QMutexLocker lock(&mutex); + if (cache.size() >= 16384) cache.clear(); + cache.insert(input, digest); + return digest; +} +} diff --git a/tray/src/JsonParse.cpp b/tray/src/JsonParse.cpp new file mode 100644 index 000000000..eaeedb06a --- /dev/null +++ b/tray/src/JsonParse.cpp @@ -0,0 +1,93 @@ +#include "JsonParse.h" + +#include +#include +#include +#include +#include + +#include + +namespace JsonParse { +namespace { + +std::atomic s_violations{0}; +QMutex s_mutex; +std::function s_handler; +QSet s_reported; + +bool fatalChecks() +{ + static const bool fatal = qEnvironmentVariable("ZERUS_GUI_THREAD_CHECKS") == QLatin1String("fatal"); + return fatal; +} + +void violation(const char *file, int line) +{ + ++s_violations; + QMutexLocker lock(&s_mutex); + if (s_handler) { const auto handler = s_handler; lock.unlock(); handler(file, line); return; } + if (fatalChecks()) qFatal("JSON parsed on the GUI thread at %s:%d (JsonParse.h)", file, line); + const QByteArray site = QByteArray(file) + ':' + QByteArray::number(line); + if (s_reported.size() < 256 && !s_reported.contains(site)) { + s_reported.insert(site); + qWarning("JSON parsed on the GUI thread at %s", site.constData()); + } +} + +QJsonDocument decode(const QByteArray &bytes, QString *error) +{ + QJsonParseError parseError; + QJsonDocument document = QJsonDocument::fromJson(bytes, &parseError); + if (error) *error = parseError.error == QJsonParseError::NoError ? QString() : parseError.errorString(); + return document; +} + +} + +bool onGuiThread() +{ + const auto *application = QCoreApplication::instance(); + return application && QThread::currentThread() == application->thread() && application->inherits("QGuiApplication"); +} + +QJsonDocument parse(const QByteArray &bytes, QString *error, const char *file, int line) +{ + if (onGuiThread()) violation(file, line); + return decode(bytes, error); +} + +QJsonObject object(const QByteArray &bytes, QString *error, const char *file, int line) +{ + QString parseError; const auto document = parse(bytes, &parseError, file, line); + if (parseError.isEmpty() && !document.isObject()) parseError = QStringLiteral("expected a JSON object"); + if (error) *error = parseError; + return parseError.isEmpty() ? document.object() : QJsonObject(); +} + +QJsonArray array(const QByteArray &bytes, QString *error, const char *file, int line) +{ + QString parseError; const auto document = parse(bytes, &parseError, file, line); + if (parseError.isEmpty() && !document.isArray()) parseError = QStringLiteral("expected a JSON array"); + if (error) *error = parseError; + return parseError.isEmpty() ? document.array() : QJsonArray(); +} + +QJsonDocument local(const QByteArray &bytes, QString *error, const char *file, int line) +{ + if (bytes.size() > LocalLimit && onGuiThread()) violation(file, line); + return decode(bytes, error); +} + +QJsonObject localObject(const QByteArray &bytes, const char *file, int line) { return local(bytes, nullptr, file, line).object(); } +QJsonArray localArray(const QByteArray &bytes, const char *file, int line) { return local(bytes, nullptr, file, line).array(); } + +int violations() { return s_violations; } + +void setViolationHandler(std::function handler) +{ + QMutexLocker lock(&s_mutex); + s_handler = std::move(handler); +} + +} diff --git a/tray/src/JsonParse.h b/tray/src/JsonParse.h new file mode 100644 index 000000000..1ebb1aeb7 --- /dev/null +++ b/tray/src/JsonParse.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +// The tray's only JSON parser: scripts/ci/check-source.py rejects +// QJsonDocument::fromJson elsewhere in tray/src. Data from hgs, the event +// stream and files is parsed off the GUI thread, in a ProcessRunner prepare +// step or on another worker, and the GUI thread applies the finished value. +// +// A parse() on the application thread of a GUI application is a violation. +// It is counted and logged once per call site; with +// ZERUS_GUI_THREAD_CHECKS=fatal, which every Qt suite sets, it aborts with that +// call site. A command-line or test program without a GUI has no GUI thread. +// +// local() is the one exception, for small values the tray wrote itself +// (preferences, drag data) that a widget needs synchronously. It is limited to +// LocalLimit bytes, and check-source.py lists the files allowed to call it and +// how often; that list only shrinks. +namespace JsonParse { + +constexpr qsizetype LocalLimit = 64 * 1024; + +// error, which may be null, receives the parse error or is cleared. object() +// and array() also report a document of the other kind as an error. +QJsonDocument parse(const QByteArray &bytes, QString *error = nullptr, + const char *file = __builtin_FILE(), int line = __builtin_LINE()); +QJsonObject object(const QByteArray &bytes, QString *error = nullptr, + const char *file = __builtin_FILE(), int line = __builtin_LINE()); +QJsonArray array(const QByteArray &bytes, QString *error = nullptr, + const char *file = __builtin_FILE(), int line = __builtin_LINE()); + +QJsonDocument local(const QByteArray &bytes, QString *error = nullptr, + const char *file = __builtin_FILE(), int line = __builtin_LINE()); +QJsonObject localObject(const QByteArray &bytes, const char *file = __builtin_FILE(), int line = __builtin_LINE()); +QJsonArray localArray(const QByteArray &bytes, const char *file = __builtin_FILE(), int line = __builtin_LINE()); + +// True on the application thread of a GUI application. +bool onGuiThread(); + +// Tests: violations so far, and a handler that replaces logging and the fatal +// check (an empty handler restores them). +int violations(); +void setViolationHandler(std::function handler); + +} diff --git a/tray/src/LinuxAttentionNotifier.cpp b/tray/src/LinuxAttentionNotifier.cpp index fd786eddb..54e5d5fe1 100644 --- a/tray/src/LinuxAttentionNotifier.cpp +++ b/tray/src/LinuxAttentionNotifier.cpp @@ -1,3 +1,4 @@ +#include "SettingsJson.h" #include "LinuxAttentionNotifier.h" #include #include @@ -20,7 +21,7 @@ LinuxAttentionNotifier::LinuxAttentionNotifier(const QDBusConnection &bus) : m_b if (m_bus.interface()) m_owner = m_bus.interface()->serviceOwner(service).value(); QSettings settings; if (!m_owner.isEmpty() && settings.value("attention/linuxOwner").toString() == m_owner) { - const auto saved = QJsonDocument::fromJson(settings.value("attention/linuxTokens").toByteArray()).object(); + const auto saved = SettingsJson::object("attention/linuxTokens"); for (auto it = saved.begin(); it != saved.end(); ++it) m_tokens[it.key().toUInt()] = it.value().toString(); } m_bus.connect(service, "/org/freedesktop/Notifications", service, "ActionInvoked", this, SLOT(action(uint,QString))); diff --git a/tray/src/NewSessionDialog.cpp b/tray/src/NewSessionDialog.cpp index 25d6c9629..93cb6ea51 100644 --- a/tray/src/NewSessionDialog.cpp +++ b/tray/src/NewSessionDialog.cpp @@ -1,3 +1,4 @@ +#include "SettingsJson.h" #include "NewSessionDialog.h" #include "RepositoryClone.h" #include "WorkspaceIcons.h" @@ -470,7 +471,7 @@ NewSessionDialog::NewSessionDialog(const QString &hgsPath,const FleetState &flee m_creationCatalog["worktrees"]=entries;m_verifiedWorktrees[host()+'\n'+path]=m_creationCatalog; m_autoPreferred=false;selectPath(path);launch(path,m_creationFolder); }); - setGroups(SessionOrganization(QJsonDocument::fromJson(QSettings().value("workspace/organization").toByteArray()).object()),project); + setGroups(SessionOrganization(SettingsJson::object("workspace/organization")),project); } void NewSessionDialog::reject(){if(!m_creation&&(!m_repository||!m_repository->cloning())){m_cloneLaunchPending=false;if(m_repository)m_repository->cancelCheck();QDialog::reject();}} QString NewSessionDialog::host() const {return m_machine->currentData().toString();} @@ -845,7 +846,7 @@ void NewSessionDialog::launch(const QString &target,const QString &addFolder) // The next dialog starts with this agent and, per machine and agent, this account. settings.setValue("workspace/launchAgent",agent()); if(const auto selectedAccount=account();!selectedAccount.isEmpty()){ - auto accounts=QJsonDocument::fromJson(settings.value("workspace/launchAccounts").toByteArray()).object(); + auto accounts=SettingsJson::object("workspace/launchAccounts"); auto machine=accounts.value(host()).toObject();machine[agent()]=selectedAccount;accounts[host()]=machine; settings.setValue("workspace/launchAccounts",QJsonDocument(accounts).toJson(QJsonDocument::Compact)); } @@ -893,7 +894,7 @@ void NewSessionDialog::updateAccounts() } m_accountGroups=AccountCatalog::group(localProfiles); } - const auto remembered=QJsonDocument::fromJson(QSettings().value("workspace/launchAccounts").toByteArray()).object().value(host()).toObject(); + const auto remembered=SettingsJson::object("workspace/launchAccounts").value(host()).toObject(); const auto current=m_account->currentItem()?m_account->currentItem()->data(ProfileRole).toJsonObject():QJsonObject(); const auto currentId=current["host"].toString()==host()&¤t["provider"]==m_preferredAgent?current["id"].toString():QString(); QList profiles;QList members; @@ -918,7 +919,7 @@ void NewSessionDialog::updateAccounts() const bool dark=m_account->property("hgsDark").toBool(); const QColor iconColor(dark?"#a1adbb":"#647386"); int defaultIndex=-1,lastIndex=-1,explicitIndex=-1,currentIndex=-1; - const auto lastAccount=QJsonDocument::fromJson(QSettings().value("workspace/launchAccounts").toByteArray()).object() + const auto lastAccount=SettingsJson::object("workspace/launchAccounts") .value(host()).toObject().value(m_preferredAgent).toString(); for(int row=0;row prepare; }; struct Pending { QPointer owner; - std::function done; + std::function apply; QMetaObject::Connection destroyed; }; +std::atomic s_preparationThread{nullptr}; + class Runner { public: Runner() @@ -297,18 +300,38 @@ class Runner { m_worker = new QObject; m_worker->moveToThread(&m_thread); QObject::connect(&m_thread, &QThread::finished, m_worker, &QObject::deleteLater); m_thread.start(); + // One serial thread keeps results in the order their children finished. + m_prepareThread.setObjectName(QStringLiteral("hgs-prepare")); + m_preparer = new QObject; m_preparer->moveToThread(&m_prepareThread); + QObject::connect(&m_prepareThread, &QThread::finished, m_preparer, &QObject::deleteLater); + s_preparationThread = &m_prepareThread; + m_prepareThread.start(); } - void run(Request request, QObject *owner, std::function done) + void run(Request request, QObject *owner, std::function prepare, std::function apply) { const quint64 id = ++m_next; - Pending pending{owner, std::move(done), {}}; + Pending pending{owner, std::move(apply), {}}; // Owner destruction kills the child synchronously, as the old per-object // QProcess children did, so fixtures and sockets are not used afterwards. pending.destroyed = QObject::connect(owner, &QObject::destroyed, &m_front, [this, id] { cancel(id); }, Qt::DirectConnection); m_pending.insert(id, std::move(pending)); - QMetaObject::invokeMethod(m_worker, [this, id, request = std::move(request)]() mutable { start(id, std::move(request)); }, - Qt::QueuedConnection); + QMetaObject::invokeMethod(m_worker, [this, id, request = std::move(request), prepare = std::move(prepare)]() mutable { + start(id, std::move(request), std::move(prepare)); + }, Qt::QueuedConnection); + } + void post(QObject *owner, std::function prepare, std::function apply) + { + const quint64 id = ++m_next; + Pending pending{owner, std::move(apply), {}}; + pending.destroyed = QObject::connect(owner, &QObject::destroyed, &m_front, [this, id] { m_pending.remove(id); }, + Qt::DirectConnection); + m_pending.insert(id, std::move(pending)); + QMetaObject::invokeMethod(m_preparer, [this, id, prepare = std::move(prepare)]() mutable { + Prepared value = prepare(); + QMetaObject::invokeMethod(&m_front, [this, id, value = std::move(value)]() mutable { deliver(id, std::move(value)); }, + Qt::QueuedConnection); + }, Qt::QueuedConnection); } void stop() { @@ -321,14 +344,18 @@ class Runner { qDeleteAll(m_jobs); m_jobs.clear(); }, Qt::BlockingQueuedConnection); m_thread.quit(); m_thread.wait(); + // Preparations already queued only compute values; nothing applies them. + m_prepareThread.quit(); m_prepareThread.wait(); + s_preparationThread = nullptr; } QThread *thread() { return &m_thread; } + QThread *preparationThread() { return &m_prepareThread; } void setObserver(std::function observer) { QMutexLocker lock(&m_observerMutex); m_observer = std::move(observer); } private: - void start(quint64 id, Request request) // worker thread + void start(quint64 id, Request request, std::function prepare) // worker thread { - auto *job = new Job; m_jobs.insert(id, job); + auto *job = new Job; m_jobs.insert(id, job); job->prepare = std::move(prepare); auto *process = job->process = new Child(m_worker); process->setProgram(request.program); process->setArguments(request.arguments); if (request.environment) process->setProcessEnvironment(*request.environment); @@ -411,7 +438,11 @@ class Runner { if (job->reported) return; job->reported = true; Result result = outcome == Result::TimedOut ? Result{} : job->result; result.outcome = outcome; - QMetaObject::invokeMethod(&m_front, [this, id, result = std::move(result)] { deliver(id, result); }, Qt::QueuedConnection); + QMetaObject::invokeMethod(m_preparer, [this, id, prepare = std::move(job->prepare), result = std::move(result)]() mutable { + Prepared value = prepare(std::move(result)); + QMetaObject::invokeMethod(&m_front, [this, id, value = std::move(value)]() mutable { deliver(id, std::move(value)); }, + Qt::QueuedConnection); + }, Qt::QueuedConnection); } void finish(quint64 id, Job *job) // worker thread { @@ -428,13 +459,13 @@ class Runner { if (process->state() != QProcess::NotRunning) { process->kill(); process->waitForFinished(1000); } delete process; } - void deliver(quint64 id, const Result &result) // application thread + void deliver(quint64 id, Prepared value) // application thread { auto it = m_pending.find(id); if (it == m_pending.end()) return; Pending pending = std::move(*it); m_pending.erase(it); QObject::disconnect(pending.destroyed); - if (pending.owner) pending.done(result); + if (pending.owner) pending.apply(std::move(value)); } void cancel(quint64 id) // application thread, while the owner is destroyed { @@ -446,8 +477,8 @@ class Runner { }, Qt::BlockingQueuedConnection); } - QThread m_thread; - QObject *m_worker = nullptr; + QThread m_thread, m_prepareThread; + QObject *m_worker = nullptr, *m_preparer = nullptr; QObject m_front; // receives results on the application thread QHash m_pending; QHash m_jobs; @@ -479,13 +510,27 @@ Runner *runner() } -void run(Request request, QObject *owner, std::function done) +void runPrepared(Request request, QObject *owner, std::function prepare, std::function apply) { Q_ASSERT(owner && owner->thread() == QThread::currentThread()); // After the application's post routines nothing can receive a result. - if (auto *instance = runner()) instance->run(std::move(request), owner, std::move(done)); + if (auto *instance = runner()) instance->run(std::move(request), owner, std::move(prepare), std::move(apply)); +} + +void run(Request request, QObject *owner, std::function done) +{ + runPrepared(std::move(request), owner, [](Result result) -> Prepared { return std::make_shared(std::move(result)); }, + [done = std::move(done)](Prepared value) { done(*std::static_pointer_cast(value)); }); +} + +void postPrepared(QObject *owner, std::function prepare, std::function apply) +{ + Q_ASSERT(owner && owner->thread() == QThread::currentThread()); + if (auto *instance = runner()) instance->post(owner, std::move(prepare), std::move(apply)); } +bool isPreparationThread() { return QThread::currentThread() == s_preparationThread.load(); } + #ifdef Q_OS_MACOS const char *launchBackend() { return "posix_spawn"; } #else @@ -493,6 +538,7 @@ const char *launchBackend() { return "QProcess"; } #endif QThread *workerThread() { auto *instance = runner(); return instance ? instance->thread() : nullptr; } +QThread *preparationThread() { auto *instance = runner(); return instance ? instance->preparationThread() : nullptr; } void setLaunchObserver(std::function observer) { if (auto *instance = runner()) instance->setObserver(std::move(observer)); } diff --git a/tray/src/ProcessRunner.h b/tray/src/ProcessRunner.h index a3b11e33a..f46c2df7c 100644 --- a/tray/src/ProcessRunner.h +++ b/tray/src/ProcessRunner.h @@ -7,7 +7,10 @@ #include #include +#include #include +#include +#include class QObject; class QThread; @@ -16,6 +19,11 @@ class QThread; // the large GUI process, pipe I/O and child reaping then never stall painting: // the GUI thread only posts a request and later receives one result. Polls start // about one hgs child per second, and on macOS each fork() cost ~4 ms there. +// +// Every result then passes one shared preparation thread before it reaches its +// owner. A caller with a prepare step parses and shapes the output there, so the +// owner's thread only applies a finished value. Results keep the order in which +// their children finished, with or without a prepare step. namespace ProcessRunner { struct Request { @@ -49,8 +57,46 @@ struct Result { // kills and reaps its running children before returning and drops their results. void run(Request request, QObject *owner, std::function done); -// Tests: the worker thread, and a hook run there right before each start. +// A prepared value, type-erased between the two threads. +using Prepared = std::shared_ptr; +void runPrepared(Request request, QObject *owner, std::function prepare, + std::function apply); + +// As run(), but prepare(Result) runs on the preparation thread and its value goes +// to apply on owner's thread. prepare must not touch owner or any other +// application-thread object: it sees only the result and what it captured by +// value. A destroyed owner still drops the value instead of applying it. +template +void run(Request request, QObject *owner, Prepare prepare, Apply apply) +{ + using Value = std::decay_t>; + runPrepared(std::move(request), owner, + [prepare = std::move(prepare)](Result result) mutable -> Prepared { + return std::make_shared(prepare(std::move(result))); + }, + [apply = std::move(apply)](Prepared value) mutable { apply(std::move(*std::static_pointer_cast(value))); }); +} + +// Prepares data that did not come from a child of run(), such as a direct +// QProcess result or a file, the same way: prepare() on the preparation thread, +// then apply(value) on owner's thread unless owner is destroyed first. +void postPrepared(QObject *owner, std::function prepare, std::function apply); +template +void post(QObject *owner, Prepare prepare, Apply apply) +{ + using Value = std::decay_t>; + postPrepared(owner, + [prepare = std::move(prepare)]() mutable -> Prepared { return std::make_shared(prepare()); }, + [apply = std::move(apply)](Prepared value) mutable { apply(std::move(*std::static_pointer_cast(value))); }); +} + +// True on the shared preparation thread. +bool isPreparationThread(); + +// Tests: the worker and preparation threads, and a hook run on the worker +// right before each start. QThread *workerThread(); +QThread *preparationThread(); // Tests: how children start, "posix_spawn" on macOS and "QProcess" elsewhere. const char *launchBackend(); void setLaunchObserver(std::function observer); diff --git a/tray/src/ProjectsDialog.cpp b/tray/src/ProjectsDialog.cpp index 34268f5b8..ede032bc7 100644 --- a/tray/src/ProjectsDialog.cpp +++ b/tray/src/ProjectsDialog.cpp @@ -1,3 +1,4 @@ +#include "SettingsJson.h" #include "ProjectsDialog.h" #include "CloneRepositoryDialog.h" #include "WorktreePanel.h" @@ -90,7 +91,7 @@ class ProjectDelegate : public QStyledItemDelegate { ProjectsDialog::ProjectsDialog(HgsClient *client, QWidget *parent, bool embedded, SessionOrganization *projects) : QDialog(parent), m_client(client), - m_ownedProjects(QJsonDocument::fromJson(QSettings().value("workspace/organization").toByteArray()).object()), + m_ownedProjects(SettingsJson::object("workspace/organization")), m_projects(projects ? projects : &m_ownedProjects) { setObjectName("projectsPage"); setWindowTitle(tr("Projects")); resize(850, 560); @@ -309,7 +310,10 @@ void ProjectsDialog::showProject() } void ProjectsDialog::changed() { - if (m_projects == &m_ownedProjects) QSettings().setValue("workspace/organization",QJsonDocument(m_projects->toJson()).toJson(QJsonDocument::Compact)); + if (m_projects == &m_ownedProjects) { + const QJsonDocument document(m_projects->toJson()); const auto bytes = document.toJson(QJsonDocument::Compact); + QSettings().setValue("workspace/organization", bytes); SettingsJson::remember("workspace/organization", bytes, document); + } emit organizationChanged(); refresh(); m_status->setText(tr("Saved")); } void ProjectsDialog::selectProject(const QString &id) diff --git a/tray/src/QuestionReply.h b/tray/src/QuestionReply.h index 989559bf2..9cfeaec59 100644 --- a/tray/src/QuestionReply.h +++ b/tray/src/QuestionReply.h @@ -1,5 +1,6 @@ #pragma once +#include "JsonParse.h" #include #include #include @@ -19,9 +20,9 @@ inline QJsonArray parse(QString text) const QString start = ""; const QString end = ""; if (!text.startsWith(start) || !text.endsWith(end)) return {}; - QJsonParseError error; - const auto document = QJsonDocument::fromJson(text.mid(start.size(), text.size() - start.size() - end.size()).toUtf8(), &error); - if (error.error != QJsonParseError::NoError) return {}; + QString error; + const auto document = JsonParse::local(text.mid(start.size(), text.size() - start.size() - end.size()).toUtf8(), &error); + if (!error.isEmpty()) return {}; const auto replies = document.isObject() ? QJsonArray{document.object()} : document.array(); for (const auto &value : replies) { if (!value.isObject()) return {}; diff --git a/tray/src/ReadMarks.h b/tray/src/ReadMarks.h index 88fbe4199..960e6b479 100644 --- a/tray/src/ReadMarks.h +++ b/tray/src/ReadMarks.h @@ -1,5 +1,6 @@ #pragma once +#include "SettingsJson.h" #include #include #include @@ -51,7 +52,7 @@ struct ReadMark { // restart keeps retrying; the root agent and the workspace window share it. namespace ReadMarkStore { inline QJsonObject load() { - return QJsonDocument::fromJson(QSettings().value("attention/pendingReads").toByteArray()).object(); + return SettingsJson::object("attention/pendingReads"); } inline void save(const QJsonObject &marks) { QSettings().setValue("attention/pendingReads", QJsonDocument(marks).toJson(QJsonDocument::Compact)); @@ -73,7 +74,7 @@ inline void remove(const QList &marks) { save(stored); } inline QJsonArray migratedHosts() { - return QJsonDocument::fromJson(QSettings().value("attention/sharedReadMigrated").toByteArray()).array(); + return SettingsJson::array("attention/sharedReadMigrated"); } inline bool migrated(const QString &host) { return migratedHosts().contains(host); } inline void setMigrated(const QString &host) { diff --git a/tray/src/RecoverySettings.h b/tray/src/RecoverySettings.h index afcd8f61d..c04023316 100644 --- a/tray/src/RecoverySettings.h +++ b/tray/src/RecoverySettings.h @@ -1,4 +1,5 @@ #pragma once +#include "SettingsJson.h" #include "HgsClient.h" #include #include @@ -20,7 +21,7 @@ class RecoverySync : public QObject { public: std::function changed; explicit RecoverySync(const QString &executable,QObject *parent):QObject(parent),client(executable,this) { - best=QJsonDocument::fromJson(QSettings().value("recovery/sharedPolicy").toByteArray()).object(); + best=SettingsJson::object("recovery/sharedPolicy"); if(!best.value("policy").isObject()||best.value("order").toArray().size()!=2)best={}; connect(&client,&HgsClient::recoveryFinished,this,[this](quint64 id,bool ok,const QJsonObject &data,const QString &error){ if(!pending.contains(id))return;const auto request=pending.take(id); diff --git a/tray/src/RelayConfiguration.h b/tray/src/RelayConfiguration.h index e9a7e0a67..ca6262e6c 100644 --- a/tray/src/RelayConfiguration.h +++ b/tray/src/RelayConfiguration.h @@ -1,4 +1,5 @@ #pragma once +#include "JsonParse.h" #include #include #include @@ -210,8 +211,8 @@ class ConfigStore { struct stat after{};const bool stable=::fstat(fd,&after)==0&&after.st_size==st.st_size&&after.st_mtime==st.st_mtime; ::close(fd); if(n<0||!stable||bytes.size()>65536)return fail(QStringLiteral("Configuration changed while reading. Reload before saving.")); - QJsonParseError parse{};const auto doc=QJsonDocument::fromJson(bytes,&parse); - if(parse.error!=QJsonParseError::NoError||!doc.isObject())return fail(QStringLiteral("Connector configuration is not a JSON object.")); + QString parse;const auto doc=JsonParse::local(bytes,&parse); + if(!parse.isEmpty()||!doc.isObject())return fail(QStringLiteral("Connector configuration is not a JSON object.")); *out={true,doc.object(),bytes,quint64(st.st_dev),quint64(st.st_ino)};return true; } #endif diff --git a/tray/src/RepositoryClone.cpp b/tray/src/RepositoryClone.cpp index cec545030..277216325 100644 --- a/tray/src/RepositoryClone.cpp +++ b/tray/src/RepositoryClone.cpp @@ -1,3 +1,4 @@ +#include "SettingsJson.h" #include "RepositoryClone.h" #include "HgsClient.h" #include @@ -29,7 +30,7 @@ RepositoryClone::RepositoryClone(HgsClient *client,const QString &host,MachinePl { connect(m_client,&HgsClient::repositoryFinished,this,&RepositoryClone::finish); m_parent=QSettings().value(settingsKey()+"/parent","~").toString(); - const auto saved=QJsonDocument::fromJson(QSettings().value(settingsKey()+"/pending").toByteArray()).object(); + const auto saved=SettingsJson::object(settingsKey()+"/pending"); if(!saved.isEmpty()){ m_url=saved["url"].toString();m_parent=saved["parent"].toString();m_name=saved["name"].toString(); m_token=saved["request_id"].toString();m_expected=saved["path"].toString();m_uncertain=true; diff --git a/tray/src/SessionCreationController.cpp b/tray/src/SessionCreationController.cpp index c8bb0bec0..302ca565b 100644 --- a/tray/src/SessionCreationController.cpp +++ b/tray/src/SessionCreationController.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "SessionCreationController.h" #include "MachinePlatform.h" #include @@ -67,9 +68,8 @@ SessionCreationController::SessionCreationController(const QString &hgs,QObject if(file.exists()){ if(!file.open(QIODevice::ReadOnly)||file.size()>JournalLimit)error=tr("Could not read the creation journal. Existing requests are kept."); else{ - QJsonParseError parse;const auto document=QJsonDocument::fromJson(file.readAll(),&parse); - saved=document.object(); - if(parse.error!=QJsonParseError::NoError||saved.value("schema")!=1||!saved.value("operations").isObject() + QString parse;saved=JsonParse::object(file.readAll(),&parse); + if(!parse.isEmpty()||saved.value("schema")!=1||!saved.value("operations").isObject() ||saved.value("operations").toObject().size()>128)error=tr("Unsupported creation journal. Existing requests are kept."); } } @@ -243,7 +243,7 @@ void SessionCreationController::persist(const QString &id,std::function QSettings settings;settings.setValue("workspace/launchOpenTerminal",choices.value("terminal").toBool()); settings.setValue("workspace/launchAgent",choices.value("cmd").toString()); if(!choices.value("account").toString().isEmpty()){ - auto accounts=QJsonDocument::fromJson(settings.value("workspace/launchAccounts").toByteArray()).object(); + auto accounts=JsonParse::object(settings.value("workspace/launchAccounts").toByteArray()); auto machine=accounts[choices.value("host").toString()].toObject();machine[choices.value("cmd").toString()]=choices.value("account"); accounts[choices.value("host").toString()]=machine;settings.setValue("workspace/launchAccounts",QJsonDocument(accounts).toJson(QJsonDocument::Compact)); } diff --git a/tray/src/SessionList.cpp b/tray/src/SessionList.cpp index 3c593b08e..0c7b70705 100644 --- a/tray/src/SessionList.cpp +++ b/tray/src/SessionList.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "SessionList.h" #include "SessionCardDelegate.h" #include @@ -455,7 +456,7 @@ QMimeData *SessionList::mimeData(const QList &items) const bool SessionList::validDrop(const QMimeData *mime) const { if (!mime->hasFormat(Mime)) return false; - const auto data = QJsonDocument::fromJson(mime->data(Mime)).object(); + const auto data = JsonParse::localObject(mime->data(Mime)); if (data.value("token").toString() != m_token || data.value("id").toString().isEmpty()) return false; const int role = data.value("group").toBool() ? SessionRoles::Group : SessionRoles::Identity; for (int row = 0; row < count(); ++row) if (item(row)->data(role).toString() == data.value("id").toString()) return true; @@ -540,7 +541,7 @@ void SessionList::dragMoveEvent(QDragMoveEvent *e) if (!validDrop(e->mimeData())) { e->ignore(); return; } // Retain Qt's edge auto-scroll; the base drop mutation is deliberately unused. QListWidget::dragMoveEvent(e); - const auto target = placement(e->position().toPoint(), QJsonDocument::fromJson(e->mimeData()->data(Mime)).object().value("group").toBool()); + const auto target = placement(e->position().toPoint(), JsonParse::localObject(e->mimeData()->data(Mime)).value("group").toBool()); m_indicator = target.indicator; m_onGroup = target.onGroup; viewport()->update(); e->setDropAction(Qt::MoveAction); e->accept(); } @@ -562,7 +563,7 @@ void SessionList::dropEvent(QDropEvent *e) return; } if (!validDrop(e->mimeData())) { e->ignore(); return; } - const auto data = QJsonDocument::fromJson(e->mimeData()->data(Mime)).object(); + const auto data = JsonParse::localObject(e->mimeData()->data(Mime)); const auto target = placement(e->position().toPoint(), data.value("group").toBool()); m_indicator = {}; viewport()->update(); if (data.value("group").toBool()) emit groupMoved(data.value("id").toString(), target.before); diff --git a/tray/src/SessionRowPreferences.h b/tray/src/SessionRowPreferences.h index 1b629a704..351caeff5 100644 --- a/tray/src/SessionRowPreferences.h +++ b/tray/src/SessionRowPreferences.h @@ -1,7 +1,7 @@ #pragma once #include "HgsClient.h" -#include +#include "IdentityHash.h" #include #include #include @@ -21,7 +21,7 @@ class SessionRowPreferences { const QJsonArray identity = session.conversationId.isEmpty() || fallback ? QJsonArray{host, session.cmd, session.archiveId, session.name, session.created} : QJsonArray{host, session.cmd, session.archiveId, session.conversationId}; - return QCryptographicHash::hash(QJsonDocument(identity).toJson(QJsonDocument::Compact), QCryptographicHash::Sha256); + return IdentityHash::sha256(QJsonDocument(identity).toJson(QJsonDocument::Compact)); } bool collapsed(const QString &host, const SessionInfo &session) const { return m_collapsed.contains(key(host, session)); diff --git a/tray/src/SessionsWindow.cpp b/tray/src/SessionsWindow.cpp index 21410bb4e..255f8e233 100644 --- a/tray/src/SessionsWindow.cpp +++ b/tray/src/SessionsWindow.cpp @@ -1,3 +1,5 @@ +#include "SettingsJson.h" +#include "JsonParse.h" #include "ArchiveBrowser.h" #include "NativeSessionCapabilities.h" #include "SessionsWindow.h" @@ -306,11 +308,11 @@ SessionsWindow::SessionsWindow(const QString &hgsPath, QWidget *parent) m_expandedSessions = QSet(expanded.begin(), expanded.end()); m_sessionRowPreferences = SessionRowPreferences(settings.value("workspace/collapsedSessionRows").toByteArray()); m_savedOrganization = settings.value("workspace/organization").toByteArray(); - const auto savedProjects = QJsonDocument::fromJson(m_savedOrganization).object(); + const auto savedProjects = SettingsJson::object("workspace/organization"); if (!savedProjects.isEmpty() && !savedProjects.contains("projects") && !settings.contains("workspace/organizationBeforeProjects")) settings.setValue("workspace/organizationBeforeProjects", m_savedOrganization); m_organization = SessionOrganization(savedProjects); - m_pendingLaunches = QJsonDocument::fromJson(settings.value("workspace/pendingLaunches").toByteArray()).object(); + m_pendingLaunches = SettingsJson::object("workspace/pendingLaunches"); m_creations=new SessionCreationController(m_client.executable(),this); if (!settings.value("sessions/geometry").toByteArray().isEmpty()) restoreGeometry(settings.value("sessions/geometry").toByteArray()); auto *root = new QHBoxLayout(this); root->setContentsMargins(0, 0, 0, 0); root->setSpacing(0); @@ -1406,7 +1408,7 @@ SessionsWindow::SessionsWindow(const QString &hgsPath, QWidget *parent) m_inspector->addTab(m_children, tr("Plan && agents")); m_inspector->addTab(detailsPage, tr("Details")); m_inspector->addTab(m_worktrees, tr("Worktrees")); const auto taskLaunchBytes = settings.value("tasks/pendingExecutorLaunches").toByteArray(); - const auto taskLaunchDoc = taskLaunchBytes.size() <= 65536 ? QJsonDocument::fromJson(taskLaunchBytes) : QJsonDocument(); + const auto taskLaunchDoc = taskLaunchBytes.size() <= 65536 ? JsonParse::local(taskLaunchBytes) : QJsonDocument(); m_taskLaunches = taskLaunchDoc.object(); bool validLaunches = taskLaunchBytes.isEmpty() || taskLaunchDoc.isObject(); if (m_taskLaunches.size() > 16) validLaunches = false; @@ -2498,6 +2500,7 @@ void SessionsWindow::saveOrganization() const auto data = QJsonDocument(m_organization.toJson()).toJson(QJsonDocument::Compact); if (data == m_savedOrganization) return; QSettings settings; settings.setValue("workspace/organization", data); settings.sync(); + SettingsJson::remember("workspace/organization", data, QJsonDocument(m_organization.toJson())); if (settings.status() == QSettings::NoError) m_savedOrganization = data; else showNotice(tr("Could not save the session layout. Check the settings file permissions."), true); } @@ -2687,7 +2690,7 @@ void SessionsWindow::setFleet(const FleetState &fleet) { if (auto *launch=qobject_cast(QApplication::activeModalWidget()); launch && launch->parentWidget()==this) launch->setFleet(fleet); m_settingsPage->setLocalSessions(fleet.local().localSessions, LocalSessionHost::canEnable(fleet.local())); m_settingsPage->setPeers(fleet.peerNames()); m_settingsPage->setMobileGateway(fleet.local().localSessions||MachinePlatform::of(fleet.local().metrics,true)==MachinePlatform::Style::Windows); - m_fleet = fleet; m_fleet.setAttentionMarks(QJsonDocument::fromJson(QSettings().value("attention/sessionMarks").toByteArray()).object()); m_fleet.setPendingReads(ReadMarkStore::load()); reconcileTerminations();m_usageStore->setFleet(fleet); m_accountsPage->setFleet(fleet); updateDashboard(); m_projectsPage->setFleet(fleet); m_machinesPage->setFleet(fleet); if (isVisible()) rebuild(); applyQueuedModelSettings(); + m_fleet = fleet; m_fleet.setAttentionMarks(SettingsJson::object("attention/sessionMarks")); m_fleet.setPendingReads(ReadMarkStore::load()); reconcileTerminations();m_usageStore->setFleet(fleet); m_accountsPage->setFleet(fleet); updateDashboard(); m_projectsPage->setFleet(fleet); m_machinesPage->setFleet(fleet); if (isVisible()) rebuild(); applyQueuedModelSettings(); updateConnectionStatus(); if (!m_nativeLaunchToOpen.isEmpty()) { for (const auto &entry : m_entries) if (entry.session.launchId==m_nativeLaunchToOpen) { diff --git a/tray/src/SettingsJson.cpp b/tray/src/SettingsJson.cpp new file mode 100644 index 000000000..5f1133d0a --- /dev/null +++ b/tray/src/SettingsJson.cpp @@ -0,0 +1,61 @@ +#include "SettingsJson.h" + +#include "JsonParse.h" +#include "ProcessRunner.h" + +#include +#include +#include +#include + +namespace SettingsJson { +namespace { + +struct Entry { + QByteArray bytes; + QJsonDocument document; +}; + +QMutex s_mutex; +QHash s_entries; + +QJsonDocument read(const QString &key, qsizetype maximumBytes = 0) +{ + const QByteArray bytes = QSettings().value(key).toByteArray(); + if (maximumBytes > 0 && bytes.size() > maximumBytes) return {}; + { + QMutexLocker lock(&s_mutex); + const auto found = s_entries.constFind(key); + if (found != s_entries.cend() && found->bytes == bytes) return found->document; + } + const QJsonDocument document = JsonParse::local(bytes); + QMutexLocker lock(&s_mutex); + s_entries.insert(key, {bytes, document}); + return document; +} + +} + +QJsonObject object(const QString &key, qsizetype maximumBytes) { return read(key, maximumBytes).object(); } +QJsonArray array(const QString &key) { return read(key).array(); } + +void remember(const QString &key, const QByteArray &bytes, const QJsonDocument &document) +{ + QMutexLocker lock(&s_mutex); + s_entries.insert(key, {bytes, document}); +} + +void warm(QObject *owner, const QStringList &keys) +{ + ProcessRunner::post(owner, [keys] { + for (const auto &key : keys) { + const QByteArray bytes = QSettings().value(key).toByteArray(); + const QJsonDocument document = JsonParse::parse(bytes); + QMutexLocker lock(&s_mutex); + s_entries.insert(key, {bytes, document}); + } + return true; + }, [](bool) {}); +} + +} diff --git a/tray/src/SettingsJson.h b/tray/src/SettingsJson.h new file mode 100644 index 000000000..1eb711bd7 --- /dev/null +++ b/tray/src/SettingsJson.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include +#include +#include + +class QObject; + +// JSON values the desktop keeps in QSettings, parsed once per change: a read +// whose stored bytes equal the previous read's returns the parsed value again +// without parsing. warm() parses keys on the preparation thread +// (ProcessRunner.h) ahead of their first read, so the GUI thread normally +// parses only values it has just changed. A parse that is still needed on the +// GUI thread goes through JsonParse::local() and its size limit. +namespace SettingsJson { + +// A stored value over maximumBytes (0: any size) reads as empty. +QJsonObject object(const QString &key, qsizetype maximumBytes = 0); +QJsonArray array(const QString &key); +void warm(QObject *owner, const QStringList &keys); +// A writer that has just stored bytes, serialized from document, under key +// records them, so the next read needs no parse. +void remember(const QString &key, const QByteArray &bytes, const QJsonDocument &document); + +} diff --git a/tray/src/SwarmController.cpp b/tray/src/SwarmController.cpp index 45e85e85e..167cc4781 100644 --- a/tray/src/SwarmController.cpp +++ b/tray/src/SwarmController.cpp @@ -1,3 +1,4 @@ +#include "SettingsJson.h" #include "SwarmController.h" #include #include @@ -77,8 +78,8 @@ void SwarmController::start(const QJsonObject &organization) if(m_started)return;m_started=true;m_desired=organization; QSettings settings; if(!settings.contains("workspace/organizationBeforeSwarm"))settings.setValue("workspace/organizationBeforeSwarm",QJsonDocument(organization).toJson(QJsonDocument::Compact)); - m_base=QJsonDocument::fromJson(settings.value("swarm/observedOrganization").toByteArray()).object(); - m_versions=QJsonDocument::fromJson(settings.value("swarm/observedVersions").toByteArray()).object(); + m_base=SettingsJson::object("swarm/observedOrganization"); + m_versions=SettingsJson::object("swarm/observedVersions"); m_ready=!m_base.isEmpty(); if(!m_ready)m_base=organization; m_poll.start(); @@ -119,6 +120,8 @@ void SwarmController::persist() QSettings settings; settings.setValue("swarm/observedOrganization", organization); settings.setValue("swarm/observedVersions", versions); + SettingsJson::remember("swarm/observedOrganization", organization, QJsonDocument(m_base)); + SettingsJson::remember("swarm/observedVersions", versions, QJsonDocument(m_versions)); settings.sync(); if (settings.status() == QSettings::NoError) { m_persistedOrganization = organization; diff --git a/tray/src/TaskDrag.h b/tray/src/TaskDrag.h index b0ca945be..4ce9873fe 100644 --- a/tray/src/TaskDrag.h +++ b/tray/src/TaskDrag.h @@ -1,4 +1,5 @@ #pragma once +#include "JsonParse.h" #include "TaskBoard.h" #include #include @@ -32,7 +33,7 @@ struct TaskDrag { } static std::optional from(const QMimeData *mime) { if (!mime || !mime->hasFormat(mimeType())) return std::nullopt; - const auto data = QJsonDocument::fromJson(mime->data(mimeType())).object(); + const auto data = JsonParse::localObject(mime->data(mimeType())); TaskDrag drag{data.value("project").toString(), data.value("uuid").toString(), data.value("id").toString(), data.value("stage").toString(), data.value("host").toString(), data.value("session").toString(), data.value("known").toBool(true)}; diff --git a/tray/src/TaskInspector.cpp b/tray/src/TaskInspector.cpp index 4d5b33fd4..6a68b2953 100644 --- a/tray/src/TaskInspector.cpp +++ b/tray/src/TaskInspector.cpp @@ -1,3 +1,4 @@ +#include "SettingsJson.h" #include "TaskInspector.h" #include #include "ActivityWidth.h" @@ -1026,7 +1027,7 @@ QString TaskInspector::inputKey(const QJsonObject &task) const { } void TaskInspector::restoreNewCard() { QSettings settings; const auto key = "tasks/newCard/" + m_project; - m_creation = QJsonDocument::fromJson(settings.value(key + "/request").toByteArray()).object(); + m_creation = SettingsJson::object(key + "/request"); m_newTitle->setText(m_creation.isEmpty() ? settings.value(key + "/title").toString() : m_creation.value("title").toString()); m_newTitle->setReadOnly(!m_creation.isEmpty()); m_creationHeader->setCurrentWidget(m_newTitle->text().isEmpty() ? static_cast(m_scope) : m_newTitle); m_add->setEnabled(!m_project.isEmpty()); m_createError->hide(); diff --git a/tray/src/TaskMessageOutbox.h b/tray/src/TaskMessageOutbox.h index 50e18ffac..073138501 100644 --- a/tray/src/TaskMessageOutbox.h +++ b/tray/src/TaskMessageOutbox.h @@ -1,4 +1,5 @@ #pragma once +#include "JsonParse.h" #include "ComposerDraftStore.h" #include #include @@ -23,7 +24,7 @@ inline Request read(const QString &file) { const QFileInfo info(file); QFile input(file); if (info.isSymLink() || !info.isFile() || info.size() > 32 * 1024 * 1024 || !input.open(QIODevice::ReadOnly)) return {{}, QObject::tr("The original message recovery snapshot is unavailable. This check sent nothing.")}; - const auto record = QJsonDocument::fromJson(input.readAll()).object(); + const auto record = JsonParse::parse(input.readAll()).object(); if (QUuid(record.value("id").toString()).isNull() || !record.value("intent").isObject()) return {{}, QObject::tr("The message recovery snapshot is invalid. Nothing was submitted.")}; return {record, {}}; diff --git a/tray/src/TrayAgent.cpp b/tray/src/TrayAgent.cpp index cb965eb4b..c976ae385 100644 --- a/tray/src/TrayAgent.cpp +++ b/tray/src/TrayAgent.cpp @@ -1,4 +1,6 @@ #include "TrayAgent.h" +#include "ProcessRunner.h" +#include "SettingsJson.h" #include "QuestionNotificationSettings.h" #include "AutomaticApprovals.h" @@ -144,7 +146,7 @@ TrayAgent::TrayAgent(const AppConfig &cfg, QObject *parent) , m_notifier(makeAttentionNotifier()) #endif , m_badge(makeApplicationBadge()) - , m_attention(QJsonDocument::fromJson(QSettings().value("attention/observed").toByteArray()).object()) + , m_attention(SettingsJson::object("attention/observed")) , m_localTimer(this) , m_peerTimer(this) { @@ -284,6 +286,13 @@ TrayAgent::TrayAgent(const AppConfig &cfg, QObject *parent) pollAllKnownPeers(m_client, m_state); }); + // Parse the large saved values on the preparation thread before their first use. + SettingsJson::warm(this, {QStringLiteral("attention/readReplies"), QStringLiteral("attention/sessionMarks"), + QStringLiteral("attention/pendingReads"), QStringLiteral("attention/sharedReadMigrated"), + QStringLiteral("workspace/organization"), QStringLiteral("workspace/pendingLaunches"), + QStringLiteral("workspace/launchAccounts"), QStringLiteral("swarm/observedOrganization"), + QStringLiteral("swarm/observedVersions"), QStringLiteral("workspace/worktreeSnapshots")}); + auto *discover = new QProcess(this); discover->setProgram(cfg.hgsPath.isEmpty() ? QStringLiteral("hgs") : cfg.hgsPath); discover->setArguments({QStringLiteral("ls"), QStringLiteral("--json")}); @@ -328,10 +337,13 @@ TrayAgent::TrayAgent(const AppConfig &cfg, QObject *parent) *done = true; if (status == QProcess::NormalExit && exitCode == 0) { - QString err; - const QList fleet = - HgsClient::parseFleet(discover->readAllStandardOutput(), &err); - if (err.isEmpty()) { + // The whole fleet's listing is parsed off the GUI thread (ProcessRunner.h). + struct Fleet { QList boxes; QString error; }; + ProcessRunner::post(this, [bytes = discover->readAllStandardOutput()] { + Fleet parsed; parsed.boxes = HgsClient::parseFleet(bytes, &parsed.error); return parsed; + }, [this](const Fleet &parsed) { + const QList &fleet = parsed.boxes; + if (parsed.error.isEmpty()) { // The first box is this computer (onLocalReady brings its state) and // its "peers" are the aliases to poll; requestPeer() below brings their // sessions. A reachable peer's own entry carries the name it reports @@ -347,6 +359,7 @@ TrayAgent::TrayAgent(const AppConfig &cfg, QObject *parent) m_peerTimer.start(); pollAllKnownPeers(m_client, m_state); } + }); } discover->deleteLater(); }); @@ -369,8 +382,8 @@ TrayAgent::~TrayAgent() void TrayAgent::onLocalReady(const BoxState &box) { m_localReadError = box.ok ? QString() : (box.error.isEmpty() ? tr("Could not read local sessions") : box.error); - m_state.setReadReplies(QJsonDocument::fromJson(QSettings().value("attention/readReplies").toByteArray()).object()); - m_state.setAttentionMarks(QJsonDocument::fromJson(QSettings().value("attention/sessionMarks").toByteArray()).object()); + m_state.setReadReplies(SettingsJson::object("attention/readReplies")); + m_state.setAttentionMarks(SettingsJson::object("attention/sessionMarks")); m_state.setPendingReads(ReadMarkStore::load()); m_lastError.clear(); const auto previousPeers = m_state.peerNames(); @@ -404,8 +417,8 @@ void TrayAgent::onPeerReady(const BoxState &box) { // box.host is the alias this answer was polled under (see HgsClient::requestPeer). if (m_sessionsWindow) m_sessionsWindow->setPollingHosts(m_client.pollingHosts()); - m_state.setReadReplies(QJsonDocument::fromJson(QSettings().value("attention/readReplies").toByteArray()).object()); - m_state.setAttentionMarks(QJsonDocument::fromJson(QSettings().value("attention/sessionMarks").toByteArray()).object()); + m_state.setReadReplies(SettingsJson::object("attention/readReplies")); + m_state.setAttentionMarks(SettingsJson::object("attention/sessionMarks")); m_state.setPendingReads(ReadMarkStore::load()); if (m_state.local().peersKnown && !m_state.local().peers.contains(box.host)) return; m_lastError.clear(); diff --git a/tray/src/UpdateController.cpp b/tray/src/UpdateController.cpp index 9b4112d91..098ae367b 100644 --- a/tray/src/UpdateController.cpp +++ b/tray/src/UpdateController.cpp @@ -1,4 +1,6 @@ #include "UpdateController.h" +#include "JsonParse.h" +#include "ProcessRunner.h" #include "ChildProcess.h" #include #include @@ -85,8 +87,8 @@ QHash bundleValues(const QString &path){ DesktopUpdateFeed parseDev(const QByteArray &data){ DesktopUpdateFeed result; if(data.size()>256*1024)return result; - QJsonParseError error;auto doc=QJsonDocument::fromJson(data,&error); - if(error.error!=QJsonParseError::NoError||!doc.isObject())return result; + QString error;auto doc=JsonParse::local(data,&error); + if(!error.isEmpty()||!doc.isObject())return result; auto raw=doc.object(); if(!integer(raw["schemaVersion"],2,2)||raw["product"]!="zerus"||raw["platform"]!="desktop"||raw["channel"]!="dev"||raw["channelTag"]!="dev"||raw.contains("sourceDirty")||!integer(raw["buildId"],1,9007199254740991LL)||!integer(raw["buildNumber"],1,9007199254740991LL))return result; auto id=raw["buildId"].toInteger();auto commit=raw["sourceCommit"].toString();auto version=raw["versionName"].toString(); @@ -172,7 +174,7 @@ DesktopUpdateFeed UpdateController::parseFeed(const QByteArray &data,const QStri if(channel=="dev")return parseDev(data); DesktopUpdateFeed result; if(data.size()>256*1024||!(channel=="stable"||channel=="nightly"))return result; - QJsonParseError error;auto doc=QJsonDocument::fromJson(data,&error);if(error.error!=QJsonParseError::NoError||!doc.isObject())return result; + QString error;auto doc=JsonParse::local(data,&error);if(!error.isEmpty()||!doc.isObject())return result; auto raw=doc.object(); if(!integer(raw["schemaVersion"],1,1)||raw["product"]!="zerus"||raw["platform"]!="desktop"||raw["channel"]!=channel||raw.contains("sourceDirty")||!integer(raw["releaseId"],1,9007199254740991LL))return result; QString version=raw["versionName"].toString(),release=raw["releaseUrl"].toString(); @@ -308,8 +310,8 @@ QString UpdateController::wingetPackage(const QString &executable){ } bool UpdateController::installerRecord(const QByteArray &data){ if(data.isEmpty()||data.size()>installerRecordLimit)return false; - QJsonParseError error;const auto document=QJsonDocument::fromJson(data,&error); - if(error.error!=QJsonParseError::NoError||!document.isObject())return false; + QString error;const auto document=JsonParse::local(data,&error); + if(!error.isEmpty()||!document.isObject())return false; const auto record=document.object(); return integer(record["schema"],1,1)&&record["kind"]=="installer"&&record["product"]=="Zerus"; } @@ -591,12 +593,17 @@ void UpdateController::startCheck(bool automatic){ auto *manager=m_fetch?nullptr:new QNetworkAccessManager(this);QNetworkRequest request(QUrl("https://zerus.dev/updates/"+QString(selected=="dev"?"v2":"v1")+"/desktop/"+selected+".json"));request.setAttribute(QNetworkRequest::RedirectPolicyAttribute,QNetworkRequest::ManualRedirectPolicy);request.setTransferTimeout(10000); auto *reply=m_fetch?m_fetch(request):manager->get(request);m_activeReply=reply;auto data=std::make_shared();auto oversized=std::make_shared(false);auto *timer=new QTimer(reply);timer->setSingleShot(true);connect(timer,&QTimer::timeout,reply,&QNetworkReply::abort);timer->start(10000); connect(reply,&QNetworkReply::readyRead,this,[reply,data,oversized]{auto bytes=reply->readAll();if(data->size()+bytes.size()>256*1024){*oversized=true;reply->abort();}else *data+=bytes;}); - connect(reply,&QNetworkReply::finished,this,[this,reply,manager,timer,data,oversized,generation,selected]{timer->stop();if(generation==m_checkGeneration){ - m_busy=false;m_activeReply=nullptr;bool mayNotify=!m_automaticInFlight||automaticChecks();m_automaticInFlight=false;DesktopUpdateFeed verified;if(!*oversized&&reply->error()==QNetworkReply::NoError&&reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()==200)verified=parseFeed(*data,selected); + connect(reply,&QNetworkReply::finished,this,[this,reply,manager,timer,data,oversized,generation,selected]{timer->stop(); + // The downloaded catalog is parsed off the GUI thread (ProcessRunner.h). + const bool fetched=generation==m_checkGeneration&&!*oversized&&reply->error()==QNetworkReply::NoError&&reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()==200; + if(generation==m_checkGeneration)m_activeReply=nullptr; + reply->deleteLater();if(manager)manager->deleteLater(); + ProcessRunner::post(this,[fetched,bytes=*data,selected]{return fetched?parseFeed(bytes,selected):DesktopUpdateFeed{};},[this,data,generation,selected](const DesktopUpdateFeed &verified){if(generation==m_checkGeneration){ + m_busy=false;m_activeReply=nullptr;bool mayNotify=!m_automaticInFlight||automaticChecks();m_automaticInFlight=false; if(verified.valid){m_freshForNotification=mayNotify;m_cached=false;m_available=false;m_feed=verified;m_lastSuccess=QDateTime::currentSecsSinceEpoch();QSettings settings;settings.setValue("updates/catalog/"+selected,*data);settings.setValue("updates/lastSuccess/"+selected,m_lastSuccess);m_notice="Last successful check: "+QDateTime::fromSecsSinceEpoch(m_lastSuccess).toString(Qt::ISODate)+".";} else {m_error=tr("Check failed. Try again later.");m_cached=m_feed.valid;m_notice=m_feed.valid?"Update check failed. Showing the last verified catalog ("+QDateTime::fromSecsSinceEpoch(m_lastSuccess).toString(Qt::ISODate)+").":"Update check failed or the catalog was invalid. Try again later.";} present(m_feed.valid?comparisonMessage(m_installation,m_feed,QCoreApplication::applicationVersion(),0,false):QString());emit changed();compare(); - }reply->deleteLater();if(manager)manager->deleteLater();}); + }});}); } void UpdateController::completeComparison(bool newer,const QString &message){ m_available=newer;present(message);emit changed(); diff --git a/tray/src/WindowsMobileSettings.h b/tray/src/WindowsMobileSettings.h index c03b63a92..ed25ce21e 100644 --- a/tray/src/WindowsMobileSettings.h +++ b/tray/src/WindowsMobileSettings.h @@ -1,4 +1,6 @@ #pragma once +#include "ProcessRunner.h" +#include "JsonParse.h" #include "ChildProcess.h" #include "RelayConfiguration.h" #include @@ -105,11 +107,14 @@ class Panel : public QWidget { auto *deadline=new QTimer(p);deadline->setSingleShot(true); auto output=std::make_shared(); const auto previous=snapshot; - auto finish=[this,p,deadline,output,action,payload,previous](bool success){ - if(process!=p)return; - deadline->stop();process=nullptr; + auto finishing=std::make_shared(false); + auto finish=[this,p,deadline,output,action,payload,previous,finishing](bool success){ + if(process!=p||*finishing)return;*finishing=true; + deadline->stop(); const auto bytes=*output+p->readAllStandardOutput(); - const auto doc=QJsonDocument::fromJson(bytes);const auto result=doc.object(); + // The reply is parsed off the GUI thread; this request stays current until then. + ProcessRunner::post(this,[bytes]{return JsonParse::parse(bytes);},[this,p,action,payload,previous,success,bytes](const QJsonDocument &doc){ + process=nullptr;const auto result=doc.object(); if(!success||!doc.isObject()||bytes.size()>65536||!confirmed(action,result,payload,previous)){ notice->setText(tr("Connector operation was not confirmed. Reload status before retrying. Check that the portable ZIP includes zerus-mobile-connector.exe and private configuration is available.")); state->setText(tr("Connector: status unknown")); @@ -127,7 +132,7 @@ class Panel : public QWidget { state->setText(result.value("running").toBool()?tr("Connector: running. Relay connectivity is not verified here."):tr("Connector: stopped")); notice->setText(action=="save"?tr("Configuration saved privately. Start the connector, then refresh Phones below."):QString());controls(false); } - p->deleteLater(); + p->deleteLater();}); }; connect(deadline,&QTimer::timeout,p,[p]{p->kill();}); connect(p,&QProcess::readyReadStandardOutput,p,[p,output]{*output+=p->readAllStandardOutput();if(output->size()>65536)p->kill();}); diff --git a/tray/src/WorkspaceSettings.h b/tray/src/WorkspaceSettings.h index d4f80f5ce..2261253c3 100644 --- a/tray/src/WorkspaceSettings.h +++ b/tray/src/WorkspaceSettings.h @@ -1,4 +1,6 @@ #pragma once +#include "ProcessRunner.h" +#include "JsonParse.h" #include "ChildProcess.h" #include "RelayConfiguration.h" #include @@ -266,9 +268,13 @@ class Panel : public QWidget { for(const auto &key:{"HGS_SESSION","HGS_RUN_ID","HGS_EXPECTED_ID","HGS_AGENT","HGS_HOST_PANE","HGS_HOST_SESSION","TMUX","TMUX_PANE"})environment.remove(QLatin1String(key)); p->setProcessEnvironment(environment);p->setProcessChannelMode(QProcess::SeparateChannels); auto *deadline=new QTimer(p);deadline->setSingleShot(true);auto output=std::make_shared(); - auto finish=[this,p,deadline,output,action,payload,done](bool success){ - if(process!=p)return;deadline->stop();process=nullptr; - const auto bytes=*output+p->readAllStandardOutput();const auto doc=QJsonDocument::fromJson(bytes);const auto result=doc.object(); + auto finishing=std::make_shared(false); + auto finish=[this,p,deadline,output,action,payload,done,finishing](bool success){ + if(process!=p||*finishing)return;*finishing=true;deadline->stop(); + const auto bytes=*output+p->readAllStandardOutput(); + // The reply is parsed off the GUI thread; this request stays current until then. + ProcessRunner::post(this,[bytes]{return JsonParse::parse(bytes);},[this,p,action,payload,done,success,bytes](const QJsonDocument &doc){ + process=nullptr;const auto result=doc.object(); if(!success||!doc.isObject()||bytes.size()>65536||!confirmed(action,result,payload)){ enrollmentPoll->stop(); continueEnrollment=false; @@ -290,7 +296,7 @@ class Panel : public QWidget { } else if(done)done(result); } - p->deleteLater(); + p->deleteLater();}); }; connect(deadline,&QTimer::timeout,p,[p]{p->kill();}); connect(p,&QProcess::readyReadStandardOutput,p,[p,output]{*output+=p->readAllStandardOutput();if(output->size()>65536)p->kill();}); diff --git a/tray/src/WorktreeSettings.h b/tray/src/WorktreeSettings.h index 9de5b46a1..dc458e819 100644 --- a/tray/src/WorktreeSettings.h +++ b/tray/src/WorktreeSettings.h @@ -1,4 +1,5 @@ #pragma once +#include "JsonParse.h" #include "ProcessRunner.h" #include #include @@ -67,9 +68,10 @@ class WorktreeSettings : public QWidget { ProcessRunner::run(command({"--help"}),this,[this,action,policy](const ProcessRunner::Result &probe){ if(!success(probe)||!probe.standardOutput.contains("worktree-maintenance-v1")){fail(tr("This computer is unavailable or needs an updated hgs CLI for cleanup settings."));return;} auto r=command({"worktrees","policy",action,"--json"});if(action=="set")r.input=QJsonDocument(policy).toJson(QJsonDocument::Compact); - ProcessRunner::run(std::move(r),this,[this,action](const ProcessRunner::Result &result){ - if(!success(result)){fail(action=="set"?tr("Save was not confirmed. Refresh before changing settings again."):tr("Could not read cleanup settings."));return;} - const auto object=QJsonDocument::fromJson(result.standardOutput).object();const auto p=object.value("policy").toObject(); + ProcessRunner::run(std::move(r),this,[](ProcessRunner::Result result){return std::make_pair(success(result),success(result)?JsonParse::object(result.standardOutput):QJsonObject());}, + [this,action](const std::pair &reply){ + if(!reply.first){fail(action=="set"?tr("Save was not confirmed. Refresh before changing settings again."):tr("Could not read cleanup settings."));return;} + const auto &object=reply.second;const auto p=object.value("policy").toObject(); bool valid=object.value("schema").toInt()==1&&p.value("version").toInt()==1&&p.value("revision").isDouble()&&p.value("grace_hours").toInt()>=1&&p.value("grace_hours").toInt()<=8760; for(const auto *key:{"enabled","after_session","unused_integrated","missing"})valid=valid&&p.value(QLatin1String(key)).isBool(); if(!valid){fail(tr("Computer returned invalid cleanup settings."));return;} diff --git a/tray/src/WorktreesDialog.cpp b/tray/src/WorktreesDialog.cpp index 09b4585ad..750994e56 100644 --- a/tray/src/WorktreesDialog.cpp +++ b/tray/src/WorktreesDialog.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "WorktreesDialog.h" #include "DialogButtons.h" #include "WorktreeBatchCleanupDialog.h" @@ -599,7 +600,7 @@ void WorktreesDialog::invalidateReview(const QString &repository) { m_reviews.remove(repository); m_reviewErrors.remove(repository); m_reviewAttempted.remove(repository); for (auto it = m_reviewAttempted.begin(); it != m_reviewAttempted.end();) { - const auto parts = QJsonDocument::fromJson(it->toUtf8()).array(); + const auto parts = JsonParse::localArray(it->toUtf8()); if (!parts.isEmpty() && parts.first().toString() == repository) it = m_reviewAttempted.erase(it); else ++it; } } diff --git a/tray/src/speech/SpeechClient.cpp b/tray/src/speech/SpeechClient.cpp index 36366e0c7..523019c89 100644 --- a/tray/src/speech/SpeechClient.cpp +++ b/tray/src/speech/SpeechClient.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "SpeechClient.h" #include "ProcessRunner.h" #include @@ -82,14 +83,19 @@ void SpeechClient::dispatch(const QString &host, const QJsonObject &payload, Rep auto request = command(m_executable, host, session.isEmpty() ? QStringList{"speech", "--json"} : QStringList{"voice", session, "--json"}); auto start = [this, request, done, redact=management(payload)](QByteArray input) mutable { auto command=request;command.input=std::move(input); - ProcessRunner::run(std::move(command), this, [done, redact](const ProcessRunner::Result &result) { - const auto error = failure(result, redact); - if (!error.isEmpty()) { done({}, error); return; } - QJsonParseError parse; - const auto json = QJsonDocument::fromJson(result.standardOutput, &parse); - if (result.standardOutput.size() > 1024 * 1024 || parse.error != QJsonParseError::NoError || !json.isObject() - || json.object().value("schema").toInt() != 1) { done({}, tr("The speech computer returned an invalid response.")); return; } - auto reply=json.object();reply.remove("client_dispatch_started");done(reply, {}); + struct Prepared { QString error; bool valid = false; QJsonObject reply; }; + ProcessRunner::run(std::move(command), this, [redact](ProcessRunner::Result result) { + Prepared prepared; prepared.error = failure(result, redact); + if (!prepared.error.isEmpty() || result.standardOutput.size() > 1024 * 1024) return prepared; + QString parse; + const auto json = JsonParse::parse(result.standardOutput, &parse); + prepared.valid = parse.isEmpty() && json.isObject() && json.object().value("schema").toInt() == 1; + if (prepared.valid) { prepared.reply = json.object(); prepared.reply.remove("client_dispatch_started"); } + return prepared; + }, [done](const Prepared &prepared) { + if (!prepared.error.isEmpty()) { done({}, prepared.error); return; } + if (!prepared.valid) { done({}, tr("The speech computer returned an invalid response.")); return; } + done(prepared.reply, {}); }); }; if(!payload.contains("video")){start(QJsonDocument(payload).toJson(QJsonDocument::Compact));return;} diff --git a/tray/src/speech/SpeechSettings.cpp b/tray/src/speech/SpeechSettings.cpp index a57b4f379..1652a3d33 100644 --- a/tray/src/speech/SpeechSettings.cpp +++ b/tray/src/speech/SpeechSettings.cpp @@ -1,3 +1,4 @@ +#include "SettingsJson.h" #include "SpeechSettings.h" #include "SpeechClient.h" #include @@ -214,7 +215,7 @@ SpeechSettings::SpeechSettings(const QString &executable, QWidget *parent, std:: m_microphone = new QComboBox; m_microphone->setObjectName("speechMicrophone"); layout->addWidget(m_microphone); layout->addWidget(label(tr("Provider settings and credentials stay on their processing computer. Recordings are limited to two minutes. Original audio and recognized text remain in conversation history; your typed draft stays separate."))); m_statistics = label({}); m_statistics->setObjectName("speechStatistics"); layout->addWidget(m_statistics); layout->addStretch(); - m_pending = QJsonDocument::fromJson(QSettings().value(pendingKey).toByteArray()).object(); + m_pending = SettingsJson::object(pendingKey); if (!validMachine(m_pending.value("machine_id").toString()) || QUuid(m_pending.value("request_id").toString()).isNull()) m_pending = {}; setPeers({}); microphones(); controls(); connect(m_host, &QComboBox::currentIndexChanged, this, [this] { QSettings().setValue("speech/manage_host", m_host->currentData()); refresh(); }); diff --git a/tray/tests/CMakeLists.txt b/tray/tests/CMakeLists.txt index 004d3e8be..e938a38a9 100644 --- a/tray/tests/CMakeLists.txt +++ b/tray/tests/CMakeLists.txt @@ -124,15 +124,24 @@ function(zerus_check_tests) endfunction() # Windows test programs find the Qt DLLs through PATH. The offscreen platform # reads fonts from QT_QPA_FONTDIR there; without it text has no glyphs or size. +# Every suite fails on JSON parsed on the GUI thread (JsonParse.h), whatever +# ENVIRONMENT it sets itself. function(zerus_test_environment) - if(NOT WIN32) + if(CMAKE_VERSION VERSION_LESS 3.22) + if(WIN32) + message(FATAL_ERROR "Windows tests need CMake 3.22 or later (ENVIRONMENT_MODIFICATION)") + endif() + message(WARNING "CMake 3.22 or later runs the Qt suites with ZERUS_GUI_THREAD_CHECKS=fatal; scripts/ci/gui.sh sets it too") return() endif() - if(CMAKE_VERSION VERSION_LESS 3.22) - message(FATAL_ERROR "Windows tests need CMake 3.22 or later (ENVIRONMENT_MODIFICATION)") + get_property(tests DIRECTORY PROPERTY TESTS) + foreach(test IN LISTS tests) + set_property(TEST ${test} APPEND PROPERTY ENVIRONMENT_MODIFICATION "ZERUS_GUI_THREAD_CHECKS=set:fatal") + endforeach() + if(NOT WIN32) + return() endif() file(TO_CMAKE_PATH "$ENV{SystemRoot}/Fonts" fonts) - get_property(tests DIRECTORY PROPERTY TESTS) foreach(test IN LISTS tests) set_property(TEST ${test} APPEND PROPERTY ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$" "QT_QPA_FONTDIR=set:${fonts}") @@ -245,6 +254,11 @@ target_include_directories(test_readmarksync PRIVATE ../src) target_link_libraries(test_readmarksync PRIVATE Qt6::Test Qt6::Core hgs-tray-core) zerus_add_test(readmarksync test_readmarksync) +add_executable(test_jsonparse test_jsonparse.cpp) +target_include_directories(test_jsonparse PRIVATE ../src) +target_link_libraries(test_jsonparse PRIVATE Qt6::Test Qt6::Gui hgs-tray-core) +zerus_add_test(jsonparse test_jsonparse) + add_executable(test_processrunner test_processrunner.cpp) target_include_directories(test_processrunner PRIVATE ../src) target_link_libraries(test_processrunner PRIVATE Qt6::Test Qt6::Core hgs-tray-core) @@ -653,19 +667,19 @@ target_link_libraries(test_traymenu PRIVATE hgs-updates) add_executable(test_windowsmobilesettings test_windowsmobilesettings.cpp) target_include_directories(test_windowsmobilesettings PRIVATE ../src) -target_link_libraries(test_windowsmobilesettings PRIVATE Qt6::Test Qt6::Widgets) +target_link_libraries(test_windowsmobilesettings PRIVATE Qt6::Test Qt6::Widgets hgs-tray-core) zerus_add_test(windowsmobilesettings test_windowsmobilesettings) set_tests_properties(windowsmobilesettings PROPERTIES ENVIRONMENT "QT_QPA_PLATFORM=offscreen") add_executable(test_relaysettings test_relaysettings.cpp) target_include_directories(test_relaysettings PRIVATE ../src) -target_link_libraries(test_relaysettings PRIVATE Qt6::Test Qt6::Widgets) +target_link_libraries(test_relaysettings PRIVATE Qt6::Test Qt6::Widgets hgs-tray-core) zerus_add_test(relaysettings test_relaysettings) set_tests_properties(relaysettings PROPERTIES ENVIRONMENT "QT_QPA_PLATFORM=offscreen") add_executable(test_workspacesettings test_workspacesettings.cpp) target_include_directories(test_workspacesettings PRIVATE ../src) -target_link_libraries(test_workspacesettings PRIVATE Qt6::Test Qt6::Widgets) +target_link_libraries(test_workspacesettings PRIVATE Qt6::Test Qt6::Widgets hgs-tray-core) zerus_add_test(workspacesettings test_workspacesettings) set_tests_properties(workspacesettings PROPERTIES ENVIRONMENT "QT_QPA_PLATFORM=offscreen") diff --git a/tray/tests/support/OffGuiThread.h b/tray/tests/support/OffGuiThread.h new file mode 100644 index 000000000..905aa1d81 --- /dev/null +++ b/tray/tests/support/OffGuiThread.h @@ -0,0 +1,15 @@ +#pragma once + +#include +#include + +// Runs f on a separate thread and returns its result. GUI suites call parsers +// that production runs on workers through this: JsonParse.h fails a parse on +// the GUI thread (ZERUS_GUI_THREAD_CHECKS=fatal in every suite). +template +auto offGuiThread(F f) +{ + std::invoke_result_t result{}; + std::thread([&] { result = f(); }).join(); + return result; +} diff --git a/tray/tests/test_analytics.cpp b/tray/tests/test_analytics.cpp index c6c92c9d5..1fb3929d8 100644 --- a/tray/tests/test_analytics.cpp +++ b/tray/tests/test_analytics.cpp @@ -6,6 +6,7 @@ // native-preference and service panels in this isolated component suite. #include "SettingsPage.h" #include "FakeCli.h" +#include "OffGuiThread.h" #include #include #include @@ -22,6 +23,14 @@ #include #include #include + +namespace { +// The client parses analytics on a worker; these call the same parsers off the GUI thread. +std::optional offGuiSummary(const QByteArray &bytes, QDate from, QDate through) +{ return offGuiThread([&] { return AnalyticsData::summary(bytes, from, through); }); } +std::optional offGuiSpeech(const QByteArray &bytes, QDate from, QDate through) +{ return offGuiThread([&] { return AnalyticsData::speech(bytes, from, through); }); } +} #include namespace { @@ -140,21 +149,21 @@ void TestAnalytics::parserKeepsOnlyReviewedFieldsAndUnknowns() { auto rows=raw.value("buckets").toArray();auto row=rows[0].toObject();row.insert("path","private-path-sentinel"); auto counts=row.value("counters").toObject();counts.insert("arbitrary_counter_sentinel",123);row.insert("counters",counts);rows[0]=row;raw.insert("buckets",rows); auto coverage=raw.value("coverage").toObject();coverage.insert("gaps",QJsonArray{"private-error-sentinel"});raw.insert("coverage",coverage); - const auto parsed=AnalyticsData::summary(bytes(raw),first,last);QVERIFY(parsed); + const auto parsed=offGuiSummary(bytes(raw),first,last);QVERIFY(parsed); QVERIFY(!bytes(*parsed).contains("sentinel"));QCOMPARE(parsed->value("coverage").toObject().value("gaps").toArray(),QJsonArray{"unknown"}); const auto kept=parsed->value("buckets").toArray()[0].toObject().value("counters").toObject();QCOMPARE(kept.value("input_tokens").toInt(),17);QVERIFY(!kept.contains("output_tokens")); - raw.insert("source_epoch","00000000-0000-0000-0000-000000000000");QVERIFY(AnalyticsData::summary(bytes(raw),first,last)); + raw.insert("source_epoch","00000000-0000-0000-0000-000000000000");QVERIFY(offGuiSummary(bytes(raw),first,last)); } void TestAnalytics::parserRejectsInvalidAndUnboundedSummaries() { - QVERIFY(!AnalyticsData::summary(QByteArray(AnalyticsData::MaximumBytes+1,'x'),first,last)); - QVERIFY(!AnalyticsData::summary(bytes(summary()),first,first.addDays(90))); - for(const auto &key:{"machine_id","source_epoch","generated_at","revision"}){auto invalid=summary();invalid.insert(key,"private");QVERIFY2(!AnalyticsData::summary(bytes(invalid),first,last),key);} - auto duplicate=summary(QJsonArray{bucket(),bucket()});QVERIFY(!AnalyticsData::summary(bytes(duplicate),first,last)); - auto bad=bucket();auto c=bad.value("counters").toObject();c.insert("work_ms",-1);bad.insert("counters",c);QVERIFY(!AnalyticsData::summary(bytes(summary(QJsonArray{bad})),first,last)); - bad=bucket();bad.insert("provider","private-account-name");QVERIFY(!AnalyticsData::summary(bytes(summary(QJsonArray{bad})),first,last)); - auto histogram=bucket().value("histograms").toObject();auto work=histogram.value("work_ms").toObject();work.insert("count",2);histogram.insert("work_ms",work);bad=bucket();bad.insert("histograms",histogram);QVERIFY(!AnalyticsData::summary(bytes(summary(QJsonArray{bad})),first,last)); - QJsonArray excessive;for(int n=0;n<=AnalyticsData::MaximumBuckets;++n)excessive.append(bucket());QVERIFY(!AnalyticsData::summary(bytes(summary(excessive)),first,last)); + QVERIFY(!offGuiSummary(QByteArray(AnalyticsData::MaximumBytes+1,'x'),first,last)); + QVERIFY(!offGuiSummary(bytes(summary()),first,first.addDays(90))); + for(const auto &key:{"machine_id","source_epoch","generated_at","revision"}){auto invalid=summary();invalid.insert(key,"private");QVERIFY2(!offGuiSummary(bytes(invalid),first,last),key);} + auto duplicate=summary(QJsonArray{bucket(),bucket()});QVERIFY(!offGuiSummary(bytes(duplicate),first,last)); + auto bad=bucket();auto c=bad.value("counters").toObject();c.insert("work_ms",-1);bad.insert("counters",c);QVERIFY(!offGuiSummary(bytes(summary(QJsonArray{bad})),first,last)); + bad=bucket();bad.insert("provider","private-account-name");QVERIFY(!offGuiSummary(bytes(summary(QJsonArray{bad})),first,last)); + auto histogram=bucket().value("histograms").toObject();auto work=histogram.value("work_ms").toObject();work.insert("count",2);histogram.insert("work_ms",work);bad=bucket();bad.insert("histograms",histogram);QVERIFY(!offGuiSummary(bytes(summary(QJsonArray{bad})),first,last)); + QJsonArray excessive;for(int n=0;n<=AnalyticsData::MaximumBuckets;++n)excessive.append(bucket());QVERIFY(!offGuiSummary(bytes(summary(excessive)),first,last)); } void TestAnalytics::fleetReplacesSnapshotsAndEpochsInsteadOfSumming() { @@ -198,21 +207,21 @@ void TestAnalytics::invalidMissingAndOverflowingHistogramsStayUnavailable() { bad=validHistogram;bad.insert("counts",QJsonArray{0,0,0,0,0,0});bad.insert("count",0);malformed.append(bad); for(const auto &h:malformed) { auto invalid=valid;invalid.insert("histograms",QJsonObject{{"work_ms",h}}); - QVERIFY(!AnalyticsData::summary(bytes(summary({invalid})),first,last)); + QVERIFY(!offGuiSummary(bytes(summary({invalid})),first,last)); QVERIFY(!AnalyticsData::histogramTotals({valid,invalid,valid}).contains("work_ms")); } auto missing=valid;missing.insert("histograms",QJsonObject{}); - QVERIFY(AnalyticsData::summary(bytes(summary({missing})),first,last)); + QVERIFY(offGuiSummary(bytes(summary({missing})),first,last)); QVERIFY(!AnalyticsData::histogramTotals({valid,missing,valid}).contains("work_ms")); QVERIFY(!AnalyticsData::histogramTotals({missing,valid}).contains("work_ms")); constexpr qint64 maximum=9007199254740991LL; const auto countLimit=workBucket({maximum,0,0,0,0,0},maximum,0); - QVERIFY(AnalyticsData::summary(bytes(summary({countLimit})),first,last)); + QVERIFY(offGuiSummary(bytes(summary({countLimit})),first,last)); // Both same-bin overflow and total-count overflow across different bins fail closed. QVERIFY(!AnalyticsData::histogramTotals({countLimit,valid}).contains("work_ms")); QVERIFY(!AnalyticsData::histogramTotals({countLimit,workBucket({0,1,0,0,0,0},1,101)}).contains("work_ms")); const auto sumLimit=workBucket({0,0,0,0,0,1},1,maximum); - QVERIFY(AnalyticsData::summary(bytes(summary({sumLimit})),first,last)); + QVERIFY(offGuiSummary(bytes(summary({sumLimit})),first,last)); QVERIFY(!AnalyticsData::histogramTotals({sumLimit,valid}).contains("work_ms")); const auto justBelow=workBucket({0,0,0,0,0,1},1,maximum-100); QCOMPARE(AnalyticsData::histogramTotals({justBelow,valid}).value("work_ms").toObject().value("sum").toInteger(),maximum); @@ -220,9 +229,9 @@ void TestAnalytics::invalidMissingAndOverflowingHistogramsStayUnavailable() { void TestAnalytics::voiceCollapsesPrivateProfilesAndPreservesUnits() { auto raw=voice();auto days=raw.value("days").toArray();auto row=days[0].toObject();row.insert("profile_id","second-private-profile");row.insert("queue_ms",1.5);row.insert("error_codes",QJsonObject{{"private-provider-message",2}});days.append(row);raw.insert("days",days); - const auto parsed=AnalyticsData::speech(bytes(raw),first,last);QVERIFY(parsed);QVERIFY(!bytes(*parsed).contains("private"));QCOMPARE(parsed->value("days").toArray().size(),1); + const auto parsed=offGuiSpeech(bytes(raw),first,last);QVERIFY(parsed);QVERIFY(!bytes(*parsed).contains("private"));QCOMPARE(parsed->value("days").toArray().size(),1); const auto result=parsed->value("days").toArray()[0].toObject();QCOMPARE(result.value("submitted").toInt(),4);QCOMPARE(result.value("queue_ms").toDouble(),60001.5);QCOMPARE(result.value("error_codes").toObject().value("unknown").toInt(),2); - days.append(days[0]);raw.insert("days",days);QVERIFY(!AnalyticsData::speech(bytes(raw),first,last)); + days.append(days[0]);raw.insert("days",days);QVERIFY(!offGuiSpeech(bytes(raw),first,last)); } void TestAnalytics::retiredEpochNeverRevivesWithNewTimestamp() { @@ -236,7 +245,7 @@ void TestAnalytics::retiredEpochNeverRevivesWithNewTimestamp() { } void TestAnalytics::diagnosticsExcludeIdentityContentAndDetailedHistory() { - const auto agent=AnalyticsData::summary(bytes(summary()),first,last),v=AnalyticsData::speech(bytes(voice()),first,last);QVERIFY(agent&&v); + const auto agent=offGuiSummary(bytes(summary()),first,last),v=offGuiSpeech(bytes(voice()),first,last);QVERIFY(agent&&v); const auto exported=bytes(AnalyticsData::diagnostic({{"private-host-sentinel",*agent,*v,true,true}},first,last)); QVERIFY(!exported.contains(machine.toUtf8()));QVERIFY(!exported.contains(epoch.toUtf8()));QVERIFY(!exported.contains("private"));QVERIFY(!exported.contains("profile_id"));QVERIFY(!exported.contains("buckets"));QVERIFY(!exported.contains("\"days\":"));QVERIFY(exported.contains("reported_totals")); } diff --git a/tray/tests/test_dashboard.cpp b/tray/tests/test_dashboard.cpp index 3548d13f5..c7ad29785 100644 --- a/tray/tests/test_dashboard.cpp +++ b/tray/tests/test_dashboard.cpp @@ -67,10 +67,9 @@ private slots: for (const auto &value : contract.value("cases").toArray()) { const auto example = value.toObject(); auto raw = example.value("raw").toObject(); raw["name"] = "codex/contract/" + example.value("name").toString(); - QString error; - const auto box = HgsClient::parseBox(QJsonDocument(QJsonObject{{"host", "fixture"}, {"ok", true}, - {"sessions", QJsonArray{raw}}}).toJson(), &error); - QVERIFY2(error.isEmpty(), qPrintable(error)); QCOMPARE(box.sessions.size(), 1); + const auto box = HgsClient::box(QJsonObject{{"host", "fixture"}, {"ok", true}, + {"sessions", QJsonArray{raw}}}); + QCOMPARE(box.sessions.size(), 1); auto session = box.sessions.first(); const bool online = example.value("online").toBool(); // Desktop overlays these device/read marks after parsing CLI rows. session.reviewLater = raw.value("review_later").toBool(); @@ -92,15 +91,14 @@ private slots: const auto contract = QJsonDocument::fromJson(file.readAll()).object(); const auto parse = [](QJsonObject raw, const QString &name) { raw["name"] = "codex/contract/" + name; - QString error; - const auto box = HgsClient::parseBox(QJsonDocument(QJsonObject{{"host", "fixture"}, {"ok", true}, - {"sessions", QJsonArray{raw}}}).toJson(), &error); - return std::pair{box, error}; + const auto box = HgsClient::box(QJsonObject{{"host", "fixture"}, {"ok", true}, + {"sessions", QJsonArray{raw}}}); + return std::pair{box, QString()}; }; for (const auto &value : contract.value("cases").toArray()) { const auto example = value.toObject(); const auto name = example.value("name").toString(); const auto [box, error] = parse(example.value("raw").toObject(), name); - QVERIFY2(error.isEmpty(), qPrintable(error)); QCOMPARE(box.sessions.size(), 1); + QCOMPARE(box.sessions.size(), 1); const int count = SessionPresentation::optionalQuestionCount(box.sessions.first(), example.value("online").toBool()); QVERIFY2(count == example.value("count").toInt(), qPrintable(name)); const QString label = count > 0 ? OptionalQuestionBadge::countText(count) : QString(); diff --git a/tray/tests/test_externalfile.cpp b/tray/tests/test_externalfile.cpp index f53f1a766..69552a532 100644 --- a/tray/tests/test_externalfile.cpp +++ b/tray/tests/test_externalfile.cpp @@ -3,6 +3,7 @@ #include #include "ExternalFile.h" #include "DownloadCache.h" +#include "OffGuiThread.h" namespace { QString write(const QTemporaryDir &dir, const QString &name, const QByteArray &bytes) @@ -114,7 +115,7 @@ private slots: #if defined(Q_OS_WIN) || defined(Q_OS_MACOS) QVERIFY(!marker(result.path).isEmpty()); #endif - const auto usage = DownloadCache::trim(dir.path(), 1, DownloadCache::OpeningGraceMs + 1001); + const auto usage = offGuiThread([&] { return DownloadCache::trim(dir.path(), 1, DownloadCache::OpeningGraceMs + 1001); }); QVERIFY2(usage.error.isEmpty(), qPrintable(usage.error)); QCOMPARE(usage.files, 0); QVERIFY(!QFileInfo::exists(result.path)); } diff --git a/tray/tests/test_jsonparse.cpp b/tray/tests/test_jsonparse.cpp new file mode 100644 index 000000000..7cd3600ca --- /dev/null +++ b/tray/tests/test_jsonparse.cpp @@ -0,0 +1,96 @@ +#include + +#include +#include + +#include + +#include "FakeCli.h" +#include "HgsClient.h" +#include "JsonParse.h" +#include "ProcessRunner.h" + +// The GUI thread applies values that workers prepared (JsonParse.h). Every +// suite fails on a parse there; this one checks the rule itself and that the +// client's polls parse off it. +class TestJsonParse : public QObject { + Q_OBJECT +private slots: + void init() + { + m_sites.clear(); + JsonParse::setViolationHandler([this](const char *file, int line) { + m_sites << QStringLiteral("%1:%2").arg(QFileInfo(QString::fromUtf8(file)).fileName()).arg(line); + }); + } + void cleanup() { JsonParse::setViolationHandler({}); } + void parsingOnTheGuiThreadIsAViolation(); + void workersParseWithoutViolation(); + void localValuesAreLimited(); + void clientPollsParseOffTheGuiThread(); + +private: + QStringList m_sites; +}; + +void TestJsonParse::parsingOnTheGuiThreadIsAViolation() +{ + QVERIFY(JsonParse::onGuiThread()); + const int before = JsonParse::violations(); + QString error; + const int line = __LINE__ + 1; + const auto object = JsonParse::object(R"({"a":1})", &error); + QVERIFY(error.isEmpty()); QCOMPARE(object.value("a").toInt(), 1); + QCOMPARE(JsonParse::violations(), before + 1); + QCOMPARE(m_sites, QStringList{QStringLiteral("test_jsonparse.cpp:%1").arg(line)}); + JsonParse::array(R"({"a":1})", &error); + QCOMPARE(error, QStringLiteral("expected a JSON array")); +} + +void TestJsonParse::workersParseWithoutViolation() +{ + QJsonObject object; bool guiThread = true; + std::thread([&] { guiThread = JsonParse::onGuiThread(); object = JsonParse::object(R"({"b":2})"); }).join(); + QVERIFY(!guiThread); QCOMPARE(object.value("b").toInt(), 2); + bool applied = false; QObject owner; + ProcessRunner::post(&owner, [] { return JsonParse::object(R"({"c":3})"); }, + [&applied](const QJsonObject &value) { applied = value.value("c").toInt() == 3; }); + QTRY_VERIFY(applied); + QVERIFY(m_sites.isEmpty()); +} + +void TestJsonParse::localValuesAreLimited() +{ + QCOMPARE(JsonParse::localObject(R"({"small":true})").value("small").toBool(), true); + QVERIFY(m_sites.isEmpty()); + const QByteArray large = "[\"" + QByteArray(JsonParse::LocalLimit, 'x') + "\"]"; + QCOMPARE(JsonParse::localArray(large).size(), 1); + QCOMPARE(m_sites.size(), 1); +} + +void TestJsonParse::clientPollsParseOffTheGuiThread() +{ + QTemporaryDir dir; QVERIFY(dir.isValid()); + const auto fixture = dir.filePath("hgs"); + QVERIFY(FakeCli::install(fixture, R"PY(#!/usr/bin/env python3 +import json, sys +args = sys.argv[1:] +if args[:1] == ["ls"]: + print(json.dumps({"host": "fixture", "ok": True, "sessions": [{"name": "codex/demo", "cmd": "codex", "activity": "idle"}]})) +elif args[:1] == ["inspect"]: + print(json.dumps({"cursor": 7, "events": [{"seq": 7, "kind": "Stop"}]})) +else: + sys.exit(2) +)PY")); + HgsClient client(FakeCli::program(fixture)); + QSignalSpy local(&client, &HgsClient::localReady), inspected(&client, &HgsClient::inspectionReady); + client.requestLocal(); + QVERIFY(client.requestInspection({}, QStringLiteral("codex/demo"))); + QTRY_COMPARE(local.size(), 1); QTRY_COMPARE(inspected.size(), 1); + QCOMPARE(local.first().first().value().sessions.first().name, QStringLiteral("codex/demo")); + QCOMPARE(inspected.first().at(2).toJsonObject().value("cursor").toInt(), 7); + QVERIFY2(m_sites.isEmpty(), qPrintable(m_sites.join(", "))); +} + +QTEST_MAIN(TestJsonParse) +#include "test_jsonparse.moc" diff --git a/tray/tests/test_processrunner.cpp b/tray/tests/test_processrunner.cpp index c3aa0aeb0..4150810e8 100644 --- a/tray/tests/test_processrunner.cpp +++ b/tray/tests/test_processrunner.cpp @@ -31,6 +31,9 @@ class TestProcessRunner : public QObject { private slots: void cleanup() { ProcessRunner::setLaunchObserver({}); } void launchesOnWorkerAndReportsOnOwnerThread(); + void preparesOnPreparationThreadAndAppliesOnOwnerThread(); + void preparedAndPlainResultsKeepCompletionOrder(); + void destroyedOwnerDropsPreparedValue(); void capturesOutputErrorAndExitCode(); void capsCombinedOutputWhileDraining(); void outputCapKillsOnlyTheDirectChild(); @@ -185,6 +188,57 @@ void TestProcessRunner::launchesOnWorkerAndReportsOnOwnerThread() QCOMPARE(result.standardOutput, QByteArray("hello")); } +void TestProcessRunner::preparesOnPreparationThreadAndAppliesOnOwnerThread() +{ + QObject owner; std::atomic prepareThread{nullptr}; QThread *applyThread = nullptr; + int applied = 0; QByteArray value; + ProcessRunner::run(child(QStringLiteral("printf hello"), {"out", "hello"}), &owner, + [&prepareThread](ProcessRunner::Result result) { + prepareThread = QThread::currentThread(); + return ProcessRunner::isPreparationThread() ? result.standardOutput.toUpper() : QByteArray(); + }, + [&](QByteArray prepared) { ++applied; applyThread = QThread::currentThread(); value = prepared; }); + QTRY_COMPARE(applied, 1); + QCOMPARE(prepareThread.load(), ProcessRunner::preparationThread()); + QVERIFY(prepareThread.load() != QThread::currentThread()); + QVERIFY(prepareThread.load() != ProcessRunner::workerThread()); + QVERIFY(!ProcessRunner::isPreparationThread()); + QCOMPARE(applyThread, QThread::currentThread()); + QCOMPARE(value, QByteArray("HELLO")); +} + +void TestProcessRunner::preparedAndPlainResultsKeepCompletionOrder() +{ + // The first child finishes first but prepares slowly. The second, plain + // result still reaches its owner after it: one serial thread passes both. + QObject owner; QStringList order; + ProcessRunner::run(child(QStringLiteral("printf first"), {"out", "first"}), &owner, + [](ProcessRunner::Result result) { QThread::msleep(400); return result.standardOutput; }, + [&order](QByteArray prepared) { order << QString::fromLatin1(prepared); }); + ProcessRunner::run(child(QStringLiteral("sleep 0.1; printf second"), {"sleep", "0.1", "out", "second"}), &owner, + [&order](const ProcessRunner::Result &result) { order << QString::fromLatin1(result.standardOutput); }); + QTRY_COMPARE(order.size(), 2); + QCOMPARE(order, QStringList({QStringLiteral("first"), QStringLiteral("second")})); +} + +void TestProcessRunner::destroyedOwnerDropsPreparedValue() +{ + auto owner = std::make_unique(); std::atomic preparing{false}, release{false}, prepared{false}; + int applied = 0; + ProcessRunner::run(child(QStringLiteral("printf done"), {"out", "done"}), owner.get(), + [&](ProcessRunner::Result result) { + preparing = true; + while (!release) QThread::msleep(5); + prepared = true; return result.standardOutput; + }, + [&applied](QByteArray) { ++applied; }); + QTRY_VERIFY(preparing.load()); + owner.reset(); release = true; + QTRY_VERIFY(prepared.load()); + QTest::qWait(200); + QCOMPARE(applied, 0); +} + void TestProcessRunner::capturesOutputErrorAndExitCode() { QObject owner; int results = 0; ProcessRunner::Result result; diff --git a/tray/tests/test_sessionswindow.cpp b/tray/tests/test_sessionswindow.cpp index 9b43f23e3..2b877dd8b 100644 --- a/tray/tests/test_sessionswindow.cpp +++ b/tray/tests/test_sessionswindow.cpp @@ -108,6 +108,7 @@ #include "LaunchWarnings.h" #include "LocalSessionHost.h" #include "ProcessRunner.h" +#include "SettingsJson.h" #include "SessionCreationController.h" #include "ProcessSettings.h" #include "QuestionNotificationSettings.h" @@ -1894,6 +1895,11 @@ void TestSessionsWindow::rebuildKeepsProjectOrderAtScale() {"sessions", storedOrder ? QJsonArray::fromStringList(ordered) : QJsonArray{}}}}}}); organization.markImported(box.host); QSettings().setValue("workspace/organization", QJsonDocument(organization.toJson()).toJson(QJsonDocument::Compact)); + // As at desktop startup (TrayAgent), the large saved layout is parsed off the GUI thread first. + QObject warming; bool warmed = false; + SettingsJson::warm(&warming, {QStringLiteral("workspace/organization")}); + ProcessRunner::post(&warming, [] { return true; }, [&warmed](bool) { warmed = true; }); + QTRY_VERIFY(warmed); FleetState state; state.setLocal(box, QDateTime::currentMSecsSinceEpoch()); SessionsWindow window(program()); keepDetailsPollsOut(window); window.show(); QElapsedTimer elapsed; elapsed.start(); window.setFleet(state); From c0ee836a7ac408bb69943b25ae14283acbee5b85 Mon Sep 17 00:00:00 2001 From: ufna <271578+ufna@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:52:16 +0300 Subject: [PATCH 2/7] docs: keep the watch-protocol crate note for its own change Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 179d1b3d5..1880164d1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,8 +70,6 @@ stable release, and raising them is routine maintenance. Run The replica crate has its own lockfile: run `cargo +1.99.1 test --locked --manifest-path crates/replica/Cargo.toml`; its golden vectors must keep decoding, and `ZERUS_BLESS=1` only adds new ones. -The event stream's frame crate has one too: run -`cargo +1.99.1 test --locked --manifest-path crates/watch-protocol/Cargo.toml --target-dir target/watch-protocol`. On Linux, the three worktree modules run through `scripts/ci/worktree-tests.py` in a fresh PID/mount namespace, including their live process blockers and private From 34f95851a486c9c18c933a1923d322f331387310 Mon Sep 17 00:00:00 2001 From: ufna <271578+ufna@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:52:36 +0300 Subject: [PATCH 3/7] feat(cli): stream one computer's sessions with hgs watch --json Event data plane M3 (zerus-604d.3). `hgs [@host] watch --json` is a long-lived NDJSON stream: hello, a bounded catalog snapshot of the live listing (rows exactly as ls --json --local --no-archives, keyed run:), deltas with removal reasons, catalog.meta, and Activity subscriptions over stdin whose cursor is the events.sqlite3 seq that inspect --after takes. Heartbeats, gap/snapshot resync, a 4 MiB coalescing writer queue and 1 MiB frames bound it; EOF on stdin ends it. `hgs -h` prints the marker `Event stream ABI: watch-v1`; @host runs it over one SSH connection with keepalives, guarded so an older peer never reads it as a launch. Change detection uses no new dependency: a 250 ms pulse stats the listing's inputs and polls PRAGMA data_version on a held journal connection, with a 5 s safety listing for changes no file shows. Volatile observation fields are masked, so deltas carry real changes only. The serde frame types live in crates/watch-protocol (no Loro or Qt) for the desktop, the Tauri client and later the connector. The inspection's journal query is shared, not forked. Co-Authored-By: Claude Opus 5.5 --- .github/dependabot.yml | 2 +- .github/workflows/checks.yml | 5 + CONTRIBUTING.md | 2 + Cargo.lock | 9 + Cargo.toml | 1 + crates/watch-protocol/.cargo/config.toml | 2 + crates/watch-protocol/Cargo.lock | 107 +++ crates/watch-protocol/Cargo.toml | 13 + crates/watch-protocol/src/lib.rs | 645 +++++++++++++++ docs/reference.md | 83 ++ mac-install.sh | 3 +- src/cli.rs | 27 +- src/main.rs | 1 + src/state/journal.rs | 52 +- src/state/mod.rs | 1 + src/state/watch_source.rs | 143 ++++ src/watch/activity.rs | 229 ++++++ src/watch/catalog.rs | 226 ++++++ src/watch/mod.rs | 984 +++++++++++++++++++++++ src/watch/stamps.rs | 127 +++ src/watch/writer.rs | 316 ++++++++ tests/test_ci_security.py | 3 + tests/test_mac_install.py | 4 +- tests/test_watch.py | 334 ++++++++ 24 files changed, 3298 insertions(+), 21 deletions(-) create mode 100644 crates/watch-protocol/.cargo/config.toml create mode 100644 crates/watch-protocol/Cargo.lock create mode 100644 crates/watch-protocol/Cargo.toml create mode 100644 crates/watch-protocol/src/lib.rs create mode 100644 src/state/watch_source.rs create mode 100644 src/watch/activity.rs create mode 100644 src/watch/catalog.rs create mode 100644 src/watch/mod.rs create mode 100644 src/watch/stamps.rs create mode 100644 src/watch/writer.rs create mode 100644 tests/test_watch.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index eb73a0f38..15d1b2c96 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,7 +14,7 @@ updates: applies-to: security-updates patterns: ["*"] - package-ecosystem: cargo - directories: ["/", "/services/mobile/relay", "/services/mobile/connector", "/crates/replica", "/crates/replica-ffi"] + directories: ["/", "/services/mobile/relay", "/services/mobile/connector", "/crates/replica", "/crates/replica-ffi", "/crates/watch-protocol"] schedule: interval: monthly open-pull-requests-limit: 1 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 28f17994a..f9339eec8 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -160,6 +160,11 @@ jobs: cargo +1.99.1 fmt --check --manifest-path crates/replica-ffi/Cargo.toml cargo +1.99.1 clippy --locked --manifest-path crates/replica-ffi/Cargo.toml --target-dir target/replica-ffi --all-targets -- -D warnings cargo +1.99.1 test --locked --manifest-path crates/replica-ffi/Cargo.toml --target-dir target/replica-ffi + - name: Watch protocol crate (format, lints, tests) + run: | + cargo +1.99.1 fmt --check --manifest-path crates/watch-protocol/Cargo.toml + cargo +1.99.1 clippy --locked --manifest-path crates/watch-protocol/Cargo.toml --target-dir target/watch-protocol --all-targets -- -D warnings + cargo +1.99.1 test --locked --manifest-path crates/watch-protocol/Cargo.toml --target-dir target/watch-protocol # Packages fail without a notice for every shipped crate; find out here. - name: Rust license notices run: python3 scripts/collect-licenses.py "$RUNNER_TEMP/rust-licenses" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1880164d1..179d1b3d5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,6 +70,8 @@ stable release, and raising them is routine maintenance. Run The replica crate has its own lockfile: run `cargo +1.99.1 test --locked --manifest-path crates/replica/Cargo.toml`; its golden vectors must keep decoding, and `ZERUS_BLESS=1` only adds new ones. +The event stream's frame crate has one too: run +`cargo +1.99.1 test --locked --manifest-path crates/watch-protocol/Cargo.toml --target-dir target/watch-protocol`. On Linux, the three worktree modules run through `scripts/ci/worktree-tests.py` in a fresh PID/mount namespace, including their live process blockers and private diff --git a/Cargo.lock b/Cargo.lock index 781e5b9f0..2adf14dca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -770,6 +770,7 @@ dependencies = [ "uuid", "windows-sys", "zerus-replica", + "zerus-watch-protocol", ] [[package]] @@ -2299,6 +2300,14 @@ dependencies = [ "uuid", ] +[[package]] +name = "zerus-watch-protocol" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 3ba3c4450..8b8c87ed7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ toml_edit = "0.25" unicode-width = "0.2" uuid = { version = "1", features = ["v4"] } zerus-replica = { path = "crates/replica" } +zerus-watch-protocol = { path = "crates/watch-protocol" } [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61", features = [ diff --git a/crates/watch-protocol/.cargo/config.toml b/crates/watch-protocol/.cargo/config.toml new file mode 100644 index 000000000..4a6a1abdd --- /dev/null +++ b/crates/watch-protocol/.cargo/config.toml @@ -0,0 +1,2 @@ +[resolver] +incompatible-rust-versions = "fallback" diff --git a/crates/watch-protocol/Cargo.lock b/crates/watch-protocol/Cargo.lock new file mode 100644 index 000000000..7a6956195 --- /dev/null +++ b/crates/watch-protocol/Cargo.lock @@ -0,0 +1,107 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "zerus-watch-protocol" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/crates/watch-protocol/Cargo.toml b/crates/watch-protocol/Cargo.toml new file mode 100644 index 000000000..fdc8f2260 --- /dev/null +++ b/crates/watch-protocol/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "zerus-watch-protocol" +version = "0.1.0" +edition = "2021" +rust-version = "1.99" +license = "MIT" +description = "Frames and control operations of the hgs watch event stream (watch-v1)" +publish = false + +# Shared by hgs and its clients. Keep it to serde: no Loro, Qt or async runtime. +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/crates/watch-protocol/src/lib.rs b/crates/watch-protocol/src/lib.rs new file mode 100644 index 000000000..792f161d9 --- /dev/null +++ b/crates/watch-protocol/src/lib.rs @@ -0,0 +1,645 @@ +//! Frames and control operations of `hgs [@host] watch --json` (ABI `watch-v1`). +//! +//! The server writes one JSON frame per line (NDJSON, UTF-8) on stdout and reads +//! one control operation per line on stdin. EOF on stdin ends the stream. Every +//! frame carries `v` (always [`VERSION`]), `t` (its type) and `seq`, which grows +//! by one per frame within a stream. A frame line, including its newline, is at +//! most [`MAX_FRAME_BYTES`]. +//! +//! Clients ignore fields they do not know. A frame type or topic they do not know +//! is skipped, except that `gap` and a fatal `error` always apply. +//! +//! See the "Event stream" section of `docs/reference.md` for the contract. +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// The `v` of every frame. +pub const VERSION: u32 = 1; +/// The ABI name a client looks for before it starts the stream. +pub const ABI: &str = "watch-v1"; +/// The line `hgs -h` prints when this ABI is available. +pub const HELP_MARKER: &str = "Event stream ABI: watch-v1"; +/// The largest frame line, including its newline. +pub const MAX_FRAME_BYTES: usize = 1 << 20; +/// The largest snapshot or delta page the server builds. +pub const PAGE_BYTES: usize = 512 << 10; +/// The server's outgoing queue; beyond it a topic receives `gap`. +pub const QUEUE_BYTES: usize = 4 << 20; +/// A heartbeat follows this much time without other frames. +pub const HEARTBEAT_MS: u64 = 10_000; +/// Concurrent Activity subscriptions per stream. +pub const ACTIVITY_SUBSCRIPTIONS: usize = 16; +/// The most events an Activity tail or tail snapshot carries. +pub const ACTIVITY_TAIL: usize = 100; +/// The most journal rows one Activity read returns. +pub const ACTIVITY_READ: usize = 200; +/// The longest control line the server accepts, without its newline. +pub const CONTROL_LINE_BYTES: usize = 64 << 10; + +/// Topic names. +pub mod topic { + /// Live session rows, keyed by [`super::catalog_key`]. + pub const CATALOG: &str = "catalog"; + /// The listing fields other than `sessions`. + pub const CATALOG_META: &str = "catalog.meta"; + /// One session's Activity events and inspection summary. + pub const ACTIVITY: &str = "activity"; +} + +/// Why a catalog key was removed. Clients treat an unknown reason as unknown. +pub mod reason { + /// The session ended and left no saved row. + pub const ENDED: &str = "ended"; + /// The run moved to Archive. + pub const ARCHIVED: &str = "archived"; + /// The saved binding was removed. + pub const FORGOTTEN: &str = "forgotten"; + /// The same session name now lists under a new run. + pub const REPLACED: &str = "replaced"; + /// The client fell behind and the server dropped frames. + pub const OVERFLOW: &str = "overflow"; + /// The Activity cursor is older than the journal's retention. + pub const RETENTION: &str = "retention"; + /// The Activity cursor is newer than the journal (a replaced journal). + pub const RESET: &str = "reset"; +} + +/// One line of the stream. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Frame { + pub v: u32, + pub seq: u64, + #[serde(flatten)] + pub body: Body, +} + +/// A frame's type (`t`) and fields. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "t", rename_all = "snake_case")] +pub enum Body { + Hello(Hello), + Snapshot(Snapshot), + Delta(Delta), + Events(Events), + Event(EventUpdate), + State(State), + Heartbeat(Heartbeat), + Gap(Gap), + Error(ErrorFrame), + Pong(Pong), +} + +/// The first frame of a stream. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Hello { + /// The computer's native identity; null when unavailable. + pub machine_id: Option, + pub host: String, + pub hgs_version: String, + /// A random identity of this stream. + pub stream: String, + pub heartbeat_ms: u64, + pub max_frame_bytes: usize, + pub topics: Vec, + pub limits: Limits, +} + +/// Bounds the server applies to this stream. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Limits { + pub activity_subscriptions: usize, + pub page_bytes: usize, + pub queue_bytes: usize, + pub activity_tail: usize, + pub activity_read: usize, + pub control_line_bytes: usize, +} + +impl Default for Limits { + fn default() -> Self { + Self { + activity_subscriptions: ACTIVITY_SUBSCRIPTIONS, + page_bytes: PAGE_BYTES, + queue_bytes: QUEUE_BYTES, + activity_tail: ACTIVITY_TAIL, + activity_read: ACTIVITY_READ, + control_line_bytes: CONTROL_LINE_BYTES, + } + } +} + +/// A bounded replacement of one topic, or of one key of it. A client keeps +/// showing what it had until the page with `last` arrives, then replaces it. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Snapshot { + pub topic: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub key: Option, + /// Keyed collections: catalog rows as [`CatalogItem`] objects, Activity + /// events as `inspect` returns them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub items: Option>, + /// Single-object topics such as `catalog.meta`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub value: Option, + pub page: u32, + pub last: bool, + /// The catalog revision, or the Activity journal `seq`. + pub cursor: i64, +} + +/// A keyed change of the catalog. Upserts carry whole rows. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Delta { + pub topic: String, + #[serde(default)] + pub upsert: Vec, + #[serde(default)] + pub remove: Vec, + pub cursor: i64, +} + +/// One catalog row and its key. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct CatalogItem { + pub key: String, + pub row: Value, +} + +/// A removed catalog key; `reason` is one of [`reason`] when known. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Removal { + pub key: String, + #[serde(default)] + pub reason: Option, +} + +/// Activity events after the previous cursor, oldest first. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Events { + pub topic: String, + pub key: String, + pub events: Vec, + /// The journal `seq` to resume after. + pub cursor: i64, +} + +/// A replacement of one open event by its stable ID; a higher `rev` wins. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct EventUpdate { + pub topic: String, + pub key: String, + pub id: String, + pub rev: u64, + pub event: Value, +} + +/// A replacement of one keyed object, such as a session's inspection summary. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct State { + pub topic: String, + pub key: String, + pub value: Value, + pub rev: u64, +} + +/// Sent after [`HEARTBEAT_MS`] without other frames. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Heartbeat { + /// Seconds since the Unix epoch. + pub at: f64, +} + +/// The client discards what it holds for the topic (or key); a snapshot follows. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Gap { + pub topic: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub key: Option, + pub reason: String, +} + +/// A refused operation or failed source. A fatal error ends the stream. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ErrorFrame { + pub code: String, + pub message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub topic: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub key: Option, + pub fatal: bool, +} + +/// The answer to a `ping`, echoing its `id`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Pong { + #[serde(default)] + pub id: Option, +} + +/// One control line from the client. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum Control { + Subscribe(Subscription), + Unsubscribe(Subscription), + Ping(Ping), +} + +/// A topic and its key. Activity takes `session`; later topics take `path` +/// (git) or `project` (tasks). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Subscription { + pub topic: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub project: Option, + /// Resume Activity after this journal `seq`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub after: Option, + /// How many recent events a subscription without `after` starts with. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tail: Option, +} + +/// The exact session a subscription follows. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct SessionRef { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub run_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub conversation_id: Option, +} + +impl SessionRef { + /// The session's key, the same as its catalog key. + pub fn key(&self) -> String { + key_for(self.run_id.as_deref(), &self.name) + } +} + +/// A liveness probe; the server answers with `pong`. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Ping { + #[serde(default)] + pub id: Option, +} + +/// `run:`, or `name:` for a session without a run. +pub fn key_for(run_id: Option<&str>, name: &str) -> String { + match run_id.filter(|run| !run.is_empty()) { + Some(run) => format!("run:{run}"), + None => format!("name:{name}"), + } +} + +/// The catalog key of an `ls --json --local` row; None without a name. +pub fn catalog_key(row: &Value) -> Option { + let name = row.get("name")?.as_str()?; + Some(key_for(row.get("run_id").and_then(Value::as_str), name)) +} + +/// A frame that cannot be written or read. +#[derive(Debug)] +pub enum FrameError { + /// The line, with its newline, would exceed [`MAX_FRAME_BYTES`]. + TooLarge(usize), + /// A line that does not start like a serialized [`Body`]. + Malformed, + Json(serde_json::Error), +} + +impl std::fmt::Display for FrameError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TooLarge(bytes) => write!(f, "frame of {bytes} bytes exceeds {MAX_FRAME_BYTES}"), + Self::Malformed => f.write_str("malformed frame body"), + Self::Json(error) => error.fmt(f), + } + } +} + +impl std::error::Error for FrameError {} + +impl From for FrameError { + fn from(error: serde_json::Error) -> Self { + Self::Json(error) + } +} + +/// Whether a line of `bytes` (without its newline) fits in a frame. +pub fn fits(bytes: usize) -> bool { + bytes < MAX_FRAME_BYTES +} + +/// Fails unless `line` (without its newline) fits in a frame. +pub fn check_size(line: &str) -> Result<(), FrameError> { + if fits(line.len()) { + Ok(()) + } else { + Err(FrameError::TooLarge(line.len() + 1)) + } +} + +/// The most bytes [`envelope`] adds to a body. +pub const ENVELOPE_BYTES: usize = 40; + +/// A body as JSON without `v` and `seq`, for a writer that numbers frames +/// only when it writes them. [`envelope`] completes it. +pub fn encode_body(body: &Body) -> Result { + let text = serde_json::to_string(body)?; + if !fits(text.len() + ENVELOPE_BYTES) { + return Err(FrameError::TooLarge(text.len() + ENVELOPE_BYTES + 1)); + } + Ok(text) +} + +/// The frame line (without its newline) of a body from [`encode_body`]. +pub fn envelope(seq: u64, body: &str) -> Result { + let rest = body.strip_prefix("{\"t\":").ok_or(FrameError::Malformed)?; + Ok(format!("{{\"v\":{VERSION},\"seq\":{seq},\"t\":{rest}")) +} + +/// One frame line, without its newline. +pub fn encode(frame: &Frame) -> Result { + let line = serde_json::to_string(frame)?; + check_size(&line)?; + Ok(line) +} + +/// One frame from a line, with or without its newline. +pub fn decode(line: &str) -> Result { + let line = line.strip_suffix('\n').unwrap_or(line); + check_size(line)?; + Ok(serde_json::from_str(line)?) +} + +/// One control operation from a line, with or without its newline. +pub fn decode_control(line: &str) -> Result { + let line = line.strip_suffix('\n').unwrap_or(line); + if line.len() > CONTROL_LINE_BYTES { + return Err(FrameError::TooLarge(line.len() + 1)); + } + Ok(serde_json::from_str(line)?) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn frames() -> Vec { + vec![ + Body::Hello(Hello { + machine_id: Some("00000000-0000-4000-8000-000000000001".into()), + host: "workstation".into(), + hgs_version: "1.56.0".into(), + stream: "stream-1".into(), + heartbeat_ms: HEARTBEAT_MS, + max_frame_bytes: MAX_FRAME_BYTES, + topics: vec![ + topic::CATALOG.into(), + topic::CATALOG_META.into(), + topic::ACTIVITY.into(), + ], + limits: Limits::default(), + }), + Body::Snapshot(Snapshot { + topic: topic::CATALOG.into(), + key: None, + items: Some(vec![ + json!({"key":"run:r1","row":{"name":"codex/p/t","run_id":"r1","at":1.5}}), + ]), + value: None, + page: 0, + last: true, + cursor: 1, + }), + Body::Snapshot(Snapshot { + topic: topic::CATALOG_META.into(), + key: None, + items: None, + value: Some(json!({"host":"workstation","projects":{}})), + page: 0, + last: true, + cursor: 1, + }), + Body::Delta(Delta { + topic: topic::CATALOG.into(), + upsert: vec![CatalogItem { + key: "name:sh/p".into(), + row: json!({"name":"sh/p"}), + }], + remove: vec![ + Removal { + key: "run:r0".into(), + reason: Some(reason::REPLACED.into()), + }, + Removal { + key: "run:r2".into(), + reason: None, + }, + ], + cursor: 2, + }), + Body::Events(Events { + topic: topic::ACTIVITY.into(), + key: "run:r1".into(), + events: vec![json!({"type":"Stop","seq":41,"at":1787840176.25})], + cursor: 41, + }), + Body::Event(EventUpdate { + topic: topic::ACTIVITY.into(), + key: "run:r1".into(), + id: "message-1".into(), + rev: 3, + event: json!({"text":"partial"}), + }), + Body::State(State { + topic: topic::ACTIVITY.into(), + key: "run:r1".into(), + value: json!({"phase":"idle"}), + rev: 4, + }), + Body::Heartbeat(Heartbeat { at: 1787840176.5 }), + Body::Gap(Gap { + topic: topic::ACTIVITY.into(), + key: Some("run:r1".into()), + reason: reason::RETENTION.into(), + }), + Body::Error(ErrorFrame { + code: "too_many_subscriptions".into(), + message: "at most 16".into(), + topic: Some(topic::ACTIVITY.into()), + key: None, + fatal: false, + }), + Body::Pong(Pong { id: Some(json!(7)) }), + ] + } + + #[test] + fn every_frame_round_trips_with_version_type_and_sequence() { + for (index, body) in frames().into_iter().enumerate() { + let frame = Frame { + v: VERSION, + seq: index as u64 + 1, + body, + }; + let line = encode(&frame).unwrap(); + assert!(!line.contains('\n')); + let value: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(value["v"], 1); + assert_eq!(value["seq"], index as u64 + 1); + assert!(value["t"].is_string(), "{line}"); + assert_eq!(decode(&format!("{line}\n")).unwrap(), frame); + } + } + + #[test] + fn envelope_numbers_an_encoded_body_like_a_whole_frame() { + for body in frames() { + let encoded = encode_body(&body).unwrap(); + let line = envelope(u64::MAX, &encoded).unwrap(); + let whole = encode(&Frame { + v: VERSION, + seq: u64::MAX, + body: body.clone(), + }) + .unwrap(); + assert_eq!(line, whole); + assert!(line.len() <= encoded.len() + ENVELOPE_BYTES); + } + assert!(matches!(envelope(1, "{}"), Err(FrameError::Malformed))); + } + + #[test] + fn wire_names_match_the_contract() { + let line = encode(&Frame { + v: 1, + seq: 9, + body: Body::Heartbeat(Heartbeat { at: 2.0 }), + }) + .unwrap(); + assert_eq!(line, r#"{"v":1,"seq":9,"t":"heartbeat","at":2.0}"#); + let gap = encode(&Frame { + v: 1, + seq: 10, + body: Body::Gap(Gap { + topic: "catalog".into(), + key: None, + reason: "overflow".into(), + }), + }) + .unwrap(); + assert_eq!( + gap, + r#"{"v":1,"seq":10,"t":"gap","topic":"catalog","reason":"overflow"}"# + ); + } + + #[test] + fn unknown_fields_are_ignored_for_forward_compatibility() { + let frame = decode(r#"{"v":1,"seq":3,"t":"heartbeat","at":1.0,"later":true}"#).unwrap(); + assert_eq!(frame.body, Body::Heartbeat(Heartbeat { at: 1.0 })); + let control = decode_control(r#"{"op":"ping","id":"a","later":[1]}"#).unwrap(); + assert_eq!( + control, + Control::Ping(Ping { + id: Some(json!("a")) + }) + ); + } + + #[test] + fn control_operations_parse_with_their_keys() { + let line = r#"{"op":"subscribe","topic":"activity","session":{"name":"codex/p/t","run_id":"r1","conversation_id":"c1"},"after":40,"tail":20}"#; + let Control::Subscribe(subscription) = decode_control(line).unwrap() else { + panic!() + }; + assert_eq!(subscription.topic, "activity"); + assert_eq!(subscription.after, Some(40)); + assert_eq!(subscription.tail, Some(20)); + let session = subscription.session.unwrap(); + assert_eq!(session.key(), "run:r1"); + assert_eq!( + SessionRef { + name: "sh/p".into(), + ..Default::default() + } + .key(), + "name:sh/p" + ); + let Control::Unsubscribe(gone) = + decode_control(r#"{"op":"unsubscribe","topic":"activity","session":{"name":"x"}}"#) + .unwrap() + else { + panic!() + }; + assert_eq!(gone.session.unwrap().key(), "name:x"); + assert_eq!( + decode_control(r#"{"op":"ping"}"#).unwrap(), + Control::Ping(Ping { id: None }) + ); + assert!(decode_control(r#"{"op":"launch"}"#).is_err()); + assert!(decode_control(r#"{"topic":"activity"}"#).is_err()); + let long = format!( + r#"{{"op":"ping","id":"{}"}}"#, + "x".repeat(CONTROL_LINE_BYTES) + ); + assert!(matches!( + decode_control(&long), + Err(FrameError::TooLarge(_)) + )); + } + + #[test] + fn catalog_keys_prefer_the_run() { + assert_eq!( + catalog_key(&json!({"name":"codex/p/t","run_id":"r1"})).as_deref(), + Some("run:r1") + ); + assert_eq!( + catalog_key(&json!({"name":"sh/p","run_id":""})).as_deref(), + Some("name:sh/p") + ); + assert_eq!( + catalog_key(&json!({"name":"sh/p","run_id":null})).as_deref(), + Some("name:sh/p") + ); + assert_eq!( + catalog_key(&json!({"name":"sh/p"})).as_deref(), + Some("name:sh/p") + ); + assert_eq!(catalog_key(&json!({"run_id":"r1"})), None); + } + + #[test] + fn frames_are_capped_at_one_mebibyte_including_the_newline() { + assert!(fits(MAX_FRAME_BYTES - 1)); + assert!(!fits(MAX_FRAME_BYTES)); + let small = "x".repeat(MAX_FRAME_BYTES - 1); + assert!(check_size(&small).is_ok()); + let large = "x".repeat(MAX_FRAME_BYTES); + assert!( + matches!(check_size(&large), Err(FrameError::TooLarge(bytes)) if bytes == MAX_FRAME_BYTES + 1) + ); + let body = Body::Events(Events { + topic: topic::ACTIVITY.into(), + key: "run:r".into(), + events: vec![json!({"detail":"x".repeat(MAX_FRAME_BYTES)})], + cursor: 1, + }); + assert!(matches!(encode_body(&body), Err(FrameError::TooLarge(_)))); + assert!(matches!( + encode(&Frame { v: 1, seq: 1, body }), + Err(FrameError::TooLarge(_)) + )); + assert!(decode(&large).is_err()); + } +} diff --git a/docs/reference.md b/docs/reference.md index d344e0123..f8ae55a23 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -3948,6 +3948,89 @@ needs manual inspection before another request. `agent_id` is supported only for verified DeepSeek continuable children. Questions carry request identity/hash through `pending_questions`; answers use the exact unchanged request. +### Event stream + +```sh +hgs watch --json +hgs @mac watch --json +``` + +`watch --json` is a long-lived stream of one computer's live sessions and of the +Activity of the sessions a client follows. It is additive: every command above +keeps its output. `hgs -h` prints `Event stream ABI: watch-v1` where it exists. +A client that does not find the line keeps polling that computer. `@host` runs +the peer's stream over one SSH connection with `ServerAliveInterval=10` and +`ServerAliveCountMax=3`. An older peer answers with an error, never a launch. + +The stream writes one JSON frame per line on stdout and reads one control +operation per line on stdin. EOF on stdin ends it with exit code 0, so a closed +client or a dropped SSH link leaves no watcher behind. A closed stdout ends it +quietly. The frame types live in `crates/watch-protocol`. + +Every frame has `v` (1), `t` and `seq`, which grows by one per frame. The first +frame is `hello` with `machine_id`, `host`, `hgs_version`, a random `stream`, +`heartbeat_ms`, `max_frame_bytes`, `topics` and `limits`. A `heartbeat` with `at` +follows 10 s without other frames. + +| `t` | Fields | Meaning | +|---|---|---| +| `snapshot` | `topic`, `key?`, `items` or `value`, `page`, `last`, `cursor` | a replacement; keep the previous display until `last` | +| `delta` | `topic`, `upsert[]` of `{key, row}`, `remove[]` of `{key, reason}`, `cursor` | catalog changes | +| `events` | `topic`, `key`, `events[]`, `cursor` | new Activity events, oldest first | +| `state` | `topic`, `key`, `value`, `rev` | the latest summary of one session | +| `gap` | `topic`, `key?`, `reason` | discard the topic's state; a snapshot follows | +| `error` | `code`, `message`, `topic?`, `key?`, `fatal` | a refused operation or a failed source | +| `pong` | `id` | the answer to `ping` | + +The `catalog` topic is always on. Its rows are exactly the `sessions` of +`hgs ls --json --local --no-archives`, keyed `run:`, or `name:` for +a row without a run. The stream starts with a snapshot of them in pages of at +most 512 KiB, then sends deltas. A rename is an upsert of the same key. A removal +gives `replaced` (the name lists under a new run), `archived`, `forgotten` or +`ended`. The listing's other fields arrive as one `catalog.meta` snapshot: at +once when a lasting field changes, and at most every 10 s when only `metrics` or +`desktop_presence` changed. Archives stay outside the stream; page them with +`archives --json`. + +```json +{"op":"subscribe","topic":"activity","session":{"name":"claude/sample-project/review","run_id":"RUN","conversation_id":"CONVERSATION"},"after":123,"tail":50} +{"op":"unsubscribe","topic":"activity","session":{"name":"claude/sample-project/review","run_id":"RUN"}} +{"op":"ping","id":1} +``` + +An Activity subscription follows one session by its catalog key. Its events are +the objects `inspect` returns, and `cursor` is the journal `seq` that +`inspect --after` takes. Without `after`, the first `events` frame carries the +newest `tail` events (at most 100). With `after`, it carries the events after +that `seq`, at most 200 per frame read. When `after` is older than the journal +keeps, or newer than its newest row, the stream sends `gap` (`retention` or +`reset`) and a tail `snapshot`. DeepSeek and OpenCode sessions answer from their +own hosts: each change replaces their event window with a `snapshot`. `state` +carries the `inspect --skip-processes` summary without `events`, `cursor` and +`history_truncated`. It is sent when anything other than `goal_observed_at` +changed. A session that leaves the catalog ends its subscription with the error +`subscription_ended`. Unsubscribe stops it. A stream follows at most 16 sessions; +more fail with `too_many_subscriptions`. Other topics fail with +`unsupported_topic`; read them with their commands. + +The stream checks its sources every 250 ms. It stats the bindings, read marks, +notification preferences, waits, launches, renames, name routes, the archive +folder and the DeepSeek and OpenCode bindings, and holds a read-only connection +to the Activity journal. A change lists again after 150 ms, at most every 500 ms +and at most once per four durations of the previous listing. A listing also +runs every 5 s, for changes no file shows, such as an exited process or a tmux +rename. Followed summaries are re-read when their journal rows or catalog row +change, at most every 500 ms, and otherwise every 2 s while the agent is busy and +every 15 s while it is idle; DeepSeek and OpenCode every 2 s. + +A frame is at most 1 MiB including its newline. An event too large for a frame +is cut with `"truncated": true`, and an oversized summary loses the oldest items +of its largest lists, named in `stream_truncated`; `inspect` returns them whole. +The outgoing queue holds 4 MiB. When the client reads slowly, catalog changes +and summaries coalesce per key. When the queue overflows, the topic receives +`gap` (`overflow`) and a fresh snapshot once the client catches up. Control lines +are at most 64 KiB. + ## Validation Use [CONTRIBUTING.md](../CONTRIBUTING.md) for standard checks. Integration tests diff --git a/mac-install.sh b/mac-install.sh index e3332d978..03c68e738 100755 --- a/mac-install.sh +++ b/mac-install.sh @@ -56,7 +56,8 @@ sources=(Cargo.toml Cargo.lock build.rs .cargo src scripts install.sh hgs hgs_st services/mobile/connector/Cargo.toml services/mobile/connector/Cargo.lock services/mobile/connector/.cargo services/mobile/connector/src services/mobile/relay/Cargo.toml services/mobile/relay/src services/mobile/crypto/Cargo.toml services/mobile/crypto/src - crates/replica/Cargo.toml crates/replica/Cargo.lock crates/replica/.cargo crates/replica/src) + crates/replica/Cargo.toml crates/replica/Cargo.lock crates/replica/.cargo crates/replica/src + crates/watch-protocol/Cargo.toml crates/watch-protocol/Cargo.lock crates/watch-protocol/.cargo crates/watch-protocol/src) tar czf - -C "$here" "${sources[@]}" \ | ssh -o BatchMode=yes "$host" "cd ~/.local/src/hgs/.rust-build && rm -rf ${sources[*]} && tar xzf -" # install.sh builds the mobile connector only on request (HGS_MOBILE_CONNECTOR=1) diff --git a/src/cli.rs b/src/cli.rs index da2e27f18..1f5c5466b 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -66,6 +66,8 @@ pub(crate) const USAGE: &str = r#"usage: hgs [@host] [project] [-c [ID]] [ hgs self [--json] the calling agent's own session reference hgs [@host] guide [topic] how agents message, wait for, launch and stop sessions and use tasks hgs inspect 'zerus://session?...' resolve and inspect a referenced conversation + hgs [@host] watch --json long-lived NDJSON event stream (stdin control) + Event stream ABI: watch-v1 hgs [@host] processes [--archive ] observed shell commands as JSON hgs [@host] processes --output ID | --stop ID --run RUN --conversation ID [--generation ID] hgs [@host] send --json submit message/attachments from JSON stdin @@ -192,7 +194,7 @@ fn local_sessions() -> Result<()> { // Repository preparation owns only its checkout and receipt. Like the existing // configuration/control commands, it does not require or enable a session host. fn needs_local_sessions(command: &str) -> bool { - !matches!(command, "ls" | "machine" | "presence" | "swarm" | "project" | "account" | "speech" | "analytics" | "graph" | "wait" | "mobile" | "hooks" | "repository") + !matches!(command, "ls" | "watch" | "machine" | "presence" | "swarm" | "project" | "account" | "speech" | "analytics" | "graph" | "wait" | "mobile" | "hooks" | "repository") } /// Fails unless this process may run what the processes of a session run: @@ -628,6 +630,7 @@ pub fn dispatch(mut args: Vec) -> Result { Ok(0) } "ls" => list(&config, args, dry), + "watch" => crate::watch::run(&config, args, dry), "archives" => { let mobile = cfg!(windows) && args.last().map(String::as_str)==Some("--windows-mobile"); let args = if mobile { &args[..args.len()-1] } else { args }; @@ -966,6 +969,23 @@ fn remote( all.extend(args.clone()); all } + // One long-lived stream: keepalives detect a dropped link, and EOF on + // the local stdin reaches the peer's watcher, which then exits. + "watch" => { + flags = vec![ + "-o".into(), + "BatchMode=yes".into(), + "-o".into(), + "ConnectTimeout=5".into(), + "-o".into(), + "ServerAliveInterval=10".into(), + "-o".into(), + "ServerAliveCountMax=3".into(), + ]; + let mut all = vec![command.into()]; + all.extend(args.clone()); + all + } // Phone management reaches the relay through the gateway peer. "mobile" => { flags = vec![ @@ -1039,6 +1059,9 @@ fn remote( platform::remote_sh(&format!("if ~/.local/bin/hgs --help 2>/dev/null | grep -qF 'hooks-health-v1'; then exec ~/.local/bin/hgs {}; else echo 'hgs: update the selected computer to inspect hook health' >&2; exit 1; fi",platform::join(&remote_args))) } else if command == "analytics" { platform::remote_sh(&format!("if ~/.local/bin/hgs --help 2>/dev/null | grep -qF 'hgs [@host] analytics --json'; then exec ~/.local/bin/hgs {} --read-only; else echo 'hgs: update the selected computer to read analytics' >&2; exit 1; fi",platform::join(&remote_args))) + } else if command == "watch" { + // An older peer would read an unknown command as an agent launch. + platform::remote_sh(&format!("if ~/.local/bin/hgs --help 2>/dev/null | grep -qF '{}'; then exec ~/.local/bin/hgs {}; else echo 'hgs: update the selected computer to stream events (watch-v1)' >&2; exit 1; fi",zerus_watch_protocol::HELP_MARKER,platform::join(&remote_args))) } else if command == "guide" { platform::remote_sh(&format!("if ~/.local/bin/hgs --help 2>/dev/null | grep -qF 'hgs [@host] guide'; then exec ~/.local/bin/hgs {}; else echo 'hgs: the selected computer has no agent guide; check hgs @host -h' >&2; exit 1; fi",platform::join(&remote_args))) } else { format!("~/.local/bin/hgs {}", platform::remote_join(&remote_args)) }; @@ -1081,6 +1104,8 @@ fn remote( } pub(crate) fn local_json(config: &Config) -> Result { local_json_archives(config, true) } +/// `ls --json --local --no-archives`, which `hgs watch` streams. +pub(crate) fn local_listing(config: &Config) -> Result { local_json_archives(config, false) } fn local_json_archives(config: &Config, archives: bool) -> Result { if !platform::local_sessions() { // A control console: no tmux, session state or local projects. Absent diff --git a/src/main.rs b/src/main.rs index c545ff3a6..fcfa71bed 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,6 +22,7 @@ mod status; mod state; mod swarm; mod tasks; +mod watch; fn main() { if std::env::args().nth(1).as_deref() == Some("--macos-session-service") { diff --git a/src/state/journal.rs b/src/state/journal.rs index 034ea8ff5..1d9733b77 100644 --- a/src/state/journal.rs +++ b/src/state/journal.rs @@ -788,28 +788,17 @@ pub(super) fn inspection(name: &str, after: i64, archive_id: Option<&str>, inclu |row| row.get(0), ) .map_err(|e| e.to_string())?; - let mut statement = db.prepare(&if after > 0 { - format!("SELECT seq, payload FROM events WHERE {TIMELINE} AND seq>?4 ORDER BY seq LIMIT 200") - } else { format!("SELECT seq, payload FROM events WHERE {TIMELINE} AND seq>?4 ORDER BY seq DESC LIMIT 100") }).map_err(|e| e.to_string())?; - let mut rows: Vec<(i64, String)> = statement - .query_map(params![journal_name, conversation, earlier, after], |row| { - Ok((row.get(0)?, row.get(1)?)) - }) - .map_err(|e| e.to_string())? - .collect::>() - .map_err(|e| e.to_string())?; - if after == 0 { - rows.reverse(); - } + let rows = if after > 0 { + timeline_rows(&db, journal_name, conversation, &earlier, after, 200, false)? + } else { + timeline_rows(&db, journal_name, conversation, &earlier, after, 100, true)? + }; output["cursor"] = json!(rows.last().map(|(seq, _)| *seq).unwrap_or(after)); output["history_truncated"] = json!(after > 0 && oldest.map(|seq| after < seq - 1).unwrap_or(false)); let mut events = Vec::new(); for (seq, payload) in rows { - let mut event: Value = serde_json::from_str(&payload).map_err(|e| e.to_string())?; - event["seq"] = json!(seq); - mark_system_prompt(&record, &mut event, ""); - events.push(event); + events.push(timeline_event(&record, seq, &payload)?); } output["events"] = json!(events); let (messages, truncated) = message_events(&db, journal_name, conversation, &earlier)?; @@ -828,6 +817,35 @@ pub(super) fn inspection(name: &str, after: i64, archive_id: Option<&str>, inclu Ok(output) } +/// A session timeline's rows after `after`, oldest first: the first `limit` +/// of them, or with `newest` the last `limit`. `inspection` and `hgs watch` +/// both read Activity through it and `timeline_event`. +pub(super) fn timeline_rows(db: &Connection, journal_name: &str, conversation: Option<&str>, earlier: &str, + after: i64, limit: usize, newest: bool) -> Result> { + let mut statement = db.prepare(&if newest { + format!("SELECT seq, payload FROM events WHERE {TIMELINE} AND seq>?4 ORDER BY seq DESC LIMIT ?5") + } else { format!("SELECT seq, payload FROM events WHERE {TIMELINE} AND seq>?4 ORDER BY seq LIMIT ?5") }).map_err(|e| e.to_string())?; + let mut rows: Vec<(i64, String)> = statement + .query_map(params![journal_name, conversation, earlier, after, limit as i64], |row| { + Ok((row.get(0)?, row.get(1)?)) + }) + .map_err(|e| e.to_string())? + .collect::>() + .map_err(|e| e.to_string())?; + if newest { + rows.reverse(); + } + Ok(rows) +} + +/// One Activity event as `inspect` returns it. +pub(super) fn timeline_event(record: &Value, seq: i64, payload: &str) -> Result { + let mut event: Value = serde_json::from_str(payload).map_err(|e| e.to_string())?; + event["seq"] = json!(seq); + mark_system_prompt(record, &mut event, ""); + Ok(event) +} + pub(super) fn journal_name(record: &Value) -> &str { if string(record, "journal_name").is_empty() { string(record, "name") diff --git a/src/state/mod.rs b/src/state/mod.rs index 9184842ef..3573c1fd1 100644 --- a/src/state/mod.rs +++ b/src/state/mod.rs @@ -65,6 +65,7 @@ mod resume_input; mod scan_cache; mod schedules; mod waits; +pub(crate) mod watch_source; mod search; pub(crate) mod session_graph; mod storage; diff --git a/src/state/watch_source.rs b/src/state/watch_source.rs new file mode 100644 index 000000000..d9b6a8199 --- /dev/null +++ b/src/state/watch_source.rs @@ -0,0 +1,143 @@ +//! What `hgs watch` (src/watch) reads from session state: inspections and the +//! Activity journal exactly as `hgs inspect` reads them, as values instead of +//! printed output. Nothing here writes session state beyond what `hgs inspect` +//! and `hgs ls` already do. +use super::*; +use rusqlite::{Connection, OpenFlags}; +use std::time::Duration; + +/// The session state folder. +pub(crate) fn state_root() -> PathBuf { + root() +} + +/// Whether `name` is a DeepSeek or OpenCode session. Their Activity comes from +/// the agent's own host and each read replaces the previous window. +pub(crate) fn native(name: &str) -> bool { + dsh::exists(name) || opencode::exists(name) +} + +/// `hgs inspect NAME --after AFTER --skip-processes`. +pub(crate) fn inspect(name: &str, after: i64) -> Result { + local_sessions()?; + if opencode::exists(name) { + return opencode::inspect(name, None); + } + if dsh::exists(name) { + return dsh::inspect(name, None); + } + journal::inspection(name, after, None, false) +} + +/// A read-only connection to the Activity journal, once it exists. It never +/// creates the file or its schema, and holds no lock between statements. +pub(crate) fn journal() -> Option { + local_sessions().ok()?; + let path = root().join("events.sqlite3"); + if !path.is_file() { + return None; + } + let db = Connection::open_with_flags( + &path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .ok()?; + db.busy_timeout(Duration::from_millis(250)).ok()?; + Some(db) +} + +/// Changes whenever another connection commits to the journal. +pub(crate) fn data_version(db: &Connection) -> Result { + db.query_row("PRAGMA data_version", [], |row| row.get(0)) + .map_err(|e| e.to_string()) +} + +/// The oldest and newest retained `seq` (0 when the journal is empty). +pub(crate) fn bounds(db: &Connection) -> Result<(i64, i64)> { + db.query_row( + "SELECT COALESCE(MIN(seq),0), COALESCE(MAX(seq),0) FROM events", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .map_err(|e| e.to_string()) +} + +/// The journal names with rows after `after`, from at most `limit` rows. +pub(crate) fn changed_names(db: &Connection, after: i64, limit: usize) -> Result> { + let mut statement = db + .prepare("SELECT DISTINCT name FROM (SELECT name FROM events WHERE seq>?1 ORDER BY seq LIMIT ?2)") + .map_err(|e| e.to_string())?; + let names = statement + .query_map(rusqlite::params![after, limit as i64], |row| row.get::<_, Option>(0)) + .map_err(|e| e.to_string())? + .filter_map(|row| row.ok().flatten()) + .collect(); + Ok(names) +} + +/// The rows a tracked session's Activity shows: its journal name and the +/// conversations its timeline spans, taken from its saved binding. +pub(crate) struct Timeline { + record: Value, + earlier: String, +} + +impl Timeline { + /// The journal name, which a rename keeps. + pub(crate) fn journal(&self) -> &str { + journal::journal_name(&self.record) + } + + pub(crate) fn run_id(&self) -> &str { + string(&self.record, "run_id") + } + + /// Events after `after` as `inspect` shapes them, oldest first: the first + /// `limit`, or with `newest` the last `limit`. + pub(crate) fn events(&self, db: &Connection, after: i64, limit: usize, newest: bool) -> Result> { + let rows = journal::timeline_rows( + db, + self.journal(), + self.record["conversation_id"].as_str(), + &self.earlier, + after, + limit, + newest, + )?; + rows.into_iter() + .map(|(seq, payload)| journal::timeline_event(&self.record, seq, &payload)) + .collect() + } +} + +/// The timeline of a tracked session; None for a session without a saved +/// binding (DeepSeek, OpenCode and untracked sessions). +pub(crate) fn timeline(name: &str) -> Result> { + local_sessions()?; + if !record_path(name).exists() { + return Ok(None); + } + let record = read(name)?; + let earlier = clear_context::earlier_conversations(&record).to_string(); + Ok(Some(Timeline { record, earlier })) +} + +/// Whether a run is in Archive. Rare: only for a catalog row that disappeared. +pub(crate) fn archived(run_id: &str) -> bool { + if run_id.is_empty() || local_sessions().is_err() || !root().join("archives").is_dir() { + return false; + } + let Ok(_guard) = lock(None) else { return false }; + let Ok(db) = archive_catalog::catalog() else { return false }; + db.query_row( + "SELECT EXISTS(SELECT 1 FROM archive_rows WHERE run=?1)", + [run_id], + |row| row.get::<_, bool>(0), + ) + .unwrap_or(false) +} + +/// Whether `name` still has a saved binding of any agent. +pub(crate) fn saved(name: &str) -> bool { + local_sessions().is_ok() && (record_path(name).exists() || native(name)) +} diff --git a/src/watch/activity.rs b/src/watch/activity.rs new file mode 100644 index 000000000..57734dd2b --- /dev/null +++ b/src/watch/activity.rs @@ -0,0 +1,229 @@ +//! Activity subscriptions: the inspection summary as `state`, journal events +//! as `events`, and bounds that keep every frame under the frame limit. +use super::catalog::{self, Digest32}; +use serde_json::{json, Value}; +use std::time::Instant; +use zerus_watch_protocol as protocol; + +/// Sections of an inspection that `state` leaves out: Activity rows travel as +/// `events`, and process sets belong to a later topic. +const NOT_STATE: [&str; 6] = ["events", "processes", "process_work", "cursor", "history_truncated", "replace_events"]; +const VOLATILE_STATE: [&[&str]; 2] = [&["goal_observed_at"], &["process_work", "sampled_at"]]; + +/// One session the client follows. +pub(super) struct Subscription { + pub key: String, + /// The current name; a rename keeps the key and updates the name. + pub name: String, + pub run_id: Option, + /// DeepSeek and OpenCode: each read replaces the event window. + pub native: bool, + pub cursor: i64, + pub tail: usize, + pub rev: u64, + pub state: Option, + pub events: Option, + pub refreshed: Instant, + /// A source changed: refresh the summary soon. + pub dirty: bool, + /// The last summary showed a busy agent: refresh it more often. + pub busy: bool, + /// The last journal read was full: read again on the next pulse. + pub more: bool, + /// Frames were dropped: send `gap` and a tail snapshot once drained. + pub resync: Option<&'static str>, + /// The catalog listed the key at least once. + pub listed: bool, + /// The digest of the session's catalog row when last seen. + pub row: Option, + /// The journal name its rows carry; a rename keeps it. + pub journal: Option, +} + +impl Subscription { + pub(super) fn new(key: String, name: String, run_id: Option, native: bool, tail: usize) -> Self { + Self { + key, + name, + run_id, + native, + cursor: 0, + tail, + rev: 0, + state: None, + events: None, + refreshed: Instant::now(), + dirty: false, + busy: false, + more: false, + resync: None, + listed: false, + row: None, + journal: None, + } + } +} + +/// The inspection without its event and process sections. +pub(super) fn state_value(inspection: &Value) -> Value { + let mut value = inspection.clone(); + if let Some(object) = value.as_object_mut() { + for key in NOT_STATE { + object.remove(key); + } + } + value +} + +/// A summary's digest with its observation timestamps masked. +pub(super) fn state_digest(value: &Value) -> Digest32 { + catalog::digest(&catalog::masked(value, &VOLATILE_STATE)) +} + +pub(super) fn busy(value: &Value) -> bool { + value["activity"] == "busy" +} + +fn encoded_len(value: &Value) -> usize { + serde_json::to_string(value).map_or(usize::MAX, |text| text.len()) +} + +/// Cuts a string to at most `bytes`, on a character boundary. +fn cut(text: &str, bytes: usize) -> String { + let mut end = bytes.min(text.len()); + while !text.is_char_boundary(end) { + end -= 1; + } + format!("{}…", &text[..end]) +} + +/// Fits one event in `limit` encoded bytes by cutting its longest strings, +/// marked with `"truncated": true`; the whole event remains readable through +/// `hgs inspect` and `hgs history`. +pub(super) fn shrink_event(event: &mut Value, limit: usize) { + for _ in 0..16 { + let size = encoded_len(event); + if size <= limit { + return; + } + let Some(object) = event.as_object_mut() else { break }; + let longest = object + .iter() + .filter_map(|(key, value)| value.as_str().map(|text| (key.clone(), text.len()))) + .max_by_key(|(_, len)| *len); + let Some((key, len)) = longest.filter(|(_, len)| *len > 64) else { break }; + let keep = len.saturating_sub(size - limit + 64); + let text = cut(object[&key].as_str().unwrap_or(""), keep); + object.insert(key, json!(text)); + object.insert("truncated".into(), json!(true)); + } + if encoded_len(event) > limit { + let keep = |key: &str| event.get(key).cloned().unwrap_or(Value::Null); + *event = json!({"seq":keep("seq"),"type":keep("type"),"at":keep("at"),"truncated":true}); + } +} + +/// Fits a summary in `limit` encoded bytes: the largest lists lose their oldest +/// half until it fits, and `stream_truncated` names them. A client reads the +/// whole summary with `hgs inspect`. +pub(super) fn shrink_state(value: &mut Value, limit: usize) { + let mut cut_lists = Vec::new(); + for _ in 0..64 { + if encoded_len(value) <= limit { + break; + } + let Some(object) = value.as_object_mut() else { break }; + let largest = object + .iter() + .filter(|(_, value)| value.as_array().is_some_and(|items| !items.is_empty())) + .map(|(key, value)| (key.clone(), encoded_len(value))) + .max_by_key(|(_, len)| *len); + let Some((key, _)) = largest else { break }; + if let Some(items) = object.get_mut(&key).and_then(Value::as_array_mut) { + let drop = items.len().div_ceil(2); + items.drain(..drop); + } + if !cut_lists.contains(&key) { + cut_lists.push(key); + } + } + if encoded_len(value) > limit { + let keep = |key: &str| value.get(key).cloned().unwrap_or(Value::Null); + *value = json!({"name":keep("name"),"run_id":keep("run_id"),"conversation_id":keep("conversation_id"), + "tracked":keep("tracked"),"stream_truncated":true}); + } else if !cut_lists.is_empty() { + value["stream_truncated"] = json!(cut_lists); + } +} + +/// Splits events into frames of at most `limit` encoded bytes of events, +/// cutting any single event that alone exceeds it. +pub(super) fn event_pages(events: Vec, limit: usize) -> Vec> { + let events = events + .into_iter() + .map(|mut event| { + shrink_event(&mut event, limit); + event + }) + .collect(); + catalog::pages(events, encoded_len, limit) +} + +/// The largest event or summary payload one frame carries. +pub(super) const PAYLOAD_LIMIT: usize = protocol::MAX_FRAME_BYTES - 64 * 1024; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn state_leaves_out_events_processes_and_cursors_and_ignores_observation_times() { + let inspection = json!({"name":"codex/p/t","phase":"idle","activity":"idle","events":[{"seq":1}], + "processes":{"items":[]},"cursor":1,"history_truncated":false,"goal_observed_at":5.0,"message_events":[]}); + let value = state_value(&inspection); + assert_eq!(value, json!({"name":"codex/p/t","phase":"idle","activity":"idle","goal_observed_at":5.0,"message_events":[]})); + let mut later = value.clone(); + later["goal_observed_at"] = json!(6.0); + assert_eq!(state_digest(&value), state_digest(&later)); + later["phase"] = json!("working"); + assert_ne!(state_digest(&value), state_digest(&later)); + assert!(!busy(&value)); + assert!(busy(&json!({"activity":"busy"}))); + } + + #[test] + fn oversized_events_are_cut_with_a_marker_and_paged() { + let mut event = json!({"seq":7,"type":"Stop","at":1.0,"detail":"é".repeat(400_000)}); + shrink_event(&mut event, 10_000); + assert!(encoded_len(&event) <= 10_000); + assert_eq!(event["truncated"], true); + assert_eq!(event["seq"], 7); + assert!(event["detail"].as_str().unwrap().ends_with('…')); + let mut small = json!({"seq":8,"detail":"short"}); + shrink_event(&mut small, 10_000); + assert_eq!(small, json!({"seq":8,"detail":"short"})); + let mut bulky = json!({"seq":9,"type":"X","at":2.0,"list":vec![1; 5_000]}); + shrink_event(&mut bulky, 1_000); + assert_eq!(bulky, json!({"seq":9,"type":"X","at":2.0,"truncated":true})); + let events = (0..10).map(|seq| json!({"seq":seq,"detail":"x".repeat(300)})).collect(); + let pages = event_pages(events, 1_000); + assert!(pages.len() >= 4); + assert!(pages.iter().all(|page| page.iter().map(encoded_len).sum::() <= 1_000)); + assert_eq!(pages.concat().len(), 10); + } + + #[test] + fn oversized_summaries_drop_their_oldest_list_items() { + let messages: Vec<_> = (0..200).map(|n| json!({"n":n,"text":"m".repeat(100)})).collect(); + let mut value = json!({"name":"codex/p/t","run_id":"r","provider_messages":messages,"phase":"idle"}); + shrink_state(&mut value, 8_000); + assert!(encoded_len(&value) <= 8_000); + assert_eq!(value["stream_truncated"], json!(["provider_messages"])); + let kept = value["provider_messages"].as_array().unwrap(); + assert_eq!(kept.last().unwrap()["n"], 199); + let mut text = json!({"name":"n","run_id":"r","prompt":"p".repeat(20_000)}); + shrink_state(&mut text, 1_000); + assert_eq!(text["stream_truncated"], true); + assert_eq!(text["name"], "n"); + } +} diff --git a/src/watch/catalog.rs b/src/watch/catalog.rs new file mode 100644 index 000000000..e65a126c7 --- /dev/null +++ b/src/watch/catalog.rs @@ -0,0 +1,226 @@ +//! Catalog rows: keys, change detection with volatile observation fields +//! masked, removal reasons and paging. +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use zerus_watch_protocol::{self as protocol, reason}; + +/// Observation timestamps that change on every listing without any change in +/// the session. Change detection ignores them; the rows still carry them. +const VOLATILE_ROW: [&[&str]; 2] = [&["goal_observed_at"], &["process_work", "sampled_at"]]; +/// Meta fields that change on every listing; sent at most every 10 s. +pub(super) const VOLATILE_META: [&[&str]; 2] = [&["metrics"], &["desktop_presence"]]; + +pub(super) type Digest32 = [u8; 32]; + +/// `value` without the fields at `paths`. +pub(super) fn masked(value: &Value, paths: &[&[&str]]) -> Value { + let mut value = value.clone(); + for path in paths { + let Some((last, parents)) = path.split_last() else { continue }; + let mut node = Some(&mut value); + for parent in parents { + node = node.and_then(|node| node.get_mut(*parent)); + } + if let Some(object) = node.and_then(Value::as_object_mut) { + object.remove(*last); + } + } + value +} + +pub(super) fn digest(value: &Value) -> Digest32 { + Sha256::digest(value.to_string().as_bytes()).into() +} + +/// A row's digest with its volatile fields masked. +pub(super) fn row_digest(row: &Value) -> Digest32 { + digest(&masked(row, &VOLATILE_ROW)) +} + +/// The listing's rows with their keys, and its other fields. Two rows with the +/// same key (never expected) keep apart: the later one falls back to its name, +/// then to its position. +pub(super) fn split(mut listing: Value) -> (Vec<(String, Value)>, Value) { + let sessions = listing + .as_object_mut() + .and_then(|object| object.remove("sessions")) + .and_then(|sessions| match sessions { + Value::Array(rows) => Some(rows), + _ => None, + }) + .unwrap_or_default(); + let mut seen = BTreeSet::new(); + let mut rows = Vec::with_capacity(sessions.len()); + for (index, row) in sessions.into_iter().enumerate() { + let name = row.get("name").and_then(Value::as_str).unwrap_or("").to_owned(); + let mut key = protocol::catalog_key(&row).unwrap_or_else(|| format!("name:{name}")); + if seen.contains(&key) { + key = protocol::key_for(None, &name); + } + if seen.contains(&key) { + key = format!("name:{name}#{index}"); + } + seen.insert(key.clone()); + rows.push((key, row)); + } + (rows, listing) +} + +/// What the client holds of one row. +#[derive(Clone, Debug, PartialEq)] +pub(super) struct Sent { + pub name: String, + pub run_id: String, + pub digest: Digest32, +} + +impl Sent { + pub(super) fn of(row: &Value, digest: Digest32) -> Self { + let text = |key| row.get(key).and_then(Value::as_str).unwrap_or("").to_owned(); + Self { name: text("name"), run_id: text("run_id"), digest } + } +} + +/// Rows to upsert (indices into the current rows) and keys to remove. +#[derive(Debug, Default, PartialEq)] +pub(super) struct Change { + pub upsert: Vec, + pub remove: Vec, +} + +impl Change { + pub(super) fn is_empty(&self) -> bool { + self.upsert.is_empty() && self.remove.is_empty() + } +} + +pub(super) fn diff(sent: &BTreeMap, rows: &[(String, Value)], digests: &[Digest32]) -> Change { + let mut change = Change::default(); + for (index, ((key, _), digest)) in rows.iter().zip(digests).enumerate() { + if sent.get(key).is_none_or(|old| old.digest != *digest) { + change.upsert.push(index); + } + } + let current: BTreeSet<&str> = rows.iter().map(|(key, _)| key.as_str()).collect(); + change.remove = sent.keys().filter(|key| !current.contains(key.as_str())).cloned().collect(); + change +} + +/// Why `old` left the catalog: its name lists under another run (replaced), +/// its run is archived, its binding is gone (forgotten), or it ended. +pub(super) fn removal_reason( + old: &Sent, + listed_names: &BTreeSet<&str>, + archived: impl Fn(&str) -> bool, + saved: impl Fn(&str) -> bool, +) -> &'static str { + if listed_names.contains(old.name.as_str()) { + return reason::REPLACED; + } + if old.run_id.is_empty() { + return reason::ENDED; + } + if archived(&old.run_id) { + return reason::ARCHIVED; + } + if !saved(&old.name) { + return reason::FORGOTTEN; + } + reason::ENDED +} + +/// Splits `items` into pages of at most `limit` bytes by `size`. A single item +/// larger than `limit` gets a page of its own. +pub(super) fn pages(items: Vec, size: impl Fn(&T) -> usize, limit: usize) -> Vec> { + let mut pages = Vec::new(); + let mut page = Vec::new(); + let mut bytes = 0; + for item in items { + let item_bytes = size(&item) + 1; + if !page.is_empty() && bytes + item_bytes > limit { + pages.push(std::mem::take(&mut page)); + bytes = 0; + } + bytes += item_bytes; + page.push(item); + } + if !page.is_empty() || pages.is_empty() { + pages.push(page); + } + pages +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn row(name: &str, run: &str, phase: &str, observed: f64) -> Value { + json!({"name":name,"run_id":run,"phase":phase,"goal_observed_at":observed, + "process_work":{"count":0,"sampled_at":observed}}) + } + + fn sent_of(rows: &[(String, Value)]) -> BTreeMap { + rows.iter().map(|(key, row)| (key.clone(), Sent::of(row, row_digest(row)))).collect() + } + + #[test] + fn observation_timestamps_alone_change_nothing() { + let a = row("codex/p/t", "r1", "idle", 1.0); + let b = row("codex/p/t", "r1", "idle", 2.0); + assert_ne!(a, b); + assert_eq!(row_digest(&a), row_digest(&b)); + assert_ne!(row_digest(&a), row_digest(&row("codex/p/t", "r1", "working", 1.0))); + // The rows themselves keep the fields. + assert_eq!(masked(&a, &VOLATILE_ROW)["process_work"], json!({"count":0})); + assert_eq!(a["process_work"]["sampled_at"], 1.0); + assert_eq!(masked(&json!({"process_work":null}), &VOLATILE_ROW), json!({"process_work":null})); + } + + #[test] + fn rows_key_by_run_then_name_and_duplicates_stay_apart() { + let listing = json!({"host":"h","ok":true,"sessions":[ + {"name":"codex/p/a","run_id":"r1"},{"name":"sh/p"},{"name":"codex/p/b","run_id":"r1"},{"name":"sh/p"}]}); + let (rows, meta) = split(listing); + let keys: Vec<_> = rows.iter().map(|(key, _)| key.as_str()).collect(); + assert_eq!(keys, ["run:r1", "name:sh/p", "name:codex/p/b", "name:sh/p#3"]); + assert_eq!(meta, json!({"host":"h","ok":true})); + assert_eq!(split(json!({"ok":false})).0, vec![]); + } + + #[test] + fn diff_upserts_changed_rows_and_removes_missing_keys() { + let (before, _) = split(json!({"sessions":[row("codex/p/a","r1","idle",1.), row("codex/p/b","r2","idle",1.), row("codex/p/c","r3","idle",1.)]})); + let sent = sent_of(&before); + let (after, _) = split(json!({"sessions":[row("codex/p/a","r1","idle",9.), row("codex/p/renamed","r2","idle",9.), row("codex/p/d","r4","idle",9.)]})); + let digests: Vec<_> = after.iter().map(|(_, row)| row_digest(row)).collect(); + let change = diff(&sent, &after, &digests); + // r1 changed only its timestamps; a rename upserts the same run key. + assert_eq!(change.upsert, [1, 2]); + assert_eq!(after[1].0, "run:r2"); + assert_eq!(change.remove, ["run:r3"]); + assert!(diff(&sent_of(&after), &after, &digests).is_empty()); + } + + #[test] + fn removals_explain_replacement_archive_forgetting_and_ending() { + let old = Sent { name: "codex/p/a".into(), run_id: "r1".into(), digest: [0; 32] }; + let none = BTreeSet::new(); + let listed: BTreeSet<&str> = ["codex/p/a"].into(); + assert_eq!(removal_reason(&old, &listed, |_| true, |_| true), reason::REPLACED); + assert_eq!(removal_reason(&old, &none, |run| run == "r1", |_| true), reason::ARCHIVED); + assert_eq!(removal_reason(&old, &none, |_| false, |_| false), reason::FORGOTTEN); + assert_eq!(removal_reason(&old, &none, |_| false, |_| true), reason::ENDED); + let untracked = Sent { name: "sh/p".into(), run_id: String::new(), digest: [0; 32] }; + assert_eq!(removal_reason(&untracked, &none, |_| true, |_| false), reason::ENDED); + } + + #[test] + fn pages_respect_the_byte_limit_and_isolate_oversized_items() { + let pages = pages(vec![4, 4, 4, 20, 4], |n| *n, 10); + assert_eq!(pages, vec![vec![4, 4], vec![4], vec![20], vec![4]]); + let empty: Vec> = super::pages(Vec::new(), |n: &usize| *n, 10); + assert_eq!(empty, vec![Vec::::new()]); + } +} diff --git a/src/watch/mod.rs b/src/watch/mod.rs new file mode 100644 index 000000000..107a4c0ac --- /dev/null +++ b/src/watch/mod.rs @@ -0,0 +1,984 @@ +//! `hgs [@host] watch --json`: one long-lived NDJSON event stream per computer +//! (ABI `watch-v1`, frames in `crates/watch-protocol`). The catalog topic +//! carries the rows of `hgs ls --json --local --no-archives`; Activity carries +//! what `hgs inspect` returns for subscribed sessions. Change detection is a +//! 250 ms pulse over cheap stamps and the journal's data version, on +//! `std::thread` only. See "Event stream" in docs/reference.md. +mod activity; +mod catalog; +mod stamps; +mod writer; + +use crate::cli::{Error, Result}; +use crate::config::Config; +use crate::state::watch_source as source; +use activity::Subscription; +use catalog::{Digest32, Sent}; +use serde_json::{json, Value}; +use std::collections::{BTreeMap, BTreeSet}; +use std::io::BufRead; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{mpsc, Arc}; +use std::time::{Duration, Instant}; +use writer::{Item, Writer}; +use zerus_watch_protocol::{self as protocol, reason, topic, Body, Control}; + +/// How often sources are checked. +const PULSE: Duration = Duration::from_millis(250); +/// A listing waits this long after a change, so a burst lists once. +const DEBOUNCE: Duration = Duration::from_millis(150); +/// Listings start at least this far apart, and at least four times the +/// previous listing's duration apart. +const LISTING_GAP: Duration = Duration::from_millis(500); +/// Listing without any observed change: process exit, tmux kill or rename, +/// ended background jobs and terminal choosers report no file change. +const SAFETY: Duration = Duration::from_secs(5); +/// `catalog.meta` with only fresh metrics or presence goes out this rarely. +const META_INTERVAL: Duration = Duration::from_secs(10); +/// A changed session's summary is re-inspected at most this often. +const STATE_GAP: Duration = Duration::from_millis(500); +/// Unchanged summaries are re-inspected after this long: provider messages +/// between tools exist only in native transcripts. +const BUSY_REFRESH: Duration = Duration::from_secs(2); +const IDLE_REFRESH: Duration = Duration::from_secs(15); +/// DeepSeek and OpenCode answer from their own hosts. +const NATIVE_REFRESH: Duration = Duration::from_secs(2); +/// With more than this queued, catalog changes wait and coalesce. +const DEFER_BYTES: usize = protocol::QUEUE_BYTES / 4; +/// A repeated source failure is reported at most this often. +const ERROR_REPEAT: Duration = Duration::from_secs(60); +/// The largest page of catalog items. +const PAGE_LIMIT: usize = protocol::PAGE_BYTES - 4096; + +enum Input { + Line(String), + TooLong, + /// Control lines arrived faster than they were handled; some were dropped. + Dropped, + End, +} + +/// Control lines read but not yet handled; beyond it lines are dropped, so a +/// client that only writes cannot grow this process. +const PENDING_CONTROLS: usize = 256; + +pub(crate) fn run(config: &Config, args: &[String], dry: bool) -> Result { + if dry || args != ["--json"] { + return Err(Error::new(1, "usage: hgs [@host] watch --json (NDJSON frames on stdout, control operations on stdin)")); + } + let (inputs, received) = mpsc::channel(); + let closed = inputs.clone(); + let writer = Writer::spawn( + std::io::stdout(), + protocol::QUEUE_BYTES, + Duration::from_millis(protocol::HEARTBEAT_MS), + move || { + let _ = closed.send(Input::End); + }, + ); + let pending = Arc::new(AtomicUsize::new(0)); + let counted = Arc::clone(&pending); + let halt = writer.clone(); + std::thread::Builder::new() + .name("hgs-watch-control".into()) + // EOF stops the writer at once, so a wait for queue room ends too. + .spawn(move || read_controls(std::io::stdin().lock(), inputs, &counted, move || halt.halt())) + .map_err(|error| Error::new(1, format!("cannot read control operations: {error}")))?; + let mut watch = Watch::new(config, writer); + watch.hello(); + watch.list(Instant::now()); + watch.emit_catalog(); + let mut next_pulse = Instant::now() + PULSE; + loop { + let now = Instant::now(); + let deadline = watch.listing_due.map_or(next_pulse, |due| due.min(next_pulse)); + let input = received.recv_timeout(deadline.saturating_duration_since(now)); + if matches!(input, Ok(Input::Line(_) | Input::TooLong)) { + pending.fetch_sub(1, Ordering::SeqCst); + } + if watch.out.writer.stopped() { + return Ok(0); + } + match input { + Ok(Input::Line(text)) => watch.control(&text), + Ok(Input::TooLong) => watch.out.error( + "bad_request", + &format!("control operations are at most {} bytes", protocol::CONTROL_LINE_BYTES), + None, + None, + ), + Ok(Input::Dropped) => watch.out.error( + "bad_request", + &format!("more than {PENDING_CONTROLS} control operations were waiting; later ones were dropped"), + None, + None, + ), + Ok(Input::End) | Err(mpsc::RecvTimeoutError::Disconnected) => return Ok(0), + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + if watch.out.writer.stopped() { + return Ok(0); + } + let now = Instant::now(); + if now >= next_pulse { + next_pulse = now + PULSE; + watch.pulse(now); + } + if watch.listing_due.is_some_and(|due| now >= due) { + watch.list(now); + } + watch.emit_catalog(); + watch.resync_activity(); + } +} + +/// Reads control lines until EOF, then runs `ended`. A line longer than the +/// limit is skipped and reported, never buffered whole; so are lines beyond +/// `PENDING_CONTROLS` not yet handled. +fn read_controls(mut input: R, inputs: mpsc::Sender, pending: &AtomicUsize, ended: impl FnOnce()) { + let mut line = Vec::new(); + let mut too_long = false; + let mut dropping = false; + loop { + let buffer = match input.fill_buf() { + Ok(buffer) => buffer, + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(_) => break, + }; + if buffer.is_empty() { + break; + } + let newline = buffer.iter().position(|byte| *byte == b'\n'); + let chunk = &buffer[..newline.unwrap_or(buffer.len())]; + if !too_long && line.len() + chunk.len() > protocol::CONTROL_LINE_BYTES { + too_long = true; + line.clear(); + } + if !too_long { + line.extend_from_slice(chunk); + } + let used = chunk.len() + usize::from(newline.is_some()); + input.consume(used); + if newline.is_some() { + let message = if too_long { + Input::TooLong + } else { + Input::Line(String::from_utf8_lossy(&line).into_owned()) + }; + line.clear(); + too_long = false; + if pending.load(Ordering::SeqCst) >= PENDING_CONTROLS { + if !dropping && inputs.send(Input::Dropped).is_err() { + break; + } + dropping = true; + continue; + } + dropping = false; + pending.fetch_add(1, Ordering::SeqCst); + if inputs.send(message).is_err() { + break; + } + } + } + if !line.is_empty() && !too_long && pending.load(Ordering::SeqCst) < PENDING_CONTROLS { + pending.fetch_add(1, Ordering::SeqCst); + let _ = inputs.send(Input::Line(String::from_utf8_lossy(&line).into_owned())); + } + ended(); + let _ = inputs.send(Input::End); +} + +/// Frames out, with repeated source failures limited. +struct Out { + writer: Writer, + reported: BTreeMap, +} + +impl Out { + fn encode(body: &Body) -> Option { + protocol::encode_body(body).ok() + } + + /// Queues a frame unless the client fell behind (false). + fn offer(&self, body: &Body, coalesce: Option) -> bool { + match Self::encode(body) { + Some(body) => self.writer.offer(Item { coalesce, body }), + None => false, + } + } + + /// Queues a frame, waiting for room. + fn put(&self, body: &Body) -> bool { + match Self::encode(body) { + Some(body) => self.writer.put(Item { coalesce: None, body }), + None => false, + } + } + + fn error(&mut self, code: &str, message: &str, topic: Option<&str>, key: Option<&str>) { + let body = Body::Error(protocol::ErrorFrame { + code: code.into(), + message: message.chars().take(2048).collect(), + topic: topic.map(str::to_owned), + key: key.map(str::to_owned), + fatal: false, + }); + self.put(&body); + } + + /// An error of a source that is retried: once per minute per message. + fn failure(&mut self, code: &str, message: &str, topic: Option<&str>, key: Option<&str>) { + let id = format!("{code}\n{}\n{message}", key.unwrap_or("")); + let now = Instant::now(); + if self.reported.get(&id).is_some_and(|at| now.duration_since(*at) < ERROR_REPEAT) { + return; + } + self.reported.retain(|_, at| now.duration_since(*at) < ERROR_REPEAT); + self.reported.insert(id, now); + self.error(code, message, topic, key); + } +} + +struct Watch<'a> { + config: &'a Config, + out: Out, + root: PathBuf, + // The latest listing, and what the client holds of it. + rows: Vec<(String, Value)>, + digests: Vec, + meta: Value, + listed: bool, + pending: bool, + sent: BTreeMap, + revision: i64, + snapshot_sent: bool, + catalog_resync: bool, + meta_sent: Option<(Digest32, Digest32, Instant)>, + // Listing schedule. + stamp: stamps::Stamp, + listing_due: Option, + listed_at: Option, + listing_cost: Duration, + // Activity. + journal: Option, + data_version: Option, + journal_max: i64, + subs: Vec, +} + +impl<'a> Watch<'a> { + fn new(config: &'a Config, writer: Writer) -> Self { + Self { + config, + out: Out { writer, reported: BTreeMap::new() }, + root: source::state_root(), + rows: Vec::new(), + digests: Vec::new(), + meta: json!({}), + listed: false, + pending: false, + sent: BTreeMap::new(), + revision: 0, + snapshot_sent: false, + catalog_resync: false, + meta_sent: None, + stamp: stamps::Stamp::default(), + listing_due: None, + listed_at: None, + listing_cost: Duration::ZERO, + journal: None, + data_version: None, + journal_max: 0, + subs: Vec::new(), + } + } + + fn hello(&mut self) { + let hello = protocol::Hello { + machine_id: crate::swarm::local_node_id(self.config).ok(), + host: self.config.host().into(), + hgs_version: env!("CARGO_PKG_VERSION").into(), + stream: uuid::Uuid::new_v4().to_string(), + heartbeat_ms: protocol::HEARTBEAT_MS, + max_frame_bytes: protocol::MAX_FRAME_BYTES, + topics: [topic::CATALOG, topic::CATALOG_META, topic::ACTIVITY].map(str::to_owned).to_vec(), + limits: protocol::Limits::default(), + }; + self.out.put(&Body::Hello(hello)); + self.open_journal(); + } + + // ---- Catalog ----------------------------------------------------------- + + fn schedule(&mut self, at: Instant) { + let gap = LISTING_GAP.max(self.listing_cost * 4); + let due = self.listed_at.map_or(at, |started| at.max(started + gap)); + self.listing_due = Some(self.listing_due.map_or(due, |current| current.min(due))); + } + + /// Re-runs the `ls --json --local --no-archives` builder. + fn list(&mut self, now: Instant) { + // The stamp is taken first: a change during the listing lists again. + self.stamp = stamps::scan(&self.root); + self.listing_due = None; + self.listed_at = Some(now); + match crate::cli::local_listing(self.config) { + Ok(listing) => { + let (rows, meta) = catalog::split(listing); + self.digests = rows.iter().map(|(_, row)| catalog::row_digest(row)).collect(); + self.rows = rows; + self.meta = meta; + self.listed = true; + self.pending = true; + } + Err(error) => self.out.failure("listing_failed", &error.message, Some(topic::CATALOG), None), + } + self.listing_cost = now.elapsed(); + } + + /// Sends what changed since the client's copy, unless the client is + /// behind: then the latest listing waits, so changes coalesce per key. + fn emit_catalog(&mut self) { + if !self.listed || !(self.pending || self.catalog_resync) || self.out.writer.backlog() > DEFER_BYTES { + return; + } + let mut removed = BTreeMap::new(); + if self.catalog_resync || !self.snapshot_sent { + if self.catalog_resync { + self.out.put(&Body::Gap(protocol::Gap { topic: topic::CATALOG.into(), key: None, reason: reason::OVERFLOW.into() })); + } + self.catalog_resync = false; + self.snapshot(); + self.snapshot_sent = true; + } else { + let change = catalog::diff(&self.sent, &self.rows, &self.digests); + if !change.is_empty() { + removed = self.delta(change); + } + self.send_meta(false); + } + self.pending = false; + self.follow_catalog(&removed); + } + + fn item(&self, index: usize) -> Value { + let (key, row) = &self.rows[index]; + json!({"key": key, "row": row}) + } + + /// Rows too large for any frame are left out with an error: read them + /// with `hgs ls --json --local`. + fn oversized(&mut self, index: usize, size: usize) -> bool { + if size <= activity::PAYLOAD_LIMIT { + return false; + } + let key = self.rows[index].0.clone(); + self.out.failure( + "row_too_large", + &format!("catalog row exceeds the frame limit ({size} bytes); read it with hgs ls --json --local"), + Some(topic::CATALOG), + Some(&key), + ); + true + } + + fn snapshot(&mut self) { + let mut items = Vec::new(); + for index in 0..self.rows.len() { + let item = self.item(index); + let size = item.to_string().len(); + if !self.oversized(index, size) { + items.push((item, size)); + } + } + self.revision += 1; + let pages = catalog::pages(items, |(_, size)| *size, PAGE_LIMIT); + let count = pages.len(); + for (page, items) in pages.into_iter().enumerate() { + let body = Body::Snapshot(protocol::Snapshot { + topic: topic::CATALOG.into(), + key: None, + items: Some(items.into_iter().map(|(item, _)| item).collect()), + value: None, + page: page as u32, + last: page + 1 == count, + cursor: self.revision, + }); + if !self.out.put(&body) { + return; + } + } + self.sent = self + .rows + .iter() + .zip(&self.digests) + .map(|((key, row), digest)| (key.clone(), Sent::of(row, *digest))) + .collect(); + self.send_meta(true); + } + + /// Sends a delta; returns the removed keys with their reasons. + fn delta(&mut self, change: catalog::Change) -> BTreeMap { + /// One entry of a delta: a removal with its reason, or a row. + enum Entry { + Remove(String, &'static str), + Upsert(usize), + } + let listed: BTreeSet<&str> = self.rows.iter().filter_map(|(_, row)| row["name"].as_str()).collect(); + let mut entries = Vec::new(); + let mut removed = BTreeMap::new(); + for key in change.remove { + let Some(old) = self.sent.get(&key) else { continue }; + let why = catalog::removal_reason(old, &listed, source::archived, source::saved); + removed.insert(key.clone(), why); + entries.push((Entry::Remove(key.clone(), why), key.len() + 48)); + } + for index in change.upsert { + let size = self.item(index).to_string().len(); + if self.oversized(index, size) { + let (key, row) = &self.rows[index]; + self.sent.insert(key.clone(), Sent::of(row, self.digests[index])); + } else { + entries.push((Entry::Upsert(index), size)); + } + } + for page in catalog::pages(entries, |(_, size)| *size, PAGE_LIMIT) { + if page.is_empty() { + continue; + } + let mut upsert = Vec::new(); + let mut remove = Vec::new(); + for (entry, _) in &page { + match entry { + Entry::Remove(key, why) => remove.push(protocol::Removal { key: key.clone(), reason: Some((*why).into()) }), + Entry::Upsert(index) => upsert.push(protocol::CatalogItem { + key: self.rows[*index].0.clone(), + row: self.rows[*index].1.clone(), + }), + } + } + self.revision += 1; + let body = Body::Delta(protocol::Delta { topic: topic::CATALOG.into(), upsert, remove, cursor: self.revision }); + if !self.out.offer(&body, None) { + self.catalog_resync = true; + return removed; + } + for (entry, _) in page { + match entry { + Entry::Remove(key, _) => { + self.sent.remove(&key); + } + Entry::Upsert(index) => { + let (key, row) = &self.rows[index]; + self.sent.insert(key.clone(), Sent::of(row, self.digests[index])); + } + } + } + } + removed + } + + /// `catalog.meta`: at once when a lasting field changed, at most every + /// 10 s when only metrics or presence did. + fn send_meta(&mut self, force: bool) { + let lasting = catalog::digest(&catalog::masked(&self.meta, &catalog::VOLATILE_META)); + let full = catalog::digest(&self.meta); + let due = match self.meta_sent { + None => true, + Some((sent_lasting, sent_full, at)) => { + sent_lasting != lasting || (sent_full != full && at.elapsed() >= META_INTERVAL) + } + }; + if !(force || due) { + return; + } + let size = self.meta.to_string().len(); + if size > activity::PAYLOAD_LIMIT { + // Never expected; a resync would only meet the same limit again. + self.out.failure( + "meta_too_large", + &format!("catalog.meta exceeds the frame limit ({size} bytes); read it with hgs ls --json --local"), + Some(topic::CATALOG_META), + None, + ); + self.meta_sent = Some((lasting, full, Instant::now())); + return; + } + let body = Body::Snapshot(protocol::Snapshot { + topic: topic::CATALOG_META.into(), + key: None, + items: None, + value: Some(self.meta.clone()), + page: 0, + last: true, + cursor: self.revision, + }); + if self.out.offer(&body, Some(format!("snapshot|{}", topic::CATALOG_META))) { + self.meta_sent = Some((lasting, full, Instant::now())); + } else { + self.catalog_resync = true; + } + } + + /// Activity subscriptions follow their catalog rows: a rename changes the + /// name, a changed row refreshes the summary, a removed row ends them. + fn follow_catalog(&mut self, removed: &BTreeMap) { + let rows: BTreeMap<&str, usize> = self.rows.iter().enumerate().map(|(index, (key, _))| (key.as_str(), index)).collect(); + let mut ended = Vec::new(); + for (position, sub) in self.subs.iter_mut().enumerate() { + match rows.get(sub.key.as_str()) { + Some(index) => { + let row = &self.rows[*index].1; + sub.listed = true; + if let Some(name) = row["name"].as_str().filter(|name| *name != sub.name) { + sub.name = name.to_owned(); + sub.dirty = true; + } + if sub.row != Some(self.digests[*index]) { + sub.row = Some(self.digests[*index]); + sub.dirty = true; + } + sub.busy = activity::busy(row); + } + None if sub.listed => ended.push((position, removed.get(&sub.key).copied().unwrap_or(reason::ENDED))), + None => {} + } + } + for (position, why) in ended.into_iter().rev() { + let sub = self.subs.remove(position); + self.out.error( + "subscription_ended", + &format!("{} left the catalog ({why})", sub.name), + Some(topic::ACTIVITY), + Some(&sub.key), + ); + } + } + + // ---- Pulse ------------------------------------------------------------- + + fn pulse(&mut self, now: Instant) { + let journal_changed = self.poll_journal(); + let stamp = stamps::scan(&self.root); + if stamp != self.stamp || journal_changed { + self.schedule(now + DEBOUNCE); + } + if self.listed_at.is_none_or(|at| now >= at + SAFETY) { + self.schedule(now); + } + self.activity(now); + } + + fn open_journal(&mut self) { + if self.journal.is_some() { + return; + } + self.journal = source::journal(); + if let Some(db) = &self.journal { + self.data_version = source::data_version(db).ok(); + self.journal_max = source::bounds(db).map_or(0, |(_, max)| max); + // Rows written before this connection opened: read after each cursor. + for sub in self.subs.iter_mut().filter(|sub| !sub.native) { + sub.more = true; + } + } + } + + /// Whether the journal changed. Subscriptions with new rows read them. + fn poll_journal(&mut self) -> bool { + self.open_journal(); + let Some(db) = &self.journal else { return false }; + let version = match source::data_version(db) { + Ok(version) => version, + Err(_) => { + self.journal = None; + return false; + } + }; + if Some(version) == self.data_version { + return false; + } + self.data_version = Some(version); + let Ok((_, max)) = source::bounds(db) else { return true }; + if max < self.journal_max { + // A replaced journal: every cursor is meaningless now. + for sub in self.subs.iter_mut().filter(|sub| !sub.native) { + sub.resync = Some(reason::RESET); + } + self.journal_max = max; + return true; + } + if max == self.journal_max { + return false; + } + const SCAN: usize = 10_000; + let names: Option> = if max - self.journal_max > SCAN as i64 { + None + } else { + source::changed_names(db, self.journal_max, SCAN).ok().map(|names| names.into_iter().collect()) + }; + for sub in self.subs.iter_mut().filter(|sub| !sub.native) { + let touched = match (&names, &sub.journal) { + (Some(names), Some(journal)) => names.contains(journal), + _ => true, + }; + if touched { + sub.more = true; + } + } + self.journal_max = max; + true + } + + fn activity(&mut self, now: Instant) { + for index in 0..self.subs.len() { + if self.subs[index].resync.is_some() { + continue; + } + if self.subs[index].more { + self.read_events(index); + } + let sub = &self.subs[index]; + let interval = if sub.native { + NATIVE_REFRESH + } else if sub.busy { + BUSY_REFRESH + } else { + IDLE_REFRESH + }; + let since = now.saturating_duration_since(sub.refreshed); + if (sub.dirty && since >= STATE_GAP) || since >= interval { + self.refresh(index); + } + } + } + + /// Journal rows after the subscription's cursor, as `inspect` shapes them. + fn read_events(&mut self, index: usize) { + let Some(db) = &self.journal else { return }; + let sub = &mut self.subs[index]; + sub.more = false; + let timeline = match source::timeline(&sub.name) { + Ok(Some(timeline)) => timeline, + Ok(None) => return, + Err(error) => { + self.out.failure("inspect_failed", &error, Some(topic::ACTIVITY), Some(&sub.key)); + return; + } + }; + if sub.run_id.as_deref().is_some_and(|run| run != timeline.run_id()) { + // The name now holds another run; the catalog removes or renames ours. + sub.dirty = true; + return; + } + sub.journal = Some(timeline.journal().to_owned()); + let events = match timeline.events(db, sub.cursor, protocol::ACTIVITY_READ, false) { + Ok(events) => events, + Err(error) => { + self.out.failure("journal_failed", &error, Some(topic::ACTIVITY), Some(&sub.key)); + return; + } + }; + sub.more = events.len() == protocol::ACTIVITY_READ; + if events.is_empty() { + return; + } + sub.dirty = true; + for page in activity::event_pages(events, activity::PAYLOAD_LIMIT) { + let cursor = page.last().and_then(|event| event["seq"].as_i64()).unwrap_or(sub.cursor); + let body = Body::Events(protocol::Events { + topic: topic::ACTIVITY.into(), + key: sub.key.clone(), + events: page, + cursor, + }); + if !self.out.offer(&body, None) { + sub.resync = Some(reason::OVERFLOW); + sub.more = false; + return; + } + sub.cursor = cursor; + } + } + + /// Re-inspects one session; sends `state` when its summary changed, and + /// for DeepSeek and OpenCode a replacement of their event window. + fn refresh(&mut self, index: usize) { + let after = if self.subs[index].native { 0 } else { self.journal_max.max(self.subs[index].cursor) }; + let sub = &mut self.subs[index]; + sub.refreshed = Instant::now(); + sub.dirty = false; + let inspection = match source::inspect(&sub.name, after) { + Ok(inspection) => inspection, + Err(error) => { + self.out.failure("inspect_failed", &error, Some(topic::ACTIVITY), Some(&sub.key)); + return; + } + }; + if inspection["run_id"].as_str().zip(sub.run_id.as_deref()).is_some_and(|(seen, wanted)| seen != wanted) { + return; + } + if !sub.native { + if let Ok(Some(timeline)) = source::timeline(&sub.name) { + sub.journal = Some(timeline.journal().to_owned()); + } + } else { + let events = inspection["events"].as_array().cloned().unwrap_or_default(); + let digest = catalog::digest(&Value::Array(events.clone())); + if sub.events != Some(digest) && send_window(&self.out, sub, events, 0) { + sub.events = Some(digest); + } + } + send_state(&self.out, sub, &inspection); + } + + /// Topics the client fell behind on: `gap`, then a tail snapshot. + fn resync_activity(&mut self) { + for index in 0..self.subs.len() { + let Some(why) = self.subs[index].resync else { continue }; + if self.out.writer.backlog() > DEFER_BYTES { + return; + } + self.out.put(&Body::Gap(protocol::Gap { + topic: topic::ACTIVITY.into(), + key: Some(self.subs[index].key.clone()), + reason: why.into(), + })); + self.subs[index].resync = None; + self.subs[index].state = None; + self.subs[index].events = None; + self.start(index, 0, true); + } + } + + // ---- Control ----------------------------------------------------------- + + fn control(&mut self, text: &str) { + if text.trim().is_empty() { + return; + } + let operation = match protocol::decode_control(text) { + Ok(operation) => operation, + Err(error) => { + self.out.error("bad_request", &format!("invalid control operation: {error}"), None, None); + return; + } + }; + match operation { + Control::Ping(ping) => { + self.out.put(&Body::Pong(protocol::Pong { id: ping.id })); + } + Control::Subscribe(subscription) => self.subscribe(subscription), + Control::Unsubscribe(subscription) => { + if let Some(session) = subscription.session.filter(|_| subscription.topic == topic::ACTIVITY) { + let key = session.key(); + self.subs.retain(|sub| sub.key != key); + } + } + } + } + + fn subscribe(&mut self, request: protocol::Subscription) { + match request.topic.as_str() { + topic::CATALOG | topic::CATALOG_META => return, + topic::ACTIVITY => {} + other => { + self.out.error( + "unsupported_topic", + &format!("watch-v1 does not stream {other}; keep reading it with its command"), + Some(other), + None, + ); + return; + } + } + let Some(session) = request + .session + .filter(|session| !session.name.is_empty() && session.name.len() <= 1024 && !session.name.chars().any(char::is_control)) + else { + self.out.error("bad_request", "an Activity subscription names its session", Some(topic::ACTIVITY), None); + return; + }; + let after = request.after.unwrap_or(0); + if after < 0 { + self.out.error("bad_request", "after must be nonnegative", Some(topic::ACTIVITY), None); + return; + } + let key = session.key(); + self.subs.retain(|sub| sub.key != key); + if self.subs.len() >= protocol::ACTIVITY_SUBSCRIPTIONS { + self.out.error( + "too_many_subscriptions", + &format!("a stream follows at most {} sessions", protocol::ACTIVITY_SUBSCRIPTIONS), + Some(topic::ACTIVITY), + Some(&key), + ); + return; + } + // A renamed session keeps its run key; follow its current name. + let row = self.rows.iter().position(|(row_key, _)| *row_key == key); + let name = row + .and_then(|index| self.rows[index].1["name"].as_str()) + .unwrap_or(&session.name) + .to_owned(); + let tail = request.tail.map_or(protocol::ACTIVITY_TAIL, |tail| (tail as usize).min(protocol::ACTIVITY_TAIL)); + let mut sub = Subscription::new(key, name.clone(), session.run_id.filter(|run| !run.is_empty()), source::native(&name), tail); + if let Some(index) = row { + sub.listed = true; + sub.row = Some(self.digests[index]); + sub.busy = activity::busy(&self.rows[index].1); + } + self.subs.push(sub); + let index = self.subs.len() - 1; + self.open_journal(); + if !self.start(index, after, false) { + self.subs.pop(); + } + } + + /// The first frames of a subscription, or its resumption after a gap. + /// False when it cannot follow the requested session. + fn start(&mut self, index: usize, after: i64, resumed: bool) -> bool { + let (minimum, maximum) = self.journal.as_ref().and_then(|db| source::bounds(db).ok()).unwrap_or((0, 0)); + let sub = &mut self.subs[index]; + let gap = if resumed || after == 0 || sub.native { + None + } else if after > maximum { + Some(reason::RESET) + } else if after < minimum - 1 { + Some(reason::RETENTION) + } else { + None + }; + let inspection = match source::inspect(&sub.name, if gap.is_some() { 0 } else { after }) { + Ok(inspection) => inspection, + Err(error) => { + self.out.error("inspect_failed", &error, Some(topic::ACTIVITY), Some(&sub.key)); + return false; + } + }; + if let (Some(seen), Some(wanted)) = (inspection["run_id"].as_str(), sub.run_id.as_deref()) { + if seen != wanted { + self.out.error( + "session_changed", + &format!("{} now runs {seen}, not {wanted}", sub.name), + Some(topic::ACTIVITY), + Some(&sub.key), + ); + return false; + } + } + sub.refreshed = Instant::now(); + let mut events = inspection["events"].as_array().cloned().unwrap_or_default(); + if !sub.native { + if let Ok(Some(timeline)) = source::timeline(&sub.name) { + sub.journal = Some(timeline.journal().to_owned()); + } + } + if sub.native { + sub.events = Some(catalog::digest(&Value::Array(events.clone()))); + send_window(&self.out, sub, events, 0); + } else { + let cursor = inspection["cursor"].as_i64().unwrap_or(after); + if gap.is_some() || resumed || after == 0 { + events.drain(..events.len().saturating_sub(sub.tail)); + } + if let Some(why) = gap { + self.out.put(&Body::Gap(protocol::Gap { topic: topic::ACTIVITY.into(), key: Some(sub.key.clone()), reason: why.into() })); + } + sub.more = after > 0 && gap.is_none() && events.len() >= protocol::ACTIVITY_READ; + if gap.is_some() || resumed { + send_window(&self.out, sub, events, cursor); + } else { + for page in activity::event_pages(events, activity::PAYLOAD_LIMIT) { + let page_cursor = page.last().and_then(|event| event["seq"].as_i64()).unwrap_or(cursor); + self.out.put(&Body::Events(protocol::Events { + topic: topic::ACTIVITY.into(), + key: sub.key.clone(), + events: page, + cursor: page_cursor, + })); + } + } + sub.cursor = cursor; + } + send_state(&self.out, sub, &inspection); + true + } +} + +/// An Activity event window as a snapshot: a DeepSeek or OpenCode window +/// (cursor 0), or a journal tail after a gap. Pages share the window's cursor. +fn send_window(out: &Out, sub: &mut Subscription, events: Vec, cursor: i64) -> bool { + let pages = activity::event_pages(events, activity::PAYLOAD_LIMIT); + let count = pages.len(); + for (page, items) in pages.into_iter().enumerate() { + let body = Body::Snapshot(protocol::Snapshot { + topic: topic::ACTIVITY.into(), + key: Some(sub.key.clone()), + items: Some(items), + value: None, + page: page as u32, + last: page + 1 == count, + cursor, + }); + if !out.put(&body) { + return false; + } + } + true +} + +/// `state` when the summary changed apart from observation times. +fn send_state(out: &Out, sub: &mut Subscription, inspection: &Value) { + let mut value = activity::state_value(inspection); + sub.busy = activity::busy(&value); + let digest = activity::state_digest(&value); + if sub.state == Some(digest) { + return; + } + activity::shrink_state(&mut value, activity::PAYLOAD_LIMIT); + sub.rev += 1; + let body = Body::State(protocol::State { topic: topic::ACTIVITY.into(), key: sub.key.clone(), value, rev: sub.rev }); + if out.offer(&body, Some(format!("state|{}|{}", topic::ACTIVITY, sub.key))) { + sub.state = Some(digest); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn read(input: &[u8], pending: usize) -> Vec { + let (sender, receiver) = mpsc::channel(); + let ended = std::cell::Cell::new(false); + read_controls(input, sender, &AtomicUsize::new(pending), || ended.set(true)); + assert!(ended.get()); + receiver + .into_iter() + .map(|input| match input { + Input::Line(text) => text, + Input::TooLong => "".into(), + Input::Dropped => "".into(), + Input::End => "".into(), + }) + .collect() + } + + fn lines(input: &[u8]) -> Vec { + read(input, 0) + } + + #[test] + fn control_lines_are_framed_bounded_and_end_at_eof() { + assert_eq!(lines(b"{\"op\":\"ping\"}\n\n{\"op\":\"ping\",\"id\":2}"), ["{\"op\":\"ping\"}", "", "{\"op\":\"ping\",\"id\":2}", ""]); + let mut long = vec![b'x'; protocol::CONTROL_LINE_BYTES + 10]; + long.extend_from_slice(b"\nnext\n"); + assert_eq!(lines(&long), ["", "next", ""]); + assert_eq!(lines(b""), [""]); + // A client that writes faster than lines are handled loses the excess. + assert_eq!(read(b"a\nb\nc", PENDING_CONTROLS), ["", ""]); + } +} diff --git a/src/watch/stamps.rs b/src/watch/stamps.rs new file mode 100644 index 000000000..e78ee71c0 --- /dev/null +++ b/src/watch/stamps.rs @@ -0,0 +1,127 @@ +//! A cheap stamp over the files the live listing reads, compared on every +//! pulse. Bindings are replaced by rename, so a stamp of name, size, mtime and +//! inode sees every change; temporary `.hgs-*` files are ignored. +use std::collections::hash_map::DefaultHasher; +use std::fs; +use std::hash::{Hash, Hasher}; +use std::path::Path; + +/// Session bindings: 64 lowercase hex digits and `.json`. +fn binding(name: &str) -> bool { + name.strip_suffix(".json") + .is_some_and(|stem| stem.len() == 64 && stem.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))) +} + +fn ordinary(name: &str) -> bool { + !name.starts_with(".hgs-") && name != ".lock" && !name.ends_with(".lock") +} + +/// Root files other than bindings that change listed rows. Desktop presence +/// changes only `catalog.meta`, which the safety listing refreshes. +const ROOT_FILES: [&str; 2] = ["read-marks.json", "notifications.json"]; +/// Folders whose every entry counts. +const FOLDERS: [&str; 6] = ["bindings", "name_routes", "renames", "launches", "dsh/bindings", "opencode/bindings"]; +/// Single files and folders whose own stamp counts. Archive files are added or +/// removed, which changes the folder. +const PATHS: [&str; 4] = ["coordination/waits.sqlite3", "coordination/waits.sqlite3-wal", "coordination/waits.sqlite3-journal", "archives"]; + +/// An order-independent hash of every stamped entry, and their count. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct Stamp { + hash: u64, + entries: usize, +} + +impl Stamp { + fn add(&mut self, path: &Path, metadata: &fs::Metadata) { + let mut hasher = DefaultHasher::new(); + path.hash(&mut hasher); + metadata.len().hash(&mut hasher); + metadata.modified().ok().hash(&mut hasher); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + metadata.ino().hash(&mut hasher); + metadata.ctime().hash(&mut hasher); + metadata.ctime_nsec().hash(&mut hasher); + } + self.hash = self.hash.wrapping_add(hasher.finish()); + self.entries += 1; + } + + fn folder(&mut self, folder: &Path, keep: impl Fn(&str) -> bool) { + let Ok(entries) = fs::read_dir(folder) else { return }; + for entry in entries.flatten() { + let name = entry.file_name(); + let Some(name) = name.to_str() else { continue }; + if !keep(name) { + continue; + } + if let Ok(metadata) = entry.metadata() { + self.add(&entry.path(), &metadata); + } + } + } + + #[cfg(test)] + fn entries(&self) -> usize { + self.entries + } +} + +/// The stamp of a state folder. +pub(super) fn scan(root: &Path) -> Stamp { + let mut stamp = Stamp::default(); + stamp.folder(root, |name| binding(name) || ROOT_FILES.contains(&name)); + for folder in FOLDERS { + stamp.folder(&root.join(folder), ordinary); + } + for path in PATHS { + let path = root.join(path); + if let Ok(metadata) = fs::symlink_metadata(&path) { + stamp.add(&path, &metadata); + } + } + stamp +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bindings_and_inputs_change_the_stamp_and_temporary_files_do_not() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let first = scan(root); + let record = root.join(format!("{}.json", "a".repeat(64))); + fs::write(&record, "{}").unwrap(); + let written = scan(root); + assert_ne!(written, first); + assert_eq!(written.entries(), 1); + // Ignored: temporary files, locks, presence and unrelated files. + fs::write(root.join(".hgs-temp"), "x").unwrap(); + fs::write(root.join("desktop-presence.json"), "{}").unwrap(); + fs::write(root.join(".lock"), "").unwrap(); + fs::write(root.join("notes.json"), "{}").unwrap(); + fs::create_dir_all(root.join("launches")).unwrap(); + fs::write(root.join("launches/.hgs-partial"), "x").unwrap(); + assert_eq!(scan(root), written); + // An atomic replacement (new inode, same size) is seen. + let temp = root.join(".hgs-next"); + fs::write(&temp, "[]").unwrap(); + fs::rename(&temp, &record).unwrap(); + let replaced = scan(root); + assert_ne!(replaced, written); + for path in ["read-marks.json", "launches/1.json", "renames/r.json", "name_routes/n.json", "dsh/bindings/b.json"] { + let path = root.join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let before = scan(root); + fs::write(&path, "{}").unwrap(); + assert_ne!(scan(root), before, "{}", path.display()); + } + let before = scan(root); + fs::create_dir(root.join("archives")).unwrap(); + assert_ne!(scan(root), before); + } +} diff --git a/src/watch/writer.rs b/src/watch/writer.rs new file mode 100644 index 000000000..f86be7eca --- /dev/null +++ b/src/watch/writer.rs @@ -0,0 +1,316 @@ +//! The only owner of the stream's stdout: a bounded queue of encoded bodies, +//! frame numbering at write time, and heartbeats. +use std::collections::VecDeque; +use std::io::Write; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::time::{Duration, Instant}; +use zerus_watch_protocol as protocol; + +/// One queued frame body (`{"t":...}`, see `protocol::encode_body`). +pub(super) struct Item { + /// Bodies with the same key replace each other while queued: the latest + /// `state` of a key, or the latest `catalog.meta`. + pub coalesce: Option, + pub body: String, +} + +struct Queue { + items: VecDeque, + bytes: usize, +} + +struct Shared { + queue: Mutex, + /// Signals the writer thread that a body is queued. + ready: Condvar, + /// Signals producers that the queue has room. + space: Condvar, + capacity: usize, + stopped: AtomicBool, +} + +impl Shared { + fn lock(&self) -> MutexGuard<'_, Queue> { + self.queue.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) + } +} + +#[derive(Clone)] +pub(super) struct Writer { + shared: Arc, +} + +impl Writer { + /// Starts the thread that writes to `out`. `closed` runs once when `out` + /// fails, as when the client closed the pipe. + pub(super) fn spawn(out: W, capacity: usize, heartbeat: Duration, closed: F) -> Self + where + W: Write + Send + 'static, + F: FnOnce() + Send + 'static, + { + let shared = Arc::new(Shared { + queue: Mutex::new(Queue { items: VecDeque::new(), bytes: 0 }), + ready: Condvar::new(), + space: Condvar::new(), + capacity, + stopped: AtomicBool::new(false), + }); + let thread = Arc::clone(&shared); + std::thread::Builder::new() + .name("hgs-watch-writer".into()) + .spawn(move || { + write_frames(&thread, out, heartbeat); + thread.stopped.store(true, Ordering::SeqCst); + thread.space.notify_all(); + closed(); + }) + .expect("start the watch writer thread"); + Self { shared } + } + + /// Queues a body unless it would overflow the queue; an empty queue always + /// takes one. A body whose `coalesce` key is already queued replaces that + /// body in place. False means the client fell behind: the caller owes the + /// topic a `gap` and a fresh snapshot. + pub(super) fn offer(&self, item: Item) -> bool { + if self.stopped() { + return false; + } + let mut queue = self.shared.lock(); + if let Some(key) = item.coalesce.as_deref() { + if let Some(queued) = queue.items.iter_mut().find(|queued| queued.coalesce.as_deref() == Some(key)) { + let previous = std::mem::replace(&mut queued.body, item.body); + let current = queued.body.len(); + queue.bytes = queue.bytes - previous.len() + current; + return true; + } + } + if !queue.items.is_empty() && queue.bytes + item.body.len() > self.shared.capacity { + return false; + } + queue.bytes += item.body.len(); + queue.items.push_back(item); + self.shared.ready.notify_one(); + true + } + + /// Queues a body, waiting while the queue is full. False once the stream + /// has stopped. + pub(super) fn put(&self, item: Item) -> bool { + let mut queue = self.shared.lock(); + while !queue.items.is_empty() && queue.bytes + item.body.len() > self.shared.capacity { + if self.stopped() { + return false; + } + queue = self + .shared + .space + .wait_timeout(queue, Duration::from_millis(100)) + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .0; + } + if self.stopped() { + return false; + } + queue.bytes += item.body.len(); + queue.items.push_back(item); + self.shared.ready.notify_one(); + true + } + + /// Bytes queued and not yet written. + pub(super) fn backlog(&self) -> usize { + self.shared.lock().bytes + } + + /// Stops the stream: queued frames are dropped and every wait ends. + pub(super) fn halt(&self) { + self.shared.stopped.store(true, Ordering::SeqCst); + let _queue = self.shared.lock(); + self.shared.space.notify_all(); + self.shared.ready.notify_all(); + } + + pub(super) fn stopped(&self) -> bool { + self.shared.stopped.load(Ordering::SeqCst) + } + + /// Waits up to `limit` for the queue to empty; for tests and orderly ends. + #[cfg(test)] + pub(super) fn drain(&self, limit: Duration) -> bool { + let deadline = Instant::now() + limit; + loop { + if self.backlog() == 0 || self.stopped() { + return self.backlog() == 0; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(2)); + } + } +} + +fn now() -> f64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() +} + +fn write_frames(shared: &Shared, mut out: W, heartbeat: Duration) { + let mut seq = 0u64; + let mut last = Instant::now(); + loop { + let body = { + let mut queue = shared.lock(); + loop { + if let Some(item) = queue.items.pop_front() { + queue.bytes -= item.body.len(); + shared.space.notify_all(); + break item.body; + } + let idle = last.elapsed(); + if idle >= heartbeat { + let beat = protocol::Body::Heartbeat(protocol::Heartbeat { at: now() }); + break protocol::encode_body(&beat).expect("a heartbeat encodes"); + } + queue = shared + .ready + .wait_timeout(queue, heartbeat - idle) + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .0; + } + }; + seq += 1; + let Ok(mut line) = protocol::envelope(seq, &body) else { + continue; + }; + line.push('\n'); + if out.write_all(line.as_bytes()).and_then(|()| out.flush()).is_err() { + return; + } + last = Instant::now(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io; + use std::sync::mpsc; + + /// A pipe end the test reads, optionally blocked until released. + #[derive(Clone)] + struct Sink { + data: Arc>>, + gate: Arc<(Mutex, Condvar)>, + fail: Arc, + } + impl Sink { + fn new(open: bool) -> Self { + Self { + data: Arc::default(), + gate: Arc::new((Mutex::new(open), Condvar::new())), + fail: Arc::default(), + } + } + fn open(&self) { + *self.gate.0.lock().unwrap() = true; + self.gate.1.notify_all(); + } + fn lines(&self) -> Vec { + String::from_utf8(self.data.lock().unwrap().clone()) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() + } + } + impl Write for Sink { + fn write(&mut self, buf: &[u8]) -> io::Result { + let mut open = self.gate.0.lock().unwrap(); + while !*open { + open = self.gate.1.wait(open).unwrap(); + } + if self.fail.load(Ordering::SeqCst) { + return Err(io::Error::from(io::ErrorKind::BrokenPipe)); + } + self.data.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + fn body(text: &str) -> String { + protocol::encode_body(&protocol::Body::Pong(protocol::Pong { id: Some(text.into()) })).unwrap() + } + + fn item(text: &str, coalesce: Option<&str>) -> Item { + Item { coalesce: coalesce.map(str::to_owned), body: body(text) } + } + + #[test] + fn frames_are_numbered_in_write_order_and_coalesce_while_queued() { + let sink = Sink::new(false); + let writer = Writer::spawn(sink.clone(), 1 << 20, Duration::from_secs(60), || {}); + assert!(writer.offer(item("first", None))); + // The writer thread may already hold "first"; queue behind it. + std::thread::sleep(Duration::from_millis(20)); + assert!(writer.offer(item("state-1", Some("state|activity|run:a")))); + assert!(writer.offer(item("events", None))); + assert!(writer.offer(item("state-2", Some("state|activity|run:a")))); + assert!(writer.offer(item("other", Some("state|activity|run:b")))); + sink.open(); + assert!(writer.drain(Duration::from_secs(5))); + std::thread::sleep(Duration::from_millis(20)); + let lines = sink.lines(); + let ids: Vec<_> = lines.iter().map(|line| line["id"].as_str().unwrap().to_owned()).collect(); + assert_eq!(ids, ["first", "state-2", "events", "other"]); + let seqs: Vec<_> = lines.iter().map(|line| line["seq"].as_u64().unwrap()).collect(); + assert_eq!(seqs, [1, 2, 3, 4]); + assert!(lines.iter().all(|line| line["v"] == 1 && line["t"] == "pong")); + } + + #[test] + fn a_full_queue_refuses_offers_and_put_waits_for_room() { + let sink = Sink::new(false); + let one = body("x").len(); + let writer = Writer::spawn(sink.clone(), 3 * one, Duration::from_secs(60), || {}); + assert!(writer.offer(item("x", None))); + std::thread::sleep(Duration::from_millis(20)); + let mut accepted = 0; + while writer.offer(item("x", None)) { + accepted += 1; + assert!(accepted <= 4, "the queue never filled"); + } + assert!(writer.backlog() <= 3 * one); + let (done, waited) = mpsc::channel(); + let blocked = writer.clone(); + std::thread::spawn(move || done.send(blocked.put(item("y", None))).unwrap()); + assert!(waited.recv_timeout(Duration::from_millis(100)).is_err(), "put returned while full"); + sink.open(); + assert_eq!(waited.recv_timeout(Duration::from_secs(5)), Ok(true)); + assert!(writer.drain(Duration::from_secs(5))); + } + + #[test] + fn heartbeats_follow_silence_and_a_closed_pipe_stops_the_writer() { + let sink = Sink::new(true); + let (closed, was_closed) = mpsc::channel(); + let writer = Writer::spawn(sink.clone(), 1 << 20, Duration::from_millis(50), move || closed.send(()).unwrap()); + std::thread::sleep(Duration::from_millis(180)); + let lines = sink.lines(); + assert!(lines.len() >= 2, "{lines:?}"); + assert!(lines.iter().all(|line| line["t"] == "heartbeat" && line["at"].as_f64().unwrap() > 1e9)); + sink.fail.store(true, Ordering::SeqCst); + writer.offer(item("lost", None)); + was_closed.recv_timeout(Duration::from_secs(5)).unwrap(); + assert!(writer.stopped()); + assert!(!writer.offer(item("after", None))); + assert!(!writer.put(item("after", None))); + } +} diff --git a/tests/test_ci_security.py b/tests/test_ci_security.py index 458798bb1..f6aaf9bcc 100644 --- a/tests/test_ci_security.py +++ b/tests/test_ci_security.py @@ -547,6 +547,9 @@ def test_source_only_resolves_and_every_suite_starts_from_it(self): "cargo +1.99.1 fmt --check --manifest-path crates/replica-ffi/Cargo.toml", "cargo +1.99.1 clippy --locked --manifest-path crates/replica-ffi/Cargo.toml --target-dir target/replica-ffi --all-targets -- -D warnings", "cargo +1.99.1 test --locked --manifest-path crates/replica-ffi/Cargo.toml --target-dir target/replica-ffi", + "cargo +1.99.1 fmt --check --manifest-path crates/watch-protocol/Cargo.toml", + "cargo +1.99.1 clippy --locked --manifest-path crates/watch-protocol/Cargo.toml --target-dir target/watch-protocol --all-targets -- -D warnings", + "cargo +1.99.1 test --locked --manifest-path crates/watch-protocol/Cargo.toml --target-dir target/watch-protocol", 'run: python3 scripts/collect-licenses.py "$RUNNER_TEMP/rust-licenses"'): self.assertIn(line, quick) for name in ("quick", "relay", "connector", "connector-macos", "android", "cli", "cli-macos-debug", "arch", "arch-package", diff --git a/tests/test_mac_install.py b/tests/test_mac_install.py index 943467033..3fb188760 100644 --- a/tests/test_mac_install.py +++ b/tests/test_mac_install.py @@ -69,7 +69,9 @@ def test_mobile_connector_sources_travel_and_its_build_stays_opt_in(self): 'services/mobile/relay/Cargo.toml', 'services/mobile/relay/src', 'services/mobile/crypto/Cargo.toml', 'services/mobile/crypto/src', 'crates/replica/Cargo.toml', 'crates/replica/Cargo.lock', - 'crates/replica/.cargo', 'crates/replica/src'): + 'crates/replica/.cargo', 'crates/replica/src', + 'crates/watch-protocol/Cargo.toml', 'crates/watch-protocol/Cargo.lock', + 'crates/watch-protocol/.cargo', 'crates/watch-protocol/src'): self.assertIn(path, sent) self.assertFalse(any('target' in path.split('/') for path in sent), sent) builds = [line for line in (self.root / 'transport.log').read_text().splitlines() diff --git a/tests/test_watch.py b/tests/test_watch.py new file mode 100644 index 000000000..1ce40e61b --- /dev/null +++ b/tests/test_watch.py @@ -0,0 +1,334 @@ +"""`hgs [@host] watch --json`: the NDJSON event stream (watch-v1). + +A private tmux socket, a scripted agent and a synthetic state folder only. +""" +import json +import os +import queue +import sqlite3 +import subprocess +import threading +import time +import unittest +import uuid + +import test_input as fixtures + +HGS = fixtures.HGS +VOLATILE = ('goal_observed_at',) + + +def steady(row): + """A row without its observation timestamps.""" + row = {key: value for key, value in row.items() if key not in VOLATILE} + if isinstance(row.get('process_work'), dict): + row['process_work'] = {k: v for k, v in row['process_work'].items() if k != 'sampled_at'} + return row + + +class Stream: + """One running `hgs watch --json` and the frames it has written.""" + + def __init__(self, test, *args): + self.test = test + self.process = subprocess.Popen([str(HGS), *args, 'watch', '--json'], env=test.env, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + self.frames = queue.Queue() + self.seen = [] + self.raw = [] + threading.Thread(target=self.read, daemon=True).start() + test.addCleanup(self.stop) + + def read(self): + for line in self.process.stdout: + self.raw.append(line) + self.frames.put(json.loads(line)) + self.frames.put(None) + + def next(self, predicate=lambda frame: True, timeout=10): + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + self.test.fail(f'no matching frame within {timeout}s; seen {[f["t"] for f in self.seen]}') + try: + frame = self.frames.get(timeout=remaining) + except queue.Empty: + continue + if frame is None: + self.test.fail('the stream ended: ' + self.process.stderr.read()) + self.seen.append(frame) + if predicate(frame): + return frame + + def quiet(self, predicate, seconds): + """Fails if a matching frame arrives within `seconds`.""" + deadline = time.monotonic() + seconds + while (remaining := deadline - time.monotonic()) > 0: + try: + frame = self.frames.get(timeout=remaining) + except queue.Empty: + return + if frame is None: + return + self.seen.append(frame) + self.test.assertFalse(predicate(frame), frame) + + def send(self, **operation): + self.process.stdin.write(json.dumps(operation) + '\n') + self.process.stdin.flush() + + def close(self, timeout=5): + self.process.stdin.close() + return self.process.wait(timeout=timeout) + + def stop(self): + if self.process.poll() is None: + self.process.kill() + self.process.wait(timeout=5) + for stream in (self.process.stdout, self.process.stderr, self.process.stdin): + if stream and not stream.closed: + stream.close() + + +@unittest.skipUnless(fixtures.TMUX, 'private tmux fixture requires tmux') +class WatchStream(unittest.TestCase): + setUp = fixtures.InputTransport.setUp + tearDown = fixtures.InputTransport.tearDown + script = fixtures.InputTransport.script + tmux = fixtures.InputTransport.tmux + wait_for = fixtures.InputTransport.wait_for + write_record = fixtures.InputTransport.write_record + + def key(self): + return 'run:' + self.run_id + + def session(self): + return dict(name=self.name, run_id=self.run_id, conversation_id=self.conversation_id) + + def listing(self): + result = subprocess.run([str(HGS), 'ls', '--json', '--local', '--no-archives'], env=self.env, + text=True, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + return json.loads(result.stdout) + + def journal(self, kind, **fields): + """Appends one Activity row as a hook would; returns its seq.""" + payload = dict(type=kind, at=time.time(), agent_id='', tool='', detail='') + payload.update(fields) + db = sqlite3.connect(self.state / 'events.sqlite3') + with db: + db.execute('CREATE TABLE IF NOT EXISTS events (seq INTEGER PRIMARY KEY, name TEXT, conversation TEXT, payload TEXT)') + cursor = db.execute('INSERT INTO events(name, conversation, payload) VALUES (?, ?, ?)', + (self.name, self.conversation_id, json.dumps(payload))) + seq = cursor.lastrowid + db.close() + return seq + + def inspect_events(self, after): + result = subprocess.run([str(HGS), 'inspect', self.name, '--after', str(after), '--skip-processes'], + env=self.env, text=True, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + return json.loads(result.stdout)['events'] + + def start(self): + stream = Stream(self) + hello = stream.next() + snapshot = stream.next(lambda f: f['t'] == 'snapshot' and f['topic'] == 'catalog' and f['last']) + meta = stream.next(lambda f: f['t'] == 'snapshot' and f['topic'] == 'catalog.meta') + return stream, hello, snapshot, meta + + def test_hello_catalog_snapshot_and_meta_match_the_listing_then_eof_ends_the_stream(self): + stream, hello, snapshot, meta = self.start() + self.assertEqual((hello['v'], hello['t'], hello['seq']), (1, 'hello', 1)) + self.assertEqual(hello['host'], 'test') + self.assertEqual((hello['heartbeat_ms'], hello['max_frame_bytes']), (10000, 1048576)) + self.assertEqual(hello['topics'], ['catalog', 'catalog.meta', 'activity']) + self.assertEqual(hello['limits']['activity_subscriptions'], 16) + uuid.UUID(hello['stream']) + self.assertEqual([frame['seq'] for frame in stream.seen], list(range(1, len(stream.seen) + 1))) + self.assertTrue(all(frame['v'] == 1 for frame in stream.seen)) + self.assertEqual(snapshot['page'], 0) + listing = self.listing() + rows = {item['key']: item['row'] for item in snapshot['items']} + self.assertEqual(set(rows), {self.key()}) + expected = next(row for row in listing['sessions'] if row['name'] == self.name) + self.assertEqual(steady(rows[self.key()]), steady(expected)) + self.assertNotIn('sessions', meta['value']) + self.assertEqual(set(meta['value']), set(listing) - {'sessions'}) + self.assertEqual(meta['value']['host'], 'test') + self.assertTrue(all(len(line.encode()) <= 1 << 20 for line in stream.raw)) + started = time.monotonic() + self.assertEqual(stream.close(), 0) + self.assertLess(time.monotonic() - started, 3) + self.assertEqual(stream.process.stderr.read(), '') + + def test_binding_changes_and_a_rename_are_upserts_on_the_run_key(self): + stream, *_ = self.start() + self.record.update(phase='working', activity='busy', prompt='Synthetic request', last_event_at=time.time()) + self.write_record() + delta = stream.next(lambda f: f['t'] == 'delta' and any( + item['key'] == self.key() and item['row'].get('prompt') == 'Synthetic request' for item in f['upsert']), timeout=8) + self.assertEqual(delta['topic'], 'catalog') + self.assertEqual(delta['remove'], []) + renamed = 'codex/input-test/renamed' + result = subprocess.run([str(HGS), 'rename', self.name, renamed], env=self.env, + text=True, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + delta = stream.next(lambda f: f['t'] == 'delta' and any( + item['key'] == self.key() and item['row']['name'] == renamed for item in f['upsert']), timeout=8) + self.assertEqual(delta['remove'], []) + # Let the rename settle; then observation timestamps alone, which + # change on every listing, produce no delta across a safety listing. + stream.quiet(lambda f: False, 1.5) + stream.quiet(lambda f: f['t'] == 'delta', 6) + self.assertEqual(stream.close(), 0) + + def test_a_forgotten_session_is_removed_with_its_reason_and_ends_its_subscription(self): + stream, *_ = self.start() + stream.send(op='subscribe', topic='activity', session=self.session()) + stream.next(lambda f: f['t'] == 'state') + result = subprocess.run([str(HGS), 'kill', self.name], env=self.env, text=True, + capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + delta = stream.next(lambda f: f['t'] == 'delta' and f['remove'], timeout=10) + self.assertEqual(delta['remove'], [dict(key=self.key(), reason='forgotten')]) + ended = stream.next(lambda f: f['t'] == 'error') + self.assertEqual((ended['code'], ended['topic'], ended['key'], ended['fatal']), + ('subscription_ended', 'activity', self.key(), False)) + self.assertIn('forgotten', ended['message']) + self.assertEqual(stream.close(), 0) + + def test_activity_streams_new_journal_rows_and_summaries_until_unsubscribed(self): + first = self.journal('UserPromptSubmit', detail='Synthetic prompt') + stream, *_ = self.start() + stream.send(op='subscribe', topic='activity', session=self.session()) + initial = stream.next(lambda f: f['t'] == 'events') + self.assertEqual((initial['topic'], initial['key'], initial['cursor']), ('activity', self.key(), first)) + self.assertEqual(initial['events'], self.inspect_events(0)[-len(initial['events']):]) + state = stream.next(lambda f: f['t'] == 'state') + self.assertEqual((state['topic'], state['key'], state['rev']), ('activity', self.key(), 1)) + self.assertEqual(state['value']['name'], self.name) + for section in ('events', 'processes', 'cursor'): + self.assertNotIn(section, state['value']) + reply = self.journal('Stop', detail='Synthetic reply') + events = stream.next(lambda f: f['t'] == 'events' and f['cursor'] == reply, timeout=5) + self.assertEqual(events['events'], self.inspect_events(first)) + self.assertEqual([event['seq'] for event in events['events']], [reply]) + stream.send(op='ping', id='after-reply') + self.assertEqual(stream.next(lambda f: f['t'] == 'pong')['id'], 'after-reply') + stream.send(op='unsubscribe', topic='activity', session=self.session()) + stream.send(op='ping', id='unsubscribed') + stream.next(lambda f: f['t'] == 'pong' and f['id'] == 'unsubscribed') + self.journal('Stop', detail='Unwatched reply') + stream.quiet(lambda f: f['t'] in ('events', 'state'), 2) + self.assertEqual(stream.close(), 0) + + def test_resume_after_a_cursor_and_gap_below_retention(self): + rows = [self.journal('Stop', detail=f'reply {n}') for n in range(5)] + stream, *_ = self.start() + stream.send(op='subscribe', topic='activity', session=self.session(), after=rows[2]) + resumed = stream.next(lambda f: f['t'] == 'events') + self.assertEqual([event['seq'] for event in resumed['events']], rows[3:]) + self.assertEqual(resumed['cursor'], rows[-1]) + # Retention removed rows after the client's cursor. + db = sqlite3.connect(self.state / 'events.sqlite3') + with db: + db.execute('DELETE FROM events WHERE seq <= ?', (rows[2],)) + db.close() + stream.send(op='subscribe', topic='activity', session=self.session(), after=rows[0], tail=1) + gap = stream.next(lambda f: f['t'] == 'gap') + self.assertEqual((gap['topic'], gap['key'], gap['reason']), ('activity', self.key(), 'retention')) + tail = stream.next(lambda f: f['t'] == 'snapshot' and f['topic'] == 'activity') + self.assertEqual([event['seq'] for event in tail['items']], rows[-1:]) + self.assertEqual((tail['key'], tail['cursor'], tail['last']), (self.key(), rows[-1], True)) + # A cursor beyond the journal belongs to a replaced journal. + stream.send(op='subscribe', topic='activity', session=self.session(), after=rows[-1] + 1000) + self.assertEqual(stream.next(lambda f: f['t'] == 'gap')['reason'], 'reset') + self.assertEqual(stream.close(), 0) + + def test_subscription_limit_unknown_topics_and_bad_requests_are_errors(self): + stream, *_ = self.start() + for index in range(16): + stream.send(op='subscribe', topic='activity', session=dict(name=f'sh/limit-{index}'), tail=0) + stream.send(op='subscribe', topic='activity', session=dict(name='sh/limit-16')) + error = stream.next(lambda f: f['t'] == 'error', timeout=30) + self.assertEqual((error['code'], error['key'], error['fatal']), ('too_many_subscriptions', 'name:sh/limit-16', False)) + self.assertEqual(sum(1 for f in stream.seen if f['t'] == 'events'), 16) + stream.send(op='subscribe', topic='processes', session=self.session()) + self.assertEqual(stream.next(lambda f: f['t'] == 'error')['code'], 'unsupported_topic') + stream.process.stdin.write('{"op":"launch"}\nnot json\n') + stream.process.stdin.flush() + self.assertEqual(stream.next(lambda f: f['t'] == 'error')['code'], 'bad_request') + self.assertEqual(stream.next(lambda f: f['t'] == 'error')['code'], 'bad_request') + stream.send(op='subscribe', topic='activity') + self.assertEqual(stream.next(lambda f: f['t'] == 'error')['code'], 'bad_request') + self.assertEqual(stream.close(), 0) + + def test_closed_output_ends_the_stream_quietly(self): + process = subprocess.Popen([str(HGS), 'watch', '--json'], env=self.env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(lambda: process.poll() is None and process.kill()) + process.stdout.readline() + process.stdout.close() + # The next frame (snapshot, meta or a pong) meets a closed pipe. + try: + process.stdin.write(b'{"op":"ping"}\n' * 3) + process.stdin.flush() + except BrokenPipeError: + pass + self.assertEqual(process.wait(timeout=10), 0) + self.assertEqual(process.stderr.read(), b'') + process.stdin.close() + process.stderr.close() + + def test_usage_marker_and_dry_run(self): + result = subprocess.run([str(HGS), '-h'], env=self.env, text=True, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0) + self.assertIn('\n Event stream ABI: watch-v1\n', result.stdout) + self.assertIn('hgs [@host] watch --json long-lived NDJSON event stream (stdin control)', result.stdout) + result = subprocess.run([str(HGS), 'watch'], env=self.env, text=True, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 1) + self.assertIn('usage: hgs [@host] watch --json', result.stderr) + result = subprocess.run([str(HGS), '--dry-run', '@remote', 'watch', '--json'], env=self.env, + text=True, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('-o ServerAliveInterval=10 -o ServerAliveCountMax=3', result.stdout) + self.assertIn('grep -qF', result.stdout) + self.assertIn('Event stream ABI: watch-v1', result.stdout) + self.assertIn('then exec ~/.local/bin/hgs watch --json', result.stdout) + + def test_old_peer_is_never_launched_and_new_peer_streams_through_ssh(self): + ssh_log = self.root / 'ssh-args' + self.script('ssh', '#!/bin/sh\nprintf "%s\\n" "$@" > "' + str(ssh_log) + '"\n' + 'for argument do last="$argument"; done\nexec /bin/sh -c "$last"\n') + peer = self.root / '.local/bin/hgs' + peer.unlink() + launched = self.root / 'old-peer-launched' + peer.write_text('#!/bin/sh\nif [ "$1" = --help ]; then echo "usage: hgs [@host] "; exit 0; fi\n' + 'touch "' + str(launched) + '"\n') + peer.chmod(0o755) + old = subprocess.run([str(HGS), '@remote', 'watch', '--json'], env=self.env, input='', + text=True, capture_output=True, timeout=10) + self.assertEqual(old.returncode, 1) + self.assertIn('update the selected computer to stream events', old.stderr) + self.assertFalse(launched.exists()) + self.assertEqual(old.stdout, '') + peer.unlink() + peer.symlink_to(HGS) + stream = Stream(self, '@remote') + hello = stream.next() + self.assertEqual(hello['t'], 'hello') + stream.next(lambda f: f['t'] == 'snapshot' and f['topic'] == 'catalog' and f['last']) + stream.send(op='ping', id='peer') + self.assertEqual(stream.next(lambda f: f['t'] == 'pong')['id'], 'peer') + self.assertEqual(stream.close(), 0) + arguments = ssh_log.read_text().splitlines() + for option in ('ServerAliveInterval=10', 'ServerAliveCountMax=3', 'BatchMode=yes'): + self.assertIn(option, arguments) + self.assertNotIn('-t', arguments) + + +if __name__ == '__main__': + unittest.main() From 226704a4c5580fda1184abc0f8817b85b5ffac6f Mon Sep 17 00:00:00 2001 From: ufna <271578+ufna@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:09:18 +0300 Subject: [PATCH 4/7] feat(tray): follow each computer through its event stream Event data plane M4 (zerus-604d.4). EventStream runs one `hgs [@host] watch --json` per computer whose CLI prints the watch-v1 marker. The child, NDJSON framing, JSON parsing and the computer's catalog and Activity store live on the stream's own thread; the GUI thread receives the same BoxState a listing poll delivers and inspect-shaped Activity increments, so the existing apply paths stay unchanged. While a computer's stream is live, the tray skips its periodic listing polls and the Sessions window follows the selected session's Activity through it, polling only the Processes section at its own interval. Listings are applied at most every 2 s until M2 makes applying one cheap; Activity within 50 ms. A failed or silent stream restarts after 5 s, 15 s, 60 s and then every 5 minutes, and polling continues meanwhile. A computer without the marker keeps polling. ZERUS_EVENT_STREAM=0 or dataPlane/eventStream=false opts out. On arch, following a busy session: event lag p50 0.20 s, p90 0.28 s against 2.34 s and 2.70 s for today's 2.5 s inspect polling. Co-Authored-By: Claude Opus 5.5 --- docs/reference.md | 9 + .../2026-10-27-event-data-plane-design.md | 6 +- tray/CMakeLists.txt | 2 +- tray/src/EventStream.cpp | 428 ++++++++++++++++++ tray/src/EventStream.h | 108 +++++ tray/src/HgsClient.cpp | 1 + tray/src/HgsClient.h | 5 + tray/src/SessionsWindow.cpp | 26 +- tray/src/SessionsWindow.h | 2 + tray/src/TrayAgent.cpp | 26 +- tray/src/TrayAgent.h | 1 + tray/tests/CMakeLists.txt | 5 + tray/tests/test_eventstream.cpp | 187 ++++++++ 13 files changed, 798 insertions(+), 8 deletions(-) create mode 100644 tray/src/EventStream.cpp create mode 100644 tray/src/EventStream.h create mode 100644 tray/tests/test_eventstream.cpp diff --git a/docs/reference.md b/docs/reference.md index f8ae55a23..07f798888 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -4031,6 +4031,15 @@ and summaries coalesce per key. When the queue overflows, the topic receives `gap` (`overflow`) and a fresh snapshot once the client catches up. Control lines are at most 64 KiB. +The desktop runs one stream for each computer whose CLI prints the marker. +While a computer's stream is live, the desktop stops polling that computer's +listing and follows the selected session's Activity through the stream; the +Processes tab keeps its own poll interval. Listings are applied at most every +2 s, Activity within 50 ms. A stream that fails or stays silent for 30 s +restarts after 5 s, 15 s, 60 s and then every 5 minutes, and the desktop polls +meanwhile. `ZERUS_EVENT_STREAM=0` or the setting `dataPlane/eventStream=false` +keeps the desktop on polling. + ## Validation Use [CONTRIBUTING.md](../CONTRIBUTING.md) for standard checks. Integration tests diff --git a/docs/superpowers/specs/2026-10-27-event-data-plane-design.md b/docs/superpowers/specs/2026-10-27-event-data-plane-design.md index c0e17c66c..7e06907e4 100644 --- a/docs/superpowers/specs/2026-10-27-event-data-plane-design.md +++ b/docs/superpowers/specs/2026-10-27-event-data-plane-design.md @@ -252,8 +252,10 @@ the same frames. The protocol does not depend on how changes are detected. - **Qt desktop**: one `EventStream` per computer. Its reader, line framing, JSON parsing and per-computer store run on a worker thread. The store - publishes immutable prepared deltas to the GUI, coalesced to at most one - application per 50 ms. While a computer's stream is healthy, the desktop + publishes immutable prepared values to the GUI: Activity within 50 ms, and + listings at most every 2 s until M2 makes applying one cheap. A listing + arrives as the same `BoxState` a poll delivers, and Activity as an + inspect-shaped increment, so the existing apply paths stay unchanged. While a computer's stream is healthy, the desktop stops its `ls`, inspection, `git-status` and `processes` polls for it. A missing marker, or three stream failures within 60 s, falls back to today's polling, and the desktop retries the stream every 5 minutes. diff --git a/tray/CMakeLists.txt b/tray/CMakeLists.txt index 384f12ce2..adbb84d3a 100644 --- a/tray/CMakeLists.txt +++ b/tray/CMakeLists.txt @@ -143,7 +143,7 @@ include(cmake/BuildVersion.cmake) # link; tests/CMakeLists.txt fails when a suite lists a source that one of them # builds. Each layer adds Qt modules, so a suite that links the lowest layer # covering its code keeps its QTEST_MAIN application type. -add_library(hgs-tray-core STATIC src/ProcessRunner.cpp src/JsonParse.cpp src/SettingsJson.cpp src/HgsClient.cpp src/FleetState.cpp +add_library(hgs-tray-core STATIC src/ProcessRunner.cpp src/JsonParse.cpp src/SettingsJson.cpp src/HgsClient.cpp src/EventStream.cpp src/FleetState.cpp src/SessionCreationController.cpp src/AnalyticsClient.cpp src/AnalyticsData.cpp src/ReadMarkSync.cpp src/AppConfig.cpp src/AccountUsageStore.cpp src/ComposerDraftStore.cpp diff --git a/tray/src/EventStream.cpp b/tray/src/EventStream.cpp new file mode 100644 index 000000000..8fa05848a --- /dev/null +++ b/tray/src/EventStream.cpp @@ -0,0 +1,428 @@ +#include "EventStream.h" + +#include "ChildProcess.h" +#include "JsonParse.h" +#include "ProcessRunner.h" + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace { +constexpr qsizetype MaximumLine = 1024 * 1024 + 64; +// Until fleet preparation leaves the GUI thread (M2), applying a listing costs +// a rebuild there (about 150 ms on a 2026-10-27 Mac sample); listings keep the +// 2 s polling cadence at most, while Activity arrives within PublishMs. +std::atomic ListingGapMs{2000}; +// Callers date a computer's data by its last listing: automatic approvals act +// on a peer listed within 30 s, and 3 minutes mark it stale. An unchanged +// listing is applied again after this long. +std::atomic ListingRefreshMs{10000}; +const QByteArray Marker = QByteArrayLiteral("Event stream ABI: watch-v1"); +EventStreams *s_instance = nullptr; + +QString sessionKey(const QString &name, const QString &runId) +{ + return runId.isEmpty() ? QStringLiteral("name:") + name : QStringLiteral("run:") + runId; +} + +QByteArray line(const QJsonObject &object) +{ + return QJsonDocument(object).toJson(QJsonDocument::Compact) + '\n'; +} +} + +// Everything below runs on the stream's thread. It talks to its EventStream +// only through queued calls, which Qt drops once the stream is gone. +class EventStreamWorker : public QObject { +public: + EventStreamWorker(EventStream *front, QString host) : m_front(front), m_host(std::move(host)) {} + ~EventStreamWorker() override { stopProcess(); } + + void begin(const ProcessRunner::Request &request) + { + stopProcess(); reset(); + if (!m_publish) { + m_publish = new QTimer(this); m_publish->setSingleShot(true); + QObject::connect(m_publish, &QTimer::timeout, this, [this] { publish(); }); + m_silence = new QTimer(this); m_silence->setInterval(qMin(5000, ListingRefreshMs.load())); + QObject::connect(m_silence, &QTimer::timeout, this, [this] { + const auto now = QDateTime::currentMSecsSinceEpoch(); + if (now - m_lastFrame > EventStream::SilenceMs) return fail(QStringLiteral("no frame for 30 s")); + if (m_ready && now - m_lastListing >= ListingRefreshMs.load()) { m_listingDirty = true; schedule(); } + }); + } + m_publish->stop(); + auto *process = m_process = new QProcess(this); + process->setProgram(request.program); process->setArguments(request.arguments); + if (request.environment) process->setProcessEnvironment(*request.environment); + ChildProcess::inheritStdioOnly(*process); + QObject::connect(process, &QProcess::readyReadStandardOutput, this, [this] { read(); }); + QObject::connect(process, &QProcess::readyReadStandardError, this, [this] { + if (m_process) m_stderr = (m_stderr + m_process->readAllStandardError()).right(4000); + }); + QObject::connect(process, &QProcess::finished, this, [this] { + read(); + fail(m_stderr.trimmed().isEmpty() ? QStringLiteral("the stream ended") : QString::fromUtf8(m_stderr.trimmed())); + }); + QObject::connect(process, &QProcess::errorOccurred, this, [this](QProcess::ProcessError error) { + if (error == QProcess::FailedToStart) fail(QStringLiteral("cannot start the stream")); + }); + m_lastFrame = QDateTime::currentMSecsSinceEpoch(); + m_silence->start(); + process->start(); + } + void finish() { stopProcess(); reset(); } + + void follow(const QString &name, const QString &runId, const QString &conversationId, qint64 after) + { + if (!m_activity.key.isEmpty()) { + write(QJsonObject{{"op", "unsubscribe"}, {"topic", "activity"}, + {"session", QJsonObject{{"name", m_activity.name}, {"run_id", m_activity.run}}}}); + } + m_activity = {}; + if (name.isEmpty()) return; + m_activity.key = sessionKey(name, runId); m_activity.name = name; m_activity.run = runId; + QJsonObject session{{"name", name}}; + if (!runId.isEmpty()) session["run_id"] = runId; + if (!conversationId.isEmpty()) session["conversation_id"] = conversationId; + QJsonObject subscribe{{"op", "subscribe"}, {"topic", "activity"}, {"session", session}}; + if (after > 0) subscribe["after"] = after; else subscribe["tail"] = 100; + write(subscribe); + } + +private: + struct Activity { + QString key, name, run; + QJsonObject state; + QJsonArray events; + qint64 cursor = 0; + bool replace = false, dirty = false; + }; + + void reset() + { + m_buffer.clear(); m_stderr.clear(); m_rows.clear(); m_pending.clear(); m_meta = {}; + m_catalog = m_haveMeta = m_listingDirty = m_ready = m_failed = false; m_activity = {}; + } + void stopProcess() + { + if (!m_process) return; + auto *process = std::exchange(m_process, nullptr); + process->disconnect(this); + if (process->state() != QProcess::NotRunning) { + // EOF ends the stream; a peer's ssh closes with it. + process->closeWriteChannel(); + if (!process->waitForFinished(1000)) { process->kill(); process->waitForFinished(1000); } + } + delete process; + if (m_silence) m_silence->stop(); + } + void write(const QJsonObject &operation) + { + if (m_process && m_process->state() == QProcess::Running) m_process->write(line(operation)); + } + void fail(const QString &reason) + { + if (m_failed) return; + // Nothing this stream prepared reaches the GUI after its failure. + m_failed = true; m_publish->stop(); m_listingDirty = m_activity.dirty = false; + stopProcess(); + QMetaObject::invokeMethod(m_front, [front = m_front, reason] { front->failed(reason); }, Qt::QueuedConnection); + } + void read() + { + if (!m_process || m_failed) return; + m_buffer += m_process->readAllStandardOutput(); + qsizetype start = 0; + for (;;) { + const auto end = m_buffer.indexOf('\n', start); + if (end < 0) break; + if (end - start > MaximumLine) return fail(QStringLiteral("a frame exceeds 1 MiB")); + QString error; + const auto frame = JsonParse::object(m_buffer.mid(start, end - start), &error); + start = end + 1; + if (!error.isEmpty()) return fail(QStringLiteral("invalid frame: ") + error); + m_lastFrame = QDateTime::currentMSecsSinceEpoch(); + handle(frame); + if (m_failed) return; + } + m_buffer.remove(0, start); + if (m_buffer.size() > MaximumLine) fail(QStringLiteral("a frame exceeds 1 MiB")); + } + void handle(const QJsonObject &frame) + { + if (frame.value("v").toInt() != 1) return fail(QStringLiteral("unsupported stream version")); + const auto type = frame.value("t").toString(), topic = frame.value("topic").toString(); + if (type == "snapshot" && topic == "catalog") { + if (frame.value("page").toInt() == 0) m_pending.clear(); + for (const auto &value : frame.value("items").toArray()) { + const auto item = value.toObject(); + m_pending.insert(item.value("key").toString(), item.value("row").toObject()); + } + if (frame.value("last").toBool()) { m_rows = std::exchange(m_pending, {}); m_catalog = m_listingDirty = true; } + } else if (type == "delta" && topic == "catalog") { + for (const auto &value : frame.value("upsert").toArray()) { + const auto item = value.toObject(); + m_rows.insert(item.value("key").toString(), item.value("row").toObject()); + } + for (const auto &value : frame.value("remove").toArray()) m_rows.remove(value.toObject().value("key").toString()); + m_listingDirty = true; + } else if (type == "snapshot" && topic == "catalog.meta") { + m_meta = frame.value("value").toObject(); m_haveMeta = m_listingDirty = true; + } else if (topic == "activity" && frame.value("key").toString() == m_activity.key && !m_activity.key.isEmpty()) { + if (type == "events") { + for (const auto &event : frame.value("events").toArray()) m_activity.events.append(event); + m_activity.cursor = frame.value("cursor").toInteger(); m_activity.dirty = true; + } else if (type == "snapshot") { + m_activity.events = frame.value("items").toArray(); m_activity.replace = true; + m_activity.cursor = frame.value("cursor").toInteger(); m_activity.dirty = true; + } else if (type == "state") { + m_activity.state = frame.value("value").toObject(); m_activity.dirty = true; + } else if (type == "gap") { + m_activity.events = {}; m_activity.dirty = false; + } + } else if (type == "error") { + if (frame.value("fatal").toBool()) return fail(frame.value("message").toString()); + const auto code = frame.value("code").toString(); + if ((code == "subscription_ended" || code == "session_changed") && frame.value("key").toString() == m_activity.key + && !m_activity.key.isEmpty()) { + const auto name = m_activity.name; m_activity = {}; + QMetaObject::invokeMethod(m_front, [front = m_front, name] { front->m_followedName.clear(); emit front->activityEnded(name); }, + Qt::QueuedConnection); + } + } + // hello, heartbeat, pong and catalog gaps only prove the stream is alive; + // after a catalog gap the previous rows stay until the next snapshot. + schedule(); + } + // Activity goes out within PublishMs; a listing waits out ListingGapMs + // after the previous one, except the first. + qint64 listingWait() const + { + if (!m_ready) return 0; + return qMax(0, ListingGapMs.load() - (QDateTime::currentMSecsSinceEpoch() - m_lastListing)); + } + void schedule() + { + if (m_failed || m_publish->isActive()) return; + if (m_activity.dirty) m_publish->start(EventStream::PublishMs); + else if (m_listingDirty) m_publish->start(int(qMax(EventStream::PublishMs, listingWait()))); + } + void publish() + { + if (m_failed) return; + if (m_listingDirty && m_catalog && m_haveMeta && listingWait() == 0) { + // Rows in listing order, as ls --json sorts them. + QList rows = m_rows.values(); + std::sort(rows.begin(), rows.end(), [](const QJsonObject &a, const QJsonObject &b) { + const auto left = a.value("name").toString(), right = b.value("name").toString(); + const int folded = QString::compare(left, right, Qt::CaseInsensitive); + return folded != 0 ? folded < 0 : left < right; + }); + QJsonArray sessions; for (const auto &row : std::as_const(rows)) sessions.append(row); + auto listing = m_meta; listing["sessions"] = sessions; + BoxState box = HgsClient::box(listing); + // A peer's listing names the computer as it calls itself (requestPeer). + if (!m_host.isEmpty()) box.reportedHost = std::exchange(box.host, m_host); + m_listingDirty = false; m_lastListing = QDateTime::currentMSecsSinceEpoch(); + const bool first = !std::exchange(m_ready, true); + QMetaObject::invokeMethod(m_front, [front = m_front, box, first] { + if (first) front->setState(EventStream::State::Live); + emit front->listingReady(box); + }, Qt::QueuedConnection); + } + if (m_activity.dirty && !m_activity.state.isEmpty()) { + auto inspection = m_activity.state; + inspection["events"] = std::exchange(m_activity.events, {}); + inspection["cursor"] = m_activity.cursor; + if (std::exchange(m_activity.replace, false)) inspection["replace_events"] = true; + m_activity.dirty = false; + QMetaObject::invokeMethod(m_front, [front = m_front, name = m_activity.name, inspection] { + if (front->m_followedName == name) { + front->m_followedAfter = inspection.value("cursor").toInteger(); + emit front->activityReady(name, inspection); + } + }, Qt::QueuedConnection); + } + schedule(); + } + + EventStream *m_front; + QString m_host; + QProcess *m_process = nullptr; + QTimer *m_publish = nullptr, *m_silence = nullptr; + QByteArray m_buffer, m_stderr; + qint64 m_lastFrame = 0, m_lastListing = 0; + QHash m_rows, m_pending; + QJsonObject m_meta; + bool m_catalog = false, m_haveMeta = false, m_listingDirty = false, m_ready = false, m_failed = false; + Activity m_activity; +}; + +EventStream::EventStream(const QString &hgs, const QString &host, QObject *parent) + : QObject(parent), m_hgs(hgs), m_host(host) +{ + m_thread.setObjectName(host.isEmpty() ? QStringLiteral("hgs-stream") : QStringLiteral("hgs-stream-") + host); + m_worker = new EventStreamWorker(this, host); + m_worker->moveToThread(&m_thread); + connect(&m_thread, &QThread::finished, m_worker, &QObject::deleteLater); + m_thread.start(); + m_restart.setSingleShot(true); + connect(&m_restart, &QTimer::timeout, this, [this] { start(); }); +} + +EventStream::~EventStream() +{ + m_restart.stop(); + QMetaObject::invokeMethod(m_worker, [worker = m_worker] { worker->finish(); }, Qt::BlockingQueuedConnection); + m_thread.quit(); m_thread.wait(); +} + +void EventStream::setListingTimingForTests(int gapMs, int refreshMs) +{ + ListingGapMs = gapMs; ListingRefreshMs = refreshMs; +} + +int EventStream::restartDelayMs(int failures) +{ + static const int delays[] = {5000, 15000, 60000, 300000}; + return delays[qBound(0, failures - 1, 3)]; +} + +void EventStream::setState(State state) +{ + if (m_state == state) return; + m_state = state; + if (state == State::Live) sendFollow(); + emit stateChanged(state); +} + +void EventStream::start() +{ + if (m_state == State::Probing || m_state == State::Starting || m_state == State::Live) return; + setState(State::Probing); + QStringList help{QStringLiteral("--help")}; + if (!m_host.isEmpty()) help.prepend(QLatin1Char('@') + m_host); + auto request = HgsClient::commandRequest(m_hgs, help); request.timeoutMs = m_host.isEmpty() ? 5000 : 15000; + ProcessRunner::run(std::move(request), this, + [](ProcessRunner::Result result) { + return result.outcome == ProcessRunner::Result::Finished && result.exitCode == 0 + ? (result.standardOutput.contains(Marker) ? 1 : 0) : -1; + }, + [this](int supported) { + if (m_state != State::Probing) return; + if (supported == 1) return launch(); + if (supported == 0) { setState(State::Unsupported); m_restart.start(600000); return; } + failed(QStringLiteral("the computer did not answer")); + }); +} + +void EventStream::launch() +{ + setState(State::Starting); + QStringList args{QStringLiteral("watch"), QStringLiteral("--json")}; + if (!m_host.isEmpty()) args.prepend(QLatin1Char('@') + m_host); + const auto request = HgsClient::commandRequest(m_hgs, args); + QMetaObject::invokeMethod(m_worker, [worker = m_worker, request] { worker->begin(request); }, Qt::QueuedConnection); +} + +void EventStream::stop() +{ + m_restart.stop(); + QMetaObject::invokeMethod(m_worker, [worker = m_worker] { worker->finish(); }, Qt::QueuedConnection); + setState(State::Stopped); +} + +void EventStream::failed(const QString &reason) +{ + if (m_state == State::Stopped) return; + const auto now = QDateTime::currentMSecsSinceEpoch(); + // Failures more than ten minutes apart start the delays over. + m_failures = now - m_lastFailure > 600000 ? 1 : m_failures + 1; + m_lastFailure = now; + qWarning().noquote() << "hgs zerus: event stream" << (m_host.isEmpty() ? QStringLiteral("(local)") : m_host) << "stopped:" << reason; + setState(State::Failed); + m_restart.start(restartDelayMs(m_failures)); +} + +void EventStream::follow(const QString &name, const QString &runId, const QString &conversationId, qint64 after) +{ + if (name == m_followedName && runId == m_followedRun && conversationId == m_followedConversation) return; + m_followedName = name; m_followedRun = runId; m_followedConversation = conversationId; m_followedAfter = after; + if (live()) sendFollow(); +} + +void EventStream::unfollow() +{ + if (m_followedName.isEmpty()) return; + m_followedName.clear(); m_followedRun.clear(); m_followedConversation.clear(); m_followedAfter = 0; + if (live()) sendFollow(); +} + +void EventStream::sendFollow() +{ + QMetaObject::invokeMethod(m_worker, [worker = m_worker, name = m_followedName, run = m_followedRun, + conversation = m_followedConversation, after = m_followedAfter] { + worker->follow(name, run, conversation, after); + }, Qt::QueuedConnection); +} + +EventStreams::EventStreams(const QString &hgs, QObject *parent) : QObject(parent), m_hgs(hgs) +{ + Q_ASSERT(!s_instance); + s_instance = this; +} + +EventStreams::~EventStreams() +{ + if (s_instance == this) s_instance = nullptr; + qDeleteAll(std::exchange(m_streams, {})); +} + +EventStreams *EventStreams::instance() { return s_instance; } + +bool EventStreams::enabled() +{ + if (qEnvironmentVariable("ZERUS_EVENT_STREAM") == QLatin1String("0")) return false; + return QSettings().value(QStringLiteral("dataPlane/eventStream"), true).toBool(); +} + +void EventStreams::setHosts(const QStringList &hosts) +{ + for (auto it = m_streams.begin(); it != m_streams.end();) { + if (hosts.contains(it.key())) { ++it; continue; } + delete it.value(); it = m_streams.erase(it); + } + for (const auto &host : hosts) { + if (m_streams.contains(host)) continue; + auto *stream = new EventStream(m_hgs, host, this); + m_streams.insert(host, stream); + connect(stream, &EventStream::listingReady, this, [this, host](const BoxState &box) { emit listingReady(host, box); }); + connect(stream, &EventStream::activityReady, this, [this, host](const QString &name, const QJsonObject &inspection) { + emit activityReady(host, name, inspection); + }); + connect(stream, &EventStream::activityEnded, this, [this, host](const QString &name) { emit activityEnded(host, name); }); + connect(stream, &EventStream::stateChanged, this, [this, host](EventStream::State state) { emit stateChanged(host, state); }); + stream->start(); + } +} + +void EventStreams::follow(const QString &host, const QString &name, const QString &runId, const QString &conversationId, qint64 after) +{ + for (auto it = m_streams.cbegin(); it != m_streams.cend(); ++it) { + if (it.key() == host) it.value()->follow(name, runId, conversationId, after); + else it.value()->unfollow(); + } +} + +void EventStreams::unfollowAll() +{ + for (auto *stream : std::as_const(m_streams)) stream->unfollow(); +} diff --git a/tray/src/EventStream.h b/tray/src/EventStream.h new file mode 100644 index 000000000..280db1ef7 --- /dev/null +++ b/tray/src/EventStream.h @@ -0,0 +1,108 @@ +#pragma once + +#include "HgsClient.h" + +#include +#include +#include +#include +#include +#include + +#include + +class EventStreamWorker; + +// One computer's `hgs [@host] watch --json` (docs/reference.md, Event stream). +// The child process, NDJSON framing, JSON parsing and the computer's catalog +// and Activity state live on the stream's own thread. The GUI thread receives +// finished values: the same BoxState a listing poll delivers, and +// inspect-shaped Activity increments, at most one of each every 50 ms. +// +// A computer whose CLI lacks the watch-v1 marker stays Unsupported and keeps +// polling. A stream that fails restarts with a growing delay; while it is not +// Live, callers poll as before. +class EventStream : public QObject { + Q_OBJECT +public: + enum class State { Stopped, Probing, Starting, Live, Unsupported, Failed }; + Q_ENUM(State) + + EventStream(const QString &hgs, const QString &host, QObject *parent = nullptr); + ~EventStream() override; + + QString host() const { return m_host; } + State state() const { return m_state; } + bool live() const { return m_state == State::Live; } + void start(); + void stop(); + + // Follows one session's Activity; a later call for another session of this + // computer replaces it. after is the journal seq to resume after (0: tail). + void follow(const QString &name, const QString &runId, const QString &conversationId, qint64 after); + void unfollow(); + QString followed() const { return m_followedName; } + + // Tests: frames and the delay before a restart. + static constexpr int PublishMs = 50; + static constexpr int SilenceMs = 30000; + static int restartDelayMs(int failures); + // Tests: the least time between applied listings (2 s) and the time after + // which an unchanged listing is applied again (10 s). + static void setListingTimingForTests(int gapMs, int refreshMs); + +signals: + void stateChanged(EventStream::State state); + void listingReady(const BoxState &box); + void activityReady(const QString &name, const QJsonObject &inspection); + // The followed session left the catalog or changed run; poll it instead. + void activityEnded(const QString &name); + +private: + friend class EventStreamWorker; + void setState(State state); + void launch(); + void failed(const QString &reason); + void sendFollow(); + + QString m_hgs, m_host; + State m_state = State::Stopped; + QThread m_thread; + EventStreamWorker *m_worker = nullptr; + QTimer m_restart; + int m_failures = 0; + qint64 m_lastFailure = 0; + QString m_followedName, m_followedRun, m_followedConversation; + qint64 m_followedAfter = 0; +}; + +// The desktop's streams by computer alias ("" is this computer), owned by the +// tray agent. Windows read it to follow Activity and to skip polls. +class EventStreams : public QObject { + Q_OBJECT +public: + explicit EventStreams(const QString &hgs, QObject *parent = nullptr); + ~EventStreams() override; + static EventStreams *instance(); + // Off with ZERUS_EVENT_STREAM=0 or the setting dataPlane/eventStream=false. + static bool enabled(); + + EventStream *stream(const QString &host) const { return m_streams.value(host); } + bool live(const QString &host) const { const auto *s = stream(host); return s && s->live(); } + // Starts streams for these computers and stops the others. + void setHosts(const QStringList &hosts); + // Follows one session's Activity on its computer's stream and stops + // following on every other stream. + void follow(const QString &host, const QString &name, const QString &runId, const QString &conversationId, qint64 after); + void unfollowAll(); + +signals: + void listingReady(const QString &host, const BoxState &box); + void activityReady(const QString &host, const QString &name, const QJsonObject &inspection); + void activityEnded(const QString &host, const QString &name); + void stateChanged(const QString &host, EventStream::State state); + +private: + QString m_hgs; + QHash m_streams; +}; diff --git a/tray/src/HgsClient.cpp b/tray/src/HgsClient.cpp index e1187f7f8..2be034ef5 100644 --- a/tray/src/HgsClient.cpp +++ b/tray/src/HgsClient.cpp @@ -330,6 +330,7 @@ HgsClient::~HgsClient() } BoxState HgsClient::box(const QJsonObject &object) { return boxFromObject(object); } +ProcessRunner::Request HgsClient::commandRequest(const QString &hgs, const QStringList &arguments) { return processRequest(hgs, arguments); } QList HgsClient::parseSessions(const QJsonArray &rows) { QJsonObject box{{"host","archive"},{"ok",true},{"sessions",rows}}; diff --git a/tray/src/HgsClient.h b/tray/src/HgsClient.h index 46b2e9bd3..d1558d620 100644 --- a/tray/src/HgsClient.h +++ b/tray/src/HgsClient.h @@ -1,5 +1,7 @@ #pragma once +#include "ProcessRunner.h" + #include #include #include @@ -300,6 +302,9 @@ class HgsClient : public QObject { static BoxState parseBox(const QByteArray &json, QString *error); // A listing that is already an object, as tests and prepared values hold it. static BoxState box(const QJsonObject &object); + // How the client starts hgs with these arguments: its environment without + // an agent's attribution, and macOS argument handling. + static ProcessRunner::Request commandRequest(const QString &hgs, const QStringList &arguments); static QList parseFleet(const QByteArray &json, QString *error); static QList parseProjects(const QByteArray &json, QString *error); diff --git a/tray/src/SessionsWindow.cpp b/tray/src/SessionsWindow.cpp index 255f8e233..cdd1e3ad9 100644 --- a/tray/src/SessionsWindow.cpp +++ b/tray/src/SessionsWindow.cpp @@ -3,6 +3,7 @@ #include "ArchiveBrowser.h" #include "NativeSessionCapabilities.h" #include "SessionsWindow.h" +#include "EventStream.h" #include #include "WorktreeBatchCleanupDialog.h" #include "WorktreeCleanupStatus.h" @@ -1903,6 +1904,17 @@ SessionsWindow::SessionsWindow(const QString &hgsPath, QWidget *parent) emit refreshRequested(); }); connect(&m_client, &HgsClient::inspectionReady, this, &SessionsWindow::acceptInspection); + if (auto *streams = EventStreams::instance()) { + connect(streams, &EventStreams::activityReady, this, [this](const QString &host, const QString &name, const QJsonObject &inspection) { + acceptInspection(host, name, inspection, {}); + }); + // The stream stopped following this run (it ended or changed): poll it instead. + connect(streams, &EventStreams::activityEnded, this, [this](const QString &host, const QString &name) { + if (const auto *entry = selected(); entry && entry->host == host && entry->session.name == name) { + m_streamEnded.insert(host + '\n' + name + '\n' + entry->session.runId); inspect(); + } + }); + } connect(&m_client, &HgsClient::subagentInspectionReady, this, [this](const QString &host,const QString &name,const QString &id,const QString &archive,const QJsonObject &data) { const auto *entry = selected(); @@ -3786,7 +3798,19 @@ void SessionsWindow::inspect() if (!isVisible() || m_pages->currentIndex() != 0 || !entry || !entry->online) return; const int processInterval=m_detailTabs->currentWidget()==m_processes?ProcessSettings::activeMs():ProcessSettings::backgroundMs(); const bool processes=ProcessSettings::enabled() && (!m_processPollAge.isValid() || m_processPollAge.elapsed()>=processInterval); - if(m_client.requestInspection(entry->host, entry->session.name, m_cursor, entry->session.archiveId,processes)) { + // A live event stream brings Activity as it happens (EventStream.h); only + // the process section is still polled, at its own interval. + auto *streams = EventStreams::instance(); + const bool streamed = streams && streams->live(entry->host) && entry->session.archiveId.isEmpty() + && !m_streamEnded.contains(entry->host + '\n' + entry->session.name + '\n' + entry->session.runId); + if (streams) { + if (streamed) streams->follow(entry->host, entry->session.name, entry->session.runId, entry->session.conversationId, m_cursor); + else streams->unfollowAll(); + } + if (streamed) { + m_inspectionAge.start(); + if (processes && m_client.requestInspection(entry->host, entry->session.name, m_cursor, entry->session.archiveId, true)) m_processPollAge.start(); + } else if(m_client.requestInspection(entry->host, entry->session.name, m_cursor, entry->session.archiveId,processes)) { m_inspectionAge.start();if(processes)m_processPollAge.start(); } refreshAccountUsage(); diff --git a/tray/src/SessionsWindow.h b/tray/src/SessionsWindow.h index 9cdbee1bd..30a11f5b1 100644 --- a/tray/src/SessionsWindow.h +++ b/tray/src/SessionsWindow.h @@ -461,6 +461,8 @@ class SessionsWindow : public QWidget { QHash m_remotePollAges; BusyIndicator *m_sessionLoading = nullptr; QElapsedTimer m_processPollAge,m_inspectionAge; + // Runs whose event-stream subscription ended; they are polled (EventStream.h). + QSet m_streamEnded; SessionList *m_sessions; SessionSearch *m_searchResults; QStackedWidget *m_listStack; diff --git a/tray/src/TrayAgent.cpp b/tray/src/TrayAgent.cpp index c976ae385..26481a9eb 100644 --- a/tray/src/TrayAgent.cpp +++ b/tray/src/TrayAgent.cpp @@ -1,4 +1,5 @@ #include "TrayAgent.h" +#include "EventStream.h" #include "ProcessRunner.h" #include "SettingsJson.h" #include "QuestionNotificationSettings.h" @@ -60,10 +61,17 @@ std::unique_ptr makeTrayIcon() // совпал с открытием меню) -- дублирующий вызов requestPeer() для уже летящего запроса // не создаёт второй процесс: HgsClient сам игнорирует его (m_peerInFlight), эта функция // вызывающего кода не гейтит. +// A computer with a live event stream (EventStream.h) needs no periodic poll. void pollAllKnownPeers(HgsClient &client, const FleetState &state) { + const auto *streams = EventStreams::instance(); for (const QString &alias : state.peerNames()) - client.requestPeer(alias); + if (!streams || !streams->live(alias)) client.requestPeer(alias); +} +void pollLocal(HgsClient &client) +{ + const auto *streams = EventStreams::instance(); + if (!streams || !streams->live({})) client.requestLocal(); } // Терминал выбирается один раз на запуск. Явный шаблон в конфиге побеждает выбор @@ -206,7 +214,7 @@ TrayAgent::TrayAgent(const AppConfig &cfg, QObject *parent) connect(&m_menu, &TrayMenu::openModeChosen, this, &TrayAgent::setOpenMode); connect(&m_menu, &TrayMenu::sessionsRequested, this, &TrayAgent::showSessions); connect(&m_menu, &TrayMenu::refreshRequested, this, [this]() { - m_client.requestLocal(); + pollLocal(m_client); pollAllKnownPeers(m_client, m_state); }); connect(&m_menu, &TrayMenu::quitRequested, qApp, &QCoreApplication::quit); @@ -245,9 +253,18 @@ TrayAgent::TrayAgent(const AppConfig &cfg, QObject *parent) // Локальный тик: опрашивает только этот бокс, никакого ssh, миллисекунды. Стартует // сразу же, не дожидаясь ни peerPollMs, ни обнаружения пиров ниже -- та часть, что // отвечает "тик и первая отрисовка не должны ждать пиров" из задания. + // One long-lived event stream per computer replaces its periodic polls + // while live; polling continues for a computer without one. + if (EventStreams::enabled()) { + m_streams = new EventStreams(m_client.executable(), this); + connect(m_streams, &EventStreams::listingReady, this, [this](const QString &host, const BoxState &box) { + if (host.isEmpty()) onLocalReady(box); else onPeerReady(box); + }); + m_streams->setHosts({QString()}); + } m_localTimer.setInterval(cfg.localPollMs); connect(&m_localTimer, &QTimer::timeout, this, [this] { - m_client.requestLocal(); + pollLocal(m_client); // Short-lived remote permission prompts must not wait for the ordinary // minute-long peer cadence. The client coalesces reads already in flight. const int interval = AutomaticApprovalPolicy::enabled() ? qMin(m_cfg.peerPollMs, 5000) : m_cfg.peerPollMs; @@ -395,6 +412,7 @@ void TrayAgent::onLocalReady(const BoxState &box) } for (const auto &peer : m_state.peerNames()) if (!previousPeers.contains(peer)) m_client.requestPeer(peer); + if (m_streams && previousPeers != m_state.peerNames()) m_streams->setHosts(QStringList{QString()} + m_state.peerNames()); if (box.peersKnown && !m_peerTimer.isActive()) m_peerTimer.start(); // A control-only computer reports no sessions; its answer only carries the // machine list and metrics. Every poll is a process start on Windows, so it @@ -702,7 +720,7 @@ void TrayAgent::showSessions() else m_client.requestPeer(host); }); connect(m_sessionsWindow, &SessionsWindow::refreshRequested, this, [this]() { - m_client.requestLocal(); pollAllKnownPeers(m_client, m_state); + pollLocal(m_client); pollAllKnownPeers(m_client, m_state); }); } m_sessionsWindow->setFleet(m_state); diff --git a/tray/src/TrayAgent.h b/tray/src/TrayAgent.h index 97127b78a..efb50eb33 100644 --- a/tray/src/TrayAgent.h +++ b/tray/src/TrayAgent.h @@ -113,6 +113,7 @@ private slots: SessionsWindow *m_sessionsWindow = nullptr; // Records desktop activity for phone alerts; see DesktopPresence.h. std::unique_ptr m_presence; + class EventStreams *m_streams = nullptr; QString m_presenceError; #ifdef Q_OS_WIN // The tray runs `hgs swarm worker` itself on Windows (SwarmWorkerSupervisor.h). diff --git a/tray/tests/CMakeLists.txt b/tray/tests/CMakeLists.txt index e938a38a9..bc8fb045e 100644 --- a/tray/tests/CMakeLists.txt +++ b/tray/tests/CMakeLists.txt @@ -254,6 +254,11 @@ target_include_directories(test_readmarksync PRIVATE ../src) target_link_libraries(test_readmarksync PRIVATE Qt6::Test Qt6::Core hgs-tray-core) zerus_add_test(readmarksync test_readmarksync) +add_executable(test_eventstream test_eventstream.cpp) +target_include_directories(test_eventstream PRIVATE ../src) +target_link_libraries(test_eventstream PRIVATE Qt6::Test Qt6::Gui hgs-tray-core) +zerus_add_test(eventstream test_eventstream) + add_executable(test_jsonparse test_jsonparse.cpp) target_include_directories(test_jsonparse PRIVATE ../src) target_link_libraries(test_jsonparse PRIVATE Qt6::Test Qt6::Gui hgs-tray-core) diff --git a/tray/tests/test_eventstream.cpp b/tray/tests/test_eventstream.cpp new file mode 100644 index 000000000..700a03acd --- /dev/null +++ b/tray/tests/test_eventstream.cpp @@ -0,0 +1,187 @@ +#include + +#include +#include + +#include "EventStream.h" +#include "FakeCli.h" +#include "JsonParse.h" + +// The desktop's consumer of `hgs watch --json` (EventStream.h), against a fake +// CLI that speaks the stream: listings and Activity arrive as the values polls +// deliver, parsed off the GUI thread, and a computer without the marker polls. +class TestEventStream : public QObject { + Q_OBJECT +private slots: + void init() { QVERIFY(m_dir.isValid()); } + void listingAndActivityArriveFromTheStream(); + void peerListingsKeepTheAlias(); + void computerWithoutTheMarkerIsUnsupported(); + void endedStreamFailsAndRestarts(); + void streamEndingBeforePublicationNeverGoesLive(); + void unchangedListingIsAppliedAgain(); + +private: + QString install(const QString &script); + QTemporaryDir m_dir; +}; + +namespace { +const char *Stream = R"PY(#!/usr/bin/env python3 +import json, sys +args = [a for a in sys.argv[1:] if not a.startswith('@')] +host = next((a[1:] for a in sys.argv[1:] if a.startswith('@')), '') +if args == ['--help']: + print('usage: hgs ...\n Event stream ABI: watch-v1') + sys.exit(0) +if args != ['watch', '--json']: + sys.exit(2) +seq = 0 +def send(t, **fields): + global seq + seq += 1 + print(json.dumps(dict(v=1, seq=seq, t=t, **fields)), flush=True) +row = lambda name, run, activity: {'name': name, 'cmd': 'codex', 'run_id': run, 'conversation_id': 'c-' + run, 'activity': activity} +send('hello', machine_id=None, host='fixture', hgs_version='1.57.0', stream='s', heartbeat_ms=10000, max_frame_bytes=1048576, + topics=['catalog', 'catalog.meta', 'activity'], limits={}) +send('snapshot', topic='catalog', items=[{'key': 'run:r2', 'row': row('codex/zeta', 'r2', 'idle')}, + {'key': 'run:r1', 'row': row('codex/Alpha', 'r1', 'idle')}], page=0, last=True, cursor=1) +send('snapshot', topic='catalog.meta', value={'host': 'fixture', 'ok': True, 'peers': [], 'hgs_version': '1.57.0'}, page=0, last=True, cursor=1) +for text in sys.stdin: + op = json.loads(text) + if op.get('op') == 'subscribe': + key = 'run:' + op['session']['run_id'] + send('events', topic='activity', key=key, events=[{'seq': 41, 'type': 'UserPromptSubmit', 'detail': 'go'}], cursor=41) + send('state', topic='activity', key=key, value={'run_id': op['session']['run_id'], 'conversation_id': 'c-r1', 'activity': 'busy'}, rev=1) + send('delta', topic='catalog', upsert=[{'key': 'run:r1', 'row': row('codex/Alpha', 'r1', 'busy')}], remove=[{'key': 'run:r2', 'reason': 'archived'}], cursor=2) + send('events', topic='activity', key=key, events=[{'seq': 42, 'type': 'Stop', 'detail': 'done'}], cursor=42) +)PY"; +} + +QString TestEventStream::install(const QString &script) +{ + const auto path = m_dir.filePath(QStringLiteral("hgs-%1").arg(QUuid::createUuid().toString(QUuid::Id128))); + if (!FakeCli::install(path, script.toUtf8())) return {}; + return FakeCli::program(path); +} + +void TestEventStream::listingAndActivityArriveFromTheStream() +{ + int guiThreadParses = 0; + JsonParse::setViolationHandler([&guiThreadParses](const char *, int) { ++guiThreadParses; }); + EventStream stream(install(QString::fromUtf8(Stream)), {}); + QSignalSpy listings(&stream, &EventStream::listingReady), activity(&stream, &EventStream::activityReady); + stream.start(); + QTRY_VERIFY(stream.live()); + QTRY_VERIFY(!listings.isEmpty()); + auto box = listings.first().first().value(); + QCOMPARE(box.host, QStringLiteral("fixture")); + QCOMPARE(box.sessions.size(), 2); + // Listing order, as ls --json sorts names. + QCOMPARE(box.sessions.at(0).name, QStringLiteral("codex/Alpha")); + QCOMPARE(box.sessions.at(1).name, QStringLiteral("codex/zeta")); + + stream.follow(QStringLiteral("codex/Alpha"), QStringLiteral("r1"), QStringLiteral("c-r1"), 0); + QTRY_VERIFY(activity.size() >= 1); + QTRY_COMPARE(listings.last().first().value().sessions.size(), 1); + box = listings.last().first().value(); + QCOMPARE(box.sessions.first().activity, QStringLiteral("busy")); + // Increments are inspect-shaped: the summary with the new events and cursor. + const auto received = [&activity] { + qsizetype count = 0; + for (const auto &arguments : activity) count += arguments.at(1).toJsonObject().value("events").toArray().size(); + return count; + }; + QTRY_COMPARE(received(), 2); + QJsonArray events; qint64 cursor = 0; + for (const auto &arguments : activity) { + const auto inspection = arguments.at(1).toJsonObject(); + QCOMPARE(arguments.at(0).toString(), QStringLiteral("codex/Alpha")); + QCOMPARE(inspection.value("run_id").toString(), QStringLiteral("r1")); + for (const auto &event : inspection.value("events").toArray()) events.append(event); + cursor = inspection.value("cursor").toInteger(); + } + QCOMPARE(events.at(0).toObject().value("seq").toInt(), 41); + QCOMPARE(events.at(1).toObject().value("seq").toInt(), 42); + QCOMPARE(cursor, 42); + QCOMPARE(guiThreadParses, 0); + JsonParse::setViolationHandler({}); + stream.stop(); + QCOMPARE(stream.state(), EventStream::State::Stopped); +} + +void TestEventStream::peerListingsKeepTheAlias() +{ + EventStream stream(install(QString::fromUtf8(Stream)), QStringLiteral("laptop")); + QSignalSpy listings(&stream, &EventStream::listingReady); + stream.start(); + QTRY_VERIFY(!listings.isEmpty()); + const auto box = listings.first().first().value(); + // A peer is found, polled and addressed by its saved alias. + QCOMPARE(box.host, QStringLiteral("laptop")); + QCOMPARE(box.reportedHost, QStringLiteral("fixture")); +} + +void TestEventStream::computerWithoutTheMarkerIsUnsupported() +{ + EventStream stream(install(QStringLiteral("#!/bin/sh\necho 'usage: hgs ...'\n")), {}); + stream.start(); + QTRY_COMPARE(stream.state(), EventStream::State::Unsupported); + QVERIFY(!stream.live()); +} + +void TestEventStream::endedStreamFailsAndRestarts() +{ + const QString once = QStringLiteral(R"PY(#!/usr/bin/env python3 +import json, sys +if sys.argv[1:] == ['--help']: + print('Event stream ABI: watch-v1'); sys.exit(0) +print(json.dumps({'v': 1, 'seq': 1, 't': 'hello', 'host': 'fixture'}), flush=True) +print(json.dumps({'v': 1, 'seq': 2, 't': 'snapshot', 'topic': 'catalog', 'items': [], 'page': 0, 'last': True, 'cursor': 1}), flush=True) +print(json.dumps({'v': 1, 'seq': 3, 't': 'snapshot', 'topic': 'catalog.meta', 'value': {'host': 'fixture', 'ok': True}, 'page': 0, 'last': True, 'cursor': 1}), flush=True) +import time; time.sleep(1) +)PY"); + EventStream stream(install(once), {}); + QSignalSpy states(&stream, &EventStream::stateChanged); + stream.start(); + QTRY_VERIFY(states.contains(QVariantList{QVariant::fromValue(EventStream::State::Live)})); + QTRY_COMPARE(stream.state(), EventStream::State::Failed); + QCOMPARE(EventStream::restartDelayMs(1), 5000); + QCOMPARE(EventStream::restartDelayMs(9), 300000); +} + +void TestEventStream::streamEndingBeforePublicationNeverGoesLive() +{ + // Frames prepared just before the child exits are dropped with it: a failed + // stream never reports Live afterwards, so callers keep polling. + const QString brief = QStringLiteral(R"PY(#!/usr/bin/env python3 +import json, sys +if sys.argv[1:] == ['--help']: + print('Event stream ABI: watch-v1'); sys.exit(0) +print(json.dumps({'v': 1, 'seq': 1, 't': 'snapshot', 'topic': 'catalog', 'items': [], 'page': 0, 'last': True, 'cursor': 1}), flush=True) +print(json.dumps({'v': 1, 'seq': 2, 't': 'snapshot', 'topic': 'catalog.meta', 'value': {'ok': True}, 'page': 0, 'last': True, 'cursor': 1}), flush=True) +)PY"); + EventStream stream(install(brief), {}); + QSignalSpy states(&stream, &EventStream::stateChanged); + stream.start(); + QTRY_COMPARE(stream.state(), EventStream::State::Failed); + QTest::qWait(200); + QCOMPARE(stream.state(), EventStream::State::Failed); + QVERIFY(!states.contains(QVariantList{QVariant::fromValue(EventStream::State::Live)})); +} + +void TestEventStream::unchangedListingIsAppliedAgain() +{ + // Callers date a computer's data by its last listing (automatic approvals + // need a peer listed within 30 s), so a quiet stream re-applies it. + EventStream::setListingTimingForTests(100, 300); + EventStream stream(install(QString::fromUtf8(Stream)), QStringLiteral("laptop")); + QSignalSpy listings(&stream, &EventStream::listingReady); + stream.start(); + QTRY_VERIFY(!listings.isEmpty()); + QTRY_VERIFY_WITH_TIMEOUT(listings.size() >= 3, 3000); + EventStream::setListingTimingForTests(2000, 10000); +} + +QTEST_MAIN(TestEventStream) +#include "test_eventstream.moc" From a98b5a666f1f24d31142628bff87429294b29bf9 Mon Sep 17 00:00:00 2001 From: ufna <271578+ufna@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:43:24 +0300 Subject: [PATCH 5/7] test(watch): wait for writer output with a deadline A loaded macOS runner produced one heartbeat in the fixed 180 ms the test slept (PR #369, CLI unit job). Both writer tests now wait for the expected frames with a 5 s deadline instead of sleeping a fixed time. Co-Authored-By: Claude Opus 5.5 --- src/watch/writer.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/watch/writer.rs b/src/watch/writer.rs index f86be7eca..76ad985d9 100644 --- a/src/watch/writer.rs +++ b/src/watch/writer.rs @@ -266,7 +266,10 @@ mod tests { assert!(writer.offer(item("other", Some("state|activity|run:b")))); sink.open(); assert!(writer.drain(Duration::from_secs(5))); - std::thread::sleep(Duration::from_millis(20)); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while sink.lines().len() < 4 && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } let lines = sink.lines(); let ids: Vec<_> = lines.iter().map(|line| line["id"].as_str().unwrap().to_owned()).collect(); assert_eq!(ids, ["first", "state-2", "events", "other"]); @@ -302,7 +305,12 @@ mod tests { let sink = Sink::new(true); let (closed, was_closed) = mpsc::channel(); let writer = Writer::spawn(sink.clone(), 1 << 20, Duration::from_millis(50), move || closed.send(()).unwrap()); - std::thread::sleep(Duration::from_millis(180)); + // A loaded runner can delay the writer thread; wait for two heartbeats + // rather than for a fixed time. + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while sink.lines().len() < 2 && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } let lines = sink.lines(); assert!(lines.len() >= 2, "{lines:?}"); assert!(lines.iter().all(|line| line["t"] == "heartbeat" && line["at"].as_f64().unwrap() > 1e9)); From 8648b5ee43a562a262cbe1ec7f3c6ce971cf192d Mon Sep 17 00:00:00 2001 From: ufna <271578+ufna@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:43:24 +0300 Subject: [PATCH 6/7] perf(cli): refresh only changed rows in hgs watch zerus-604d.6 and zerus-604d.7. The watcher re-derives only a session whose binding changed or whose journal rows arrived, through the listing's own row steps under the same lock, and lists all rows again when anything else changes and every 5 s. A 5 s probe covers what no file shows: tmux sessions that start, end, are renamed, attached or detached, exited agent processes, ended background work, sessions without a hook event yet, and DeepSeek and OpenCode hosts. A listing captured each live pane several times: once for the effort footer and once per eligible terminal chooser (usage limit, resume, Kimi cache hint, approval), 62 tmux capture-pane calls on arch for 35 panes. Inside the read-only status observation those readers now share one capture per pane; answers, resizes and key sends still capture afresh. On arch: a listing dropped from about 0.25 s to 0.21 s CPU, and the watcher from 12.9 % to 6.1 % of a core over 60 s. With a 15 s whole listing it measured 4.3 %, but a chooser without a hook would show up to 15 s late; that tradeoff waits for the owner. ls and inspect output are unchanged (105/105 stable rows and 831/831 stable inspect fields compared). Co-Authored-By: Claude Opus 5.5 --- docs/reference.md | 18 ++- src/cli.rs | 14 +- src/state/effort.rs | 3 +- src/state/journal.rs | 6 + src/state/kimi_tui_choice.rs | 94 ++++++++++++- src/state/mod.rs | 78 ++++++++--- src/state/watch_source.rs | 171 ++++++++++++++++++++++++ src/watch/catalog.rs | 7 +- src/watch/mod.rs | 246 +++++++++++++++++++++++++++++++---- src/watch/stamps.rs | 120 ++++++++++++----- tests/test_watch.py | 31 +++++ 11 files changed, 686 insertions(+), 102 deletions(-) diff --git a/docs/reference.md b/docs/reference.md index 07f798888..3fb0607af 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -4016,12 +4016,18 @@ more fail with `too_many_subscriptions`. Other topics fail with The stream checks its sources every 250 ms. It stats the bindings, read marks, notification preferences, waits, launches, renames, name routes, the archive folder and the DeepSeek and OpenCode bindings, and holds a read-only connection -to the Activity journal. A change lists again after 150 ms, at most every 500 ms -and at most once per four durations of the previous listing. A listing also -runs every 5 s, for changes no file shows, such as an exited process or a tmux -rename. Followed summaries are re-read when their journal rows or catalog row -change, at most every 500 ms, and otherwise every 2 s while the agent is busy and -every 15 s while it is idle; DeepSeek and OpenCode every 2 s. +to the Activity journal. A changed binding, or new journal rows of its session, +derives only that session's row again, through the steps and under the lock of +the listing. Any other change lists all rows again. Both wait 150 ms after a +change and start at most every 500 ms and at most once per four durations of +the previous one. Every 5 s the stream probes what no file shows: tmux sessions +that start, end, are renamed, attached or detached, exited agent processes, +ended background work, sessions that have not reported a hook event yet and the +DeepSeek and OpenCode hosts. All rows are listed again every 5 s, for terminal +choosers and status lines of sessions that already reported hooks. Followed +summaries are re-read when their journal rows or catalog row change, at most +every 500 ms, and otherwise every 2 s while the agent is busy and every 15 s +while it is idle; DeepSeek and OpenCode every 2 s. A frame is at most 1 MiB including its newline. An event too large for a frame is cut with `"truncated": true`, and an oversized summary loses the oldest items diff --git a/src/cli.rs b/src/cli.rs index 1f5c5466b..7f9df20f3 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1130,13 +1130,7 @@ fn local_json_archives(config: &Config, archives: bool) -> Result { let launches = state::launches::Evidence::read(); // A listing that fails is an error: no sessions would show live ones as stopped. let listing = terminal().listing().map_err(|e| Error::new(1, e))?; - let sessions: Vec = listing.sessions.iter().map(|row| { - let name = row.name.as_str(); - let mut parts = name.splitn(3, '/'); let cmd = parts.next().unwrap_or(""); - let project = parts.next().filter(|s| !s.is_empty()); let tag = parts.next().filter(|s| !s.is_empty()); - let ttys: Vec<&str> = listing.clients.iter().filter_map(|client| if client.session == name { Some(client.terminal.as_str()) } else { None }).collect(); - json!({"name":name,"cmd":cmd,"project":project,"tag":tag,"attached":row.attached,"clients":ttys,"created":row.created,"launch_id":row.launch_id}) - }).collect(); + let sessions = state::listed_rows(&listing); let mut raw = json!({"host":config.host(),"ok":true,"hgs_version":env!("CARGO_PKG_VERSION"),"projects":projects,"sessions":sessions,"peers":config.peers,"metrics":crate::metrics::snapshot(&config.state)}); // Phone alerts wait while the owner is active here; see src/presence.rs. if let Some(presence) = crate::presence::listing(&config.state, crate::presence::now()) { @@ -1150,11 +1144,7 @@ fn local_json_archives(config: &Config, archives: bool) -> Result { raw["local_host"] = crate::host::admin::local_host(&config.state); } // A session renamed before merge_listing reads the panes keeps its ID. - let ids = listing - .sessions - .iter() - .map(|row| (row.name.clone(), row.id.clone())) - .collect(); + let ids = state::listed_ids(&listing); let mut merged = state::merge_listing_archives(raw, &ids, archives).map_err(|e| Error::new(1, e))?; merged["archive_paging"] = json!(true); merged["archive_revision"] = json!(state::archive_catalog::revision().map_err(|e|Error::new(1,e))?); diff --git a/src/state/effort.rs b/src/state/effort.rs index 79ae2424b..9bc7bcd30 100644 --- a/src/state/effort.rs +++ b/src/state/effort.rs @@ -406,7 +406,8 @@ fn capture(record: &Value) -> Result { { return Err("invalid tracked pane".into()); } - let output = backend::current().capture(pane, backend::Capture::Plain)?; + // A status summary shares this screen with its terminal choosers. + let output = kimi_tui_choice::plain(pane)?; Ok(String::from_utf8_lossy(&output).into_owned()) } diff --git a/src/state/journal.rs b/src/state/journal.rs index 1d9733b77..14ead0d48 100644 --- a/src/state/journal.rs +++ b/src/state/journal.rs @@ -413,7 +413,13 @@ fn kimi_reply(db: &Connection, record: &Value, skip_scheduled: bool) -> Result Value { + kimi_tui_choice::observing(|| observed_summary(record, live_pane)) +} + +fn observed_summary(record: &Value, live_pane: bool) -> Value { let mut output = normalized_activity(record, live_pane); output["conversation_id"] = record["conversation_id"].clone(); output diff --git a/src/state/kimi_tui_choice.rs b/src/state/kimi_tui_choice.rs index 8a400fb4d..03a4d026f 100644 --- a/src/state/kimi_tui_choice.rs +++ b/src/state/kimi_tui_choice.rs @@ -17,8 +17,57 @@ pub(super) fn launcher_notices(rows: &[&str], agent: &str) -> bool { .contains(&prelude) } +thread_local! { + /// Plain screens captured during one read-only observation, by pane. + static OBSERVED: std::cell::RefCell>>> = + const { std::cell::RefCell::new(None) }; +} + +/// Runs a read-only status observation in which each pane is captured at most +/// once. A summary reads its pane for the effort footer and again for every +/// eligible terminal chooser; they now share one screen instead of starting a +/// `tmux capture-pane` each (zerus-604d.7). Nothing that sends keys, resizes a +/// pane or answers a chooser may run inside: those always capture afresh. +pub(super) fn observing(observe: impl FnOnce() -> T) -> T { + struct Reset(bool); + impl Drop for Reset { + fn drop(&mut self) { + if self.0 { + OBSERVED.with(|screens| screens.borrow_mut().take()); + } + } + } + // A nested observation shares the outer one's screens. + let outer = OBSERVED.with(|screens| { + let mut screens = screens.borrow_mut(); + let outer = screens.is_none(); + if outer { + *screens = Some(Default::default()); + } + outer + }); + let _reset = Reset(outer); + observe() +} + +/// The pane's plain screen; within `observing`, the one captured first. +/// Failures are not kept, so a later reader tries again. +pub(super) fn plain(pane: &str) -> Result> { + let shared = OBSERVED.with(|screens| screens.borrow().as_ref().and_then(|screens| screens.get(pane).cloned())); + if let Some(screen) = shared { + return Ok(screen); + } + let screen = backend::current().capture(pane, backend::Capture::Plain)?; + OBSERVED.with(|screens| { + if let Some(screens) = screens.borrow_mut().as_mut() { + screens.insert(pane.to_owned(), screen.clone()); + } + }); + Ok(screen) +} + pub(super) fn capture(record: &Value) -> Result { - let out = backend::current().capture(string(record, "pane"), backend::Capture::Plain)?; + let out = plain(string(record, "pane"))?; if out.len() > 512 * 1024 { return Err("Terminal snapshot is too large".into()); } @@ -293,3 +342,46 @@ fn navigate( } }) } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; + + static CAPTURES: AtomicUsize = AtomicUsize::new(0); + struct Counting; + impl backend::TerminalBackend for Counting { + fn capture(&self, pane: &str, _: backend::Capture) -> Result> { + let n = CAPTURES.fetch_add(1, SeqCst); + if pane == "%9" { + return Err("no such pane".into()); + } + Ok(format!("{pane} screen {n}").into_bytes()) + } + } + + #[test] + fn an_observation_captures_each_pane_once_and_nothing_else_shares() { + let record = |pane: &str| json!({"pane": pane}); + backend::with_fake(&Counting, || { + CAPTURES.store(0, SeqCst); + // Outside an observation every read is fresh. + assert_eq!(capture(&record("%1")).unwrap(), "%1 screen 0"); + assert_eq!(capture(&record("%1")).unwrap(), "%1 screen 1"); + let seen = observing(|| { + let first = capture(&record("%1")).unwrap(); + // A nested observation, and the effort footer's reader, share it. + let nested = observing(|| String::from_utf8(plain("%1").unwrap()).unwrap()); + let other = capture(&record("%2")).unwrap(); + // A failure is not kept: the next reader tries again. + assert!(capture(&record("%9")).is_err()); + assert!(plain("%9").is_err()); + (first, nested, other, capture(&record("%1")).unwrap()) + }); + assert_eq!(seen, ("%1 screen 2".into(), "%1 screen 2".into(), "%2 screen 3".into(), "%1 screen 2".into())); + assert_eq!(CAPTURES.load(SeqCst), 6); + // The observation ended: reads are fresh again. + assert_eq!(capture(&record("%1")).unwrap(), "%1 screen 6"); + }); + } +} diff --git a/src/state/mod.rs b/src/state/mod.rs index 3573c1fd1..a05a35f30 100644 --- a/src/state/mod.rs +++ b/src/state/mod.rs @@ -409,7 +409,7 @@ pub fn merge_listing_archives(mut box_value: Value, ids: &BTreeMap { + marks: &'a Value, + preferences: Option<&'a notifications::Store>, + /// Bounds question indexing for the whole listing. + budget: std::time::Instant, +} + +/// The row of a live session whose pane runs its saved binding `record`. +/// `hgs watch` re-derives single rows through this same step +/// (`watch_source::rows`). +fn live_row(session: &mut Value, record: Value, shared: &Shared, process_work: &mut processes::WorkSample) -> Result<()> { + // An approval answered in Terminal sends no hook. + let record = claude_approval::reconcile(record); + session["resumable"] = json!( + !string(&record, "conversation_id").is_empty() + && string(&record, "error").is_empty() + ); + let summary = journal::summary(&record, true); + session["process_work"] = process_work.summary(&record, summary["activity"] == "idle" && summary["phase"] == "idle"); + session + .as_object_mut() + .ok_or("invalid session")? + .extend(summary.as_object().ok_or("invalid summary")?.clone()); + let questions = read_marks::pending_questions(&record, session, true, shared.budget); + read_marks::enrich_listing(session, shared.marks, journal::journal_name(&record), questions); + notifications::enrich(session, shared.preferences, journal::journal_name(&record)); + Ok(()) +} + +/// The last steps of every listed row. `names` holds the session names of the +/// pane snapshot by session ID. +fn finish_rows(sessions: &mut [Value], marks: &Value, names: &BTreeMap) { // DeepSeek and untracked live sessions are owned by their names. for session in sessions.iter_mut() { if session.get("attention_signature").is_none() { let owner = string(session, "name").to_owned(); let questions=if session["agent"]=="opencode" {session["pending_questions"].as_array().cloned().unwrap_or_default()}else{Vec::new()}; - read_marks::enrich_listing(session, &marks, &owner, Ok(Some(questions))); + read_marks::enrich_listing(session, marks, &owner, Ok(Some(questions))); } } @@ -497,9 +519,25 @@ pub fn merge_listing_archives(mut box_value: Value, ids: &BTreeMap Vec { + listing.sessions.iter().map(|row| { + let name = row.name.as_str(); + let mut parts = name.splitn(3, '/'); let cmd = parts.next().unwrap_or(""); + let project = parts.next().filter(|s| !s.is_empty()); let tag = parts.next().filter(|s| !s.is_empty()); + let ttys: Vec<&str> = listing.clients.iter().filter_map(|client| if client.session == name { Some(client.terminal.as_str()) } else { None }).collect(); + json!({"name":name,"cmd":cmd,"project":project,"tag":tag,"attached":row.attached,"clients":ttys,"created":row.created,"launch_id":row.launch_id}) + }).collect() +} + +/// Each listed session's immutable ID (`$N`) by its listed name. +pub(crate) fn listed_ids(listing: &backend::Listing) -> BTreeMap { + listing.sessions.iter().map(|row| (row.name.clone(), row.id.clone())).collect() } /// Select a tracked conversation or explicit pending request by exact ID/home. diff --git a/src/state/watch_source.rs b/src/state/watch_source.rs index d9b6a8199..bdc43c264 100644 --- a/src/state/watch_source.rs +++ b/src/state/watch_source.rs @@ -4,6 +4,7 @@ //! and `hgs ls` already do. use super::*; use rusqlite::{Connection, OpenFlags}; +use std::collections::BTreeSet; use std::time::Duration; /// The session state folder. @@ -141,3 +142,173 @@ pub(crate) fn archived(run_id: &str) -> bool { pub(crate) fn saved(name: &str) -> bool { local_sessions().is_ok() && (record_path(name).exists() || native(name)) } + +// ---- Rows between listings (zerus-604d.6) ------------------------------------ +// +// A whole `ls --json --local --no-archives` summarizes every session. Between +// such listings `hgs watch` derives again only the live tracked rows whose +// inputs changed, through the same steps (`live_row`, `finish_rows`) and under +// the same lock, and probes what no file reports. + +/// What the terminal showed a derivation: its listed sessions with their IDs, +/// attached clients, creation and launch, and every pane with its run marker. +/// A session that starts, ends, is renamed, attached or detached changes it. +fn structure(sessions: &[Value], ids: &BTreeMap, snapshot: &Snapshot, names: &BTreeMap) -> u64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + json!([sessions, ids, snapshot, names]).to_string().hash(&mut hasher); + hasher.finish() +} + +/// The terminal's rows and session IDs, as a listing starts from them. +fn terminal() -> Result<(Vec, BTreeMap)> { + let listing = backend::current().listing()?; + Ok((listed_rows(&listing), listed_ids(&listing))) +} + +/// The terminal's structure now. A listing that starts later sees this one or +/// a newer one, so taking it first never hides a change. +pub(crate) fn structure_now() -> Result { + local_sessions()?; + let (sessions, ids) = terminal()?; + let (snapshot, names) = live_sessions()?; + Ok(structure(&sessions, &ids, &snapshot, &names)) +} + +/// The rows `ls --json --local --no-archives` lists now for the live tracked +/// sessions in `wanted`, with the terminal structure they saw. None when only +/// a whole listing can tell: the panes no longer show exactly the listed +/// sessions under their listed names (one started, ended or was renamed), or +/// a wanted session does not run its saved binding in its pane. +pub(crate) fn rows(wanted: &BTreeSet) -> Result, u64)>> { + local_sessions()?; + if !root().exists() { + return Ok(None); + } + let (sessions, ids) = terminal()?; + // As in a listing: the panes and the bindings are read under the lock, so + // a rename is seen before it began or after it finished (zerus-gd02). + let (snapshot, names, records) = { + let _guard = lock(None)?; + let (snapshot, names) = live_sessions()?; + let mut records = Vec::with_capacity(wanted.len()); + for name in wanted { + match read(name) { + Ok(record) if record["rename_shadow"] != true => records.push(record), + _ => return Ok(None), + } + } + (snapshot, names, records) + }; + let shape = structure(&sessions, &ids, &snapshot, &names); + let consistent = sessions.len() == snapshot.len() + && sessions.iter().all(|session| { + let listed = string(session, "name"); + match ids.get(listed).and_then(|id| names.get(id)) { + Some(name) => name == listed, + None => snapshot.contains_key(listed), + } + }); + if !consistent { + return Ok(None); + } + let marks = read_marks::load(); + let preferences = notifications::load(); + let shared = Shared { marks: &marks, preferences: preferences.as_ref(), budget: read_marks::question_budget() }; + let mut process_work = processes::WorkSample::default(); + let mut rows = Vec::with_capacity(records.len()); + for record in records { + let name = string(&record, "name"); + let Some(session) = sessions.iter().find(|session| string(session, "name") == name) else { + return Ok(None); + }; + if !matches(&record, snapshot.get(name)) { + return Ok(None); + } + let mut session = session.clone(); + session + .as_object_mut() + .ok_or("invalid session")? + .extend(telemetry::unavailable().as_object().unwrap().clone()); + live_row(&mut session, record, &shared, &mut process_work)?; + rows.push(session); + } + finish_rows(&mut rows, &marks, &names); + Ok(Some((rows, shape))) +} + +/// The DeepSeek and OpenCode rows of `ls --json --local --no-archives`. Their +/// hosts answer them, so no file shows their changes. +pub(crate) fn native_rows() -> Result> { + local_sessions()?; + let mut rows = dsh::snapshots()?; + rows.extend(opencode::snapshots()?); + if !rows.is_empty() { + let (_, names) = live_sessions()?; + finish_rows(&mut rows, &read_marks::load(), &names); + } + Ok(rows) +} + +/// What a probe between listings found. +pub(crate) struct Probe { + /// The terminal's structure (`structure_now`). + pub(crate) structure: u64, + /// Live tracked rows to derive again. + pub(crate) stale: Vec, +} + +/// `process_work` without its sampling time. +fn work(mut value: Value) -> Value { + if let Some(fields) = value.as_object_mut() { + fields.remove("sampled_at"); + } + value +} + +/// Compares the listed `rows` with what no file reports: the terminal's +/// structure and, for each live tracked session, its agent process and the +/// work that process owns, from one process sample. A session that has not +/// reported a hook event yet is always derived again: startup choosers have +/// no hook. It takes no lock; what it finds only asks for rows that do. +pub(crate) fn probe(rows: &[&Value]) -> Result { + local_sessions()?; + let (sessions, ids) = terminal()?; + let (snapshot, names) = live_sessions()?; + let mut sample = processes::WorkSample::default(); + let mut stale = Vec::new(); + for row in rows { + let name = string(row, "name"); + if row["tracked"] != true || !snapshot.contains_key(name) || native(name) { + continue; + } + let Ok(record) = read(name) else { + stale.push(name.to_owned()); + continue; + }; + let quiet = string(&record, "conversation_id").is_empty() + || record["last_event_at"].as_f64().unwrap_or(0.) == 0.; + let idle = row["activity"] == "idle" && row["phase"] == "idle"; + // As the summary decides `terminal_supported` of a live pane. + let terminal = string(&record, "archive_id").is_empty() && process_alive(&record); + if quiet + || !matches(&record, snapshot.get(name)) + || row["terminal_supported"] != json!(terminal) + || work(row["process_work"].clone()) != work(sample.summary(&record, idle)) + { + stale.push(name.to_owned()); + } + } + Ok(Probe { structure: structure(&sessions, &ids, &snapshot, &names), stale }) +} + +/// The session a binding file holds and the journal name its Activity rows +/// carry; None for a rename's shadow or a file that is no binding. +pub(crate) fn binding_identity(path: &Path) -> Option<(String, String)> { + let value: Value = serde_json::from_slice(&std::fs::read(path).ok()?).ok()?; + let name = string(&value, "name"); + if value["rename_shadow"] == true || name.is_empty() { + return None; + } + Some((name.to_owned(), journal::journal_name(&value).to_owned())) +} diff --git a/src/watch/catalog.rs b/src/watch/catalog.rs index e65a126c7..0e32f531c 100644 --- a/src/watch/catalog.rs +++ b/src/watch/catalog.rs @@ -38,6 +38,11 @@ pub(super) fn row_digest(row: &Value) -> Digest32 { digest(&masked(row, &VOLATILE_ROW)) } +/// A row's key: its run, or its name for a row without one. +pub(super) fn key(row: &Value) -> String { + protocol::catalog_key(row).unwrap_or_else(|| format!("name:{}", row.get("name").and_then(Value::as_str).unwrap_or(""))) +} + /// The listing's rows with their keys, and its other fields. Two rows with the /// same key (never expected) keep apart: the later one falls back to its name, /// then to its position. @@ -54,7 +59,7 @@ pub(super) fn split(mut listing: Value) -> (Vec<(String, Value)>, Value) { let mut rows = Vec::with_capacity(sessions.len()); for (index, row) in sessions.into_iter().enumerate() { let name = row.get("name").and_then(Value::as_str).unwrap_or("").to_owned(); - let mut key = protocol::catalog_key(&row).unwrap_or_else(|| format!("name:{name}")); + let mut key = key(&row); if seen.contains(&key) { key = protocol::key_for(None, &name); } diff --git a/src/watch/mod.rs b/src/watch/mod.rs index 107a4c0ac..482621411 100644 --- a/src/watch/mod.rs +++ b/src/watch/mod.rs @@ -3,7 +3,9 @@ //! carries the rows of `hgs ls --json --local --no-archives`; Activity carries //! what `hgs inspect` returns for subscribed sessions. Change detection is a //! 250 ms pulse over cheap stamps and the journal's data version, on -//! `std::thread` only. See "Event stream" in docs/reference.md. +//! `std::thread` only. A changed binding or journal row derives only its +//! session's row again; a 5 s probe watches what no file shows and the whole +//! listing runs every 5 s. See "Event stream" in docs/reference.md. mod activity; mod catalog; mod stamps; @@ -26,14 +28,23 @@ use zerus_watch_protocol::{self as protocol, reason, topic, Body, Control}; /// How often sources are checked. const PULSE: Duration = Duration::from_millis(250); -/// A listing waits this long after a change, so a burst lists once. +/// A listing or a row refresh waits this long after a change, so a burst +/// derives once. const DEBOUNCE: Duration = Duration::from_millis(150); -/// Listings start at least this far apart, and at least four times the -/// previous listing's duration apart. +/// Listings, and row refreshes, start at least this far apart, and at least +/// four times the previous one's duration apart. const LISTING_GAP: Duration = Duration::from_millis(500); -/// Listing without any observed change: process exit, tmux kill or rename, -/// ended background jobs and terminal choosers report no file change. -const SAFETY: Duration = Duration::from_secs(5); +/// What no file shows is probed this often: a session that starts, ends, is +/// killed, renamed, attached or detached, an exited agent process, ended +/// background work, a session that has not reported a hook event yet +/// (startup choosers) and DeepSeek and OpenCode hosts. +const PROBE: Duration = Duration::from_secs(5); +/// The whole listing runs at least this often, for what neither a file nor +/// the probe shows: terminal choosers and status lines of sessions that have +/// reported hooks, and native transcripts between hook events. 15 s would +/// halve the watcher's CPU on a busy computer but show such a chooser up to +/// 15 s late; that tradeoff waits for the owner (zerus-604d.6). +const FULL: Duration = Duration::from_secs(5); /// `catalog.meta` with only fresh metrics or presence goes out this rarely. const META_INTERVAL: Duration = Duration::from_secs(10); /// A changed session's summary is re-inspected at most this often. @@ -92,7 +103,7 @@ pub(crate) fn run(config: &Config, args: &[String], dry: bool) -> Result { let mut next_pulse = Instant::now() + PULSE; loop { let now = Instant::now(); - let deadline = watch.listing_due.map_or(next_pulse, |due| due.min(next_pulse)); + let deadline = [watch.listing_due, watch.refresh_due].into_iter().flatten().fold(next_pulse, Instant::min); let input = received.recv_timeout(deadline.saturating_duration_since(now)); if matches!(input, Ok(Input::Line(_) | Input::TooLong)) { pending.fetch_sub(1, Ordering::SeqCst); @@ -128,6 +139,9 @@ pub(crate) fn run(config: &Config, args: &[String], dry: bool) -> Result { if watch.listing_due.is_some_and(|due| now >= due) { watch.list(now); } + if watch.refresh_due.is_some_and(|due| now >= due) { + watch.refresh_rows(now); + } watch.emit_catalog(); watch.resync_activity(); } @@ -190,6 +204,15 @@ fn read_controls(mut input: R, inputs: mpsc::Sender, pending: let _ = inputs.send(Input::End); } +/// What changed in the Activity journal since the previous pulse. +enum Journal { + Unchanged, + /// Rows of these journal names were added. + Names(BTreeSet), + /// Rows were added or the journal was replaced, for names not known. + Unknown, +} + /// Frames out, with repeated source failures limited. struct Out { writer: Writer, @@ -257,10 +280,21 @@ struct Watch<'a> { catalog_resync: bool, meta_sent: Option<(Digest32, Digest32, Instant)>, // Listing schedule. - stamp: stamps::Stamp, + scan: stamps::Scan, + /// Each binding file's session and journal names (None: no binding). + identities: BTreeMap>, + /// The terminal's structure when the rows were derived. + structure: Option, listing_due: Option, listed_at: Option, listing_cost: Duration, + // Row refreshes between listings. + stale: BTreeSet, + stale_native: bool, + refresh_due: Option, + refreshed_at: Option, + refresh_cost: Duration, + probed_at: Option, // Activity. journal: Option, data_version: Option, @@ -270,10 +304,14 @@ struct Watch<'a> { impl<'a> Watch<'a> { fn new(config: &'a Config, writer: Writer) -> Self { + let root = source::state_root(); + // Stamps are taken before the first listing: a change during it lists again. + let scan = stamps::scan(&root); + let identities = scan.bindings.keys().map(|path| (path.clone(), source::binding_identity(path))).collect(); Self { config, out: Out { writer, reported: BTreeMap::new() }, - root: source::state_root(), + root, rows: Vec::new(), digests: Vec::new(), meta: json!({}), @@ -284,10 +322,18 @@ impl<'a> Watch<'a> { snapshot_sent: false, catalog_resync: false, meta_sent: None, - stamp: stamps::Stamp::default(), + scan, + identities, + structure: None, listing_due: None, listed_at: None, listing_cost: Duration::ZERO, + stale: BTreeSet::new(), + stale_native: false, + refresh_due: None, + refreshed_at: None, + refresh_cost: Duration::ZERO, + probed_at: None, journal: None, data_version: None, journal_max: 0, @@ -318,12 +364,23 @@ impl<'a> Watch<'a> { self.listing_due = Some(self.listing_due.map_or(due, |current| current.min(due))); } + fn schedule_refresh(&mut self, at: Instant) { + let gap = LISTING_GAP.max(self.refresh_cost * 4); + let due = self.refreshed_at.map_or(at, |started| at.max(started + gap)); + self.refresh_due = Some(self.refresh_due.map_or(due, |current| current.min(due))); + } + /// Re-runs the `ls --json --local --no-archives` builder. fn list(&mut self, now: Instant) { - // The stamp is taken first: a change during the listing lists again. - self.stamp = stamps::scan(&self.root); self.listing_due = None; self.listed_at = Some(now); + // The listing covers every row refresh and probe asked for so far. + self.probed_at = Some(now); + self.stale.clear(); + self.stale_native = false; + self.refresh_due = None; + // The structure is taken first: a change during the listing lists again. + self.structure = source::structure_now().ok(); match crate::cli::local_listing(self.config) { Ok(listing) => { let (rows, meta) = catalog::split(listing); @@ -338,6 +395,109 @@ impl<'a> Watch<'a> { self.listing_cost = now.elapsed(); } + /// Derives again the rows whose inputs changed since they were listed; a + /// whole listing follows when only it can tell. + fn refresh_rows(&mut self, now: Instant) { + self.refresh_due = None; + self.refreshed_at = Some(now); + let wanted = std::mem::take(&mut self.stale); + let native = std::mem::take(&mut self.stale_native); + match self.derive(&wanted, native) { + Ok(true) => {} + Ok(false) => self.schedule(now), + Err(error) => { + self.out.failure("listing_failed", &error, Some(topic::CATALOG), None); + self.schedule(now); + } + } + self.refresh_cost = now.elapsed(); + } + + /// Replaces the rows of the live tracked sessions in `wanted` and, with + /// `native`, the DeepSeek and OpenCode rows. False when a whole listing + /// must decide: the terminal changed, a session is no longer live under + /// its binding, or a row would change its key or its place. + fn derive(&mut self, wanted: &BTreeSet, native: bool) -> std::result::Result { + if !self.listed { + return Ok(false); + } + let mut rows = Vec::new(); + if !wanted.is_empty() { + let Some((derived, structure)) = source::rows(wanted)? else { return Ok(false) }; + if Some(structure) != self.structure { + return Ok(false); + } + rows = derived; + } + if native { + let derived = source::native_rows()?; + let listed: BTreeSet<&str> = self + .rows + .iter() + .filter(|(_, row)| row["name"].as_str().is_some_and(source::native)) + .map(|(key, _)| key.as_str()) + .collect(); + let keys: Vec = derived.iter().map(catalog::key).collect(); + if keys.len() != listed.len() || keys.iter().any(|key| !listed.contains(key.as_str())) { + return Ok(false); + } + rows.extend(derived); + } + let mut updates = Vec::with_capacity(rows.len()); + for row in rows { + let key = catalog::key(&row); + let Some(index) = self.rows.iter().position(|(listed, _)| *listed == key) else { return Ok(false) }; + if self.rows[index].1["name"] != row["name"] { + return Ok(false); + } + updates.push((index, row)); + } + for (index, row) in updates { + self.digests[index] = catalog::row_digest(&row); + self.rows[index].1 = row; + } + self.pending = true; + Ok(true) + } + + /// What no file shows: the terminal's structure, agent processes and + /// their work, sessions without a hook event yet and DeepSeek and + /// OpenCode hosts. The listing's metrics and presence are refreshed too. + fn probe(&mut self, now: Instant) { + self.probed_at = Some(now); + if !self.listed { + // The last listing failed: try again. + self.schedule(now); + return; + } + let rows: Vec<&Value> = self.rows.iter().map(|(_, row)| row).collect(); + let probe = match source::probe(&rows) { + Ok(probe) if Some(probe.structure) == self.structure => probe, + // A changed terminal, or one this probe cannot read: list it whole. + _ => { + self.schedule(now); + return; + } + }; + let native = self.rows.iter().any(|(_, row)| row["name"].as_str().is_some_and(source::native)); + if !probe.stale.is_empty() || native { + self.stale.extend(probe.stale); + self.stale_native |= native; + self.schedule_refresh(now); + } + // `metrics` and `desktop_presence` as the listing reports them now. + self.meta["metrics"] = crate::metrics::snapshot(&self.config.state); + match crate::presence::listing(&self.config.state, crate::presence::now()) { + Some(presence) => self.meta["desktop_presence"] = presence, + None => { + if let Some(meta) = self.meta.as_object_mut() { + meta.remove("desktop_presence"); + } + } + } + self.pending = true; + } + /// Sends what changed since the client's copy, unless the client is /// behind: then the latest listing waits, so changes coalesce per key. fn emit_catalog(&mut self) { @@ -560,12 +720,46 @@ impl<'a> Watch<'a> { // ---- Pulse ------------------------------------------------------------- fn pulse(&mut self, now: Instant) { - let journal_changed = self.poll_journal(); - let stamp = stamps::scan(&self.root); - if stamp != self.stamp || journal_changed { + let journal = self.poll_journal(); + let scan = stamps::scan(&self.root); + let (changed, removed) = scan.changed_bindings(&self.scan); + // Inputs other than bindings, and added or removed bindings, change + // the whole listing; a changed binding changes its own row. + let mut whole = scan.global != self.scan.global || removed || matches!(journal, Journal::Unknown); + let mut names = BTreeSet::new(); + for path in changed { + let identity = source::binding_identity(&path); + match &identity { + Some((name, _)) if self.scan.bindings.contains_key(&path) => { + names.insert(name.clone()); + } + _ => whole = true, + } + self.identities.insert(path, identity); + } + if removed { + self.identities.retain(|path, _| scan.bindings.contains_key(path)); + } + // Journal rows change the rows of the sessions they belong to. Rows + // of other names belong to no listed binding. + if let Journal::Names(journals) = &journal { + for (name, journal) in self.identities.values().flatten() { + if journals.contains(journal) { + names.insert(name.clone()); + } + } + } + self.scan = scan; + if whole { self.schedule(now + DEBOUNCE); + } else if !names.is_empty() { + self.stale.extend(names); + self.schedule_refresh(now + DEBOUNCE); } - if self.listed_at.is_none_or(|at| now >= at + SAFETY) { + if self.probed_at.is_none_or(|at| now >= at + PROBE) { + self.probe(now); + } + if self.listed_at.is_none_or(|at| now >= at + FULL) { self.schedule(now); } self.activity(now); @@ -586,32 +780,32 @@ impl<'a> Watch<'a> { } } - /// Whether the journal changed. Subscriptions with new rows read them. - fn poll_journal(&mut self) -> bool { + /// The journal names with new rows. Subscriptions with new rows read them. + fn poll_journal(&mut self) -> Journal { self.open_journal(); - let Some(db) = &self.journal else { return false }; + let Some(db) = &self.journal else { return Journal::Unchanged }; let version = match source::data_version(db) { Ok(version) => version, Err(_) => { self.journal = None; - return false; + return Journal::Unchanged; } }; if Some(version) == self.data_version { - return false; + return Journal::Unchanged; } self.data_version = Some(version); - let Ok((_, max)) = source::bounds(db) else { return true }; + let Ok((_, max)) = source::bounds(db) else { return Journal::Unknown }; if max < self.journal_max { // A replaced journal: every cursor is meaningless now. for sub in self.subs.iter_mut().filter(|sub| !sub.native) { sub.resync = Some(reason::RESET); } self.journal_max = max; - return true; + return Journal::Unknown; } if max == self.journal_max { - return false; + return Journal::Unchanged; } const SCAN: usize = 10_000; let names: Option> = if max - self.journal_max > SCAN as i64 { @@ -629,7 +823,7 @@ impl<'a> Watch<'a> { } } self.journal_max = max; - true + names.map_or(Journal::Unknown, Journal::Names) } fn activity(&mut self, now: Instant) { diff --git a/src/watch/stamps.rs b/src/watch/stamps.rs index e78ee71c0..b43957470 100644 --- a/src/watch/stamps.rs +++ b/src/watch/stamps.rs @@ -1,10 +1,12 @@ //! A cheap stamp over the files the live listing reads, compared on every //! pulse. Bindings are replaced by rename, so a stamp of name, size, mtime and -//! inode sees every change; temporary `.hgs-*` files are ignored. +//! inode sees every change; temporary `.hgs-*` files are ignored. Each session +//! binding has a stamp of its own, so a change re-derives only its row. use std::collections::hash_map::DefaultHasher; +use std::collections::BTreeMap; use std::fs; use std::hash::{Hash, Hasher}; -use std::path::Path; +use std::path::{Path, PathBuf}; /// Session bindings: 64 lowercase hex digits and `.json`. fn binding(name: &str) -> bool { @@ -17,14 +19,31 @@ fn ordinary(name: &str) -> bool { } /// Root files other than bindings that change listed rows. Desktop presence -/// changes only `catalog.meta`, which the safety listing refreshes. +/// changes only `catalog.meta`, which the probe refreshes. const ROOT_FILES: [&str; 2] = ["read-marks.json", "notifications.json"]; +/// The folder that holds routed bindings next to the root's own. +const BINDINGS: &str = "bindings"; /// Folders whose every entry counts. -const FOLDERS: [&str; 6] = ["bindings", "name_routes", "renames", "launches", "dsh/bindings", "opencode/bindings"]; +const FOLDERS: [&str; 6] = [BINDINGS, "name_routes", "renames", "launches", "dsh/bindings", "opencode/bindings"]; /// Single files and folders whose own stamp counts. Archive files are added or /// removed, which changes the folder. const PATHS: [&str; 4] = ["coordination/waits.sqlite3", "coordination/waits.sqlite3-wal", "coordination/waits.sqlite3-journal", "archives"]; +fn entry(path: &Path, metadata: &fs::Metadata) -> u64 { + let mut hasher = DefaultHasher::new(); + path.hash(&mut hasher); + metadata.len().hash(&mut hasher); + metadata.modified().ok().hash(&mut hasher); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + metadata.ino().hash(&mut hasher); + metadata.ctime().hash(&mut hasher); + metadata.ctime_nsec().hash(&mut hasher); + } + hasher.finish() +} + /// An order-independent hash of every stamped entry, and their count. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(super) struct Stamp { @@ -34,55 +53,71 @@ pub(super) struct Stamp { impl Stamp { fn add(&mut self, path: &Path, metadata: &fs::Metadata) { - let mut hasher = DefaultHasher::new(); - path.hash(&mut hasher); - metadata.len().hash(&mut hasher); - metadata.modified().ok().hash(&mut hasher); - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt; - metadata.ino().hash(&mut hasher); - metadata.ctime().hash(&mut hasher); - metadata.ctime_nsec().hash(&mut hasher); - } - self.hash = self.hash.wrapping_add(hasher.finish()); + self.hash = self.hash.wrapping_add(entry(path, metadata)); self.entries += 1; } - fn folder(&mut self, folder: &Path, keep: impl Fn(&str) -> bool) { + #[cfg(test)] + fn entries(&self) -> usize { + self.entries + } +} + +/// The stamps of a state folder: one for every input but session bindings, +/// and one per binding file. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(super) struct Scan { + pub(super) global: Stamp, + pub(super) bindings: BTreeMap, +} + +impl Scan { + fn folder(&mut self, folder: &Path, keep: impl Fn(&str) -> bool, bindings: bool) { let Ok(entries) = fs::read_dir(folder) else { return }; - for entry in entries.flatten() { - let name = entry.file_name(); + for found in entries.flatten() { + let name = found.file_name(); let Some(name) = name.to_str() else { continue }; if !keep(name) { continue; } - if let Ok(metadata) = entry.metadata() { - self.add(&entry.path(), &metadata); + let Ok(metadata) = found.metadata() else { continue }; + let path = found.path(); + if bindings && binding(name) { + self.bindings.insert(path.clone(), entry(&path, &metadata)); + } else { + self.global.add(&path, &metadata); } } } - #[cfg(test)] - fn entries(&self) -> usize { - self.entries + /// Binding files whose stamp changed or that appeared, and whether any + /// disappeared, since `before`. + pub(super) fn changed_bindings(&self, before: &Scan) -> (Vec, bool) { + let changed = self + .bindings + .iter() + .filter(|(path, stamp)| before.bindings.get(*path) != Some(stamp)) + .map(|(path, _)| path.clone()) + .collect(); + let removed = before.bindings.keys().any(|path| !self.bindings.contains_key(path)); + (changed, removed) } } -/// The stamp of a state folder. -pub(super) fn scan(root: &Path) -> Stamp { - let mut stamp = Stamp::default(); - stamp.folder(root, |name| binding(name) || ROOT_FILES.contains(&name)); +/// The stamps of a state folder. +pub(super) fn scan(root: &Path) -> Scan { + let mut scan = Scan::default(); + scan.folder(root, |name| binding(name) || ROOT_FILES.contains(&name), true); for folder in FOLDERS { - stamp.folder(&root.join(folder), ordinary); + scan.folder(&root.join(folder), ordinary, folder == BINDINGS); } for path in PATHS { let path = root.join(path); if let Ok(metadata) = fs::symlink_metadata(&path) { - stamp.add(&path, &metadata); + scan.global.add(&path, &metadata); } } - stamp + scan } #[cfg(test)] @@ -98,7 +133,9 @@ mod tests { fs::write(&record, "{}").unwrap(); let written = scan(root); assert_ne!(written, first); - assert_eq!(written.entries(), 1); + // A binding has a stamp of its own; every other input stays unchanged. + assert_eq!((written.global, written.bindings.len()), (first.global, 1)); + assert_eq!(written.changed_bindings(&first), (vec![record.clone()], false)); // Ignored: temporary files, locks, presence and unrelated files. fs::write(root.join(".hgs-temp"), "x").unwrap(); fs::write(root.join("desktop-presence.json"), "{}").unwrap(); @@ -113,15 +150,28 @@ mod tests { fs::rename(&temp, &record).unwrap(); let replaced = scan(root); assert_ne!(replaced, written); - for path in ["read-marks.json", "launches/1.json", "renames/r.json", "name_routes/n.json", "dsh/bindings/b.json"] { + assert_eq!(replaced.global, written.global); + assert_eq!(replaced.changed_bindings(&written), (vec![record.clone()], false)); + // A routed binding is a binding too. + let routed = root.join(format!("bindings/{}.json", "b".repeat(64))); + fs::create_dir_all(routed.parent().unwrap()).unwrap(); + fs::write(&routed, "{}").unwrap(); + let both = scan(root); + assert_eq!((both.global, both.changed_bindings(&replaced)), (replaced.global, (vec![routed.clone()], false))); + fs::remove_file(&record).unwrap(); + assert_eq!(scan(root).changed_bindings(&both), (vec![], true)); + for path in ["read-marks.json", "launches/1.json", "renames/r.json", "name_routes/n.json", "dsh/bindings/b.json", "bindings/notes.txt"] { let path = root.join(path); fs::create_dir_all(path.parent().unwrap()).unwrap(); let before = scan(root); fs::write(&path, "{}").unwrap(); - assert_ne!(scan(root), before, "{}", path.display()); + let after = scan(root); + assert_ne!(after.global, before.global, "{}", path.display()); + assert_eq!(after.bindings, before.bindings, "{}", path.display()); } let before = scan(root); fs::create_dir(root.join("archives")).unwrap(); - assert_ne!(scan(root), before); + assert_ne!(scan(root).global, before.global); + assert_eq!(before.global.entries(), 6); } } diff --git a/tests/test_watch.py b/tests/test_watch.py index 1ce40e61b..4c0aeb3fa 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -185,6 +185,37 @@ def test_binding_changes_and_a_rename_are_upserts_on_the_run_key(self): stream.quiet(lambda f: f['t'] == 'delta', 6) self.assertEqual(stream.close(), 0) + def listed_row(self): + return next(row for row in self.listing()['sessions'] if row['name'] == self.name) + + def upserted(self, predicate, timeout): + """The row of the first delta that upserts this session as `predicate` wants.""" + delta = self.stream.next(lambda f: f['t'] == 'delta' and any( + item['key'] == self.key() and predicate(item['row']) for item in f['upsert']), timeout=timeout) + return next(item['row'] for item in delta['upsert'] if item['key'] == self.key()) + + def test_a_changed_binding_or_journal_row_derives_the_row_the_listing_shows(self): + # Between whole listings (every 5 s) only the changed session's row is + # derived again, through the listing's own steps. + self.stream, *_ = self.start() + self.record.update(prompt='Derived request', last_event_at=time.time()) + self.write_record() + row = self.upserted(lambda row: row.get('prompt') == 'Derived request', timeout=4) + self.assertEqual(steady(row), steady(self.listed_row())) + # A journal row alone, such as a reply, changes the reply identity. + reply = self.journal('Stop', detail='Derived reply') + row = self.upserted(lambda row: str(row.get('reply_id', '')).startswith(f'{reply}:'), timeout=4) + self.assertEqual(steady(row), steady(self.listed_row())) + self.assertEqual(self.stream.close(), 0) + + def test_a_session_killed_in_tmux_is_noticed_without_any_file_change(self): + self.stream, *_ = self.start() + self.tmux('kill-session', '-t', '=' + self.name) + # The probe sees the terminal change within 5 s and lists it whole. + row = self.upserted(lambda row: row.get('state') == 'stopped', timeout=8) + self.assertEqual(steady(row), steady(self.listed_row())) + self.assertEqual(self.stream.close(), 0) + def test_a_forgotten_session_is_removed_with_its_reason_and_ends_its_subscription(self): stream, *_ = self.start() stream.send(op='subscribe', topic='activity', session=self.session()) From 675c4d8fef06b5892fa4b33ade0be78dee87a04d Mon Sep 17 00:00:00 2001 From: ufna <271578+ufna@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:21:44 +0300 Subject: [PATCH 7/7] test(tray): use a Python fake CLI for the marker-less computer The Windows fake CLI runs Python scripts; a /bin/sh fake never answered there, so the stream failed instead of reporting Unsupported (#369 Windows job). Co-Authored-By: Claude Opus 5.5 --- tray/tests/test_eventstream.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tray/tests/test_eventstream.cpp b/tray/tests/test_eventstream.cpp index 700a03acd..29eb2fa01 100644 --- a/tray/tests/test_eventstream.cpp +++ b/tray/tests/test_eventstream.cpp @@ -124,7 +124,8 @@ void TestEventStream::peerListingsKeepTheAlias() void TestEventStream::computerWithoutTheMarkerIsUnsupported() { - EventStream stream(install(QStringLiteral("#!/bin/sh\necho 'usage: hgs ...'\n")), {}); + // Fake CLIs are Python scripts, which the Windows fake runs too. + EventStream stream(install(QStringLiteral("#!/usr/bin/env python3\nprint('usage: hgs ...')\n")), {}); stream.start(); QTRY_COMPARE(stream.state(), EventStream::State::Unsupported); QVERIFY(!stream.live());