diff --git a/.github/dependabot.yml b/.github/dependabot.yml index eb73a0f38..15d1b2c96 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,7 +14,7 @@ updates: applies-to: security-updates patterns: ["*"] - package-ecosystem: cargo - directories: ["/", "/services/mobile/relay", "/services/mobile/connector", "/crates/replica", "/crates/replica-ffi"] + directories: ["/", "/services/mobile/relay", "/services/mobile/connector", "/crates/replica", "/crates/replica-ffi", "/crates/watch-protocol"] schedule: interval: monthly open-pull-requests-limit: 1 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 28f17994a..f9339eec8 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -160,6 +160,11 @@ jobs: cargo +1.99.1 fmt --check --manifest-path crates/replica-ffi/Cargo.toml cargo +1.99.1 clippy --locked --manifest-path crates/replica-ffi/Cargo.toml --target-dir target/replica-ffi --all-targets -- -D warnings cargo +1.99.1 test --locked --manifest-path crates/replica-ffi/Cargo.toml --target-dir target/replica-ffi + - name: Watch protocol crate (format, lints, tests) + run: | + cargo +1.99.1 fmt --check --manifest-path crates/watch-protocol/Cargo.toml + cargo +1.99.1 clippy --locked --manifest-path crates/watch-protocol/Cargo.toml --target-dir target/watch-protocol --all-targets -- -D warnings + cargo +1.99.1 test --locked --manifest-path crates/watch-protocol/Cargo.toml --target-dir target/watch-protocol # Packages fail without a notice for every shipped crate; find out here. - name: Rust license notices run: python3 scripts/collect-licenses.py "$RUNNER_TEMP/rust-licenses" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 641b97955..179d1b3d5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,6 +70,8 @@ stable release, and raising them is routine maintenance. Run The replica crate has its own lockfile: run `cargo +1.99.1 test --locked --manifest-path crates/replica/Cargo.toml`; its golden vectors must keep decoding, and `ZERUS_BLESS=1` only adds new ones. +The event stream's frame crate has one too: run +`cargo +1.99.1 test --locked --manifest-path crates/watch-protocol/Cargo.toml --target-dir target/watch-protocol`. On Linux, the three worktree modules run through `scripts/ci/worktree-tests.py` in a fresh PID/mount namespace, including their live process blockers and private @@ -110,6 +112,16 @@ class) and before it starts. `CHILD_PROCESS_EXEMPT` in the script names the files that may start children otherwise, each with its reason. +The desktop parses JSON only through [JsonParse.h](tray/src/JsonParse.h), off +the GUI thread: `hgs` output in a `ProcessRunner` prepare step, other data +through `ProcessRunner::post` or an existing worker, and QSettings values +through `SettingsJson`. `check-source.py` rejects `QJsonDocument::fromJson` in +`tray/src` outside `JsonParse.cpp`, and lists the files that may still call +`JsonParse::local()` on the GUI thread and how often; that list only shrinks. +Every Qt suite runs with `ZERUS_GUI_THREAD_CHECKS=fatal`, so a parse on the GUI +thread aborts the test with its call site. A GUI suite that checks a parser +directly calls it through `offGuiThread()` (`tray/tests/support/OffGuiThread.h`). + A change to tmux or `ps` call sites must keep Unix behaviour unchanged: compare a [tmux census](docs/ci-and-aur.md#unix-tmux-census) of the branch with one of main. `scripts/ci/check-windows-target.sh` type-checks the Windows target from Linux or diff --git a/Cargo.lock b/Cargo.lock index 781e5b9f0..2adf14dca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -770,6 +770,7 @@ dependencies = [ "uuid", "windows-sys", "zerus-replica", + "zerus-watch-protocol", ] [[package]] @@ -2299,6 +2300,14 @@ dependencies = [ "uuid", ] +[[package]] +name = "zerus-watch-protocol" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 3ba3c4450..8b8c87ed7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ toml_edit = "0.25" unicode-width = "0.2" uuid = { version = "1", features = ["v4"] } zerus-replica = { path = "crates/replica" } +zerus-watch-protocol = { path = "crates/watch-protocol" } [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61", features = [ diff --git a/crates/watch-protocol/.cargo/config.toml b/crates/watch-protocol/.cargo/config.toml new file mode 100644 index 000000000..4a6a1abdd --- /dev/null +++ b/crates/watch-protocol/.cargo/config.toml @@ -0,0 +1,2 @@ +[resolver] +incompatible-rust-versions = "fallback" diff --git a/crates/watch-protocol/Cargo.lock b/crates/watch-protocol/Cargo.lock new file mode 100644 index 000000000..7a6956195 --- /dev/null +++ b/crates/watch-protocol/Cargo.lock @@ -0,0 +1,107 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "zerus-watch-protocol" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/crates/watch-protocol/Cargo.toml b/crates/watch-protocol/Cargo.toml new file mode 100644 index 000000000..fdc8f2260 --- /dev/null +++ b/crates/watch-protocol/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "zerus-watch-protocol" +version = "0.1.0" +edition = "2021" +rust-version = "1.99" +license = "MIT" +description = "Frames and control operations of the hgs watch event stream (watch-v1)" +publish = false + +# Shared by hgs and its clients. Keep it to serde: no Loro, Qt or async runtime. +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/crates/watch-protocol/src/lib.rs b/crates/watch-protocol/src/lib.rs new file mode 100644 index 000000000..792f161d9 --- /dev/null +++ b/crates/watch-protocol/src/lib.rs @@ -0,0 +1,645 @@ +//! Frames and control operations of `hgs [@host] watch --json` (ABI `watch-v1`). +//! +//! The server writes one JSON frame per line (NDJSON, UTF-8) on stdout and reads +//! one control operation per line on stdin. EOF on stdin ends the stream. Every +//! frame carries `v` (always [`VERSION`]), `t` (its type) and `seq`, which grows +//! by one per frame within a stream. A frame line, including its newline, is at +//! most [`MAX_FRAME_BYTES`]. +//! +//! Clients ignore fields they do not know. A frame type or topic they do not know +//! is skipped, except that `gap` and a fatal `error` always apply. +//! +//! See the "Event stream" section of `docs/reference.md` for the contract. +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// The `v` of every frame. +pub const VERSION: u32 = 1; +/// The ABI name a client looks for before it starts the stream. +pub const ABI: &str = "watch-v1"; +/// The line `hgs -h` prints when this ABI is available. +pub const HELP_MARKER: &str = "Event stream ABI: watch-v1"; +/// The largest frame line, including its newline. +pub const MAX_FRAME_BYTES: usize = 1 << 20; +/// The largest snapshot or delta page the server builds. +pub const PAGE_BYTES: usize = 512 << 10; +/// The server's outgoing queue; beyond it a topic receives `gap`. +pub const QUEUE_BYTES: usize = 4 << 20; +/// A heartbeat follows this much time without other frames. +pub const HEARTBEAT_MS: u64 = 10_000; +/// Concurrent Activity subscriptions per stream. +pub const ACTIVITY_SUBSCRIPTIONS: usize = 16; +/// The most events an Activity tail or tail snapshot carries. +pub const ACTIVITY_TAIL: usize = 100; +/// The most journal rows one Activity read returns. +pub const ACTIVITY_READ: usize = 200; +/// The longest control line the server accepts, without its newline. +pub const CONTROL_LINE_BYTES: usize = 64 << 10; + +/// Topic names. +pub mod topic { + /// Live session rows, keyed by [`super::catalog_key`]. + pub const CATALOG: &str = "catalog"; + /// The listing fields other than `sessions`. + pub const CATALOG_META: &str = "catalog.meta"; + /// One session's Activity events and inspection summary. + pub const ACTIVITY: &str = "activity"; +} + +/// Why a catalog key was removed. Clients treat an unknown reason as unknown. +pub mod reason { + /// The session ended and left no saved row. + pub const ENDED: &str = "ended"; + /// The run moved to Archive. + pub const ARCHIVED: &str = "archived"; + /// The saved binding was removed. + pub const FORGOTTEN: &str = "forgotten"; + /// The same session name now lists under a new run. + pub const REPLACED: &str = "replaced"; + /// The client fell behind and the server dropped frames. + pub const OVERFLOW: &str = "overflow"; + /// The Activity cursor is older than the journal's retention. + pub const RETENTION: &str = "retention"; + /// The Activity cursor is newer than the journal (a replaced journal). + pub const RESET: &str = "reset"; +} + +/// One line of the stream. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Frame { + pub v: u32, + pub seq: u64, + #[serde(flatten)] + pub body: Body, +} + +/// A frame's type (`t`) and fields. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "t", rename_all = "snake_case")] +pub enum Body { + Hello(Hello), + Snapshot(Snapshot), + Delta(Delta), + Events(Events), + Event(EventUpdate), + State(State), + Heartbeat(Heartbeat), + Gap(Gap), + Error(ErrorFrame), + Pong(Pong), +} + +/// The first frame of a stream. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Hello { + /// The computer's native identity; null when unavailable. + pub machine_id: Option, + pub host: String, + pub hgs_version: String, + /// A random identity of this stream. + pub stream: String, + pub heartbeat_ms: u64, + pub max_frame_bytes: usize, + pub topics: Vec, + pub limits: Limits, +} + +/// Bounds the server applies to this stream. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Limits { + pub activity_subscriptions: usize, + pub page_bytes: usize, + pub queue_bytes: usize, + pub activity_tail: usize, + pub activity_read: usize, + pub control_line_bytes: usize, +} + +impl Default for Limits { + fn default() -> Self { + Self { + activity_subscriptions: ACTIVITY_SUBSCRIPTIONS, + page_bytes: PAGE_BYTES, + queue_bytes: QUEUE_BYTES, + activity_tail: ACTIVITY_TAIL, + activity_read: ACTIVITY_READ, + control_line_bytes: CONTROL_LINE_BYTES, + } + } +} + +/// A bounded replacement of one topic, or of one key of it. A client keeps +/// showing what it had until the page with `last` arrives, then replaces it. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Snapshot { + pub topic: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub key: Option, + /// Keyed collections: catalog rows as [`CatalogItem`] objects, Activity + /// events as `inspect` returns them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub items: Option>, + /// Single-object topics such as `catalog.meta`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub value: Option, + pub page: u32, + pub last: bool, + /// The catalog revision, or the Activity journal `seq`. + pub cursor: i64, +} + +/// A keyed change of the catalog. Upserts carry whole rows. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Delta { + pub topic: String, + #[serde(default)] + pub upsert: Vec, + #[serde(default)] + pub remove: Vec, + pub cursor: i64, +} + +/// One catalog row and its key. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct CatalogItem { + pub key: String, + pub row: Value, +} + +/// A removed catalog key; `reason` is one of [`reason`] when known. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Removal { + pub key: String, + #[serde(default)] + pub reason: Option, +} + +/// Activity events after the previous cursor, oldest first. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Events { + pub topic: String, + pub key: String, + pub events: Vec, + /// The journal `seq` to resume after. + pub cursor: i64, +} + +/// A replacement of one open event by its stable ID; a higher `rev` wins. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct EventUpdate { + pub topic: String, + pub key: String, + pub id: String, + pub rev: u64, + pub event: Value, +} + +/// A replacement of one keyed object, such as a session's inspection summary. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct State { + pub topic: String, + pub key: String, + pub value: Value, + pub rev: u64, +} + +/// Sent after [`HEARTBEAT_MS`] without other frames. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Heartbeat { + /// Seconds since the Unix epoch. + pub at: f64, +} + +/// The client discards what it holds for the topic (or key); a snapshot follows. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Gap { + pub topic: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub key: Option, + pub reason: String, +} + +/// A refused operation or failed source. A fatal error ends the stream. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ErrorFrame { + pub code: String, + pub message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub topic: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub key: Option, + pub fatal: bool, +} + +/// The answer to a `ping`, echoing its `id`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Pong { + #[serde(default)] + pub id: Option, +} + +/// One control line from the client. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum Control { + Subscribe(Subscription), + Unsubscribe(Subscription), + Ping(Ping), +} + +/// A topic and its key. Activity takes `session`; later topics take `path` +/// (git) or `project` (tasks). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Subscription { + pub topic: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub project: Option, + /// Resume Activity after this journal `seq`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub after: Option, + /// How many recent events a subscription without `after` starts with. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tail: Option, +} + +/// The exact session a subscription follows. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct SessionRef { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub run_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub conversation_id: Option, +} + +impl SessionRef { + /// The session's key, the same as its catalog key. + pub fn key(&self) -> String { + key_for(self.run_id.as_deref(), &self.name) + } +} + +/// A liveness probe; the server answers with `pong`. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Ping { + #[serde(default)] + pub id: Option, +} + +/// `run:`, or `name:` for a session without a run. +pub fn key_for(run_id: Option<&str>, name: &str) -> String { + match run_id.filter(|run| !run.is_empty()) { + Some(run) => format!("run:{run}"), + None => format!("name:{name}"), + } +} + +/// The catalog key of an `ls --json --local` row; None without a name. +pub fn catalog_key(row: &Value) -> Option { + let name = row.get("name")?.as_str()?; + Some(key_for(row.get("run_id").and_then(Value::as_str), name)) +} + +/// A frame that cannot be written or read. +#[derive(Debug)] +pub enum FrameError { + /// The line, with its newline, would exceed [`MAX_FRAME_BYTES`]. + TooLarge(usize), + /// A line that does not start like a serialized [`Body`]. + Malformed, + Json(serde_json::Error), +} + +impl std::fmt::Display for FrameError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TooLarge(bytes) => write!(f, "frame of {bytes} bytes exceeds {MAX_FRAME_BYTES}"), + Self::Malformed => f.write_str("malformed frame body"), + Self::Json(error) => error.fmt(f), + } + } +} + +impl std::error::Error for FrameError {} + +impl From for FrameError { + fn from(error: serde_json::Error) -> Self { + Self::Json(error) + } +} + +/// Whether a line of `bytes` (without its newline) fits in a frame. +pub fn fits(bytes: usize) -> bool { + bytes < MAX_FRAME_BYTES +} + +/// Fails unless `line` (without its newline) fits in a frame. +pub fn check_size(line: &str) -> Result<(), FrameError> { + if fits(line.len()) { + Ok(()) + } else { + Err(FrameError::TooLarge(line.len() + 1)) + } +} + +/// The most bytes [`envelope`] adds to a body. +pub const ENVELOPE_BYTES: usize = 40; + +/// A body as JSON without `v` and `seq`, for a writer that numbers frames +/// only when it writes them. [`envelope`] completes it. +pub fn encode_body(body: &Body) -> Result { + let text = serde_json::to_string(body)?; + if !fits(text.len() + ENVELOPE_BYTES) { + return Err(FrameError::TooLarge(text.len() + ENVELOPE_BYTES + 1)); + } + Ok(text) +} + +/// The frame line (without its newline) of a body from [`encode_body`]. +pub fn envelope(seq: u64, body: &str) -> Result { + let rest = body.strip_prefix("{\"t\":").ok_or(FrameError::Malformed)?; + Ok(format!("{{\"v\":{VERSION},\"seq\":{seq},\"t\":{rest}")) +} + +/// One frame line, without its newline. +pub fn encode(frame: &Frame) -> Result { + let line = serde_json::to_string(frame)?; + check_size(&line)?; + Ok(line) +} + +/// One frame from a line, with or without its newline. +pub fn decode(line: &str) -> Result { + let line = line.strip_suffix('\n').unwrap_or(line); + check_size(line)?; + Ok(serde_json::from_str(line)?) +} + +/// One control operation from a line, with or without its newline. +pub fn decode_control(line: &str) -> Result { + let line = line.strip_suffix('\n').unwrap_or(line); + if line.len() > CONTROL_LINE_BYTES { + return Err(FrameError::TooLarge(line.len() + 1)); + } + Ok(serde_json::from_str(line)?) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn frames() -> Vec { + vec![ + Body::Hello(Hello { + machine_id: Some("00000000-0000-4000-8000-000000000001".into()), + host: "workstation".into(), + hgs_version: "1.56.0".into(), + stream: "stream-1".into(), + heartbeat_ms: HEARTBEAT_MS, + max_frame_bytes: MAX_FRAME_BYTES, + topics: vec![ + topic::CATALOG.into(), + topic::CATALOG_META.into(), + topic::ACTIVITY.into(), + ], + limits: Limits::default(), + }), + Body::Snapshot(Snapshot { + topic: topic::CATALOG.into(), + key: None, + items: Some(vec![ + json!({"key":"run:r1","row":{"name":"codex/p/t","run_id":"r1","at":1.5}}), + ]), + value: None, + page: 0, + last: true, + cursor: 1, + }), + Body::Snapshot(Snapshot { + topic: topic::CATALOG_META.into(), + key: None, + items: None, + value: Some(json!({"host":"workstation","projects":{}})), + page: 0, + last: true, + cursor: 1, + }), + Body::Delta(Delta { + topic: topic::CATALOG.into(), + upsert: vec![CatalogItem { + key: "name:sh/p".into(), + row: json!({"name":"sh/p"}), + }], + remove: vec![ + Removal { + key: "run:r0".into(), + reason: Some(reason::REPLACED.into()), + }, + Removal { + key: "run:r2".into(), + reason: None, + }, + ], + cursor: 2, + }), + Body::Events(Events { + topic: topic::ACTIVITY.into(), + key: "run:r1".into(), + events: vec![json!({"type":"Stop","seq":41,"at":1787840176.25})], + cursor: 41, + }), + Body::Event(EventUpdate { + topic: topic::ACTIVITY.into(), + key: "run:r1".into(), + id: "message-1".into(), + rev: 3, + event: json!({"text":"partial"}), + }), + Body::State(State { + topic: topic::ACTIVITY.into(), + key: "run:r1".into(), + value: json!({"phase":"idle"}), + rev: 4, + }), + Body::Heartbeat(Heartbeat { at: 1787840176.5 }), + Body::Gap(Gap { + topic: topic::ACTIVITY.into(), + key: Some("run:r1".into()), + reason: reason::RETENTION.into(), + }), + Body::Error(ErrorFrame { + code: "too_many_subscriptions".into(), + message: "at most 16".into(), + topic: Some(topic::ACTIVITY.into()), + key: None, + fatal: false, + }), + Body::Pong(Pong { id: Some(json!(7)) }), + ] + } + + #[test] + fn every_frame_round_trips_with_version_type_and_sequence() { + for (index, body) in frames().into_iter().enumerate() { + let frame = Frame { + v: VERSION, + seq: index as u64 + 1, + body, + }; + let line = encode(&frame).unwrap(); + assert!(!line.contains('\n')); + let value: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(value["v"], 1); + assert_eq!(value["seq"], index as u64 + 1); + assert!(value["t"].is_string(), "{line}"); + assert_eq!(decode(&format!("{line}\n")).unwrap(), frame); + } + } + + #[test] + fn envelope_numbers_an_encoded_body_like_a_whole_frame() { + for body in frames() { + let encoded = encode_body(&body).unwrap(); + let line = envelope(u64::MAX, &encoded).unwrap(); + let whole = encode(&Frame { + v: VERSION, + seq: u64::MAX, + body: body.clone(), + }) + .unwrap(); + assert_eq!(line, whole); + assert!(line.len() <= encoded.len() + ENVELOPE_BYTES); + } + assert!(matches!(envelope(1, "{}"), Err(FrameError::Malformed))); + } + + #[test] + fn wire_names_match_the_contract() { + let line = encode(&Frame { + v: 1, + seq: 9, + body: Body::Heartbeat(Heartbeat { at: 2.0 }), + }) + .unwrap(); + assert_eq!(line, r#"{"v":1,"seq":9,"t":"heartbeat","at":2.0}"#); + let gap = encode(&Frame { + v: 1, + seq: 10, + body: Body::Gap(Gap { + topic: "catalog".into(), + key: None, + reason: "overflow".into(), + }), + }) + .unwrap(); + assert_eq!( + gap, + r#"{"v":1,"seq":10,"t":"gap","topic":"catalog","reason":"overflow"}"# + ); + } + + #[test] + fn unknown_fields_are_ignored_for_forward_compatibility() { + let frame = decode(r#"{"v":1,"seq":3,"t":"heartbeat","at":1.0,"later":true}"#).unwrap(); + assert_eq!(frame.body, Body::Heartbeat(Heartbeat { at: 1.0 })); + let control = decode_control(r#"{"op":"ping","id":"a","later":[1]}"#).unwrap(); + assert_eq!( + control, + Control::Ping(Ping { + id: Some(json!("a")) + }) + ); + } + + #[test] + fn control_operations_parse_with_their_keys() { + let line = r#"{"op":"subscribe","topic":"activity","session":{"name":"codex/p/t","run_id":"r1","conversation_id":"c1"},"after":40,"tail":20}"#; + let Control::Subscribe(subscription) = decode_control(line).unwrap() else { + panic!() + }; + assert_eq!(subscription.topic, "activity"); + assert_eq!(subscription.after, Some(40)); + assert_eq!(subscription.tail, Some(20)); + let session = subscription.session.unwrap(); + assert_eq!(session.key(), "run:r1"); + assert_eq!( + SessionRef { + name: "sh/p".into(), + ..Default::default() + } + .key(), + "name:sh/p" + ); + let Control::Unsubscribe(gone) = + decode_control(r#"{"op":"unsubscribe","topic":"activity","session":{"name":"x"}}"#) + .unwrap() + else { + panic!() + }; + assert_eq!(gone.session.unwrap().key(), "name:x"); + assert_eq!( + decode_control(r#"{"op":"ping"}"#).unwrap(), + Control::Ping(Ping { id: None }) + ); + assert!(decode_control(r#"{"op":"launch"}"#).is_err()); + assert!(decode_control(r#"{"topic":"activity"}"#).is_err()); + let long = format!( + r#"{{"op":"ping","id":"{}"}}"#, + "x".repeat(CONTROL_LINE_BYTES) + ); + assert!(matches!( + decode_control(&long), + Err(FrameError::TooLarge(_)) + )); + } + + #[test] + fn catalog_keys_prefer_the_run() { + assert_eq!( + catalog_key(&json!({"name":"codex/p/t","run_id":"r1"})).as_deref(), + Some("run:r1") + ); + assert_eq!( + catalog_key(&json!({"name":"sh/p","run_id":""})).as_deref(), + Some("name:sh/p") + ); + assert_eq!( + catalog_key(&json!({"name":"sh/p","run_id":null})).as_deref(), + Some("name:sh/p") + ); + assert_eq!( + catalog_key(&json!({"name":"sh/p"})).as_deref(), + Some("name:sh/p") + ); + assert_eq!(catalog_key(&json!({"run_id":"r1"})), None); + } + + #[test] + fn frames_are_capped_at_one_mebibyte_including_the_newline() { + assert!(fits(MAX_FRAME_BYTES - 1)); + assert!(!fits(MAX_FRAME_BYTES)); + let small = "x".repeat(MAX_FRAME_BYTES - 1); + assert!(check_size(&small).is_ok()); + let large = "x".repeat(MAX_FRAME_BYTES); + assert!( + matches!(check_size(&large), Err(FrameError::TooLarge(bytes)) if bytes == MAX_FRAME_BYTES + 1) + ); + let body = Body::Events(Events { + topic: topic::ACTIVITY.into(), + key: "run:r".into(), + events: vec![json!({"detail":"x".repeat(MAX_FRAME_BYTES)})], + cursor: 1, + }); + assert!(matches!(encode_body(&body), Err(FrameError::TooLarge(_)))); + assert!(matches!( + encode(&Frame { v: 1, seq: 1, body }), + Err(FrameError::TooLarge(_)) + )); + assert!(decode(&large).is_err()); + } +} diff --git a/docs/reference.md b/docs/reference.md index d344e0123..3fb0607af 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -3948,6 +3948,104 @@ needs manual inspection before another request. `agent_id` is supported only for verified DeepSeek continuable children. Questions carry request identity/hash through `pending_questions`; answers use the exact unchanged request. +### Event stream + +```sh +hgs watch --json +hgs @mac watch --json +``` + +`watch --json` is a long-lived stream of one computer's live sessions and of the +Activity of the sessions a client follows. It is additive: every command above +keeps its output. `hgs -h` prints `Event stream ABI: watch-v1` where it exists. +A client that does not find the line keeps polling that computer. `@host` runs +the peer's stream over one SSH connection with `ServerAliveInterval=10` and +`ServerAliveCountMax=3`. An older peer answers with an error, never a launch. + +The stream writes one JSON frame per line on stdout and reads one control +operation per line on stdin. EOF on stdin ends it with exit code 0, so a closed +client or a dropped SSH link leaves no watcher behind. A closed stdout ends it +quietly. The frame types live in `crates/watch-protocol`. + +Every frame has `v` (1), `t` and `seq`, which grows by one per frame. The first +frame is `hello` with `machine_id`, `host`, `hgs_version`, a random `stream`, +`heartbeat_ms`, `max_frame_bytes`, `topics` and `limits`. A `heartbeat` with `at` +follows 10 s without other frames. + +| `t` | Fields | Meaning | +|---|---|---| +| `snapshot` | `topic`, `key?`, `items` or `value`, `page`, `last`, `cursor` | a replacement; keep the previous display until `last` | +| `delta` | `topic`, `upsert[]` of `{key, row}`, `remove[]` of `{key, reason}`, `cursor` | catalog changes | +| `events` | `topic`, `key`, `events[]`, `cursor` | new Activity events, oldest first | +| `state` | `topic`, `key`, `value`, `rev` | the latest summary of one session | +| `gap` | `topic`, `key?`, `reason` | discard the topic's state; a snapshot follows | +| `error` | `code`, `message`, `topic?`, `key?`, `fatal` | a refused operation or a failed source | +| `pong` | `id` | the answer to `ping` | + +The `catalog` topic is always on. Its rows are exactly the `sessions` of +`hgs ls --json --local --no-archives`, keyed `run:`, or `name:` for +a row without a run. The stream starts with a snapshot of them in pages of at +most 512 KiB, then sends deltas. A rename is an upsert of the same key. A removal +gives `replaced` (the name lists under a new run), `archived`, `forgotten` or +`ended`. The listing's other fields arrive as one `catalog.meta` snapshot: at +once when a lasting field changes, and at most every 10 s when only `metrics` or +`desktop_presence` changed. Archives stay outside the stream; page them with +`archives --json`. + +```json +{"op":"subscribe","topic":"activity","session":{"name":"claude/sample-project/review","run_id":"RUN","conversation_id":"CONVERSATION"},"after":123,"tail":50} +{"op":"unsubscribe","topic":"activity","session":{"name":"claude/sample-project/review","run_id":"RUN"}} +{"op":"ping","id":1} +``` + +An Activity subscription follows one session by its catalog key. Its events are +the objects `inspect` returns, and `cursor` is the journal `seq` that +`inspect --after` takes. Without `after`, the first `events` frame carries the +newest `tail` events (at most 100). With `after`, it carries the events after +that `seq`, at most 200 per frame read. When `after` is older than the journal +keeps, or newer than its newest row, the stream sends `gap` (`retention` or +`reset`) and a tail `snapshot`. DeepSeek and OpenCode sessions answer from their +own hosts: each change replaces their event window with a `snapshot`. `state` +carries the `inspect --skip-processes` summary without `events`, `cursor` and +`history_truncated`. It is sent when anything other than `goal_observed_at` +changed. A session that leaves the catalog ends its subscription with the error +`subscription_ended`. Unsubscribe stops it. A stream follows at most 16 sessions; +more fail with `too_many_subscriptions`. Other topics fail with +`unsupported_topic`; read them with their commands. + +The stream checks its sources every 250 ms. It stats the bindings, read marks, +notification preferences, waits, launches, renames, name routes, the archive +folder and the DeepSeek and OpenCode bindings, and holds a read-only connection +to the Activity journal. A changed binding, or new journal rows of its session, +derives only that session's row again, through the steps and under the lock of +the listing. Any other change lists all rows again. Both wait 150 ms after a +change and start at most every 500 ms and at most once per four durations of +the previous one. Every 5 s the stream probes what no file shows: tmux sessions +that start, end, are renamed, attached or detached, exited agent processes, +ended background work, sessions that have not reported a hook event yet and the +DeepSeek and OpenCode hosts. All rows are listed again every 5 s, for terminal +choosers and status lines of sessions that already reported hooks. Followed +summaries are re-read when their journal rows or catalog row change, at most +every 500 ms, and otherwise every 2 s while the agent is busy and every 15 s +while it is idle; DeepSeek and OpenCode every 2 s. + +A frame is at most 1 MiB including its newline. An event too large for a frame +is cut with `"truncated": true`, and an oversized summary loses the oldest items +of its largest lists, named in `stream_truncated`; `inspect` returns them whole. +The outgoing queue holds 4 MiB. When the client reads slowly, catalog changes +and summaries coalesce per key. When the queue overflows, the topic receives +`gap` (`overflow`) and a fresh snapshot once the client catches up. Control lines +are at most 64 KiB. + +The desktop runs one stream for each computer whose CLI prints the marker. +While a computer's stream is live, the desktop stops polling that computer's +listing and follows the selected session's Activity through the stream; the +Processes tab keeps its own poll interval. Listings are applied at most every +2 s, Activity within 50 ms. A stream that fails or stays silent for 30 s +restarts after 5 s, 15 s, 60 s and then every 5 minutes, and the desktop polls +meanwhile. `ZERUS_EVENT_STREAM=0` or the setting `dataPlane/eventStream=false` +keeps the desktop on polling. + ## Validation Use [CONTRIBUTING.md](../CONTRIBUTING.md) for standard checks. Integration tests diff --git a/docs/superpowers/specs/2026-10-27-event-data-plane-design.md b/docs/superpowers/specs/2026-10-27-event-data-plane-design.md new file mode 100644 index 000000000..7e06907e4 --- /dev/null +++ b/docs/superpowers/specs/2026-10-27-event-data-plane-design.md @@ -0,0 +1,291 @@ +# Event data plane + +Epic: `zerus-604d`. Owner direction, 2026-10-27: an event-driven data channel +is mandatory, and parsing and similar processing must run off the GUI (main) +thread. This design replaces per-request `hgs` process polling with one +long-lived event stream per computer and moves all data preparation in the +desktop off the GUI thread. The stream contract is shared by the Qt desktop, +the planned Tauri client (`2026-10-27-tauri-web-ui-design.md`), the mobile +connector and a future plugin host's `events.subscribe`. + +## Baseline (2026-10-27) + +Measured on the owner's Mac (M1 Pro, all sessions on `arch`) and on `arch` +itself, both running Dev 0.47.1-dev.1398 with `hgs 1.56.0`. Raw logs, scripts +and hashes are kept with the review report, outside the repository. + +| Measure | arch GUI | Mac GUI | +|---|---|---| +| `hgs` children started per second (60 s idle window) | 3.63 | 3.60 | +| of which `git-status` per minute | 123 | 114 (each one SSH to arch) | +| of which local `ls` / peer `ls` / `inspect` per minute | 42 / 13 / 20 | 29 / 10 / 16 | +| GUI process CPU (self) | 2.4 % | 17.0 % | +| CPU of reaped children | 35.5 % of a core | n/a | +| RSS | 166 MiB | 139 MiB | + +Command costs (median of 3–5 runs): + +| Command | on arch | from the Mac to arch | +|---|---|---| +| `ls --json --local --no-archives` (35 live rows) | 280 ms, 294 KB | 599 ms, 294 KB | +| `inspect NAME --after CURSOR` | 117 ms, 171 KB | 524 ms, 171 KB | +| same with `--skip-processes` | 50 ms, 51 KB | — | +| `git-status --path P --json` | 21–223 ms, 0.4 KB | 500 ms | +| `ssh arch true` | — | 240 ms | + +Of the 171 KB incremental inspection, 120 KB is the `processes` section. The +desktop includes it every 2.5 s while the Processes tab is shown and every 30 s +otherwise; other polls pass `--skip-processes` (51 KB). `--after` trims only +`events`. Every other section is rebuilt and resent each time. + +JSON parsing itself is cheap. Qt `fromJson` + `boxFromObject` of the 294 KB +listing takes 1.75 ms on arch, and 6.9 ms for the 1.1 MB listing with archives. +The GUI-thread cost lies after the parse. A 15 s `sample` of the Mac GUI +(5,834 main-thread samples, about 2.6 ms each) attributes: + +- 11.4 % of the main thread to `ProcessRunner` result callbacks; +- 7.2 % to `TrayAgent::onLocalReady`, about **150 ms per local poll**: three + QSettings JSON re-parses, several full unread passes, `SessionsWindow::rebuild` + and `AccountsPage::render`; +- 3.9 % to SHA-256 through OpenSSL (`QCryptographicHash`) inside `rebuild`, for + account identity keys and row preference keys recomputed for every row on + every poll; +- about 52 ms of GUI time per applied inspection, and 87 ms per peer listing; +- 19 % to painting, which is outside this design. + +Each local listing row changes on every poll even when nothing happened: +`goal_observed_at` and `process_work.sampled_at` are observation timestamps. +Any delta protocol must mask them, or every delta becomes a full snapshot. + +Session open from the Mac is therefore at least one SSH inspection (0.4–0.5 s) +plus about 50 ms of GUI work before the first Activity render. A reply appears +on the next 2.5 s poll plus one inspection: on average about 1.75 s and at worst +about 3 s after the agent wrote it. These two values are derived from component +timings. Milestone 1 adds instrumentation that measures them end to end. + +## Rule: no data preparation on the GUI thread + +The GUI thread posts requests and applies prepared, immutable values. It never +parses JSON, merges, diffs, sorts, filters, hashes or builds models from data +that came from `hgs`, the stream or disk. + +### ProcessRunner preparation stage + +`ProcessRunner` already starts children on its worker thread. Every result now +also passes a single serial preparation thread, `hgs-prepare`, before it reaches +the owner: + +```cpp +ProcessRunner::run(request, owner, + [](ProcessRunner::Result result) { return prepareListing(result); }, // hgs-prepare + [this](PreparedListing listing) { apply(std::move(listing)); }); // owner thread +``` + +The prepare step sees only the result and values it captured. It must not touch +the owner or any other application-thread object. Results keep the order in +which their children finished, whether or not they have a prepare step, because +the legacy `run(request, owner, done)` passes the same serial thread unchanged. +A destroyed owner still drops a prepared value. + +`HgsClient` converts each `runHgs` call to a prepared variant. In the first +step, signals keep their existing types (`BoxState`, `QJsonObject`, parsed +lists). Those are built off-thread and handed over by implicit sharing, so +receivers stay unchanged. Later steps replace them with typed prepared models. + +### Enforcement + +- `tray/src/JsonParse.h` is the tray's only JSON parser. In a GUI application it + records a violation when it runs on the application thread. With + `ZERUS_GUI_THREAD_CHECKS=fatal`, which the Qt test suites set, a violation + aborts the test with the call site. +- `scripts/ci/check-source.py` rejects `QJsonDocument::fromJson` and + `QCborValue::fromCbor` in `tray/src` outside `JsonParse.cpp`. +- During migration, small local preference values that are still read on the + GUI thread go through `JsonParse::localSetting()`. It is capped at 64 KiB, and + `check-source.py` lists the files and call counts allowed to use it. The list + may only shrink, and each entry has a Beads task. Owner decision, + 2026-10-27: «согласен, оставляем исключения до M2» — the list must be empty + at the end of M2. All `hgs` and stream output is parsed off the GUI thread + from M1 on, without exception. +- JSON values kept in QSettings go through `SettingsJson`. It parses a value + only when its stored bytes changed since the last read. The tray warms the + large keys (organization, swarm observations, read marks, worktree + snapshots) on the preparation thread at startup, and writers record what + they stored, so the GUI thread does not parse them again. On arch the saved + organization is 66 KB and the swarm versions 102 KB, both above the + `local()` limit, so a cold read of either on the GUI thread counts as a + violation. +- Tests: preparation runs on `hgs-prepare` and application happens on the owner + thread; global result order is preserved; a destroyed owner drops prepared + values; a fixture listing and inspection through `HgsClient` parse off the GUI + thread; and a GUI-thread parse fails under the fatal check. + +### Model preparation off the GUI thread + +After the parse moves, the dominant cost is the work that follows each listing: +unread and attention computation, fleet merges, account catalogs, sorting, +grouping and per-row roles. Milestone 2 introduces a `FleetModel` worker. It +owns the merged fleet, read marks and organization inputs, and publishes an +immutable `PreparedFleet`: rows in display order with precomputed roles, group +headers, counts, unread and attention results, and a changed-row set. The GUI +applies only the changed rows to `SessionList` and the pages that are visible. +Read marks and attention marks are held in memory, written through on change, +and never re-parsed from QSettings per poll. Account and row-preference +identity hashes are computed once per identity, on the worker. + +## Event stream: `hgs [@host] watch --json` + +### Transport + +- One long-lived `hgs watch --json` process per computer. Local: a child of the + desktop. Peer: `hgs @host watch --json`, one SSH connection with + `ServerAliveInterval=10` and `ServerAliveCountMax=3`. The remote command is + guarded by the `watch-v1` marker, the same way other newer commands are, so + an old peer never receives an unknown command as an agent launch. +- Server to client: NDJSON on stdout, UTF-8, one frame per line. A frame is at + most 1 MiB. Larger content is paged or truncated with a reference that the + client fetches with an existing bounded command. +- Client to server: NDJSON control operations on stdin. EOF on stdin ends the + stream, so a closed desktop or a dropped SSH connection never leaves an + orphan watcher behind. +- `hgs -h` prints the line `Event stream ABI: watch-v1`. Clients that do not + find it keep today's polling for that computer. + +### Frames + +Every frame has `v` (1), `t` (type) and `seq`. `seq` is monotonic per stream, +and each stream has a random `stream` ID that appears in its `hello`. + +| `t` | Fields | Meaning | +|---|---|---| +| `hello` | `machine_id`, `host`, `hgs_version`, `stream`, `heartbeat_ms`, `max_frame_bytes`, `topics`, `limits` | first frame | +| `snapshot` | `topic`, `key?`, `items` or `value`, `page`, `last`, `cursor` | bounded replacement of one topic or key. Retain the previous display until `last` | +| `delta` | `topic`, `upsert[]`, `remove[]` with `key` and `reason`, `cursor` | keyed collection change | +| `events` | `topic: "activity"`, `key`, `events[]`, `cursor` | append-only Activity events after `cursor` | +| `event` | `topic: "activity"`, `key`, `id`, `rev`, `event` | replace one open event by stable ID; a higher `rev` wins | +| `state` | `topic`, `key`, `value`, `rev` | replace one keyed object (git status, process set, session summary) | +| `heartbeat` | `at` | sent after 10 s without other frames | +| `gap` | `topic`, `key?`, `reason` | the client must discard that topic's state; a snapshot follows | +| `error` | `code`, `message`, `topic?`, `fatal` | a fatal error ends the stream | + +Control operations: `subscribe` and `unsubscribe` with `topic` and its key +(`session` as `{name, run_id, conversation_id}`, `path` or `project`), an +optional `after` cursor and `tail` count for Activity, and `ping`. The catalog +topic is always on. + +### Topics + +| Topic | Key | Content | Resume | +|---|---|---|---| +| `catalog` | `run:`, or `name:` for rows without a run | exactly today's `ls --json --local --no-archives` row objects. A rename is an upsert on the same key. A resume or restart that starts a new run removes the old key with `reason: "replaced"`. Removals carry `ended`, `archived`, `forgotten` or `replaced` when known | a fresh bounded snapshot. Clients diff it against their retained rows | +| `catalog.meta` | — | the non-row fields of the listing (`projects`, `peers`, `preferences`, `desktop_presence`, `archive_revision`, `hgs_version`, `metrics`) | snapshot | +| `activity` | session key | `events`: the same event objects as `inspect`, cursor = journal `seq`. `state`: the inspection summary without `events` and `processes`. `event`: open-event updates such as provider messages between tools | `after` = the last `seq`. Below retention the server sends `gap` and a tail snapshot | +| `processes` | session key | `state`: the `processes` section of `inspect` | snapshot | +| `git` | checkout path | `state`: the `git-status --json` object | snapshot | +| `tasks` | project | `delta` of task rows, as `task list` | snapshot | + +Archives stay outside the stream. They remain a keyset-paged read (`archives +--json`), so the snapshot is bounded by live sessions, not by history. + +Status semantics do not change. Rows carry the same fields; Working, +Background, Scheduled, Needs attention and unread reply identity are derived +by the clients exactly as today. + +Change detection masks the volatile observation fields (`goal_observed_at`, +`process_work.sampled_at`). A row is sent when anything else in it changes. +`catalog.meta` with fresh `metrics` and presence is sent at most every 10 s. + +### Event sources + +No file watcher or async runtime exists in `hgs` today, and v1 adds no +dependency. One watcher process uses `std::thread` and cheap change detection +on a 250 ms pulse: + +- **Activity**: a held read-only connection to `events.sqlite3`. When + `PRAGMA data_version` changes, read `seq > cursor` for subscribed + conversations (at most 200 rows per read). `seq` is global and monotonic and + already serves as the `inspect --after` cursor. Hooks write it on every + native hook event, and retention keeps 50,000 rows. Text between tools exists + only in native transcripts. For subscribed sessions the watcher stats the + transcript and, when it grows, re-derives provider messages from an offset + cursor keyed on device, inode, prefix hash and offset (the existing + tail-reader pattern). DeepSeek and OpenCode keep their inspect RPCs, polled + only while subscribed. +- **Catalog**: a stamp vector over the binding directories, `renames/`, + `name_routes/`, `read-marks.json`, `notifications.json`, `waits.sqlite3`, + `launches/`, the archive revision and the provider binding directories. When + the stamp changes (150 ms debounce, at most every 500 ms), or after a 5 s + safety tick, the watcher re-runs the in-process listing builder that + `ls --json --local --no-archives` uses and diffs rows by key. The safety tick + covers what hooks never report: process exit, tmux kill or rename, ended + background jobs and terminal choosers. Bindings are replaced by rename; + temporary `.hgs-*` files are ignored. +- **Processes**: hook-recorded shell jobs (the binding stamp) plus one `ps` + work sample every 2 s per subscribed session. The full snapshot, which + re-reads the transcript tail, runs at most every 10 s. +- **Git**: stats of `HEAD`, `index`, the current branch ref, `packed-refs` and + `FETCH_HEAD` in the git directory and common directory of each subscribed + path. Activity events from file-changing tools in sessions under that path + also trigger a check. There is a 30 s safety tick, and the existing + `git-status` command (with its 60 s remote cache) runs on change. +- **Tasks**: stats of `replica//project.loro` and its `tasks/` + directory, then the existing task listing. Loro stays inside `crates/replica`. + +A later version may replace stat pulses with inotify/kqueue/FSEvents behind +the same frames. The protocol does not depend on how changes are detected. + +### Bounds and backpressure + +- Subscriptions: at most 16 Activity sessions, 8 process sets, 64 git paths + and 8 task projects per stream. Excess subscriptions fail with an `error` + frame. +- A writer thread owns stdout with a bounded queue (4 MiB). When the client + stops reading, pending `state` and `delta` frames coalesce per topic and key + (latest wins). If the queue still overflows, the affected topic receives + `gap` followed by a fresh snapshot once the client drains. +- Snapshot pages are at most 512 KiB. Activity tails are at most 100 events, + with older history read through the existing paged commands. +- Listing work is bounded exactly as today. The watcher never adds per-session + caches beyond subscribed sessions. + +### Consumers + +- **Qt desktop**: one `EventStream` per computer. Its reader, line framing, + JSON parsing and per-computer store run on a worker thread. The store + publishes immutable prepared values to the GUI: Activity within 50 ms, and + listings at most every 2 s until M2 makes applying one cheap. A listing + arrives as the same `BoxState` a poll delivers, and Activity as an + inspect-shaped increment, so the existing apply paths stay unchanged. While a computer's stream is healthy, the desktop + stops its `ls`, inspection, `git-status` and `processes` polls for it. A + missing marker, or three stream failures within 60 s, falls back to today's + polling, and the desktop retries the stream every 5 minutes. +- **Tauri client**: its Rust backend spawns the same subprocess per computer, + parses frames in Rust (never in the webview) and pushes view deltas to the + webview, coalesced per animation frame. A small serde crate + `crates/watch-protocol` defines the frame types. It is shared by `hgs` and + the Tauri crate and contains neither Loro nor Qt. +- **Mobile connector**: it keeps polling until it opts in, then consumes the + same frames. API v1 and its journals stay unchanged. +- **Plugin host**: `events.subscribe` maps onto these topics, filtered by the + manifest's permissions. + +## Compatibility + +- `hgs` CLI commands, `HGS_*` variables, state paths, API v1 and the connector + journal stay unchanged. The stream is additive. +- Older peers lack the `watch-v1` marker and keep today's polling. +- An older desktop never starts `watch`. The command runs only while a client + holds its stdin open. +- Session counts may reach tens of thousands. The live catalog is bounded by + live sessions, archives stay paged, Activity is cursor-based, and the + watcher holds no unbounded per-session state. + +## Milestones + +| | Scope | Measure afterwards | +|---|---|---| +| M1 | Preparation stage, every parse of `hgs`, connector and network output off-thread, enforcement, settings values parsed once per change, cached identity hashes | Mac `sample` main-thread share, per-poll GUI time | +| M2 | `FleetModel` worker: unread, attention, sort, group and row roles prepared off-thread; the local-value list empty; open/reply latency instrumentation | per-poll GUI time below 5 ms | +| M3 | `hgs watch` server: catalog and Activity topics, `crates/watch-protocol`, `docs/reference.md` | spawns per second, idle CPU on arch | +| M4 | Qt `EventStream` consumer with fallback, local and peers | session open and reply latency from the Mac | +| M5 | git, processes and tasks topics; connector and plugin adoption | spawns per second close to zero while idle | diff --git a/mac-install.sh b/mac-install.sh index e3332d978..03c68e738 100755 --- a/mac-install.sh +++ b/mac-install.sh @@ -56,7 +56,8 @@ sources=(Cargo.toml Cargo.lock build.rs .cargo src scripts install.sh hgs hgs_st services/mobile/connector/Cargo.toml services/mobile/connector/Cargo.lock services/mobile/connector/.cargo services/mobile/connector/src services/mobile/relay/Cargo.toml services/mobile/relay/src services/mobile/crypto/Cargo.toml services/mobile/crypto/src - crates/replica/Cargo.toml crates/replica/Cargo.lock crates/replica/.cargo crates/replica/src) + crates/replica/Cargo.toml crates/replica/Cargo.lock crates/replica/.cargo crates/replica/src + crates/watch-protocol/Cargo.toml crates/watch-protocol/Cargo.lock crates/watch-protocol/.cargo crates/watch-protocol/src) tar czf - -C "$here" "${sources[@]}" \ | ssh -o BatchMode=yes "$host" "cd ~/.local/src/hgs/.rust-build && rm -rf ${sources[*]} && tar xzf -" # install.sh builds the mobile connector only on request (HGS_MOBILE_CONNECTOR=1) diff --git a/scripts/ci/check-source.py b/scripts/ci/check-source.py index 1d67f8226..068fd9e27 100644 --- a/scripts/ci/check-source.py +++ b/scripts/ci/check-source.py @@ -21,6 +21,27 @@ "tray/src/WindowsTerminalBackend.cpp": "the app links it only on Windows, where inheritStdioOnly() does nothing", } CHILD_PROCESS_SOURCES = re.compile(r"tray/src/.+\.(?:cpp|h|mm)") +# The desktop parses JSON only through tray/src/JsonParse.h, which reports a +# parse on the GUI thread. JsonParse::local() is the exception for small values +# the tray wrote itself; these files may call it this many times. Owner +# decision, 2026-10-27: the list only shrinks and is empty at the end of M2 +# (zerus-604d.2). +JSON_PARSER = "tray/src/JsonParse.cpp" +LOCAL_JSON_CALLS = { + "tray/src/AccountFilter.cpp": 1, # binding keys the filter built + "tray/src/AttentionTracker.cpp": 1, # notification tokens the tray encoded + "tray/src/ComposerDraftStore.cpp": 1, # question draft identities + "tray/src/QuestionReply.h": 1, # the user's own answers in a prompt; M2 prepares them with the row + "tray/src/RelayConfiguration.h": 1, # private connector configuration, at most 64 KiB + "tray/src/SessionList.cpp": 3, # drag data the list wrote + "tray/src/SessionsWindow.cpp": 1, # pending task launches, at most 64 KiB + "tray/src/SettingsJson.cpp": 1, # a changed QSettings value before warm() parsed it + "tray/src/TaskDrag.h": 1, # drag data the board wrote + "tray/src/UpdateController.cpp": 3, # the saved update catalog and installer record + "tray/src/WorktreesDialog.cpp": 1, # review keys the dialog built +} +JSON_DECODER = re.compile(r"\b(?:QJsonDocument\s*::\s*fromJson|QCborValue\s*::\s*fromCbor)\b") +LOCAL_JSON = re.compile(r"\bJsonParse\s*::\s*local(?:Object|Array)?\s*\(") SEE_CHILD_PROCESS = "(tray/src/ChildProcess.h)" LEXEME = re.compile(r"//|/\*|[\"']") LITERAL = {'"': re.compile(r'"(?:\\.|[^"\\\n])*"?'), "'": re.compile(r"'(?:\\.|[^'\\\n])*'?")} @@ -249,6 +270,24 @@ def child_process_findings(files): return [f"{name}:{line}: {reason} {SEE_CHILD_PROCESS}." for name, line, reason in sorted(problems)] +def json_parse_findings(files): + """"file:line: reason" for JSON parsed outside JsonParse.h or local() calls beyond LOCAL_JSON_CALLS.""" + problems = [] + for name, text in sorted(files.items()): + code = code_only(text) + if name != JSON_PARSER: + for match in JSON_DECODER.finditer(code): + line = code.count("\n", 0, match.start()) + 1 + problems.append(f"{name}:{line}: parse JSON with JsonParse (tray/src/JsonParse.h), off the GUI thread") + calls = len(LOCAL_JSON.findall(code)) + allowed = LOCAL_JSON_CALLS.get(name, 0) + if calls > allowed: + problems.append(f"{name}: {calls} JsonParse::local() calls, {allowed} allowed; parse off the GUI thread instead") + elif calls < allowed: + problems.append(f"{name}: lower LOCAL_JSON_CALLS to {calls} in scripts/ci/check-source.py") + return problems + + def main(): paths = subprocess.check_output(["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"], cwd=ROOT).decode().split("\0") # First, so that a desktop child the unit tests would also find shows only its findings. @@ -258,6 +297,10 @@ def main(): raise SystemExit(f"Remove the child process exemption of missing {', '.join(stale)}.") if problems := child_process_findings(sources): raise SystemExit("\n".join(problems)) + if stale := sorted(set(LOCAL_JSON_CALLS) - set(sources)): + raise SystemExit(f"Remove the local JSON allowance of missing {', '.join(stale)}.") + if problems := json_parse_findings(sources): + raise SystemExit("\n".join(problems)) subprocess.run([sys.executable, "-m", "unittest", "discover", "-q", "-s", "tests", "-p", "test_source_*.py"], cwd=ROOT, check=True) subprocess.run([sys.executable, str(ROOT / "scripts/check-source-privacy.py")], check=True) subprocess.run(["git", "diff", "--check"], cwd=ROOT, check=True) @@ -316,7 +359,7 @@ def main(): ast.parse(path.read_text(), filename=name) elif path.read_bytes().split(b"\n", 1)[0].startswith((b"#!/bin/sh", b"#!/usr/bin/env bash")) or path.name in {"PKGBUILD", "PKGBUILD.in"}: subprocess.run(["bash", "-n", str(path)], check=True) - print("Source syntax, privacy, child process and release version checks passed.") + print("Source syntax, privacy, child process, JSON parsing and release version checks passed.") if __name__ == "__main__": diff --git a/scripts/ci/gui.sh b/scripts/ci/gui.sh index fbbd53ce7..840806551 100644 --- a/scripts/ci/gui.sh +++ b/scripts/ci/gui.sh @@ -63,6 +63,8 @@ if [ "$phase" = build ]; then exit 0 fi export QT_QPA_PLATFORM=offscreen QT_QUICK_BACKEND=software +# JSON parsed on the GUI thread fails the suite (tray/src/JsonParse.h). +export ZERUS_GUI_THREAD_CHECKS=fatal export QTWEBENGINE_CHROMIUM_FLAGS=--disable-gpu rm -rf artifacts/test-results/qt-functions mkdir -p artifacts/test-results/qt-functions diff --git a/src/cli.rs b/src/cli.rs index da2e27f18..7f9df20f3 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -66,6 +66,8 @@ pub(crate) const USAGE: &str = r#"usage: hgs [@host] [project] [-c [ID]] [ hgs self [--json] the calling agent's own session reference hgs [@host] guide [topic] how agents message, wait for, launch and stop sessions and use tasks hgs inspect 'zerus://session?...' resolve and inspect a referenced conversation + hgs [@host] watch --json long-lived NDJSON event stream (stdin control) + Event stream ABI: watch-v1 hgs [@host] processes [--archive ] observed shell commands as JSON hgs [@host] processes --output ID | --stop ID --run RUN --conversation ID [--generation ID] hgs [@host] send --json submit message/attachments from JSON stdin @@ -192,7 +194,7 @@ fn local_sessions() -> Result<()> { // Repository preparation owns only its checkout and receipt. Like the existing // configuration/control commands, it does not require or enable a session host. fn needs_local_sessions(command: &str) -> bool { - !matches!(command, "ls" | "machine" | "presence" | "swarm" | "project" | "account" | "speech" | "analytics" | "graph" | "wait" | "mobile" | "hooks" | "repository") + !matches!(command, "ls" | "watch" | "machine" | "presence" | "swarm" | "project" | "account" | "speech" | "analytics" | "graph" | "wait" | "mobile" | "hooks" | "repository") } /// Fails unless this process may run what the processes of a session run: @@ -628,6 +630,7 @@ pub fn dispatch(mut args: Vec) -> Result { Ok(0) } "ls" => list(&config, args, dry), + "watch" => crate::watch::run(&config, args, dry), "archives" => { let mobile = cfg!(windows) && args.last().map(String::as_str)==Some("--windows-mobile"); let args = if mobile { &args[..args.len()-1] } else { args }; @@ -966,6 +969,23 @@ fn remote( all.extend(args.clone()); all } + // One long-lived stream: keepalives detect a dropped link, and EOF on + // the local stdin reaches the peer's watcher, which then exits. + "watch" => { + flags = vec![ + "-o".into(), + "BatchMode=yes".into(), + "-o".into(), + "ConnectTimeout=5".into(), + "-o".into(), + "ServerAliveInterval=10".into(), + "-o".into(), + "ServerAliveCountMax=3".into(), + ]; + let mut all = vec![command.into()]; + all.extend(args.clone()); + all + } // Phone management reaches the relay through the gateway peer. "mobile" => { flags = vec![ @@ -1039,6 +1059,9 @@ fn remote( platform::remote_sh(&format!("if ~/.local/bin/hgs --help 2>/dev/null | grep -qF 'hooks-health-v1'; then exec ~/.local/bin/hgs {}; else echo 'hgs: update the selected computer to inspect hook health' >&2; exit 1; fi",platform::join(&remote_args))) } else if command == "analytics" { platform::remote_sh(&format!("if ~/.local/bin/hgs --help 2>/dev/null | grep -qF 'hgs [@host] analytics --json'; then exec ~/.local/bin/hgs {} --read-only; else echo 'hgs: update the selected computer to read analytics' >&2; exit 1; fi",platform::join(&remote_args))) + } else if command == "watch" { + // An older peer would read an unknown command as an agent launch. + platform::remote_sh(&format!("if ~/.local/bin/hgs --help 2>/dev/null | grep -qF '{}'; then exec ~/.local/bin/hgs {}; else echo 'hgs: update the selected computer to stream events (watch-v1)' >&2; exit 1; fi",zerus_watch_protocol::HELP_MARKER,platform::join(&remote_args))) } else if command == "guide" { platform::remote_sh(&format!("if ~/.local/bin/hgs --help 2>/dev/null | grep -qF 'hgs [@host] guide'; then exec ~/.local/bin/hgs {}; else echo 'hgs: the selected computer has no agent guide; check hgs @host -h' >&2; exit 1; fi",platform::join(&remote_args))) } else { format!("~/.local/bin/hgs {}", platform::remote_join(&remote_args)) }; @@ -1081,6 +1104,8 @@ fn remote( } pub(crate) fn local_json(config: &Config) -> Result { local_json_archives(config, true) } +/// `ls --json --local --no-archives`, which `hgs watch` streams. +pub(crate) fn local_listing(config: &Config) -> Result { local_json_archives(config, false) } fn local_json_archives(config: &Config, archives: bool) -> Result { if !platform::local_sessions() { // A control console: no tmux, session state or local projects. Absent @@ -1105,13 +1130,7 @@ fn local_json_archives(config: &Config, archives: bool) -> Result { let launches = state::launches::Evidence::read(); // A listing that fails is an error: no sessions would show live ones as stopped. let listing = terminal().listing().map_err(|e| Error::new(1, e))?; - let sessions: Vec = listing.sessions.iter().map(|row| { - let name = row.name.as_str(); - let mut parts = name.splitn(3, '/'); let cmd = parts.next().unwrap_or(""); - let project = parts.next().filter(|s| !s.is_empty()); let tag = parts.next().filter(|s| !s.is_empty()); - let ttys: Vec<&str> = listing.clients.iter().filter_map(|client| if client.session == name { Some(client.terminal.as_str()) } else { None }).collect(); - json!({"name":name,"cmd":cmd,"project":project,"tag":tag,"attached":row.attached,"clients":ttys,"created":row.created,"launch_id":row.launch_id}) - }).collect(); + let sessions = state::listed_rows(&listing); let mut raw = json!({"host":config.host(),"ok":true,"hgs_version":env!("CARGO_PKG_VERSION"),"projects":projects,"sessions":sessions,"peers":config.peers,"metrics":crate::metrics::snapshot(&config.state)}); // Phone alerts wait while the owner is active here; see src/presence.rs. if let Some(presence) = crate::presence::listing(&config.state, crate::presence::now()) { @@ -1125,11 +1144,7 @@ fn local_json_archives(config: &Config, archives: bool) -> Result { raw["local_host"] = crate::host::admin::local_host(&config.state); } // A session renamed before merge_listing reads the panes keeps its ID. - let ids = listing - .sessions - .iter() - .map(|row| (row.name.clone(), row.id.clone())) - .collect(); + let ids = state::listed_ids(&listing); let mut merged = state::merge_listing_archives(raw, &ids, archives).map_err(|e| Error::new(1, e))?; merged["archive_paging"] = json!(true); merged["archive_revision"] = json!(state::archive_catalog::revision().map_err(|e|Error::new(1,e))?); diff --git a/src/main.rs b/src/main.rs index c545ff3a6..fcfa71bed 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,6 +22,7 @@ mod status; mod state; mod swarm; mod tasks; +mod watch; fn main() { if std::env::args().nth(1).as_deref() == Some("--macos-session-service") { diff --git a/src/state/effort.rs b/src/state/effort.rs index 79ae2424b..9bc7bcd30 100644 --- a/src/state/effort.rs +++ b/src/state/effort.rs @@ -406,7 +406,8 @@ fn capture(record: &Value) -> Result { { return Err("invalid tracked pane".into()); } - let output = backend::current().capture(pane, backend::Capture::Plain)?; + // A status summary shares this screen with its terminal choosers. + let output = kimi_tui_choice::plain(pane)?; Ok(String::from_utf8_lossy(&output).into_owned()) } diff --git a/src/state/journal.rs b/src/state/journal.rs index 034ea8ff5..14ead0d48 100644 --- a/src/state/journal.rs +++ b/src/state/journal.rs @@ -413,7 +413,13 @@ fn kimi_reply(db: &Connection, record: &Value, skip_scheduled: bool) -> Result Value { + kimi_tui_choice::observing(|| observed_summary(record, live_pane)) +} + +fn observed_summary(record: &Value, live_pane: bool) -> Value { let mut output = normalized_activity(record, live_pane); output["conversation_id"] = record["conversation_id"].clone(); output @@ -788,28 +794,17 @@ pub(super) fn inspection(name: &str, after: i64, archive_id: Option<&str>, inclu |row| row.get(0), ) .map_err(|e| e.to_string())?; - let mut statement = db.prepare(&if after > 0 { - format!("SELECT seq, payload FROM events WHERE {TIMELINE} AND seq>?4 ORDER BY seq LIMIT 200") - } else { format!("SELECT seq, payload FROM events WHERE {TIMELINE} AND seq>?4 ORDER BY seq DESC LIMIT 100") }).map_err(|e| e.to_string())?; - let mut rows: Vec<(i64, String)> = statement - .query_map(params![journal_name, conversation, earlier, after], |row| { - Ok((row.get(0)?, row.get(1)?)) - }) - .map_err(|e| e.to_string())? - .collect::>() - .map_err(|e| e.to_string())?; - if after == 0 { - rows.reverse(); - } + let rows = if after > 0 { + timeline_rows(&db, journal_name, conversation, &earlier, after, 200, false)? + } else { + timeline_rows(&db, journal_name, conversation, &earlier, after, 100, true)? + }; output["cursor"] = json!(rows.last().map(|(seq, _)| *seq).unwrap_or(after)); output["history_truncated"] = json!(after > 0 && oldest.map(|seq| after < seq - 1).unwrap_or(false)); let mut events = Vec::new(); for (seq, payload) in rows { - let mut event: Value = serde_json::from_str(&payload).map_err(|e| e.to_string())?; - event["seq"] = json!(seq); - mark_system_prompt(&record, &mut event, ""); - events.push(event); + events.push(timeline_event(&record, seq, &payload)?); } output["events"] = json!(events); let (messages, truncated) = message_events(&db, journal_name, conversation, &earlier)?; @@ -828,6 +823,35 @@ pub(super) fn inspection(name: &str, after: i64, archive_id: Option<&str>, inclu Ok(output) } +/// A session timeline's rows after `after`, oldest first: the first `limit` +/// of them, or with `newest` the last `limit`. `inspection` and `hgs watch` +/// both read Activity through it and `timeline_event`. +pub(super) fn timeline_rows(db: &Connection, journal_name: &str, conversation: Option<&str>, earlier: &str, + after: i64, limit: usize, newest: bool) -> Result> { + let mut statement = db.prepare(&if newest { + format!("SELECT seq, payload FROM events WHERE {TIMELINE} AND seq>?4 ORDER BY seq DESC LIMIT ?5") + } else { format!("SELECT seq, payload FROM events WHERE {TIMELINE} AND seq>?4 ORDER BY seq LIMIT ?5") }).map_err(|e| e.to_string())?; + let mut rows: Vec<(i64, String)> = statement + .query_map(params![journal_name, conversation, earlier, after, limit as i64], |row| { + Ok((row.get(0)?, row.get(1)?)) + }) + .map_err(|e| e.to_string())? + .collect::>() + .map_err(|e| e.to_string())?; + if newest { + rows.reverse(); + } + Ok(rows) +} + +/// One Activity event as `inspect` returns it. +pub(super) fn timeline_event(record: &Value, seq: i64, payload: &str) -> Result { + let mut event: Value = serde_json::from_str(payload).map_err(|e| e.to_string())?; + event["seq"] = json!(seq); + mark_system_prompt(record, &mut event, ""); + Ok(event) +} + pub(super) fn journal_name(record: &Value) -> &str { if string(record, "journal_name").is_empty() { string(record, "name") diff --git a/src/state/kimi_tui_choice.rs b/src/state/kimi_tui_choice.rs index 8a400fb4d..03a4d026f 100644 --- a/src/state/kimi_tui_choice.rs +++ b/src/state/kimi_tui_choice.rs @@ -17,8 +17,57 @@ pub(super) fn launcher_notices(rows: &[&str], agent: &str) -> bool { .contains(&prelude) } +thread_local! { + /// Plain screens captured during one read-only observation, by pane. + static OBSERVED: std::cell::RefCell>>> = + const { std::cell::RefCell::new(None) }; +} + +/// Runs a read-only status observation in which each pane is captured at most +/// once. A summary reads its pane for the effort footer and again for every +/// eligible terminal chooser; they now share one screen instead of starting a +/// `tmux capture-pane` each (zerus-604d.7). Nothing that sends keys, resizes a +/// pane or answers a chooser may run inside: those always capture afresh. +pub(super) fn observing(observe: impl FnOnce() -> T) -> T { + struct Reset(bool); + impl Drop for Reset { + fn drop(&mut self) { + if self.0 { + OBSERVED.with(|screens| screens.borrow_mut().take()); + } + } + } + // A nested observation shares the outer one's screens. + let outer = OBSERVED.with(|screens| { + let mut screens = screens.borrow_mut(); + let outer = screens.is_none(); + if outer { + *screens = Some(Default::default()); + } + outer + }); + let _reset = Reset(outer); + observe() +} + +/// The pane's plain screen; within `observing`, the one captured first. +/// Failures are not kept, so a later reader tries again. +pub(super) fn plain(pane: &str) -> Result> { + let shared = OBSERVED.with(|screens| screens.borrow().as_ref().and_then(|screens| screens.get(pane).cloned())); + if let Some(screen) = shared { + return Ok(screen); + } + let screen = backend::current().capture(pane, backend::Capture::Plain)?; + OBSERVED.with(|screens| { + if let Some(screens) = screens.borrow_mut().as_mut() { + screens.insert(pane.to_owned(), screen.clone()); + } + }); + Ok(screen) +} + pub(super) fn capture(record: &Value) -> Result { - let out = backend::current().capture(string(record, "pane"), backend::Capture::Plain)?; + let out = plain(string(record, "pane"))?; if out.len() > 512 * 1024 { return Err("Terminal snapshot is too large".into()); } @@ -293,3 +342,46 @@ fn navigate( } }) } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; + + static CAPTURES: AtomicUsize = AtomicUsize::new(0); + struct Counting; + impl backend::TerminalBackend for Counting { + fn capture(&self, pane: &str, _: backend::Capture) -> Result> { + let n = CAPTURES.fetch_add(1, SeqCst); + if pane == "%9" { + return Err("no such pane".into()); + } + Ok(format!("{pane} screen {n}").into_bytes()) + } + } + + #[test] + fn an_observation_captures_each_pane_once_and_nothing_else_shares() { + let record = |pane: &str| json!({"pane": pane}); + backend::with_fake(&Counting, || { + CAPTURES.store(0, SeqCst); + // Outside an observation every read is fresh. + assert_eq!(capture(&record("%1")).unwrap(), "%1 screen 0"); + assert_eq!(capture(&record("%1")).unwrap(), "%1 screen 1"); + let seen = observing(|| { + let first = capture(&record("%1")).unwrap(); + // A nested observation, and the effort footer's reader, share it. + let nested = observing(|| String::from_utf8(plain("%1").unwrap()).unwrap()); + let other = capture(&record("%2")).unwrap(); + // A failure is not kept: the next reader tries again. + assert!(capture(&record("%9")).is_err()); + assert!(plain("%9").is_err()); + (first, nested, other, capture(&record("%1")).unwrap()) + }); + assert_eq!(seen, ("%1 screen 2".into(), "%1 screen 2".into(), "%2 screen 3".into(), "%1 screen 2".into())); + assert_eq!(CAPTURES.load(SeqCst), 6); + // The observation ended: reads are fresh again. + assert_eq!(capture(&record("%1")).unwrap(), "%1 screen 6"); + }); + } +} diff --git a/src/state/mod.rs b/src/state/mod.rs index 9184842ef..a05a35f30 100644 --- a/src/state/mod.rs +++ b/src/state/mod.rs @@ -65,6 +65,7 @@ mod resume_input; mod scan_cache; mod schedules; mod waits; +pub(crate) mod watch_source; mod search; pub(crate) mod session_graph; mod storage; @@ -408,7 +409,7 @@ pub fn merge_listing_archives(mut box_value: Value, ids: &BTreeMap { + marks: &'a Value, + preferences: Option<&'a notifications::Store>, + /// Bounds question indexing for the whole listing. + budget: std::time::Instant, +} + +/// The row of a live session whose pane runs its saved binding `record`. +/// `hgs watch` re-derives single rows through this same step +/// (`watch_source::rows`). +fn live_row(session: &mut Value, record: Value, shared: &Shared, process_work: &mut processes::WorkSample) -> Result<()> { + // An approval answered in Terminal sends no hook. + let record = claude_approval::reconcile(record); + session["resumable"] = json!( + !string(&record, "conversation_id").is_empty() + && string(&record, "error").is_empty() + ); + let summary = journal::summary(&record, true); + session["process_work"] = process_work.summary(&record, summary["activity"] == "idle" && summary["phase"] == "idle"); + session + .as_object_mut() + .ok_or("invalid session")? + .extend(summary.as_object().ok_or("invalid summary")?.clone()); + let questions = read_marks::pending_questions(&record, session, true, shared.budget); + read_marks::enrich_listing(session, shared.marks, journal::journal_name(&record), questions); + notifications::enrich(session, shared.preferences, journal::journal_name(&record)); + Ok(()) +} + +/// The last steps of every listed row. `names` holds the session names of the +/// pane snapshot by session ID. +fn finish_rows(sessions: &mut [Value], marks: &Value, names: &BTreeMap) { // DeepSeek and untracked live sessions are owned by their names. for session in sessions.iter_mut() { if session.get("attention_signature").is_none() { let owner = string(session, "name").to_owned(); let questions=if session["agent"]=="opencode" {session["pending_questions"].as_array().cloned().unwrap_or_default()}else{Vec::new()}; - read_marks::enrich_listing(session, &marks, &owner, Ok(Some(questions))); + read_marks::enrich_listing(session, marks, &owner, Ok(Some(questions))); } } @@ -496,9 +519,25 @@ pub fn merge_listing_archives(mut box_value: Value, ids: &BTreeMap Vec { + listing.sessions.iter().map(|row| { + let name = row.name.as_str(); + let mut parts = name.splitn(3, '/'); let cmd = parts.next().unwrap_or(""); + let project = parts.next().filter(|s| !s.is_empty()); let tag = parts.next().filter(|s| !s.is_empty()); + let ttys: Vec<&str> = listing.clients.iter().filter_map(|client| if client.session == name { Some(client.terminal.as_str()) } else { None }).collect(); + json!({"name":name,"cmd":cmd,"project":project,"tag":tag,"attached":row.attached,"clients":ttys,"created":row.created,"launch_id":row.launch_id}) + }).collect() +} + +/// Each listed session's immutable ID (`$N`) by its listed name. +pub(crate) fn listed_ids(listing: &backend::Listing) -> BTreeMap { + listing.sessions.iter().map(|row| (row.name.clone(), row.id.clone())).collect() } /// Select a tracked conversation or explicit pending request by exact ID/home. diff --git a/src/state/watch_source.rs b/src/state/watch_source.rs new file mode 100644 index 000000000..bdc43c264 --- /dev/null +++ b/src/state/watch_source.rs @@ -0,0 +1,314 @@ +//! What `hgs watch` (src/watch) reads from session state: inspections and the +//! Activity journal exactly as `hgs inspect` reads them, as values instead of +//! printed output. Nothing here writes session state beyond what `hgs inspect` +//! and `hgs ls` already do. +use super::*; +use rusqlite::{Connection, OpenFlags}; +use std::collections::BTreeSet; +use std::time::Duration; + +/// The session state folder. +pub(crate) fn state_root() -> PathBuf { + root() +} + +/// Whether `name` is a DeepSeek or OpenCode session. Their Activity comes from +/// the agent's own host and each read replaces the previous window. +pub(crate) fn native(name: &str) -> bool { + dsh::exists(name) || opencode::exists(name) +} + +/// `hgs inspect NAME --after AFTER --skip-processes`. +pub(crate) fn inspect(name: &str, after: i64) -> Result { + local_sessions()?; + if opencode::exists(name) { + return opencode::inspect(name, None); + } + if dsh::exists(name) { + return dsh::inspect(name, None); + } + journal::inspection(name, after, None, false) +} + +/// A read-only connection to the Activity journal, once it exists. It never +/// creates the file or its schema, and holds no lock between statements. +pub(crate) fn journal() -> Option { + local_sessions().ok()?; + let path = root().join("events.sqlite3"); + if !path.is_file() { + return None; + } + let db = Connection::open_with_flags( + &path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .ok()?; + db.busy_timeout(Duration::from_millis(250)).ok()?; + Some(db) +} + +/// Changes whenever another connection commits to the journal. +pub(crate) fn data_version(db: &Connection) -> Result { + db.query_row("PRAGMA data_version", [], |row| row.get(0)) + .map_err(|e| e.to_string()) +} + +/// The oldest and newest retained `seq` (0 when the journal is empty). +pub(crate) fn bounds(db: &Connection) -> Result<(i64, i64)> { + db.query_row( + "SELECT COALESCE(MIN(seq),0), COALESCE(MAX(seq),0) FROM events", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .map_err(|e| e.to_string()) +} + +/// The journal names with rows after `after`, from at most `limit` rows. +pub(crate) fn changed_names(db: &Connection, after: i64, limit: usize) -> Result> { + let mut statement = db + .prepare("SELECT DISTINCT name FROM (SELECT name FROM events WHERE seq>?1 ORDER BY seq LIMIT ?2)") + .map_err(|e| e.to_string())?; + let names = statement + .query_map(rusqlite::params![after, limit as i64], |row| row.get::<_, Option>(0)) + .map_err(|e| e.to_string())? + .filter_map(|row| row.ok().flatten()) + .collect(); + Ok(names) +} + +/// The rows a tracked session's Activity shows: its journal name and the +/// conversations its timeline spans, taken from its saved binding. +pub(crate) struct Timeline { + record: Value, + earlier: String, +} + +impl Timeline { + /// The journal name, which a rename keeps. + pub(crate) fn journal(&self) -> &str { + journal::journal_name(&self.record) + } + + pub(crate) fn run_id(&self) -> &str { + string(&self.record, "run_id") + } + + /// Events after `after` as `inspect` shapes them, oldest first: the first + /// `limit`, or with `newest` the last `limit`. + pub(crate) fn events(&self, db: &Connection, after: i64, limit: usize, newest: bool) -> Result> { + let rows = journal::timeline_rows( + db, + self.journal(), + self.record["conversation_id"].as_str(), + &self.earlier, + after, + limit, + newest, + )?; + rows.into_iter() + .map(|(seq, payload)| journal::timeline_event(&self.record, seq, &payload)) + .collect() + } +} + +/// The timeline of a tracked session; None for a session without a saved +/// binding (DeepSeek, OpenCode and untracked sessions). +pub(crate) fn timeline(name: &str) -> Result> { + local_sessions()?; + if !record_path(name).exists() { + return Ok(None); + } + let record = read(name)?; + let earlier = clear_context::earlier_conversations(&record).to_string(); + Ok(Some(Timeline { record, earlier })) +} + +/// Whether a run is in Archive. Rare: only for a catalog row that disappeared. +pub(crate) fn archived(run_id: &str) -> bool { + if run_id.is_empty() || local_sessions().is_err() || !root().join("archives").is_dir() { + return false; + } + let Ok(_guard) = lock(None) else { return false }; + let Ok(db) = archive_catalog::catalog() else { return false }; + db.query_row( + "SELECT EXISTS(SELECT 1 FROM archive_rows WHERE run=?1)", + [run_id], + |row| row.get::<_, bool>(0), + ) + .unwrap_or(false) +} + +/// Whether `name` still has a saved binding of any agent. +pub(crate) fn saved(name: &str) -> bool { + local_sessions().is_ok() && (record_path(name).exists() || native(name)) +} + +// ---- Rows between listings (zerus-604d.6) ------------------------------------ +// +// A whole `ls --json --local --no-archives` summarizes every session. Between +// such listings `hgs watch` derives again only the live tracked rows whose +// inputs changed, through the same steps (`live_row`, `finish_rows`) and under +// the same lock, and probes what no file reports. + +/// What the terminal showed a derivation: its listed sessions with their IDs, +/// attached clients, creation and launch, and every pane with its run marker. +/// A session that starts, ends, is renamed, attached or detached changes it. +fn structure(sessions: &[Value], ids: &BTreeMap, snapshot: &Snapshot, names: &BTreeMap) -> u64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + json!([sessions, ids, snapshot, names]).to_string().hash(&mut hasher); + hasher.finish() +} + +/// The terminal's rows and session IDs, as a listing starts from them. +fn terminal() -> Result<(Vec, BTreeMap)> { + let listing = backend::current().listing()?; + Ok((listed_rows(&listing), listed_ids(&listing))) +} + +/// The terminal's structure now. A listing that starts later sees this one or +/// a newer one, so taking it first never hides a change. +pub(crate) fn structure_now() -> Result { + local_sessions()?; + let (sessions, ids) = terminal()?; + let (snapshot, names) = live_sessions()?; + Ok(structure(&sessions, &ids, &snapshot, &names)) +} + +/// The rows `ls --json --local --no-archives` lists now for the live tracked +/// sessions in `wanted`, with the terminal structure they saw. None when only +/// a whole listing can tell: the panes no longer show exactly the listed +/// sessions under their listed names (one started, ended or was renamed), or +/// a wanted session does not run its saved binding in its pane. +pub(crate) fn rows(wanted: &BTreeSet) -> Result, u64)>> { + local_sessions()?; + if !root().exists() { + return Ok(None); + } + let (sessions, ids) = terminal()?; + // As in a listing: the panes and the bindings are read under the lock, so + // a rename is seen before it began or after it finished (zerus-gd02). + let (snapshot, names, records) = { + let _guard = lock(None)?; + let (snapshot, names) = live_sessions()?; + let mut records = Vec::with_capacity(wanted.len()); + for name in wanted { + match read(name) { + Ok(record) if record["rename_shadow"] != true => records.push(record), + _ => return Ok(None), + } + } + (snapshot, names, records) + }; + let shape = structure(&sessions, &ids, &snapshot, &names); + let consistent = sessions.len() == snapshot.len() + && sessions.iter().all(|session| { + let listed = string(session, "name"); + match ids.get(listed).and_then(|id| names.get(id)) { + Some(name) => name == listed, + None => snapshot.contains_key(listed), + } + }); + if !consistent { + return Ok(None); + } + let marks = read_marks::load(); + let preferences = notifications::load(); + let shared = Shared { marks: &marks, preferences: preferences.as_ref(), budget: read_marks::question_budget() }; + let mut process_work = processes::WorkSample::default(); + let mut rows = Vec::with_capacity(records.len()); + for record in records { + let name = string(&record, "name"); + let Some(session) = sessions.iter().find(|session| string(session, "name") == name) else { + return Ok(None); + }; + if !matches(&record, snapshot.get(name)) { + return Ok(None); + } + let mut session = session.clone(); + session + .as_object_mut() + .ok_or("invalid session")? + .extend(telemetry::unavailable().as_object().unwrap().clone()); + live_row(&mut session, record, &shared, &mut process_work)?; + rows.push(session); + } + finish_rows(&mut rows, &marks, &names); + Ok(Some((rows, shape))) +} + +/// The DeepSeek and OpenCode rows of `ls --json --local --no-archives`. Their +/// hosts answer them, so no file shows their changes. +pub(crate) fn native_rows() -> Result> { + local_sessions()?; + let mut rows = dsh::snapshots()?; + rows.extend(opencode::snapshots()?); + if !rows.is_empty() { + let (_, names) = live_sessions()?; + finish_rows(&mut rows, &read_marks::load(), &names); + } + Ok(rows) +} + +/// What a probe between listings found. +pub(crate) struct Probe { + /// The terminal's structure (`structure_now`). + pub(crate) structure: u64, + /// Live tracked rows to derive again. + pub(crate) stale: Vec, +} + +/// `process_work` without its sampling time. +fn work(mut value: Value) -> Value { + if let Some(fields) = value.as_object_mut() { + fields.remove("sampled_at"); + } + value +} + +/// Compares the listed `rows` with what no file reports: the terminal's +/// structure and, for each live tracked session, its agent process and the +/// work that process owns, from one process sample. A session that has not +/// reported a hook event yet is always derived again: startup choosers have +/// no hook. It takes no lock; what it finds only asks for rows that do. +pub(crate) fn probe(rows: &[&Value]) -> Result { + local_sessions()?; + let (sessions, ids) = terminal()?; + let (snapshot, names) = live_sessions()?; + let mut sample = processes::WorkSample::default(); + let mut stale = Vec::new(); + for row in rows { + let name = string(row, "name"); + if row["tracked"] != true || !snapshot.contains_key(name) || native(name) { + continue; + } + let Ok(record) = read(name) else { + stale.push(name.to_owned()); + continue; + }; + let quiet = string(&record, "conversation_id").is_empty() + || record["last_event_at"].as_f64().unwrap_or(0.) == 0.; + let idle = row["activity"] == "idle" && row["phase"] == "idle"; + // As the summary decides `terminal_supported` of a live pane. + let terminal = string(&record, "archive_id").is_empty() && process_alive(&record); + if quiet + || !matches(&record, snapshot.get(name)) + || row["terminal_supported"] != json!(terminal) + || work(row["process_work"].clone()) != work(sample.summary(&record, idle)) + { + stale.push(name.to_owned()); + } + } + Ok(Probe { structure: structure(&sessions, &ids, &snapshot, &names), stale }) +} + +/// The session a binding file holds and the journal name its Activity rows +/// carry; None for a rename's shadow or a file that is no binding. +pub(crate) fn binding_identity(path: &Path) -> Option<(String, String)> { + let value: Value = serde_json::from_slice(&std::fs::read(path).ok()?).ok()?; + let name = string(&value, "name"); + if value["rename_shadow"] == true || name.is_empty() { + return None; + } + Some((name.to_owned(), journal::journal_name(&value).to_owned())) +} diff --git a/src/watch/activity.rs b/src/watch/activity.rs new file mode 100644 index 000000000..57734dd2b --- /dev/null +++ b/src/watch/activity.rs @@ -0,0 +1,229 @@ +//! Activity subscriptions: the inspection summary as `state`, journal events +//! as `events`, and bounds that keep every frame under the frame limit. +use super::catalog::{self, Digest32}; +use serde_json::{json, Value}; +use std::time::Instant; +use zerus_watch_protocol as protocol; + +/// Sections of an inspection that `state` leaves out: Activity rows travel as +/// `events`, and process sets belong to a later topic. +const NOT_STATE: [&str; 6] = ["events", "processes", "process_work", "cursor", "history_truncated", "replace_events"]; +const VOLATILE_STATE: [&[&str]; 2] = [&["goal_observed_at"], &["process_work", "sampled_at"]]; + +/// One session the client follows. +pub(super) struct Subscription { + pub key: String, + /// The current name; a rename keeps the key and updates the name. + pub name: String, + pub run_id: Option, + /// DeepSeek and OpenCode: each read replaces the event window. + pub native: bool, + pub cursor: i64, + pub tail: usize, + pub rev: u64, + pub state: Option, + pub events: Option, + pub refreshed: Instant, + /// A source changed: refresh the summary soon. + pub dirty: bool, + /// The last summary showed a busy agent: refresh it more often. + pub busy: bool, + /// The last journal read was full: read again on the next pulse. + pub more: bool, + /// Frames were dropped: send `gap` and a tail snapshot once drained. + pub resync: Option<&'static str>, + /// The catalog listed the key at least once. + pub listed: bool, + /// The digest of the session's catalog row when last seen. + pub row: Option, + /// The journal name its rows carry; a rename keeps it. + pub journal: Option, +} + +impl Subscription { + pub(super) fn new(key: String, name: String, run_id: Option, native: bool, tail: usize) -> Self { + Self { + key, + name, + run_id, + native, + cursor: 0, + tail, + rev: 0, + state: None, + events: None, + refreshed: Instant::now(), + dirty: false, + busy: false, + more: false, + resync: None, + listed: false, + row: None, + journal: None, + } + } +} + +/// The inspection without its event and process sections. +pub(super) fn state_value(inspection: &Value) -> Value { + let mut value = inspection.clone(); + if let Some(object) = value.as_object_mut() { + for key in NOT_STATE { + object.remove(key); + } + } + value +} + +/// A summary's digest with its observation timestamps masked. +pub(super) fn state_digest(value: &Value) -> Digest32 { + catalog::digest(&catalog::masked(value, &VOLATILE_STATE)) +} + +pub(super) fn busy(value: &Value) -> bool { + value["activity"] == "busy" +} + +fn encoded_len(value: &Value) -> usize { + serde_json::to_string(value).map_or(usize::MAX, |text| text.len()) +} + +/// Cuts a string to at most `bytes`, on a character boundary. +fn cut(text: &str, bytes: usize) -> String { + let mut end = bytes.min(text.len()); + while !text.is_char_boundary(end) { + end -= 1; + } + format!("{}…", &text[..end]) +} + +/// Fits one event in `limit` encoded bytes by cutting its longest strings, +/// marked with `"truncated": true`; the whole event remains readable through +/// `hgs inspect` and `hgs history`. +pub(super) fn shrink_event(event: &mut Value, limit: usize) { + for _ in 0..16 { + let size = encoded_len(event); + if size <= limit { + return; + } + let Some(object) = event.as_object_mut() else { break }; + let longest = object + .iter() + .filter_map(|(key, value)| value.as_str().map(|text| (key.clone(), text.len()))) + .max_by_key(|(_, len)| *len); + let Some((key, len)) = longest.filter(|(_, len)| *len > 64) else { break }; + let keep = len.saturating_sub(size - limit + 64); + let text = cut(object[&key].as_str().unwrap_or(""), keep); + object.insert(key, json!(text)); + object.insert("truncated".into(), json!(true)); + } + if encoded_len(event) > limit { + let keep = |key: &str| event.get(key).cloned().unwrap_or(Value::Null); + *event = json!({"seq":keep("seq"),"type":keep("type"),"at":keep("at"),"truncated":true}); + } +} + +/// Fits a summary in `limit` encoded bytes: the largest lists lose their oldest +/// half until it fits, and `stream_truncated` names them. A client reads the +/// whole summary with `hgs inspect`. +pub(super) fn shrink_state(value: &mut Value, limit: usize) { + let mut cut_lists = Vec::new(); + for _ in 0..64 { + if encoded_len(value) <= limit { + break; + } + let Some(object) = value.as_object_mut() else { break }; + let largest = object + .iter() + .filter(|(_, value)| value.as_array().is_some_and(|items| !items.is_empty())) + .map(|(key, value)| (key.clone(), encoded_len(value))) + .max_by_key(|(_, len)| *len); + let Some((key, _)) = largest else { break }; + if let Some(items) = object.get_mut(&key).and_then(Value::as_array_mut) { + let drop = items.len().div_ceil(2); + items.drain(..drop); + } + if !cut_lists.contains(&key) { + cut_lists.push(key); + } + } + if encoded_len(value) > limit { + let keep = |key: &str| value.get(key).cloned().unwrap_or(Value::Null); + *value = json!({"name":keep("name"),"run_id":keep("run_id"),"conversation_id":keep("conversation_id"), + "tracked":keep("tracked"),"stream_truncated":true}); + } else if !cut_lists.is_empty() { + value["stream_truncated"] = json!(cut_lists); + } +} + +/// Splits events into frames of at most `limit` encoded bytes of events, +/// cutting any single event that alone exceeds it. +pub(super) fn event_pages(events: Vec, limit: usize) -> Vec> { + let events = events + .into_iter() + .map(|mut event| { + shrink_event(&mut event, limit); + event + }) + .collect(); + catalog::pages(events, encoded_len, limit) +} + +/// The largest event or summary payload one frame carries. +pub(super) const PAYLOAD_LIMIT: usize = protocol::MAX_FRAME_BYTES - 64 * 1024; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn state_leaves_out_events_processes_and_cursors_and_ignores_observation_times() { + let inspection = json!({"name":"codex/p/t","phase":"idle","activity":"idle","events":[{"seq":1}], + "processes":{"items":[]},"cursor":1,"history_truncated":false,"goal_observed_at":5.0,"message_events":[]}); + let value = state_value(&inspection); + assert_eq!(value, json!({"name":"codex/p/t","phase":"idle","activity":"idle","goal_observed_at":5.0,"message_events":[]})); + let mut later = value.clone(); + later["goal_observed_at"] = json!(6.0); + assert_eq!(state_digest(&value), state_digest(&later)); + later["phase"] = json!("working"); + assert_ne!(state_digest(&value), state_digest(&later)); + assert!(!busy(&value)); + assert!(busy(&json!({"activity":"busy"}))); + } + + #[test] + fn oversized_events_are_cut_with_a_marker_and_paged() { + let mut event = json!({"seq":7,"type":"Stop","at":1.0,"detail":"é".repeat(400_000)}); + shrink_event(&mut event, 10_000); + assert!(encoded_len(&event) <= 10_000); + assert_eq!(event["truncated"], true); + assert_eq!(event["seq"], 7); + assert!(event["detail"].as_str().unwrap().ends_with('…')); + let mut small = json!({"seq":8,"detail":"short"}); + shrink_event(&mut small, 10_000); + assert_eq!(small, json!({"seq":8,"detail":"short"})); + let mut bulky = json!({"seq":9,"type":"X","at":2.0,"list":vec![1; 5_000]}); + shrink_event(&mut bulky, 1_000); + assert_eq!(bulky, json!({"seq":9,"type":"X","at":2.0,"truncated":true})); + let events = (0..10).map(|seq| json!({"seq":seq,"detail":"x".repeat(300)})).collect(); + let pages = event_pages(events, 1_000); + assert!(pages.len() >= 4); + assert!(pages.iter().all(|page| page.iter().map(encoded_len).sum::() <= 1_000)); + assert_eq!(pages.concat().len(), 10); + } + + #[test] + fn oversized_summaries_drop_their_oldest_list_items() { + let messages: Vec<_> = (0..200).map(|n| json!({"n":n,"text":"m".repeat(100)})).collect(); + let mut value = json!({"name":"codex/p/t","run_id":"r","provider_messages":messages,"phase":"idle"}); + shrink_state(&mut value, 8_000); + assert!(encoded_len(&value) <= 8_000); + assert_eq!(value["stream_truncated"], json!(["provider_messages"])); + let kept = value["provider_messages"].as_array().unwrap(); + assert_eq!(kept.last().unwrap()["n"], 199); + let mut text = json!({"name":"n","run_id":"r","prompt":"p".repeat(20_000)}); + shrink_state(&mut text, 1_000); + assert_eq!(text["stream_truncated"], true); + assert_eq!(text["name"], "n"); + } +} diff --git a/src/watch/catalog.rs b/src/watch/catalog.rs new file mode 100644 index 000000000..0e32f531c --- /dev/null +++ b/src/watch/catalog.rs @@ -0,0 +1,231 @@ +//! Catalog rows: keys, change detection with volatile observation fields +//! masked, removal reasons and paging. +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use zerus_watch_protocol::{self as protocol, reason}; + +/// Observation timestamps that change on every listing without any change in +/// the session. Change detection ignores them; the rows still carry them. +const VOLATILE_ROW: [&[&str]; 2] = [&["goal_observed_at"], &["process_work", "sampled_at"]]; +/// Meta fields that change on every listing; sent at most every 10 s. +pub(super) const VOLATILE_META: [&[&str]; 2] = [&["metrics"], &["desktop_presence"]]; + +pub(super) type Digest32 = [u8; 32]; + +/// `value` without the fields at `paths`. +pub(super) fn masked(value: &Value, paths: &[&[&str]]) -> Value { + let mut value = value.clone(); + for path in paths { + let Some((last, parents)) = path.split_last() else { continue }; + let mut node = Some(&mut value); + for parent in parents { + node = node.and_then(|node| node.get_mut(*parent)); + } + if let Some(object) = node.and_then(Value::as_object_mut) { + object.remove(*last); + } + } + value +} + +pub(super) fn digest(value: &Value) -> Digest32 { + Sha256::digest(value.to_string().as_bytes()).into() +} + +/// A row's digest with its volatile fields masked. +pub(super) fn row_digest(row: &Value) -> Digest32 { + digest(&masked(row, &VOLATILE_ROW)) +} + +/// A row's key: its run, or its name for a row without one. +pub(super) fn key(row: &Value) -> String { + protocol::catalog_key(row).unwrap_or_else(|| format!("name:{}", row.get("name").and_then(Value::as_str).unwrap_or(""))) +} + +/// The listing's rows with their keys, and its other fields. Two rows with the +/// same key (never expected) keep apart: the later one falls back to its name, +/// then to its position. +pub(super) fn split(mut listing: Value) -> (Vec<(String, Value)>, Value) { + let sessions = listing + .as_object_mut() + .and_then(|object| object.remove("sessions")) + .and_then(|sessions| match sessions { + Value::Array(rows) => Some(rows), + _ => None, + }) + .unwrap_or_default(); + let mut seen = BTreeSet::new(); + let mut rows = Vec::with_capacity(sessions.len()); + for (index, row) in sessions.into_iter().enumerate() { + let name = row.get("name").and_then(Value::as_str).unwrap_or("").to_owned(); + let mut key = key(&row); + if seen.contains(&key) { + key = protocol::key_for(None, &name); + } + if seen.contains(&key) { + key = format!("name:{name}#{index}"); + } + seen.insert(key.clone()); + rows.push((key, row)); + } + (rows, listing) +} + +/// What the client holds of one row. +#[derive(Clone, Debug, PartialEq)] +pub(super) struct Sent { + pub name: String, + pub run_id: String, + pub digest: Digest32, +} + +impl Sent { + pub(super) fn of(row: &Value, digest: Digest32) -> Self { + let text = |key| row.get(key).and_then(Value::as_str).unwrap_or("").to_owned(); + Self { name: text("name"), run_id: text("run_id"), digest } + } +} + +/// Rows to upsert (indices into the current rows) and keys to remove. +#[derive(Debug, Default, PartialEq)] +pub(super) struct Change { + pub upsert: Vec, + pub remove: Vec, +} + +impl Change { + pub(super) fn is_empty(&self) -> bool { + self.upsert.is_empty() && self.remove.is_empty() + } +} + +pub(super) fn diff(sent: &BTreeMap, rows: &[(String, Value)], digests: &[Digest32]) -> Change { + let mut change = Change::default(); + for (index, ((key, _), digest)) in rows.iter().zip(digests).enumerate() { + if sent.get(key).is_none_or(|old| old.digest != *digest) { + change.upsert.push(index); + } + } + let current: BTreeSet<&str> = rows.iter().map(|(key, _)| key.as_str()).collect(); + change.remove = sent.keys().filter(|key| !current.contains(key.as_str())).cloned().collect(); + change +} + +/// Why `old` left the catalog: its name lists under another run (replaced), +/// its run is archived, its binding is gone (forgotten), or it ended. +pub(super) fn removal_reason( + old: &Sent, + listed_names: &BTreeSet<&str>, + archived: impl Fn(&str) -> bool, + saved: impl Fn(&str) -> bool, +) -> &'static str { + if listed_names.contains(old.name.as_str()) { + return reason::REPLACED; + } + if old.run_id.is_empty() { + return reason::ENDED; + } + if archived(&old.run_id) { + return reason::ARCHIVED; + } + if !saved(&old.name) { + return reason::FORGOTTEN; + } + reason::ENDED +} + +/// Splits `items` into pages of at most `limit` bytes by `size`. A single item +/// larger than `limit` gets a page of its own. +pub(super) fn pages(items: Vec, size: impl Fn(&T) -> usize, limit: usize) -> Vec> { + let mut pages = Vec::new(); + let mut page = Vec::new(); + let mut bytes = 0; + for item in items { + let item_bytes = size(&item) + 1; + if !page.is_empty() && bytes + item_bytes > limit { + pages.push(std::mem::take(&mut page)); + bytes = 0; + } + bytes += item_bytes; + page.push(item); + } + if !page.is_empty() || pages.is_empty() { + pages.push(page); + } + pages +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn row(name: &str, run: &str, phase: &str, observed: f64) -> Value { + json!({"name":name,"run_id":run,"phase":phase,"goal_observed_at":observed, + "process_work":{"count":0,"sampled_at":observed}}) + } + + fn sent_of(rows: &[(String, Value)]) -> BTreeMap { + rows.iter().map(|(key, row)| (key.clone(), Sent::of(row, row_digest(row)))).collect() + } + + #[test] + fn observation_timestamps_alone_change_nothing() { + let a = row("codex/p/t", "r1", "idle", 1.0); + let b = row("codex/p/t", "r1", "idle", 2.0); + assert_ne!(a, b); + assert_eq!(row_digest(&a), row_digest(&b)); + assert_ne!(row_digest(&a), row_digest(&row("codex/p/t", "r1", "working", 1.0))); + // The rows themselves keep the fields. + assert_eq!(masked(&a, &VOLATILE_ROW)["process_work"], json!({"count":0})); + assert_eq!(a["process_work"]["sampled_at"], 1.0); + assert_eq!(masked(&json!({"process_work":null}), &VOLATILE_ROW), json!({"process_work":null})); + } + + #[test] + fn rows_key_by_run_then_name_and_duplicates_stay_apart() { + let listing = json!({"host":"h","ok":true,"sessions":[ + {"name":"codex/p/a","run_id":"r1"},{"name":"sh/p"},{"name":"codex/p/b","run_id":"r1"},{"name":"sh/p"}]}); + let (rows, meta) = split(listing); + let keys: Vec<_> = rows.iter().map(|(key, _)| key.as_str()).collect(); + assert_eq!(keys, ["run:r1", "name:sh/p", "name:codex/p/b", "name:sh/p#3"]); + assert_eq!(meta, json!({"host":"h","ok":true})); + assert_eq!(split(json!({"ok":false})).0, vec![]); + } + + #[test] + fn diff_upserts_changed_rows_and_removes_missing_keys() { + let (before, _) = split(json!({"sessions":[row("codex/p/a","r1","idle",1.), row("codex/p/b","r2","idle",1.), row("codex/p/c","r3","idle",1.)]})); + let sent = sent_of(&before); + let (after, _) = split(json!({"sessions":[row("codex/p/a","r1","idle",9.), row("codex/p/renamed","r2","idle",9.), row("codex/p/d","r4","idle",9.)]})); + let digests: Vec<_> = after.iter().map(|(_, row)| row_digest(row)).collect(); + let change = diff(&sent, &after, &digests); + // r1 changed only its timestamps; a rename upserts the same run key. + assert_eq!(change.upsert, [1, 2]); + assert_eq!(after[1].0, "run:r2"); + assert_eq!(change.remove, ["run:r3"]); + assert!(diff(&sent_of(&after), &after, &digests).is_empty()); + } + + #[test] + fn removals_explain_replacement_archive_forgetting_and_ending() { + let old = Sent { name: "codex/p/a".into(), run_id: "r1".into(), digest: [0; 32] }; + let none = BTreeSet::new(); + let listed: BTreeSet<&str> = ["codex/p/a"].into(); + assert_eq!(removal_reason(&old, &listed, |_| true, |_| true), reason::REPLACED); + assert_eq!(removal_reason(&old, &none, |run| run == "r1", |_| true), reason::ARCHIVED); + assert_eq!(removal_reason(&old, &none, |_| false, |_| false), reason::FORGOTTEN); + assert_eq!(removal_reason(&old, &none, |_| false, |_| true), reason::ENDED); + let untracked = Sent { name: "sh/p".into(), run_id: String::new(), digest: [0; 32] }; + assert_eq!(removal_reason(&untracked, &none, |_| true, |_| false), reason::ENDED); + } + + #[test] + fn pages_respect_the_byte_limit_and_isolate_oversized_items() { + let pages = pages(vec![4, 4, 4, 20, 4], |n| *n, 10); + assert_eq!(pages, vec![vec![4, 4], vec![4], vec![20], vec![4]]); + let empty: Vec> = super::pages(Vec::new(), |n: &usize| *n, 10); + assert_eq!(empty, vec![Vec::::new()]); + } +} diff --git a/src/watch/mod.rs b/src/watch/mod.rs new file mode 100644 index 000000000..482621411 --- /dev/null +++ b/src/watch/mod.rs @@ -0,0 +1,1178 @@ +//! `hgs [@host] watch --json`: one long-lived NDJSON event stream per computer +//! (ABI `watch-v1`, frames in `crates/watch-protocol`). The catalog topic +//! carries the rows of `hgs ls --json --local --no-archives`; Activity carries +//! what `hgs inspect` returns for subscribed sessions. Change detection is a +//! 250 ms pulse over cheap stamps and the journal's data version, on +//! `std::thread` only. A changed binding or journal row derives only its +//! session's row again; a 5 s probe watches what no file shows and the whole +//! listing runs every 5 s. See "Event stream" in docs/reference.md. +mod activity; +mod catalog; +mod stamps; +mod writer; + +use crate::cli::{Error, Result}; +use crate::config::Config; +use crate::state::watch_source as source; +use activity::Subscription; +use catalog::{Digest32, Sent}; +use serde_json::{json, Value}; +use std::collections::{BTreeMap, BTreeSet}; +use std::io::BufRead; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{mpsc, Arc}; +use std::time::{Duration, Instant}; +use writer::{Item, Writer}; +use zerus_watch_protocol::{self as protocol, reason, topic, Body, Control}; + +/// How often sources are checked. +const PULSE: Duration = Duration::from_millis(250); +/// A listing or a row refresh waits this long after a change, so a burst +/// derives once. +const DEBOUNCE: Duration = Duration::from_millis(150); +/// Listings, and row refreshes, start at least this far apart, and at least +/// four times the previous one's duration apart. +const LISTING_GAP: Duration = Duration::from_millis(500); +/// What no file shows is probed this often: a session that starts, ends, is +/// killed, renamed, attached or detached, an exited agent process, ended +/// background work, a session that has not reported a hook event yet +/// (startup choosers) and DeepSeek and OpenCode hosts. +const PROBE: Duration = Duration::from_secs(5); +/// The whole listing runs at least this often, for what neither a file nor +/// the probe shows: terminal choosers and status lines of sessions that have +/// reported hooks, and native transcripts between hook events. 15 s would +/// halve the watcher's CPU on a busy computer but show such a chooser up to +/// 15 s late; that tradeoff waits for the owner (zerus-604d.6). +const FULL: Duration = Duration::from_secs(5); +/// `catalog.meta` with only fresh metrics or presence goes out this rarely. +const META_INTERVAL: Duration = Duration::from_secs(10); +/// A changed session's summary is re-inspected at most this often. +const STATE_GAP: Duration = Duration::from_millis(500); +/// Unchanged summaries are re-inspected after this long: provider messages +/// between tools exist only in native transcripts. +const BUSY_REFRESH: Duration = Duration::from_secs(2); +const IDLE_REFRESH: Duration = Duration::from_secs(15); +/// DeepSeek and OpenCode answer from their own hosts. +const NATIVE_REFRESH: Duration = Duration::from_secs(2); +/// With more than this queued, catalog changes wait and coalesce. +const DEFER_BYTES: usize = protocol::QUEUE_BYTES / 4; +/// A repeated source failure is reported at most this often. +const ERROR_REPEAT: Duration = Duration::from_secs(60); +/// The largest page of catalog items. +const PAGE_LIMIT: usize = protocol::PAGE_BYTES - 4096; + +enum Input { + Line(String), + TooLong, + /// Control lines arrived faster than they were handled; some were dropped. + Dropped, + End, +} + +/// Control lines read but not yet handled; beyond it lines are dropped, so a +/// client that only writes cannot grow this process. +const PENDING_CONTROLS: usize = 256; + +pub(crate) fn run(config: &Config, args: &[String], dry: bool) -> Result { + if dry || args != ["--json"] { + return Err(Error::new(1, "usage: hgs [@host] watch --json (NDJSON frames on stdout, control operations on stdin)")); + } + let (inputs, received) = mpsc::channel(); + let closed = inputs.clone(); + let writer = Writer::spawn( + std::io::stdout(), + protocol::QUEUE_BYTES, + Duration::from_millis(protocol::HEARTBEAT_MS), + move || { + let _ = closed.send(Input::End); + }, + ); + let pending = Arc::new(AtomicUsize::new(0)); + let counted = Arc::clone(&pending); + let halt = writer.clone(); + std::thread::Builder::new() + .name("hgs-watch-control".into()) + // EOF stops the writer at once, so a wait for queue room ends too. + .spawn(move || read_controls(std::io::stdin().lock(), inputs, &counted, move || halt.halt())) + .map_err(|error| Error::new(1, format!("cannot read control operations: {error}")))?; + let mut watch = Watch::new(config, writer); + watch.hello(); + watch.list(Instant::now()); + watch.emit_catalog(); + let mut next_pulse = Instant::now() + PULSE; + loop { + let now = Instant::now(); + let deadline = [watch.listing_due, watch.refresh_due].into_iter().flatten().fold(next_pulse, Instant::min); + let input = received.recv_timeout(deadline.saturating_duration_since(now)); + if matches!(input, Ok(Input::Line(_) | Input::TooLong)) { + pending.fetch_sub(1, Ordering::SeqCst); + } + if watch.out.writer.stopped() { + return Ok(0); + } + match input { + Ok(Input::Line(text)) => watch.control(&text), + Ok(Input::TooLong) => watch.out.error( + "bad_request", + &format!("control operations are at most {} bytes", protocol::CONTROL_LINE_BYTES), + None, + None, + ), + Ok(Input::Dropped) => watch.out.error( + "bad_request", + &format!("more than {PENDING_CONTROLS} control operations were waiting; later ones were dropped"), + None, + None, + ), + Ok(Input::End) | Err(mpsc::RecvTimeoutError::Disconnected) => return Ok(0), + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + if watch.out.writer.stopped() { + return Ok(0); + } + let now = Instant::now(); + if now >= next_pulse { + next_pulse = now + PULSE; + watch.pulse(now); + } + if watch.listing_due.is_some_and(|due| now >= due) { + watch.list(now); + } + if watch.refresh_due.is_some_and(|due| now >= due) { + watch.refresh_rows(now); + } + watch.emit_catalog(); + watch.resync_activity(); + } +} + +/// Reads control lines until EOF, then runs `ended`. A line longer than the +/// limit is skipped and reported, never buffered whole; so are lines beyond +/// `PENDING_CONTROLS` not yet handled. +fn read_controls(mut input: R, inputs: mpsc::Sender, pending: &AtomicUsize, ended: impl FnOnce()) { + let mut line = Vec::new(); + let mut too_long = false; + let mut dropping = false; + loop { + let buffer = match input.fill_buf() { + Ok(buffer) => buffer, + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(_) => break, + }; + if buffer.is_empty() { + break; + } + let newline = buffer.iter().position(|byte| *byte == b'\n'); + let chunk = &buffer[..newline.unwrap_or(buffer.len())]; + if !too_long && line.len() + chunk.len() > protocol::CONTROL_LINE_BYTES { + too_long = true; + line.clear(); + } + if !too_long { + line.extend_from_slice(chunk); + } + let used = chunk.len() + usize::from(newline.is_some()); + input.consume(used); + if newline.is_some() { + let message = if too_long { + Input::TooLong + } else { + Input::Line(String::from_utf8_lossy(&line).into_owned()) + }; + line.clear(); + too_long = false; + if pending.load(Ordering::SeqCst) >= PENDING_CONTROLS { + if !dropping && inputs.send(Input::Dropped).is_err() { + break; + } + dropping = true; + continue; + } + dropping = false; + pending.fetch_add(1, Ordering::SeqCst); + if inputs.send(message).is_err() { + break; + } + } + } + if !line.is_empty() && !too_long && pending.load(Ordering::SeqCst) < PENDING_CONTROLS { + pending.fetch_add(1, Ordering::SeqCst); + let _ = inputs.send(Input::Line(String::from_utf8_lossy(&line).into_owned())); + } + ended(); + let _ = inputs.send(Input::End); +} + +/// What changed in the Activity journal since the previous pulse. +enum Journal { + Unchanged, + /// Rows of these journal names were added. + Names(BTreeSet), + /// Rows were added or the journal was replaced, for names not known. + Unknown, +} + +/// Frames out, with repeated source failures limited. +struct Out { + writer: Writer, + reported: BTreeMap, +} + +impl Out { + fn encode(body: &Body) -> Option { + protocol::encode_body(body).ok() + } + + /// Queues a frame unless the client fell behind (false). + fn offer(&self, body: &Body, coalesce: Option) -> bool { + match Self::encode(body) { + Some(body) => self.writer.offer(Item { coalesce, body }), + None => false, + } + } + + /// Queues a frame, waiting for room. + fn put(&self, body: &Body) -> bool { + match Self::encode(body) { + Some(body) => self.writer.put(Item { coalesce: None, body }), + None => false, + } + } + + fn error(&mut self, code: &str, message: &str, topic: Option<&str>, key: Option<&str>) { + let body = Body::Error(protocol::ErrorFrame { + code: code.into(), + message: message.chars().take(2048).collect(), + topic: topic.map(str::to_owned), + key: key.map(str::to_owned), + fatal: false, + }); + self.put(&body); + } + + /// An error of a source that is retried: once per minute per message. + fn failure(&mut self, code: &str, message: &str, topic: Option<&str>, key: Option<&str>) { + let id = format!("{code}\n{}\n{message}", key.unwrap_or("")); + let now = Instant::now(); + if self.reported.get(&id).is_some_and(|at| now.duration_since(*at) < ERROR_REPEAT) { + return; + } + self.reported.retain(|_, at| now.duration_since(*at) < ERROR_REPEAT); + self.reported.insert(id, now); + self.error(code, message, topic, key); + } +} + +struct Watch<'a> { + config: &'a Config, + out: Out, + root: PathBuf, + // The latest listing, and what the client holds of it. + rows: Vec<(String, Value)>, + digests: Vec, + meta: Value, + listed: bool, + pending: bool, + sent: BTreeMap, + revision: i64, + snapshot_sent: bool, + catalog_resync: bool, + meta_sent: Option<(Digest32, Digest32, Instant)>, + // Listing schedule. + scan: stamps::Scan, + /// Each binding file's session and journal names (None: no binding). + identities: BTreeMap>, + /// The terminal's structure when the rows were derived. + structure: Option, + listing_due: Option, + listed_at: Option, + listing_cost: Duration, + // Row refreshes between listings. + stale: BTreeSet, + stale_native: bool, + refresh_due: Option, + refreshed_at: Option, + refresh_cost: Duration, + probed_at: Option, + // Activity. + journal: Option, + data_version: Option, + journal_max: i64, + subs: Vec, +} + +impl<'a> Watch<'a> { + fn new(config: &'a Config, writer: Writer) -> Self { + let root = source::state_root(); + // Stamps are taken before the first listing: a change during it lists again. + let scan = stamps::scan(&root); + let identities = scan.bindings.keys().map(|path| (path.clone(), source::binding_identity(path))).collect(); + Self { + config, + out: Out { writer, reported: BTreeMap::new() }, + root, + rows: Vec::new(), + digests: Vec::new(), + meta: json!({}), + listed: false, + pending: false, + sent: BTreeMap::new(), + revision: 0, + snapshot_sent: false, + catalog_resync: false, + meta_sent: None, + scan, + identities, + structure: None, + listing_due: None, + listed_at: None, + listing_cost: Duration::ZERO, + stale: BTreeSet::new(), + stale_native: false, + refresh_due: None, + refreshed_at: None, + refresh_cost: Duration::ZERO, + probed_at: None, + journal: None, + data_version: None, + journal_max: 0, + subs: Vec::new(), + } + } + + fn hello(&mut self) { + let hello = protocol::Hello { + machine_id: crate::swarm::local_node_id(self.config).ok(), + host: self.config.host().into(), + hgs_version: env!("CARGO_PKG_VERSION").into(), + stream: uuid::Uuid::new_v4().to_string(), + heartbeat_ms: protocol::HEARTBEAT_MS, + max_frame_bytes: protocol::MAX_FRAME_BYTES, + topics: [topic::CATALOG, topic::CATALOG_META, topic::ACTIVITY].map(str::to_owned).to_vec(), + limits: protocol::Limits::default(), + }; + self.out.put(&Body::Hello(hello)); + self.open_journal(); + } + + // ---- Catalog ----------------------------------------------------------- + + fn schedule(&mut self, at: Instant) { + let gap = LISTING_GAP.max(self.listing_cost * 4); + let due = self.listed_at.map_or(at, |started| at.max(started + gap)); + self.listing_due = Some(self.listing_due.map_or(due, |current| current.min(due))); + } + + fn schedule_refresh(&mut self, at: Instant) { + let gap = LISTING_GAP.max(self.refresh_cost * 4); + let due = self.refreshed_at.map_or(at, |started| at.max(started + gap)); + self.refresh_due = Some(self.refresh_due.map_or(due, |current| current.min(due))); + } + + /// Re-runs the `ls --json --local --no-archives` builder. + fn list(&mut self, now: Instant) { + self.listing_due = None; + self.listed_at = Some(now); + // The listing covers every row refresh and probe asked for so far. + self.probed_at = Some(now); + self.stale.clear(); + self.stale_native = false; + self.refresh_due = None; + // The structure is taken first: a change during the listing lists again. + self.structure = source::structure_now().ok(); + match crate::cli::local_listing(self.config) { + Ok(listing) => { + let (rows, meta) = catalog::split(listing); + self.digests = rows.iter().map(|(_, row)| catalog::row_digest(row)).collect(); + self.rows = rows; + self.meta = meta; + self.listed = true; + self.pending = true; + } + Err(error) => self.out.failure("listing_failed", &error.message, Some(topic::CATALOG), None), + } + self.listing_cost = now.elapsed(); + } + + /// Derives again the rows whose inputs changed since they were listed; a + /// whole listing follows when only it can tell. + fn refresh_rows(&mut self, now: Instant) { + self.refresh_due = None; + self.refreshed_at = Some(now); + let wanted = std::mem::take(&mut self.stale); + let native = std::mem::take(&mut self.stale_native); + match self.derive(&wanted, native) { + Ok(true) => {} + Ok(false) => self.schedule(now), + Err(error) => { + self.out.failure("listing_failed", &error, Some(topic::CATALOG), None); + self.schedule(now); + } + } + self.refresh_cost = now.elapsed(); + } + + /// Replaces the rows of the live tracked sessions in `wanted` and, with + /// `native`, the DeepSeek and OpenCode rows. False when a whole listing + /// must decide: the terminal changed, a session is no longer live under + /// its binding, or a row would change its key or its place. + fn derive(&mut self, wanted: &BTreeSet, native: bool) -> std::result::Result { + if !self.listed { + return Ok(false); + } + let mut rows = Vec::new(); + if !wanted.is_empty() { + let Some((derived, structure)) = source::rows(wanted)? else { return Ok(false) }; + if Some(structure) != self.structure { + return Ok(false); + } + rows = derived; + } + if native { + let derived = source::native_rows()?; + let listed: BTreeSet<&str> = self + .rows + .iter() + .filter(|(_, row)| row["name"].as_str().is_some_and(source::native)) + .map(|(key, _)| key.as_str()) + .collect(); + let keys: Vec = derived.iter().map(catalog::key).collect(); + if keys.len() != listed.len() || keys.iter().any(|key| !listed.contains(key.as_str())) { + return Ok(false); + } + rows.extend(derived); + } + let mut updates = Vec::with_capacity(rows.len()); + for row in rows { + let key = catalog::key(&row); + let Some(index) = self.rows.iter().position(|(listed, _)| *listed == key) else { return Ok(false) }; + if self.rows[index].1["name"] != row["name"] { + return Ok(false); + } + updates.push((index, row)); + } + for (index, row) in updates { + self.digests[index] = catalog::row_digest(&row); + self.rows[index].1 = row; + } + self.pending = true; + Ok(true) + } + + /// What no file shows: the terminal's structure, agent processes and + /// their work, sessions without a hook event yet and DeepSeek and + /// OpenCode hosts. The listing's metrics and presence are refreshed too. + fn probe(&mut self, now: Instant) { + self.probed_at = Some(now); + if !self.listed { + // The last listing failed: try again. + self.schedule(now); + return; + } + let rows: Vec<&Value> = self.rows.iter().map(|(_, row)| row).collect(); + let probe = match source::probe(&rows) { + Ok(probe) if Some(probe.structure) == self.structure => probe, + // A changed terminal, or one this probe cannot read: list it whole. + _ => { + self.schedule(now); + return; + } + }; + let native = self.rows.iter().any(|(_, row)| row["name"].as_str().is_some_and(source::native)); + if !probe.stale.is_empty() || native { + self.stale.extend(probe.stale); + self.stale_native |= native; + self.schedule_refresh(now); + } + // `metrics` and `desktop_presence` as the listing reports them now. + self.meta["metrics"] = crate::metrics::snapshot(&self.config.state); + match crate::presence::listing(&self.config.state, crate::presence::now()) { + Some(presence) => self.meta["desktop_presence"] = presence, + None => { + if let Some(meta) = self.meta.as_object_mut() { + meta.remove("desktop_presence"); + } + } + } + self.pending = true; + } + + /// Sends what changed since the client's copy, unless the client is + /// behind: then the latest listing waits, so changes coalesce per key. + fn emit_catalog(&mut self) { + if !self.listed || !(self.pending || self.catalog_resync) || self.out.writer.backlog() > DEFER_BYTES { + return; + } + let mut removed = BTreeMap::new(); + if self.catalog_resync || !self.snapshot_sent { + if self.catalog_resync { + self.out.put(&Body::Gap(protocol::Gap { topic: topic::CATALOG.into(), key: None, reason: reason::OVERFLOW.into() })); + } + self.catalog_resync = false; + self.snapshot(); + self.snapshot_sent = true; + } else { + let change = catalog::diff(&self.sent, &self.rows, &self.digests); + if !change.is_empty() { + removed = self.delta(change); + } + self.send_meta(false); + } + self.pending = false; + self.follow_catalog(&removed); + } + + fn item(&self, index: usize) -> Value { + let (key, row) = &self.rows[index]; + json!({"key": key, "row": row}) + } + + /// Rows too large for any frame are left out with an error: read them + /// with `hgs ls --json --local`. + fn oversized(&mut self, index: usize, size: usize) -> bool { + if size <= activity::PAYLOAD_LIMIT { + return false; + } + let key = self.rows[index].0.clone(); + self.out.failure( + "row_too_large", + &format!("catalog row exceeds the frame limit ({size} bytes); read it with hgs ls --json --local"), + Some(topic::CATALOG), + Some(&key), + ); + true + } + + fn snapshot(&mut self) { + let mut items = Vec::new(); + for index in 0..self.rows.len() { + let item = self.item(index); + let size = item.to_string().len(); + if !self.oversized(index, size) { + items.push((item, size)); + } + } + self.revision += 1; + let pages = catalog::pages(items, |(_, size)| *size, PAGE_LIMIT); + let count = pages.len(); + for (page, items) in pages.into_iter().enumerate() { + let body = Body::Snapshot(protocol::Snapshot { + topic: topic::CATALOG.into(), + key: None, + items: Some(items.into_iter().map(|(item, _)| item).collect()), + value: None, + page: page as u32, + last: page + 1 == count, + cursor: self.revision, + }); + if !self.out.put(&body) { + return; + } + } + self.sent = self + .rows + .iter() + .zip(&self.digests) + .map(|((key, row), digest)| (key.clone(), Sent::of(row, *digest))) + .collect(); + self.send_meta(true); + } + + /// Sends a delta; returns the removed keys with their reasons. + fn delta(&mut self, change: catalog::Change) -> BTreeMap { + /// One entry of a delta: a removal with its reason, or a row. + enum Entry { + Remove(String, &'static str), + Upsert(usize), + } + let listed: BTreeSet<&str> = self.rows.iter().filter_map(|(_, row)| row["name"].as_str()).collect(); + let mut entries = Vec::new(); + let mut removed = BTreeMap::new(); + for key in change.remove { + let Some(old) = self.sent.get(&key) else { continue }; + let why = catalog::removal_reason(old, &listed, source::archived, source::saved); + removed.insert(key.clone(), why); + entries.push((Entry::Remove(key.clone(), why), key.len() + 48)); + } + for index in change.upsert { + let size = self.item(index).to_string().len(); + if self.oversized(index, size) { + let (key, row) = &self.rows[index]; + self.sent.insert(key.clone(), Sent::of(row, self.digests[index])); + } else { + entries.push((Entry::Upsert(index), size)); + } + } + for page in catalog::pages(entries, |(_, size)| *size, PAGE_LIMIT) { + if page.is_empty() { + continue; + } + let mut upsert = Vec::new(); + let mut remove = Vec::new(); + for (entry, _) in &page { + match entry { + Entry::Remove(key, why) => remove.push(protocol::Removal { key: key.clone(), reason: Some((*why).into()) }), + Entry::Upsert(index) => upsert.push(protocol::CatalogItem { + key: self.rows[*index].0.clone(), + row: self.rows[*index].1.clone(), + }), + } + } + self.revision += 1; + let body = Body::Delta(protocol::Delta { topic: topic::CATALOG.into(), upsert, remove, cursor: self.revision }); + if !self.out.offer(&body, None) { + self.catalog_resync = true; + return removed; + } + for (entry, _) in page { + match entry { + Entry::Remove(key, _) => { + self.sent.remove(&key); + } + Entry::Upsert(index) => { + let (key, row) = &self.rows[index]; + self.sent.insert(key.clone(), Sent::of(row, self.digests[index])); + } + } + } + } + removed + } + + /// `catalog.meta`: at once when a lasting field changed, at most every + /// 10 s when only metrics or presence did. + fn send_meta(&mut self, force: bool) { + let lasting = catalog::digest(&catalog::masked(&self.meta, &catalog::VOLATILE_META)); + let full = catalog::digest(&self.meta); + let due = match self.meta_sent { + None => true, + Some((sent_lasting, sent_full, at)) => { + sent_lasting != lasting || (sent_full != full && at.elapsed() >= META_INTERVAL) + } + }; + if !(force || due) { + return; + } + let size = self.meta.to_string().len(); + if size > activity::PAYLOAD_LIMIT { + // Never expected; a resync would only meet the same limit again. + self.out.failure( + "meta_too_large", + &format!("catalog.meta exceeds the frame limit ({size} bytes); read it with hgs ls --json --local"), + Some(topic::CATALOG_META), + None, + ); + self.meta_sent = Some((lasting, full, Instant::now())); + return; + } + let body = Body::Snapshot(protocol::Snapshot { + topic: topic::CATALOG_META.into(), + key: None, + items: None, + value: Some(self.meta.clone()), + page: 0, + last: true, + cursor: self.revision, + }); + if self.out.offer(&body, Some(format!("snapshot|{}", topic::CATALOG_META))) { + self.meta_sent = Some((lasting, full, Instant::now())); + } else { + self.catalog_resync = true; + } + } + + /// Activity subscriptions follow their catalog rows: a rename changes the + /// name, a changed row refreshes the summary, a removed row ends them. + fn follow_catalog(&mut self, removed: &BTreeMap) { + let rows: BTreeMap<&str, usize> = self.rows.iter().enumerate().map(|(index, (key, _))| (key.as_str(), index)).collect(); + let mut ended = Vec::new(); + for (position, sub) in self.subs.iter_mut().enumerate() { + match rows.get(sub.key.as_str()) { + Some(index) => { + let row = &self.rows[*index].1; + sub.listed = true; + if let Some(name) = row["name"].as_str().filter(|name| *name != sub.name) { + sub.name = name.to_owned(); + sub.dirty = true; + } + if sub.row != Some(self.digests[*index]) { + sub.row = Some(self.digests[*index]); + sub.dirty = true; + } + sub.busy = activity::busy(row); + } + None if sub.listed => ended.push((position, removed.get(&sub.key).copied().unwrap_or(reason::ENDED))), + None => {} + } + } + for (position, why) in ended.into_iter().rev() { + let sub = self.subs.remove(position); + self.out.error( + "subscription_ended", + &format!("{} left the catalog ({why})", sub.name), + Some(topic::ACTIVITY), + Some(&sub.key), + ); + } + } + + // ---- Pulse ------------------------------------------------------------- + + fn pulse(&mut self, now: Instant) { + let journal = self.poll_journal(); + let scan = stamps::scan(&self.root); + let (changed, removed) = scan.changed_bindings(&self.scan); + // Inputs other than bindings, and added or removed bindings, change + // the whole listing; a changed binding changes its own row. + let mut whole = scan.global != self.scan.global || removed || matches!(journal, Journal::Unknown); + let mut names = BTreeSet::new(); + for path in changed { + let identity = source::binding_identity(&path); + match &identity { + Some((name, _)) if self.scan.bindings.contains_key(&path) => { + names.insert(name.clone()); + } + _ => whole = true, + } + self.identities.insert(path, identity); + } + if removed { + self.identities.retain(|path, _| scan.bindings.contains_key(path)); + } + // Journal rows change the rows of the sessions they belong to. Rows + // of other names belong to no listed binding. + if let Journal::Names(journals) = &journal { + for (name, journal) in self.identities.values().flatten() { + if journals.contains(journal) { + names.insert(name.clone()); + } + } + } + self.scan = scan; + if whole { + self.schedule(now + DEBOUNCE); + } else if !names.is_empty() { + self.stale.extend(names); + self.schedule_refresh(now + DEBOUNCE); + } + if self.probed_at.is_none_or(|at| now >= at + PROBE) { + self.probe(now); + } + if self.listed_at.is_none_or(|at| now >= at + FULL) { + self.schedule(now); + } + self.activity(now); + } + + fn open_journal(&mut self) { + if self.journal.is_some() { + return; + } + self.journal = source::journal(); + if let Some(db) = &self.journal { + self.data_version = source::data_version(db).ok(); + self.journal_max = source::bounds(db).map_or(0, |(_, max)| max); + // Rows written before this connection opened: read after each cursor. + for sub in self.subs.iter_mut().filter(|sub| !sub.native) { + sub.more = true; + } + } + } + + /// The journal names with new rows. Subscriptions with new rows read them. + fn poll_journal(&mut self) -> Journal { + self.open_journal(); + let Some(db) = &self.journal else { return Journal::Unchanged }; + let version = match source::data_version(db) { + Ok(version) => version, + Err(_) => { + self.journal = None; + return Journal::Unchanged; + } + }; + if Some(version) == self.data_version { + return Journal::Unchanged; + } + self.data_version = Some(version); + let Ok((_, max)) = source::bounds(db) else { return Journal::Unknown }; + if max < self.journal_max { + // A replaced journal: every cursor is meaningless now. + for sub in self.subs.iter_mut().filter(|sub| !sub.native) { + sub.resync = Some(reason::RESET); + } + self.journal_max = max; + return Journal::Unknown; + } + if max == self.journal_max { + return Journal::Unchanged; + } + const SCAN: usize = 10_000; + let names: Option> = if max - self.journal_max > SCAN as i64 { + None + } else { + source::changed_names(db, self.journal_max, SCAN).ok().map(|names| names.into_iter().collect()) + }; + for sub in self.subs.iter_mut().filter(|sub| !sub.native) { + let touched = match (&names, &sub.journal) { + (Some(names), Some(journal)) => names.contains(journal), + _ => true, + }; + if touched { + sub.more = true; + } + } + self.journal_max = max; + names.map_or(Journal::Unknown, Journal::Names) + } + + fn activity(&mut self, now: Instant) { + for index in 0..self.subs.len() { + if self.subs[index].resync.is_some() { + continue; + } + if self.subs[index].more { + self.read_events(index); + } + let sub = &self.subs[index]; + let interval = if sub.native { + NATIVE_REFRESH + } else if sub.busy { + BUSY_REFRESH + } else { + IDLE_REFRESH + }; + let since = now.saturating_duration_since(sub.refreshed); + if (sub.dirty && since >= STATE_GAP) || since >= interval { + self.refresh(index); + } + } + } + + /// Journal rows after the subscription's cursor, as `inspect` shapes them. + fn read_events(&mut self, index: usize) { + let Some(db) = &self.journal else { return }; + let sub = &mut self.subs[index]; + sub.more = false; + let timeline = match source::timeline(&sub.name) { + Ok(Some(timeline)) => timeline, + Ok(None) => return, + Err(error) => { + self.out.failure("inspect_failed", &error, Some(topic::ACTIVITY), Some(&sub.key)); + return; + } + }; + if sub.run_id.as_deref().is_some_and(|run| run != timeline.run_id()) { + // The name now holds another run; the catalog removes or renames ours. + sub.dirty = true; + return; + } + sub.journal = Some(timeline.journal().to_owned()); + let events = match timeline.events(db, sub.cursor, protocol::ACTIVITY_READ, false) { + Ok(events) => events, + Err(error) => { + self.out.failure("journal_failed", &error, Some(topic::ACTIVITY), Some(&sub.key)); + return; + } + }; + sub.more = events.len() == protocol::ACTIVITY_READ; + if events.is_empty() { + return; + } + sub.dirty = true; + for page in activity::event_pages(events, activity::PAYLOAD_LIMIT) { + let cursor = page.last().and_then(|event| event["seq"].as_i64()).unwrap_or(sub.cursor); + let body = Body::Events(protocol::Events { + topic: topic::ACTIVITY.into(), + key: sub.key.clone(), + events: page, + cursor, + }); + if !self.out.offer(&body, None) { + sub.resync = Some(reason::OVERFLOW); + sub.more = false; + return; + } + sub.cursor = cursor; + } + } + + /// Re-inspects one session; sends `state` when its summary changed, and + /// for DeepSeek and OpenCode a replacement of their event window. + fn refresh(&mut self, index: usize) { + let after = if self.subs[index].native { 0 } else { self.journal_max.max(self.subs[index].cursor) }; + let sub = &mut self.subs[index]; + sub.refreshed = Instant::now(); + sub.dirty = false; + let inspection = match source::inspect(&sub.name, after) { + Ok(inspection) => inspection, + Err(error) => { + self.out.failure("inspect_failed", &error, Some(topic::ACTIVITY), Some(&sub.key)); + return; + } + }; + if inspection["run_id"].as_str().zip(sub.run_id.as_deref()).is_some_and(|(seen, wanted)| seen != wanted) { + return; + } + if !sub.native { + if let Ok(Some(timeline)) = source::timeline(&sub.name) { + sub.journal = Some(timeline.journal().to_owned()); + } + } else { + let events = inspection["events"].as_array().cloned().unwrap_or_default(); + let digest = catalog::digest(&Value::Array(events.clone())); + if sub.events != Some(digest) && send_window(&self.out, sub, events, 0) { + sub.events = Some(digest); + } + } + send_state(&self.out, sub, &inspection); + } + + /// Topics the client fell behind on: `gap`, then a tail snapshot. + fn resync_activity(&mut self) { + for index in 0..self.subs.len() { + let Some(why) = self.subs[index].resync else { continue }; + if self.out.writer.backlog() > DEFER_BYTES { + return; + } + self.out.put(&Body::Gap(protocol::Gap { + topic: topic::ACTIVITY.into(), + key: Some(self.subs[index].key.clone()), + reason: why.into(), + })); + self.subs[index].resync = None; + self.subs[index].state = None; + self.subs[index].events = None; + self.start(index, 0, true); + } + } + + // ---- Control ----------------------------------------------------------- + + fn control(&mut self, text: &str) { + if text.trim().is_empty() { + return; + } + let operation = match protocol::decode_control(text) { + Ok(operation) => operation, + Err(error) => { + self.out.error("bad_request", &format!("invalid control operation: {error}"), None, None); + return; + } + }; + match operation { + Control::Ping(ping) => { + self.out.put(&Body::Pong(protocol::Pong { id: ping.id })); + } + Control::Subscribe(subscription) => self.subscribe(subscription), + Control::Unsubscribe(subscription) => { + if let Some(session) = subscription.session.filter(|_| subscription.topic == topic::ACTIVITY) { + let key = session.key(); + self.subs.retain(|sub| sub.key != key); + } + } + } + } + + fn subscribe(&mut self, request: protocol::Subscription) { + match request.topic.as_str() { + topic::CATALOG | topic::CATALOG_META => return, + topic::ACTIVITY => {} + other => { + self.out.error( + "unsupported_topic", + &format!("watch-v1 does not stream {other}; keep reading it with its command"), + Some(other), + None, + ); + return; + } + } + let Some(session) = request + .session + .filter(|session| !session.name.is_empty() && session.name.len() <= 1024 && !session.name.chars().any(char::is_control)) + else { + self.out.error("bad_request", "an Activity subscription names its session", Some(topic::ACTIVITY), None); + return; + }; + let after = request.after.unwrap_or(0); + if after < 0 { + self.out.error("bad_request", "after must be nonnegative", Some(topic::ACTIVITY), None); + return; + } + let key = session.key(); + self.subs.retain(|sub| sub.key != key); + if self.subs.len() >= protocol::ACTIVITY_SUBSCRIPTIONS { + self.out.error( + "too_many_subscriptions", + &format!("a stream follows at most {} sessions", protocol::ACTIVITY_SUBSCRIPTIONS), + Some(topic::ACTIVITY), + Some(&key), + ); + return; + } + // A renamed session keeps its run key; follow its current name. + let row = self.rows.iter().position(|(row_key, _)| *row_key == key); + let name = row + .and_then(|index| self.rows[index].1["name"].as_str()) + .unwrap_or(&session.name) + .to_owned(); + let tail = request.tail.map_or(protocol::ACTIVITY_TAIL, |tail| (tail as usize).min(protocol::ACTIVITY_TAIL)); + let mut sub = Subscription::new(key, name.clone(), session.run_id.filter(|run| !run.is_empty()), source::native(&name), tail); + if let Some(index) = row { + sub.listed = true; + sub.row = Some(self.digests[index]); + sub.busy = activity::busy(&self.rows[index].1); + } + self.subs.push(sub); + let index = self.subs.len() - 1; + self.open_journal(); + if !self.start(index, after, false) { + self.subs.pop(); + } + } + + /// The first frames of a subscription, or its resumption after a gap. + /// False when it cannot follow the requested session. + fn start(&mut self, index: usize, after: i64, resumed: bool) -> bool { + let (minimum, maximum) = self.journal.as_ref().and_then(|db| source::bounds(db).ok()).unwrap_or((0, 0)); + let sub = &mut self.subs[index]; + let gap = if resumed || after == 0 || sub.native { + None + } else if after > maximum { + Some(reason::RESET) + } else if after < minimum - 1 { + Some(reason::RETENTION) + } else { + None + }; + let inspection = match source::inspect(&sub.name, if gap.is_some() { 0 } else { after }) { + Ok(inspection) => inspection, + Err(error) => { + self.out.error("inspect_failed", &error, Some(topic::ACTIVITY), Some(&sub.key)); + return false; + } + }; + if let (Some(seen), Some(wanted)) = (inspection["run_id"].as_str(), sub.run_id.as_deref()) { + if seen != wanted { + self.out.error( + "session_changed", + &format!("{} now runs {seen}, not {wanted}", sub.name), + Some(topic::ACTIVITY), + Some(&sub.key), + ); + return false; + } + } + sub.refreshed = Instant::now(); + let mut events = inspection["events"].as_array().cloned().unwrap_or_default(); + if !sub.native { + if let Ok(Some(timeline)) = source::timeline(&sub.name) { + sub.journal = Some(timeline.journal().to_owned()); + } + } + if sub.native { + sub.events = Some(catalog::digest(&Value::Array(events.clone()))); + send_window(&self.out, sub, events, 0); + } else { + let cursor = inspection["cursor"].as_i64().unwrap_or(after); + if gap.is_some() || resumed || after == 0 { + events.drain(..events.len().saturating_sub(sub.tail)); + } + if let Some(why) = gap { + self.out.put(&Body::Gap(protocol::Gap { topic: topic::ACTIVITY.into(), key: Some(sub.key.clone()), reason: why.into() })); + } + sub.more = after > 0 && gap.is_none() && events.len() >= protocol::ACTIVITY_READ; + if gap.is_some() || resumed { + send_window(&self.out, sub, events, cursor); + } else { + for page in activity::event_pages(events, activity::PAYLOAD_LIMIT) { + let page_cursor = page.last().and_then(|event| event["seq"].as_i64()).unwrap_or(cursor); + self.out.put(&Body::Events(protocol::Events { + topic: topic::ACTIVITY.into(), + key: sub.key.clone(), + events: page, + cursor: page_cursor, + })); + } + } + sub.cursor = cursor; + } + send_state(&self.out, sub, &inspection); + true + } +} + +/// An Activity event window as a snapshot: a DeepSeek or OpenCode window +/// (cursor 0), or a journal tail after a gap. Pages share the window's cursor. +fn send_window(out: &Out, sub: &mut Subscription, events: Vec, cursor: i64) -> bool { + let pages = activity::event_pages(events, activity::PAYLOAD_LIMIT); + let count = pages.len(); + for (page, items) in pages.into_iter().enumerate() { + let body = Body::Snapshot(protocol::Snapshot { + topic: topic::ACTIVITY.into(), + key: Some(sub.key.clone()), + items: Some(items), + value: None, + page: page as u32, + last: page + 1 == count, + cursor, + }); + if !out.put(&body) { + return false; + } + } + true +} + +/// `state` when the summary changed apart from observation times. +fn send_state(out: &Out, sub: &mut Subscription, inspection: &Value) { + let mut value = activity::state_value(inspection); + sub.busy = activity::busy(&value); + let digest = activity::state_digest(&value); + if sub.state == Some(digest) { + return; + } + activity::shrink_state(&mut value, activity::PAYLOAD_LIMIT); + sub.rev += 1; + let body = Body::State(protocol::State { topic: topic::ACTIVITY.into(), key: sub.key.clone(), value, rev: sub.rev }); + if out.offer(&body, Some(format!("state|{}|{}", topic::ACTIVITY, sub.key))) { + sub.state = Some(digest); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn read(input: &[u8], pending: usize) -> Vec { + let (sender, receiver) = mpsc::channel(); + let ended = std::cell::Cell::new(false); + read_controls(input, sender, &AtomicUsize::new(pending), || ended.set(true)); + assert!(ended.get()); + receiver + .into_iter() + .map(|input| match input { + Input::Line(text) => text, + Input::TooLong => "".into(), + Input::Dropped => "".into(), + Input::End => "".into(), + }) + .collect() + } + + fn lines(input: &[u8]) -> Vec { + read(input, 0) + } + + #[test] + fn control_lines_are_framed_bounded_and_end_at_eof() { + assert_eq!(lines(b"{\"op\":\"ping\"}\n\n{\"op\":\"ping\",\"id\":2}"), ["{\"op\":\"ping\"}", "", "{\"op\":\"ping\",\"id\":2}", ""]); + let mut long = vec![b'x'; protocol::CONTROL_LINE_BYTES + 10]; + long.extend_from_slice(b"\nnext\n"); + assert_eq!(lines(&long), ["", "next", ""]); + assert_eq!(lines(b""), [""]); + // A client that writes faster than lines are handled loses the excess. + assert_eq!(read(b"a\nb\nc", PENDING_CONTROLS), ["", ""]); + } +} diff --git a/src/watch/stamps.rs b/src/watch/stamps.rs new file mode 100644 index 000000000..b43957470 --- /dev/null +++ b/src/watch/stamps.rs @@ -0,0 +1,177 @@ +//! A cheap stamp over the files the live listing reads, compared on every +//! pulse. Bindings are replaced by rename, so a stamp of name, size, mtime and +//! inode sees every change; temporary `.hgs-*` files are ignored. Each session +//! binding has a stamp of its own, so a change re-derives only its row. +use std::collections::hash_map::DefaultHasher; +use std::collections::BTreeMap; +use std::fs; +use std::hash::{Hash, Hasher}; +use std::path::{Path, PathBuf}; + +/// Session bindings: 64 lowercase hex digits and `.json`. +fn binding(name: &str) -> bool { + name.strip_suffix(".json") + .is_some_and(|stem| stem.len() == 64 && stem.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))) +} + +fn ordinary(name: &str) -> bool { + !name.starts_with(".hgs-") && name != ".lock" && !name.ends_with(".lock") +} + +/// Root files other than bindings that change listed rows. Desktop presence +/// changes only `catalog.meta`, which the probe refreshes. +const ROOT_FILES: [&str; 2] = ["read-marks.json", "notifications.json"]; +/// The folder that holds routed bindings next to the root's own. +const BINDINGS: &str = "bindings"; +/// Folders whose every entry counts. +const FOLDERS: [&str; 6] = [BINDINGS, "name_routes", "renames", "launches", "dsh/bindings", "opencode/bindings"]; +/// Single files and folders whose own stamp counts. Archive files are added or +/// removed, which changes the folder. +const PATHS: [&str; 4] = ["coordination/waits.sqlite3", "coordination/waits.sqlite3-wal", "coordination/waits.sqlite3-journal", "archives"]; + +fn entry(path: &Path, metadata: &fs::Metadata) -> u64 { + let mut hasher = DefaultHasher::new(); + path.hash(&mut hasher); + metadata.len().hash(&mut hasher); + metadata.modified().ok().hash(&mut hasher); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + metadata.ino().hash(&mut hasher); + metadata.ctime().hash(&mut hasher); + metadata.ctime_nsec().hash(&mut hasher); + } + hasher.finish() +} + +/// An order-independent hash of every stamped entry, and their count. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct Stamp { + hash: u64, + entries: usize, +} + +impl Stamp { + fn add(&mut self, path: &Path, metadata: &fs::Metadata) { + self.hash = self.hash.wrapping_add(entry(path, metadata)); + self.entries += 1; + } + + #[cfg(test)] + fn entries(&self) -> usize { + self.entries + } +} + +/// The stamps of a state folder: one for every input but session bindings, +/// and one per binding file. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(super) struct Scan { + pub(super) global: Stamp, + pub(super) bindings: BTreeMap, +} + +impl Scan { + fn folder(&mut self, folder: &Path, keep: impl Fn(&str) -> bool, bindings: bool) { + let Ok(entries) = fs::read_dir(folder) else { return }; + for found in entries.flatten() { + let name = found.file_name(); + let Some(name) = name.to_str() else { continue }; + if !keep(name) { + continue; + } + let Ok(metadata) = found.metadata() else { continue }; + let path = found.path(); + if bindings && binding(name) { + self.bindings.insert(path.clone(), entry(&path, &metadata)); + } else { + self.global.add(&path, &metadata); + } + } + } + + /// Binding files whose stamp changed or that appeared, and whether any + /// disappeared, since `before`. + pub(super) fn changed_bindings(&self, before: &Scan) -> (Vec, bool) { + let changed = self + .bindings + .iter() + .filter(|(path, stamp)| before.bindings.get(*path) != Some(stamp)) + .map(|(path, _)| path.clone()) + .collect(); + let removed = before.bindings.keys().any(|path| !self.bindings.contains_key(path)); + (changed, removed) + } +} + +/// The stamps of a state folder. +pub(super) fn scan(root: &Path) -> Scan { + let mut scan = Scan::default(); + scan.folder(root, |name| binding(name) || ROOT_FILES.contains(&name), true); + for folder in FOLDERS { + scan.folder(&root.join(folder), ordinary, folder == BINDINGS); + } + for path in PATHS { + let path = root.join(path); + if let Ok(metadata) = fs::symlink_metadata(&path) { + scan.global.add(&path, &metadata); + } + } + scan +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bindings_and_inputs_change_the_stamp_and_temporary_files_do_not() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let first = scan(root); + let record = root.join(format!("{}.json", "a".repeat(64))); + fs::write(&record, "{}").unwrap(); + let written = scan(root); + assert_ne!(written, first); + // A binding has a stamp of its own; every other input stays unchanged. + assert_eq!((written.global, written.bindings.len()), (first.global, 1)); + assert_eq!(written.changed_bindings(&first), (vec![record.clone()], false)); + // Ignored: temporary files, locks, presence and unrelated files. + fs::write(root.join(".hgs-temp"), "x").unwrap(); + fs::write(root.join("desktop-presence.json"), "{}").unwrap(); + fs::write(root.join(".lock"), "").unwrap(); + fs::write(root.join("notes.json"), "{}").unwrap(); + fs::create_dir_all(root.join("launches")).unwrap(); + fs::write(root.join("launches/.hgs-partial"), "x").unwrap(); + assert_eq!(scan(root), written); + // An atomic replacement (new inode, same size) is seen. + let temp = root.join(".hgs-next"); + fs::write(&temp, "[]").unwrap(); + fs::rename(&temp, &record).unwrap(); + let replaced = scan(root); + assert_ne!(replaced, written); + assert_eq!(replaced.global, written.global); + assert_eq!(replaced.changed_bindings(&written), (vec![record.clone()], false)); + // A routed binding is a binding too. + let routed = root.join(format!("bindings/{}.json", "b".repeat(64))); + fs::create_dir_all(routed.parent().unwrap()).unwrap(); + fs::write(&routed, "{}").unwrap(); + let both = scan(root); + assert_eq!((both.global, both.changed_bindings(&replaced)), (replaced.global, (vec![routed.clone()], false))); + fs::remove_file(&record).unwrap(); + assert_eq!(scan(root).changed_bindings(&both), (vec![], true)); + for path in ["read-marks.json", "launches/1.json", "renames/r.json", "name_routes/n.json", "dsh/bindings/b.json", "bindings/notes.txt"] { + let path = root.join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let before = scan(root); + fs::write(&path, "{}").unwrap(); + let after = scan(root); + assert_ne!(after.global, before.global, "{}", path.display()); + assert_eq!(after.bindings, before.bindings, "{}", path.display()); + } + let before = scan(root); + fs::create_dir(root.join("archives")).unwrap(); + assert_ne!(scan(root).global, before.global); + assert_eq!(before.global.entries(), 6); + } +} diff --git a/src/watch/writer.rs b/src/watch/writer.rs new file mode 100644 index 000000000..76ad985d9 --- /dev/null +++ b/src/watch/writer.rs @@ -0,0 +1,324 @@ +//! The only owner of the stream's stdout: a bounded queue of encoded bodies, +//! frame numbering at write time, and heartbeats. +use std::collections::VecDeque; +use std::io::Write; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::time::{Duration, Instant}; +use zerus_watch_protocol as protocol; + +/// One queued frame body (`{"t":...}`, see `protocol::encode_body`). +pub(super) struct Item { + /// Bodies with the same key replace each other while queued: the latest + /// `state` of a key, or the latest `catalog.meta`. + pub coalesce: Option, + pub body: String, +} + +struct Queue { + items: VecDeque, + bytes: usize, +} + +struct Shared { + queue: Mutex, + /// Signals the writer thread that a body is queued. + ready: Condvar, + /// Signals producers that the queue has room. + space: Condvar, + capacity: usize, + stopped: AtomicBool, +} + +impl Shared { + fn lock(&self) -> MutexGuard<'_, Queue> { + self.queue.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) + } +} + +#[derive(Clone)] +pub(super) struct Writer { + shared: Arc, +} + +impl Writer { + /// Starts the thread that writes to `out`. `closed` runs once when `out` + /// fails, as when the client closed the pipe. + pub(super) fn spawn(out: W, capacity: usize, heartbeat: Duration, closed: F) -> Self + where + W: Write + Send + 'static, + F: FnOnce() + Send + 'static, + { + let shared = Arc::new(Shared { + queue: Mutex::new(Queue { items: VecDeque::new(), bytes: 0 }), + ready: Condvar::new(), + space: Condvar::new(), + capacity, + stopped: AtomicBool::new(false), + }); + let thread = Arc::clone(&shared); + std::thread::Builder::new() + .name("hgs-watch-writer".into()) + .spawn(move || { + write_frames(&thread, out, heartbeat); + thread.stopped.store(true, Ordering::SeqCst); + thread.space.notify_all(); + closed(); + }) + .expect("start the watch writer thread"); + Self { shared } + } + + /// Queues a body unless it would overflow the queue; an empty queue always + /// takes one. A body whose `coalesce` key is already queued replaces that + /// body in place. False means the client fell behind: the caller owes the + /// topic a `gap` and a fresh snapshot. + pub(super) fn offer(&self, item: Item) -> bool { + if self.stopped() { + return false; + } + let mut queue = self.shared.lock(); + if let Some(key) = item.coalesce.as_deref() { + if let Some(queued) = queue.items.iter_mut().find(|queued| queued.coalesce.as_deref() == Some(key)) { + let previous = std::mem::replace(&mut queued.body, item.body); + let current = queued.body.len(); + queue.bytes = queue.bytes - previous.len() + current; + return true; + } + } + if !queue.items.is_empty() && queue.bytes + item.body.len() > self.shared.capacity { + return false; + } + queue.bytes += item.body.len(); + queue.items.push_back(item); + self.shared.ready.notify_one(); + true + } + + /// Queues a body, waiting while the queue is full. False once the stream + /// has stopped. + pub(super) fn put(&self, item: Item) -> bool { + let mut queue = self.shared.lock(); + while !queue.items.is_empty() && queue.bytes + item.body.len() > self.shared.capacity { + if self.stopped() { + return false; + } + queue = self + .shared + .space + .wait_timeout(queue, Duration::from_millis(100)) + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .0; + } + if self.stopped() { + return false; + } + queue.bytes += item.body.len(); + queue.items.push_back(item); + self.shared.ready.notify_one(); + true + } + + /// Bytes queued and not yet written. + pub(super) fn backlog(&self) -> usize { + self.shared.lock().bytes + } + + /// Stops the stream: queued frames are dropped and every wait ends. + pub(super) fn halt(&self) { + self.shared.stopped.store(true, Ordering::SeqCst); + let _queue = self.shared.lock(); + self.shared.space.notify_all(); + self.shared.ready.notify_all(); + } + + pub(super) fn stopped(&self) -> bool { + self.shared.stopped.load(Ordering::SeqCst) + } + + /// Waits up to `limit` for the queue to empty; for tests and orderly ends. + #[cfg(test)] + pub(super) fn drain(&self, limit: Duration) -> bool { + let deadline = Instant::now() + limit; + loop { + if self.backlog() == 0 || self.stopped() { + return self.backlog() == 0; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(2)); + } + } +} + +fn now() -> f64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() +} + +fn write_frames(shared: &Shared, mut out: W, heartbeat: Duration) { + let mut seq = 0u64; + let mut last = Instant::now(); + loop { + let body = { + let mut queue = shared.lock(); + loop { + if let Some(item) = queue.items.pop_front() { + queue.bytes -= item.body.len(); + shared.space.notify_all(); + break item.body; + } + let idle = last.elapsed(); + if idle >= heartbeat { + let beat = protocol::Body::Heartbeat(protocol::Heartbeat { at: now() }); + break protocol::encode_body(&beat).expect("a heartbeat encodes"); + } + queue = shared + .ready + .wait_timeout(queue, heartbeat - idle) + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .0; + } + }; + seq += 1; + let Ok(mut line) = protocol::envelope(seq, &body) else { + continue; + }; + line.push('\n'); + if out.write_all(line.as_bytes()).and_then(|()| out.flush()).is_err() { + return; + } + last = Instant::now(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io; + use std::sync::mpsc; + + /// A pipe end the test reads, optionally blocked until released. + #[derive(Clone)] + struct Sink { + data: Arc>>, + gate: Arc<(Mutex, Condvar)>, + fail: Arc, + } + impl Sink { + fn new(open: bool) -> Self { + Self { + data: Arc::default(), + gate: Arc::new((Mutex::new(open), Condvar::new())), + fail: Arc::default(), + } + } + fn open(&self) { + *self.gate.0.lock().unwrap() = true; + self.gate.1.notify_all(); + } + fn lines(&self) -> Vec { + String::from_utf8(self.data.lock().unwrap().clone()) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() + } + } + impl Write for Sink { + fn write(&mut self, buf: &[u8]) -> io::Result { + let mut open = self.gate.0.lock().unwrap(); + while !*open { + open = self.gate.1.wait(open).unwrap(); + } + if self.fail.load(Ordering::SeqCst) { + return Err(io::Error::from(io::ErrorKind::BrokenPipe)); + } + self.data.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + fn body(text: &str) -> String { + protocol::encode_body(&protocol::Body::Pong(protocol::Pong { id: Some(text.into()) })).unwrap() + } + + fn item(text: &str, coalesce: Option<&str>) -> Item { + Item { coalesce: coalesce.map(str::to_owned), body: body(text) } + } + + #[test] + fn frames_are_numbered_in_write_order_and_coalesce_while_queued() { + let sink = Sink::new(false); + let writer = Writer::spawn(sink.clone(), 1 << 20, Duration::from_secs(60), || {}); + assert!(writer.offer(item("first", None))); + // The writer thread may already hold "first"; queue behind it. + std::thread::sleep(Duration::from_millis(20)); + assert!(writer.offer(item("state-1", Some("state|activity|run:a")))); + assert!(writer.offer(item("events", None))); + assert!(writer.offer(item("state-2", Some("state|activity|run:a")))); + assert!(writer.offer(item("other", Some("state|activity|run:b")))); + sink.open(); + assert!(writer.drain(Duration::from_secs(5))); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while sink.lines().len() < 4 && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + let lines = sink.lines(); + let ids: Vec<_> = lines.iter().map(|line| line["id"].as_str().unwrap().to_owned()).collect(); + assert_eq!(ids, ["first", "state-2", "events", "other"]); + let seqs: Vec<_> = lines.iter().map(|line| line["seq"].as_u64().unwrap()).collect(); + assert_eq!(seqs, [1, 2, 3, 4]); + assert!(lines.iter().all(|line| line["v"] == 1 && line["t"] == "pong")); + } + + #[test] + fn a_full_queue_refuses_offers_and_put_waits_for_room() { + let sink = Sink::new(false); + let one = body("x").len(); + let writer = Writer::spawn(sink.clone(), 3 * one, Duration::from_secs(60), || {}); + assert!(writer.offer(item("x", None))); + std::thread::sleep(Duration::from_millis(20)); + let mut accepted = 0; + while writer.offer(item("x", None)) { + accepted += 1; + assert!(accepted <= 4, "the queue never filled"); + } + assert!(writer.backlog() <= 3 * one); + let (done, waited) = mpsc::channel(); + let blocked = writer.clone(); + std::thread::spawn(move || done.send(blocked.put(item("y", None))).unwrap()); + assert!(waited.recv_timeout(Duration::from_millis(100)).is_err(), "put returned while full"); + sink.open(); + assert_eq!(waited.recv_timeout(Duration::from_secs(5)), Ok(true)); + assert!(writer.drain(Duration::from_secs(5))); + } + + #[test] + fn heartbeats_follow_silence_and_a_closed_pipe_stops_the_writer() { + let sink = Sink::new(true); + let (closed, was_closed) = mpsc::channel(); + let writer = Writer::spawn(sink.clone(), 1 << 20, Duration::from_millis(50), move || closed.send(()).unwrap()); + // A loaded runner can delay the writer thread; wait for two heartbeats + // rather than for a fixed time. + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while sink.lines().len() < 2 && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let lines = sink.lines(); + assert!(lines.len() >= 2, "{lines:?}"); + assert!(lines.iter().all(|line| line["t"] == "heartbeat" && line["at"].as_f64().unwrap() > 1e9)); + sink.fail.store(true, Ordering::SeqCst); + writer.offer(item("lost", None)); + was_closed.recv_timeout(Duration::from_secs(5)).unwrap(); + assert!(writer.stopped()); + assert!(!writer.offer(item("after", None))); + assert!(!writer.put(item("after", None))); + } +} diff --git a/tests/test_ci_security.py b/tests/test_ci_security.py index 458798bb1..f6aaf9bcc 100644 --- a/tests/test_ci_security.py +++ b/tests/test_ci_security.py @@ -547,6 +547,9 @@ def test_source_only_resolves_and_every_suite_starts_from_it(self): "cargo +1.99.1 fmt --check --manifest-path crates/replica-ffi/Cargo.toml", "cargo +1.99.1 clippy --locked --manifest-path crates/replica-ffi/Cargo.toml --target-dir target/replica-ffi --all-targets -- -D warnings", "cargo +1.99.1 test --locked --manifest-path crates/replica-ffi/Cargo.toml --target-dir target/replica-ffi", + "cargo +1.99.1 fmt --check --manifest-path crates/watch-protocol/Cargo.toml", + "cargo +1.99.1 clippy --locked --manifest-path crates/watch-protocol/Cargo.toml --target-dir target/watch-protocol --all-targets -- -D warnings", + "cargo +1.99.1 test --locked --manifest-path crates/watch-protocol/Cargo.toml --target-dir target/watch-protocol", 'run: python3 scripts/collect-licenses.py "$RUNNER_TEMP/rust-licenses"'): self.assertIn(line, quick) for name in ("quick", "relay", "connector", "connector-macos", "android", "cli", "cli-macos-debug", "arch", "arch-package", diff --git a/tests/test_mac_install.py b/tests/test_mac_install.py index 943467033..3fb188760 100644 --- a/tests/test_mac_install.py +++ b/tests/test_mac_install.py @@ -69,7 +69,9 @@ def test_mobile_connector_sources_travel_and_its_build_stays_opt_in(self): 'services/mobile/relay/Cargo.toml', 'services/mobile/relay/src', 'services/mobile/crypto/Cargo.toml', 'services/mobile/crypto/src', 'crates/replica/Cargo.toml', 'crates/replica/Cargo.lock', - 'crates/replica/.cargo', 'crates/replica/src'): + 'crates/replica/.cargo', 'crates/replica/src', + 'crates/watch-protocol/Cargo.toml', 'crates/watch-protocol/Cargo.lock', + 'crates/watch-protocol/.cargo', 'crates/watch-protocol/src'): self.assertIn(path, sent) self.assertFalse(any('target' in path.split('/') for path in sent), sent) builds = [line for line in (self.root / 'transport.log').read_text().splitlines() diff --git a/tests/test_source_json_parse.py b/tests/test_source_json_parse.py new file mode 100644 index 000000000..84fa6c8aa --- /dev/null +++ b/tests/test_source_json_parse.py @@ -0,0 +1,53 @@ +"""Exercise the desktop JSON parsing rule of scripts/ci/check-source.py.""" +import importlib.util +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("check_source", ROOT / "scripts/ci/check-source.py") +check = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(check) + + +class CurrentTreeTest(unittest.TestCase): + def test_desktop_parses_json_only_through_json_parse(self): + files = {path.relative_to(ROOT).as_posix(): path.read_text(encoding="utf-8") + for path in (ROOT / "tray/src").rglob("*") if path.suffix in {".cpp", ".h", ".mm"}} + self.assertEqual(check.json_parse_findings(files), []) + self.assertLessEqual(set(check.LOCAL_JSON_CALLS), set(files), "remove allowances of deleted files") + + +class RuleTest(unittest.TestCase): + def test_direct_decoders_are_rejected_outside_the_parser(self): + problems = check.json_parse_findings({ + "tray/src/View.cpp": "void f(const QByteArray &b) {\n auto d = QJsonDocument::fromJson(b);\n}\n", + "tray/src/Cbor.cpp": "auto v = QCborValue :: fromCbor(bytes);\n", + check.JSON_PARSER: "QJsonDocument document = QJsonDocument::fromJson(bytes, &error);\n", + }) + self.assertEqual(problems, [ + "tray/src/Cbor.cpp:1: parse JSON with JsonParse (tray/src/JsonParse.h), off the GUI thread", + "tray/src/View.cpp:2: parse JSON with JsonParse (tray/src/JsonParse.h), off the GUI thread", + ]) + + def test_comments_and_strings_do_not_count(self): + problems = check.json_parse_findings({ + "tray/src/Notes.h": "// QJsonDocument::fromJson is rejected here\n" + "const char *text = \"JsonParse::local(bytes)\";\n", + }) + self.assertEqual(problems, []) + + def test_local_calls_must_match_the_allowance_exactly(self): + listed = next(iter(check.LOCAL_JSON_CALLS)) + allowed = check.LOCAL_JSON_CALLS[listed] + extra = "".join("JsonParse::localObject(a);\n" for _ in range(allowed + 1)) + fewer = "".join("JsonParse::local(a);\n" for _ in range(allowed - 1)) + self.assertEqual(check.json_parse_findings({"tray/src/New.cpp": "JsonParse::localArray(key);\n"}), + ["tray/src/New.cpp: 1 JsonParse::local() calls, 0 allowed; parse off the GUI thread instead"]) + self.assertEqual(check.json_parse_findings({listed: extra}), + [f"{listed}: {allowed + 1} JsonParse::local() calls, {allowed} allowed; parse off the GUI thread instead"]) + self.assertEqual(check.json_parse_findings({listed: fewer}), + [f"{listed}: lower LOCAL_JSON_CALLS to {allowed - 1} in scripts/ci/check-source.py"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_watch.py b/tests/test_watch.py new file mode 100644 index 000000000..4c0aeb3fa --- /dev/null +++ b/tests/test_watch.py @@ -0,0 +1,365 @@ +"""`hgs [@host] watch --json`: the NDJSON event stream (watch-v1). + +A private tmux socket, a scripted agent and a synthetic state folder only. +""" +import json +import os +import queue +import sqlite3 +import subprocess +import threading +import time +import unittest +import uuid + +import test_input as fixtures + +HGS = fixtures.HGS +VOLATILE = ('goal_observed_at',) + + +def steady(row): + """A row without its observation timestamps.""" + row = {key: value for key, value in row.items() if key not in VOLATILE} + if isinstance(row.get('process_work'), dict): + row['process_work'] = {k: v for k, v in row['process_work'].items() if k != 'sampled_at'} + return row + + +class Stream: + """One running `hgs watch --json` and the frames it has written.""" + + def __init__(self, test, *args): + self.test = test + self.process = subprocess.Popen([str(HGS), *args, 'watch', '--json'], env=test.env, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + self.frames = queue.Queue() + self.seen = [] + self.raw = [] + threading.Thread(target=self.read, daemon=True).start() + test.addCleanup(self.stop) + + def read(self): + for line in self.process.stdout: + self.raw.append(line) + self.frames.put(json.loads(line)) + self.frames.put(None) + + def next(self, predicate=lambda frame: True, timeout=10): + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + self.test.fail(f'no matching frame within {timeout}s; seen {[f["t"] for f in self.seen]}') + try: + frame = self.frames.get(timeout=remaining) + except queue.Empty: + continue + if frame is None: + self.test.fail('the stream ended: ' + self.process.stderr.read()) + self.seen.append(frame) + if predicate(frame): + return frame + + def quiet(self, predicate, seconds): + """Fails if a matching frame arrives within `seconds`.""" + deadline = time.monotonic() + seconds + while (remaining := deadline - time.monotonic()) > 0: + try: + frame = self.frames.get(timeout=remaining) + except queue.Empty: + return + if frame is None: + return + self.seen.append(frame) + self.test.assertFalse(predicate(frame), frame) + + def send(self, **operation): + self.process.stdin.write(json.dumps(operation) + '\n') + self.process.stdin.flush() + + def close(self, timeout=5): + self.process.stdin.close() + return self.process.wait(timeout=timeout) + + def stop(self): + if self.process.poll() is None: + self.process.kill() + self.process.wait(timeout=5) + for stream in (self.process.stdout, self.process.stderr, self.process.stdin): + if stream and not stream.closed: + stream.close() + + +@unittest.skipUnless(fixtures.TMUX, 'private tmux fixture requires tmux') +class WatchStream(unittest.TestCase): + setUp = fixtures.InputTransport.setUp + tearDown = fixtures.InputTransport.tearDown + script = fixtures.InputTransport.script + tmux = fixtures.InputTransport.tmux + wait_for = fixtures.InputTransport.wait_for + write_record = fixtures.InputTransport.write_record + + def key(self): + return 'run:' + self.run_id + + def session(self): + return dict(name=self.name, run_id=self.run_id, conversation_id=self.conversation_id) + + def listing(self): + result = subprocess.run([str(HGS), 'ls', '--json', '--local', '--no-archives'], env=self.env, + text=True, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + return json.loads(result.stdout) + + def journal(self, kind, **fields): + """Appends one Activity row as a hook would; returns its seq.""" + payload = dict(type=kind, at=time.time(), agent_id='', tool='', detail='') + payload.update(fields) + db = sqlite3.connect(self.state / 'events.sqlite3') + with db: + db.execute('CREATE TABLE IF NOT EXISTS events (seq INTEGER PRIMARY KEY, name TEXT, conversation TEXT, payload TEXT)') + cursor = db.execute('INSERT INTO events(name, conversation, payload) VALUES (?, ?, ?)', + (self.name, self.conversation_id, json.dumps(payload))) + seq = cursor.lastrowid + db.close() + return seq + + def inspect_events(self, after): + result = subprocess.run([str(HGS), 'inspect', self.name, '--after', str(after), '--skip-processes'], + env=self.env, text=True, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + return json.loads(result.stdout)['events'] + + def start(self): + stream = Stream(self) + hello = stream.next() + snapshot = stream.next(lambda f: f['t'] == 'snapshot' and f['topic'] == 'catalog' and f['last']) + meta = stream.next(lambda f: f['t'] == 'snapshot' and f['topic'] == 'catalog.meta') + return stream, hello, snapshot, meta + + def test_hello_catalog_snapshot_and_meta_match_the_listing_then_eof_ends_the_stream(self): + stream, hello, snapshot, meta = self.start() + self.assertEqual((hello['v'], hello['t'], hello['seq']), (1, 'hello', 1)) + self.assertEqual(hello['host'], 'test') + self.assertEqual((hello['heartbeat_ms'], hello['max_frame_bytes']), (10000, 1048576)) + self.assertEqual(hello['topics'], ['catalog', 'catalog.meta', 'activity']) + self.assertEqual(hello['limits']['activity_subscriptions'], 16) + uuid.UUID(hello['stream']) + self.assertEqual([frame['seq'] for frame in stream.seen], list(range(1, len(stream.seen) + 1))) + self.assertTrue(all(frame['v'] == 1 for frame in stream.seen)) + self.assertEqual(snapshot['page'], 0) + listing = self.listing() + rows = {item['key']: item['row'] for item in snapshot['items']} + self.assertEqual(set(rows), {self.key()}) + expected = next(row for row in listing['sessions'] if row['name'] == self.name) + self.assertEqual(steady(rows[self.key()]), steady(expected)) + self.assertNotIn('sessions', meta['value']) + self.assertEqual(set(meta['value']), set(listing) - {'sessions'}) + self.assertEqual(meta['value']['host'], 'test') + self.assertTrue(all(len(line.encode()) <= 1 << 20 for line in stream.raw)) + started = time.monotonic() + self.assertEqual(stream.close(), 0) + self.assertLess(time.monotonic() - started, 3) + self.assertEqual(stream.process.stderr.read(), '') + + def test_binding_changes_and_a_rename_are_upserts_on_the_run_key(self): + stream, *_ = self.start() + self.record.update(phase='working', activity='busy', prompt='Synthetic request', last_event_at=time.time()) + self.write_record() + delta = stream.next(lambda f: f['t'] == 'delta' and any( + item['key'] == self.key() and item['row'].get('prompt') == 'Synthetic request' for item in f['upsert']), timeout=8) + self.assertEqual(delta['topic'], 'catalog') + self.assertEqual(delta['remove'], []) + renamed = 'codex/input-test/renamed' + result = subprocess.run([str(HGS), 'rename', self.name, renamed], env=self.env, + text=True, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + delta = stream.next(lambda f: f['t'] == 'delta' and any( + item['key'] == self.key() and item['row']['name'] == renamed for item in f['upsert']), timeout=8) + self.assertEqual(delta['remove'], []) + # Let the rename settle; then observation timestamps alone, which + # change on every listing, produce no delta across a safety listing. + stream.quiet(lambda f: False, 1.5) + stream.quiet(lambda f: f['t'] == 'delta', 6) + self.assertEqual(stream.close(), 0) + + def listed_row(self): + return next(row for row in self.listing()['sessions'] if row['name'] == self.name) + + def upserted(self, predicate, timeout): + """The row of the first delta that upserts this session as `predicate` wants.""" + delta = self.stream.next(lambda f: f['t'] == 'delta' and any( + item['key'] == self.key() and predicate(item['row']) for item in f['upsert']), timeout=timeout) + return next(item['row'] for item in delta['upsert'] if item['key'] == self.key()) + + def test_a_changed_binding_or_journal_row_derives_the_row_the_listing_shows(self): + # Between whole listings (every 5 s) only the changed session's row is + # derived again, through the listing's own steps. + self.stream, *_ = self.start() + self.record.update(prompt='Derived request', last_event_at=time.time()) + self.write_record() + row = self.upserted(lambda row: row.get('prompt') == 'Derived request', timeout=4) + self.assertEqual(steady(row), steady(self.listed_row())) + # A journal row alone, such as a reply, changes the reply identity. + reply = self.journal('Stop', detail='Derived reply') + row = self.upserted(lambda row: str(row.get('reply_id', '')).startswith(f'{reply}:'), timeout=4) + self.assertEqual(steady(row), steady(self.listed_row())) + self.assertEqual(self.stream.close(), 0) + + def test_a_session_killed_in_tmux_is_noticed_without_any_file_change(self): + self.stream, *_ = self.start() + self.tmux('kill-session', '-t', '=' + self.name) + # The probe sees the terminal change within 5 s and lists it whole. + row = self.upserted(lambda row: row.get('state') == 'stopped', timeout=8) + self.assertEqual(steady(row), steady(self.listed_row())) + self.assertEqual(self.stream.close(), 0) + + def test_a_forgotten_session_is_removed_with_its_reason_and_ends_its_subscription(self): + stream, *_ = self.start() + stream.send(op='subscribe', topic='activity', session=self.session()) + stream.next(lambda f: f['t'] == 'state') + result = subprocess.run([str(HGS), 'kill', self.name], env=self.env, text=True, + capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + delta = stream.next(lambda f: f['t'] == 'delta' and f['remove'], timeout=10) + self.assertEqual(delta['remove'], [dict(key=self.key(), reason='forgotten')]) + ended = stream.next(lambda f: f['t'] == 'error') + self.assertEqual((ended['code'], ended['topic'], ended['key'], ended['fatal']), + ('subscription_ended', 'activity', self.key(), False)) + self.assertIn('forgotten', ended['message']) + self.assertEqual(stream.close(), 0) + + def test_activity_streams_new_journal_rows_and_summaries_until_unsubscribed(self): + first = self.journal('UserPromptSubmit', detail='Synthetic prompt') + stream, *_ = self.start() + stream.send(op='subscribe', topic='activity', session=self.session()) + initial = stream.next(lambda f: f['t'] == 'events') + self.assertEqual((initial['topic'], initial['key'], initial['cursor']), ('activity', self.key(), first)) + self.assertEqual(initial['events'], self.inspect_events(0)[-len(initial['events']):]) + state = stream.next(lambda f: f['t'] == 'state') + self.assertEqual((state['topic'], state['key'], state['rev']), ('activity', self.key(), 1)) + self.assertEqual(state['value']['name'], self.name) + for section in ('events', 'processes', 'cursor'): + self.assertNotIn(section, state['value']) + reply = self.journal('Stop', detail='Synthetic reply') + events = stream.next(lambda f: f['t'] == 'events' and f['cursor'] == reply, timeout=5) + self.assertEqual(events['events'], self.inspect_events(first)) + self.assertEqual([event['seq'] for event in events['events']], [reply]) + stream.send(op='ping', id='after-reply') + self.assertEqual(stream.next(lambda f: f['t'] == 'pong')['id'], 'after-reply') + stream.send(op='unsubscribe', topic='activity', session=self.session()) + stream.send(op='ping', id='unsubscribed') + stream.next(lambda f: f['t'] == 'pong' and f['id'] == 'unsubscribed') + self.journal('Stop', detail='Unwatched reply') + stream.quiet(lambda f: f['t'] in ('events', 'state'), 2) + self.assertEqual(stream.close(), 0) + + def test_resume_after_a_cursor_and_gap_below_retention(self): + rows = [self.journal('Stop', detail=f'reply {n}') for n in range(5)] + stream, *_ = self.start() + stream.send(op='subscribe', topic='activity', session=self.session(), after=rows[2]) + resumed = stream.next(lambda f: f['t'] == 'events') + self.assertEqual([event['seq'] for event in resumed['events']], rows[3:]) + self.assertEqual(resumed['cursor'], rows[-1]) + # Retention removed rows after the client's cursor. + db = sqlite3.connect(self.state / 'events.sqlite3') + with db: + db.execute('DELETE FROM events WHERE seq <= ?', (rows[2],)) + db.close() + stream.send(op='subscribe', topic='activity', session=self.session(), after=rows[0], tail=1) + gap = stream.next(lambda f: f['t'] == 'gap') + self.assertEqual((gap['topic'], gap['key'], gap['reason']), ('activity', self.key(), 'retention')) + tail = stream.next(lambda f: f['t'] == 'snapshot' and f['topic'] == 'activity') + self.assertEqual([event['seq'] for event in tail['items']], rows[-1:]) + self.assertEqual((tail['key'], tail['cursor'], tail['last']), (self.key(), rows[-1], True)) + # A cursor beyond the journal belongs to a replaced journal. + stream.send(op='subscribe', topic='activity', session=self.session(), after=rows[-1] + 1000) + self.assertEqual(stream.next(lambda f: f['t'] == 'gap')['reason'], 'reset') + self.assertEqual(stream.close(), 0) + + def test_subscription_limit_unknown_topics_and_bad_requests_are_errors(self): + stream, *_ = self.start() + for index in range(16): + stream.send(op='subscribe', topic='activity', session=dict(name=f'sh/limit-{index}'), tail=0) + stream.send(op='subscribe', topic='activity', session=dict(name='sh/limit-16')) + error = stream.next(lambda f: f['t'] == 'error', timeout=30) + self.assertEqual((error['code'], error['key'], error['fatal']), ('too_many_subscriptions', 'name:sh/limit-16', False)) + self.assertEqual(sum(1 for f in stream.seen if f['t'] == 'events'), 16) + stream.send(op='subscribe', topic='processes', session=self.session()) + self.assertEqual(stream.next(lambda f: f['t'] == 'error')['code'], 'unsupported_topic') + stream.process.stdin.write('{"op":"launch"}\nnot json\n') + stream.process.stdin.flush() + self.assertEqual(stream.next(lambda f: f['t'] == 'error')['code'], 'bad_request') + self.assertEqual(stream.next(lambda f: f['t'] == 'error')['code'], 'bad_request') + stream.send(op='subscribe', topic='activity') + self.assertEqual(stream.next(lambda f: f['t'] == 'error')['code'], 'bad_request') + self.assertEqual(stream.close(), 0) + + def test_closed_output_ends_the_stream_quietly(self): + process = subprocess.Popen([str(HGS), 'watch', '--json'], env=self.env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(lambda: process.poll() is None and process.kill()) + process.stdout.readline() + process.stdout.close() + # The next frame (snapshot, meta or a pong) meets a closed pipe. + try: + process.stdin.write(b'{"op":"ping"}\n' * 3) + process.stdin.flush() + except BrokenPipeError: + pass + self.assertEqual(process.wait(timeout=10), 0) + self.assertEqual(process.stderr.read(), b'') + process.stdin.close() + process.stderr.close() + + def test_usage_marker_and_dry_run(self): + result = subprocess.run([str(HGS), '-h'], env=self.env, text=True, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0) + self.assertIn('\n Event stream ABI: watch-v1\n', result.stdout) + self.assertIn('hgs [@host] watch --json long-lived NDJSON event stream (stdin control)', result.stdout) + result = subprocess.run([str(HGS), 'watch'], env=self.env, text=True, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 1) + self.assertIn('usage: hgs [@host] watch --json', result.stderr) + result = subprocess.run([str(HGS), '--dry-run', '@remote', 'watch', '--json'], env=self.env, + text=True, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('-o ServerAliveInterval=10 -o ServerAliveCountMax=3', result.stdout) + self.assertIn('grep -qF', result.stdout) + self.assertIn('Event stream ABI: watch-v1', result.stdout) + self.assertIn('then exec ~/.local/bin/hgs watch --json', result.stdout) + + def test_old_peer_is_never_launched_and_new_peer_streams_through_ssh(self): + ssh_log = self.root / 'ssh-args' + self.script('ssh', '#!/bin/sh\nprintf "%s\\n" "$@" > "' + str(ssh_log) + '"\n' + 'for argument do last="$argument"; done\nexec /bin/sh -c "$last"\n') + peer = self.root / '.local/bin/hgs' + peer.unlink() + launched = self.root / 'old-peer-launched' + peer.write_text('#!/bin/sh\nif [ "$1" = --help ]; then echo "usage: hgs [@host] "; exit 0; fi\n' + 'touch "' + str(launched) + '"\n') + peer.chmod(0o755) + old = subprocess.run([str(HGS), '@remote', 'watch', '--json'], env=self.env, input='', + text=True, capture_output=True, timeout=10) + self.assertEqual(old.returncode, 1) + self.assertIn('update the selected computer to stream events', old.stderr) + self.assertFalse(launched.exists()) + self.assertEqual(old.stdout, '') + peer.unlink() + peer.symlink_to(HGS) + stream = Stream(self, '@remote') + hello = stream.next() + self.assertEqual(hello['t'], 'hello') + stream.next(lambda f: f['t'] == 'snapshot' and f['topic'] == 'catalog' and f['last']) + stream.send(op='ping', id='peer') + self.assertEqual(stream.next(lambda f: f['t'] == 'pong')['id'], 'peer') + self.assertEqual(stream.close(), 0) + arguments = ssh_log.read_text().splitlines() + for option in ('ServerAliveInterval=10', 'ServerAliveCountMax=3', 'BatchMode=yes'): + self.assertIn(option, arguments) + self.assertNotIn('-t', arguments) + + +if __name__ == '__main__': + unittest.main() diff --git a/tray/CMakeLists.txt b/tray/CMakeLists.txt index 443606095..adbb84d3a 100644 --- a/tray/CMakeLists.txt +++ b/tray/CMakeLists.txt @@ -99,7 +99,7 @@ endif() add_library(hgs-updates STATIC src/UpdateController.cpp src/UpdateController.h src/WindowsUpdateTrust.cpp src/WindowsUpdateTrust.h) target_include_directories(hgs-updates PUBLIC src) -target_link_libraries(hgs-updates PUBLIC Qt6::Widgets Qt6::Network) +target_link_libraries(hgs-updates PUBLIC Qt6::Widgets Qt6::Network hgs-tray-core) if(WIN32) # WindowsUpdateTrust: CNG for the update manifest, WinVerifyTrust for installers. target_link_libraries(hgs-updates PRIVATE bcrypt wintrust crypt32) @@ -143,7 +143,7 @@ include(cmake/BuildVersion.cmake) # link; tests/CMakeLists.txt fails when a suite lists a source that one of them # builds. Each layer adds Qt modules, so a suite that links the lowest layer # covering its code keeps its QTEST_MAIN application type. -add_library(hgs-tray-core STATIC src/ProcessRunner.cpp src/HgsClient.cpp src/FleetState.cpp +add_library(hgs-tray-core STATIC src/ProcessRunner.cpp src/JsonParse.cpp src/SettingsJson.cpp src/HgsClient.cpp src/EventStream.cpp src/FleetState.cpp src/SessionCreationController.cpp src/AnalyticsClient.cpp src/AnalyticsData.cpp src/ReadMarkSync.cpp src/AppConfig.cpp src/AccountUsageStore.cpp src/ComposerDraftStore.cpp @@ -192,7 +192,7 @@ target_link_libraries(hgs-tray-ui PUBLIC hgs-tray-core Qt6::Widgets add_library(hgs-tray-activity STATIC src/ActivityView.cpp) target_include_directories(hgs-tray-activity PUBLIC src) -target_link_libraries(hgs-tray-activity PUBLIC Qt6::Widgets Qt6::Multimedia PRIVATE hgs-markdown hgs-round-video) +target_link_libraries(hgs-tray-activity PUBLIC Qt6::Widgets Qt6::Multimedia PRIVATE hgs-tray-core hgs-markdown hgs-round-video) target_link_libraries(hgs-tray-ui PRIVATE hgs-tray-activity) add_library(hgs-tray-views STATIC src/ArchiveDatePicker.cpp src/ArchiveDatePicker.h src/ArchiveBrowser.cpp src/ArchiveBrowser.h src/SessionsWindow.cpp src/SessionsWindowShortcuts.cpp src/SessionsWindowGoTo.cpp src/SessionSelection.cpp src/NativeUiView.cpp diff --git a/tray/src/AccountCatalog.h b/tray/src/AccountCatalog.h index de01c6a2c..4013e1efa 100644 --- a/tray/src/AccountCatalog.h +++ b/tray/src/AccountCatalog.h @@ -1,6 +1,6 @@ #pragma once +#include "IdentityHash.h" #include "MachinePlatform.h" -#include #include #include #include @@ -22,7 +22,7 @@ inline QString key(const QJsonObject &p) { return "profile:" + profileKey(p); const auto scope = identity["organization"].toString().trimmed().toCaseFolded(); const auto identityKey = p["provider"].toString() + '\n' + (account.isEmpty() ? "email:" + email : "id:" + account) + '\n' + scope; - return "account:" + QString::fromLatin1(QCryptographicHash::hash(identityKey.toUtf8(), QCryptographicHash::Sha256).toHex()); + return "account:" + QString::fromLatin1(IdentityHash::sha256(identityKey.toUtf8()).toHex()); } // A quota belongs to the account: select a snapshot, never add quotas from // several machines or let a failed probe replace a good snapshot. diff --git a/tray/src/AccountFilter.cpp b/tray/src/AccountFilter.cpp index 056f1c29f..65b9121fe 100644 --- a/tray/src/AccountFilter.cpp +++ b/tray/src/AccountFilter.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "AccountFilter.h" #include "ScopeFilterWidgets.h" #include "WorkspaceIcons.h" @@ -27,7 +28,7 @@ QString AccountFilter::caption(const SessionAccounts::Account &account) const { })) { QStringList profiles; for (const auto &binding : account.bindings) { - const auto fields = QJsonDocument::fromJson(binding.toUtf8()).array(); + const auto fields = JsonParse::localArray(binding.toUtf8()); if (fields.size() != 4) continue; const auto host = fields[0].toString(), profile = fields[3].toString(); profiles.append((host.isEmpty() ? tr("This computer") : host) + " / " + (profile.isEmpty() ? tr("Not recorded") : profile)); diff --git a/tray/src/AnalyticsClient.cpp b/tray/src/AnalyticsClient.cpp index bf9613168..9eb3ea709 100644 --- a/tray/src/AnalyticsClient.cpp +++ b/tray/src/AnalyticsClient.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "AnalyticsClient.h" #include "AnalyticsData.h" #include "ProcessRunner.h" @@ -58,8 +59,8 @@ void AnalyticsClient::request(const QString &host, const QString &verb, const QJ if (mutation) { // Mutations return metadata with the default range. Only a // fresh explicit summary repopulates the controls/view. - QJsonParseError error; const auto json = QJsonDocument::fromJson(bytes,&error); - if (error.error == QJsonParseError::NoError && json.isObject() && json.object().value("schema") == 1 + QString error; const auto json = JsonParse::parse(bytes,&error); + if (error.isEmpty() && json.isObject() && json.object().value("schema") == 1 && json.object().value("source") == "agent" && !json.object().contains("error")) safe = QJsonObject{{"schema",1}}; } else safe = verb == "speech" ? AnalyticsData::speech(bytes,from,through) : AnalyticsData::summary(bytes,from,through); if (guard) QMetaObject::invokeMethod(guard,[guard,safe,done,mutation] { diff --git a/tray/src/AnalyticsData.cpp b/tray/src/AnalyticsData.cpp index b61887abe..3fd7a4e65 100644 --- a/tray/src/AnalyticsData.cpp +++ b/tray/src/AnalyticsData.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "AnalyticsData.h" #include #include @@ -24,9 +25,9 @@ bool uuid(const QJsonValue &value, bool allowNil = false) { } std::optional object(const QByteArray &bytes) { if (bytes.size() > MaximumBytes) return {}; - QJsonParseError error; - const auto document = QJsonDocument::fromJson(bytes, &error); - if (error.error != QJsonParseError::NoError || !document.isObject() || document.object().value("schema") != 1) return {}; + QString error; + const auto document = JsonParse::parse(bytes, &error); + if (!error.isEmpty() || !document.isObject() || document.object().value("schema") != 1) return {}; return document.object(); } std::optional codes(const QJsonValue &value, const QStringList &allowed) { diff --git a/tray/src/AttentionTracker.cpp b/tray/src/AttentionTracker.cpp index a28c0e849..903035c7b 100644 --- a/tray/src/AttentionTracker.cpp +++ b/tray/src/AttentionTracker.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "AttentionTracker.h" #include "SessionPresentation.h" #include "SessionNotifications.h" @@ -269,7 +270,7 @@ void AttentionTracker::deliveryFailed(const QString &token, qint64 now) QJsonObject AttentionTracker::target(const QString &token) { if (token.size() > 8192) return {}; - const auto result = QJsonDocument::fromJson(QByteArray::fromBase64(token.toLatin1(), QByteArray::Base64UrlEncoding)).object(); + const auto result = JsonParse::localObject(QByteArray::fromBase64(token.toLatin1(), QByteArray::Base64UrlEncoding)); if (!result.value("host").isString() || result.value("name").toString().isEmpty() || result.value("notice").toString().isEmpty()) return {}; return result; diff --git a/tray/src/ComposerDraftStore.cpp b/tray/src/ComposerDraftStore.cpp index 3a31980ed..7dd04e7a8 100644 --- a/tray/src/ComposerDraftStore.cpp +++ b/tray/src/ComposerDraftStore.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "ComposerDraftStore.h" #include @@ -138,9 +139,13 @@ void ComposerDraftStore::saveAsync(const QString &key, const ComposerDraft &draf owner, std::move(done)); } +// The blocking fixture API reads on the writer thread too, after every queued +// write, so no draft file is parsed on the GUI thread. ComposerDraft ComposerDraftStore::load(const QString &key) const { - flushWrites(); return loadDirect(key); + const auto store = *this; ComposerDraft draft; + DraftWrites::instance()->sync([&] { draft = store.loadDirect(key); return WriteResult{true, {}}; }); + return draft; } ComposerDraft ComposerDraftStore::loadDirect(const QString &key) const @@ -152,9 +157,9 @@ ComposerDraft ComposerDraftStore::loadDirect(const QString &key) const draft.storageError = QObject::tr("Could not restore this draft completely. Saved files have been kept locally."); }; if (!file.open(QIODevice::ReadOnly)) { invalid(); return draft; } - QJsonParseError error; - const auto object = QJsonDocument::fromJson(file.readAll(), &error).object(); - if (error.error != QJsonParseError::NoError || object.value("version").toInt() != 1 || object.value("key").toString() != key) { + QString error; + const auto object = JsonParse::parse(file.readAll(), &error).object(); + if (!error.isEmpty() || object.value("version").toInt() != 1 || object.value("key").toString() != key) { invalid(); return draft; } draft.text = object.value("text").toString(); @@ -218,7 +223,9 @@ bool ComposerDraftStore::saveDirect(const QString &key, ComposerDraft &draft) co QStringList ComposerDraftStore::keys() const { - flushWrites(); auto result = indexDirect().keys(); result.sort(); return result; + const auto store = *this; QStringList result; + DraftWrites::instance()->sync([&] { result = store.indexDirect().keys(); return WriteResult{true, {}}; }); + result.sort(); return result; } QHash ComposerDraftStore::indexDirect() const @@ -227,7 +234,7 @@ QHash ComposerDraftStore::indexDirect() const for (const auto &name : QDir(root()).entryList(QDir::Dirs | QDir::NoDotAndDotDot)) { QFile file(root() + '/' + name + "/draft.json"); if (!file.open(QIODevice::ReadOnly)) continue; - const auto object = QJsonDocument::fromJson(file.readAll()).object(); + const auto object = JsonParse::parse(file.readAll()).object(); const auto key = object.value("key").toString(); const auto text = object.value("text").toString(); const auto attachments = object.value("attachments").toArray(); @@ -260,7 +267,7 @@ ComposerDraftOwner ComposerDraftStore::owner(const QString &key) } if (key.startsWith(Question)) { // A question identity is a compact JSON array, so it holds no line break. - const auto identity = QJsonDocument::fromJson(key.mid(Question.size()).section('\n', 0, 0).toUtf8()).array(); + const auto identity = JsonParse::localArray(key.mid(Question.size()).section('\n', 0, 0).toUtf8()); auto result = owner(identity.at(0).toString()); if (result.kind != ComposerDraftOwner::Kind::Unknown) { result.kind = ComposerDraftOwner::Kind::Question; return result; } } @@ -328,11 +335,11 @@ void ComposerDraftStore::renameAsync(const QString &oldKey, const QString &newKe QString ComposerDraftStore::label(const QString &key) const { - flushWrites(); - QFile file(folder(key) + "/draft.json"); - if (file.open(QIODevice::ReadOnly)) { - const auto label = QJsonDocument::fromJson(file.readAll()).object().value("label").toString(); - if (!label.isEmpty()) return label; - } - return QString(key).replace('\n', " / "); + const auto path = folder(key) + "/draft.json"; QString label; + DraftWrites::instance()->sync([&] { + QFile file(path); + if (file.open(QIODevice::ReadOnly)) label = JsonParse::parse(file.readAll()).object().value("label").toString(); + return WriteResult{true, {}}; + }); + return label.isEmpty() ? QString(key).replace('\n', " / ") : label; } diff --git a/tray/src/DownloadCache.cpp b/tray/src/DownloadCache.cpp index 54e4b211d..db473f2a5 100644 --- a/tray/src/DownloadCache.cpp +++ b/tray/src/DownloadCache.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "DownloadCache.h" #include "LocalFile.h" #include @@ -138,10 +139,9 @@ struct Ledger { QFile file(QDir(root).filePath(Index)); if (!occupied(file.fileName())) return; if (!ordinary(QFileInfo(file)) || file.size() > 32 * 1024 * 1024 || !file.open(QIODevice::ReadOnly)) { error = text("Cannot read the downloaded files index."); return; } - QJsonParseError parse; - const auto document = QJsonDocument::fromJson(file.readAll(), &parse); - const auto object = document.object(); - if (parse.error != QJsonParseError::NoError || object.value("version") != 1 || !object.value("files").isArray()) { error = text("The downloaded files index is invalid. Existing files were preserved."); return; } + QString parse; + const auto object = JsonParse::parse(file.readAll(), &parse).object(); + if (!parse.isEmpty() || object.value("version") != 1 || !object.value("files").isArray()) { error = text("The downloaded files index is invalid. Existing files were preserved."); return; } entries = object.value("files").toArray(); QSet names; for (const auto &value : entries) { diff --git a/tray/src/EventStream.cpp b/tray/src/EventStream.cpp new file mode 100644 index 000000000..8fa05848a --- /dev/null +++ b/tray/src/EventStream.cpp @@ -0,0 +1,428 @@ +#include "EventStream.h" + +#include "ChildProcess.h" +#include "JsonParse.h" +#include "ProcessRunner.h" + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace { +constexpr qsizetype MaximumLine = 1024 * 1024 + 64; +// Until fleet preparation leaves the GUI thread (M2), applying a listing costs +// a rebuild there (about 150 ms on a 2026-10-27 Mac sample); listings keep the +// 2 s polling cadence at most, while Activity arrives within PublishMs. +std::atomic ListingGapMs{2000}; +// Callers date a computer's data by its last listing: automatic approvals act +// on a peer listed within 30 s, and 3 minutes mark it stale. An unchanged +// listing is applied again after this long. +std::atomic ListingRefreshMs{10000}; +const QByteArray Marker = QByteArrayLiteral("Event stream ABI: watch-v1"); +EventStreams *s_instance = nullptr; + +QString sessionKey(const QString &name, const QString &runId) +{ + return runId.isEmpty() ? QStringLiteral("name:") + name : QStringLiteral("run:") + runId; +} + +QByteArray line(const QJsonObject &object) +{ + return QJsonDocument(object).toJson(QJsonDocument::Compact) + '\n'; +} +} + +// Everything below runs on the stream's thread. It talks to its EventStream +// only through queued calls, which Qt drops once the stream is gone. +class EventStreamWorker : public QObject { +public: + EventStreamWorker(EventStream *front, QString host) : m_front(front), m_host(std::move(host)) {} + ~EventStreamWorker() override { stopProcess(); } + + void begin(const ProcessRunner::Request &request) + { + stopProcess(); reset(); + if (!m_publish) { + m_publish = new QTimer(this); m_publish->setSingleShot(true); + QObject::connect(m_publish, &QTimer::timeout, this, [this] { publish(); }); + m_silence = new QTimer(this); m_silence->setInterval(qMin(5000, ListingRefreshMs.load())); + QObject::connect(m_silence, &QTimer::timeout, this, [this] { + const auto now = QDateTime::currentMSecsSinceEpoch(); + if (now - m_lastFrame > EventStream::SilenceMs) return fail(QStringLiteral("no frame for 30 s")); + if (m_ready && now - m_lastListing >= ListingRefreshMs.load()) { m_listingDirty = true; schedule(); } + }); + } + m_publish->stop(); + auto *process = m_process = new QProcess(this); + process->setProgram(request.program); process->setArguments(request.arguments); + if (request.environment) process->setProcessEnvironment(*request.environment); + ChildProcess::inheritStdioOnly(*process); + QObject::connect(process, &QProcess::readyReadStandardOutput, this, [this] { read(); }); + QObject::connect(process, &QProcess::readyReadStandardError, this, [this] { + if (m_process) m_stderr = (m_stderr + m_process->readAllStandardError()).right(4000); + }); + QObject::connect(process, &QProcess::finished, this, [this] { + read(); + fail(m_stderr.trimmed().isEmpty() ? QStringLiteral("the stream ended") : QString::fromUtf8(m_stderr.trimmed())); + }); + QObject::connect(process, &QProcess::errorOccurred, this, [this](QProcess::ProcessError error) { + if (error == QProcess::FailedToStart) fail(QStringLiteral("cannot start the stream")); + }); + m_lastFrame = QDateTime::currentMSecsSinceEpoch(); + m_silence->start(); + process->start(); + } + void finish() { stopProcess(); reset(); } + + void follow(const QString &name, const QString &runId, const QString &conversationId, qint64 after) + { + if (!m_activity.key.isEmpty()) { + write(QJsonObject{{"op", "unsubscribe"}, {"topic", "activity"}, + {"session", QJsonObject{{"name", m_activity.name}, {"run_id", m_activity.run}}}}); + } + m_activity = {}; + if (name.isEmpty()) return; + m_activity.key = sessionKey(name, runId); m_activity.name = name; m_activity.run = runId; + QJsonObject session{{"name", name}}; + if (!runId.isEmpty()) session["run_id"] = runId; + if (!conversationId.isEmpty()) session["conversation_id"] = conversationId; + QJsonObject subscribe{{"op", "subscribe"}, {"topic", "activity"}, {"session", session}}; + if (after > 0) subscribe["after"] = after; else subscribe["tail"] = 100; + write(subscribe); + } + +private: + struct Activity { + QString key, name, run; + QJsonObject state; + QJsonArray events; + qint64 cursor = 0; + bool replace = false, dirty = false; + }; + + void reset() + { + m_buffer.clear(); m_stderr.clear(); m_rows.clear(); m_pending.clear(); m_meta = {}; + m_catalog = m_haveMeta = m_listingDirty = m_ready = m_failed = false; m_activity = {}; + } + void stopProcess() + { + if (!m_process) return; + auto *process = std::exchange(m_process, nullptr); + process->disconnect(this); + if (process->state() != QProcess::NotRunning) { + // EOF ends the stream; a peer's ssh closes with it. + process->closeWriteChannel(); + if (!process->waitForFinished(1000)) { process->kill(); process->waitForFinished(1000); } + } + delete process; + if (m_silence) m_silence->stop(); + } + void write(const QJsonObject &operation) + { + if (m_process && m_process->state() == QProcess::Running) m_process->write(line(operation)); + } + void fail(const QString &reason) + { + if (m_failed) return; + // Nothing this stream prepared reaches the GUI after its failure. + m_failed = true; m_publish->stop(); m_listingDirty = m_activity.dirty = false; + stopProcess(); + QMetaObject::invokeMethod(m_front, [front = m_front, reason] { front->failed(reason); }, Qt::QueuedConnection); + } + void read() + { + if (!m_process || m_failed) return; + m_buffer += m_process->readAllStandardOutput(); + qsizetype start = 0; + for (;;) { + const auto end = m_buffer.indexOf('\n', start); + if (end < 0) break; + if (end - start > MaximumLine) return fail(QStringLiteral("a frame exceeds 1 MiB")); + QString error; + const auto frame = JsonParse::object(m_buffer.mid(start, end - start), &error); + start = end + 1; + if (!error.isEmpty()) return fail(QStringLiteral("invalid frame: ") + error); + m_lastFrame = QDateTime::currentMSecsSinceEpoch(); + handle(frame); + if (m_failed) return; + } + m_buffer.remove(0, start); + if (m_buffer.size() > MaximumLine) fail(QStringLiteral("a frame exceeds 1 MiB")); + } + void handle(const QJsonObject &frame) + { + if (frame.value("v").toInt() != 1) return fail(QStringLiteral("unsupported stream version")); + const auto type = frame.value("t").toString(), topic = frame.value("topic").toString(); + if (type == "snapshot" && topic == "catalog") { + if (frame.value("page").toInt() == 0) m_pending.clear(); + for (const auto &value : frame.value("items").toArray()) { + const auto item = value.toObject(); + m_pending.insert(item.value("key").toString(), item.value("row").toObject()); + } + if (frame.value("last").toBool()) { m_rows = std::exchange(m_pending, {}); m_catalog = m_listingDirty = true; } + } else if (type == "delta" && topic == "catalog") { + for (const auto &value : frame.value("upsert").toArray()) { + const auto item = value.toObject(); + m_rows.insert(item.value("key").toString(), item.value("row").toObject()); + } + for (const auto &value : frame.value("remove").toArray()) m_rows.remove(value.toObject().value("key").toString()); + m_listingDirty = true; + } else if (type == "snapshot" && topic == "catalog.meta") { + m_meta = frame.value("value").toObject(); m_haveMeta = m_listingDirty = true; + } else if (topic == "activity" && frame.value("key").toString() == m_activity.key && !m_activity.key.isEmpty()) { + if (type == "events") { + for (const auto &event : frame.value("events").toArray()) m_activity.events.append(event); + m_activity.cursor = frame.value("cursor").toInteger(); m_activity.dirty = true; + } else if (type == "snapshot") { + m_activity.events = frame.value("items").toArray(); m_activity.replace = true; + m_activity.cursor = frame.value("cursor").toInteger(); m_activity.dirty = true; + } else if (type == "state") { + m_activity.state = frame.value("value").toObject(); m_activity.dirty = true; + } else if (type == "gap") { + m_activity.events = {}; m_activity.dirty = false; + } + } else if (type == "error") { + if (frame.value("fatal").toBool()) return fail(frame.value("message").toString()); + const auto code = frame.value("code").toString(); + if ((code == "subscription_ended" || code == "session_changed") && frame.value("key").toString() == m_activity.key + && !m_activity.key.isEmpty()) { + const auto name = m_activity.name; m_activity = {}; + QMetaObject::invokeMethod(m_front, [front = m_front, name] { front->m_followedName.clear(); emit front->activityEnded(name); }, + Qt::QueuedConnection); + } + } + // hello, heartbeat, pong and catalog gaps only prove the stream is alive; + // after a catalog gap the previous rows stay until the next snapshot. + schedule(); + } + // Activity goes out within PublishMs; a listing waits out ListingGapMs + // after the previous one, except the first. + qint64 listingWait() const + { + if (!m_ready) return 0; + return qMax(0, ListingGapMs.load() - (QDateTime::currentMSecsSinceEpoch() - m_lastListing)); + } + void schedule() + { + if (m_failed || m_publish->isActive()) return; + if (m_activity.dirty) m_publish->start(EventStream::PublishMs); + else if (m_listingDirty) m_publish->start(int(qMax(EventStream::PublishMs, listingWait()))); + } + void publish() + { + if (m_failed) return; + if (m_listingDirty && m_catalog && m_haveMeta && listingWait() == 0) { + // Rows in listing order, as ls --json sorts them. + QList rows = m_rows.values(); + std::sort(rows.begin(), rows.end(), [](const QJsonObject &a, const QJsonObject &b) { + const auto left = a.value("name").toString(), right = b.value("name").toString(); + const int folded = QString::compare(left, right, Qt::CaseInsensitive); + return folded != 0 ? folded < 0 : left < right; + }); + QJsonArray sessions; for (const auto &row : std::as_const(rows)) sessions.append(row); + auto listing = m_meta; listing["sessions"] = sessions; + BoxState box = HgsClient::box(listing); + // A peer's listing names the computer as it calls itself (requestPeer). + if (!m_host.isEmpty()) box.reportedHost = std::exchange(box.host, m_host); + m_listingDirty = false; m_lastListing = QDateTime::currentMSecsSinceEpoch(); + const bool first = !std::exchange(m_ready, true); + QMetaObject::invokeMethod(m_front, [front = m_front, box, first] { + if (first) front->setState(EventStream::State::Live); + emit front->listingReady(box); + }, Qt::QueuedConnection); + } + if (m_activity.dirty && !m_activity.state.isEmpty()) { + auto inspection = m_activity.state; + inspection["events"] = std::exchange(m_activity.events, {}); + inspection["cursor"] = m_activity.cursor; + if (std::exchange(m_activity.replace, false)) inspection["replace_events"] = true; + m_activity.dirty = false; + QMetaObject::invokeMethod(m_front, [front = m_front, name = m_activity.name, inspection] { + if (front->m_followedName == name) { + front->m_followedAfter = inspection.value("cursor").toInteger(); + emit front->activityReady(name, inspection); + } + }, Qt::QueuedConnection); + } + schedule(); + } + + EventStream *m_front; + QString m_host; + QProcess *m_process = nullptr; + QTimer *m_publish = nullptr, *m_silence = nullptr; + QByteArray m_buffer, m_stderr; + qint64 m_lastFrame = 0, m_lastListing = 0; + QHash m_rows, m_pending; + QJsonObject m_meta; + bool m_catalog = false, m_haveMeta = false, m_listingDirty = false, m_ready = false, m_failed = false; + Activity m_activity; +}; + +EventStream::EventStream(const QString &hgs, const QString &host, QObject *parent) + : QObject(parent), m_hgs(hgs), m_host(host) +{ + m_thread.setObjectName(host.isEmpty() ? QStringLiteral("hgs-stream") : QStringLiteral("hgs-stream-") + host); + m_worker = new EventStreamWorker(this, host); + m_worker->moveToThread(&m_thread); + connect(&m_thread, &QThread::finished, m_worker, &QObject::deleteLater); + m_thread.start(); + m_restart.setSingleShot(true); + connect(&m_restart, &QTimer::timeout, this, [this] { start(); }); +} + +EventStream::~EventStream() +{ + m_restart.stop(); + QMetaObject::invokeMethod(m_worker, [worker = m_worker] { worker->finish(); }, Qt::BlockingQueuedConnection); + m_thread.quit(); m_thread.wait(); +} + +void EventStream::setListingTimingForTests(int gapMs, int refreshMs) +{ + ListingGapMs = gapMs; ListingRefreshMs = refreshMs; +} + +int EventStream::restartDelayMs(int failures) +{ + static const int delays[] = {5000, 15000, 60000, 300000}; + return delays[qBound(0, failures - 1, 3)]; +} + +void EventStream::setState(State state) +{ + if (m_state == state) return; + m_state = state; + if (state == State::Live) sendFollow(); + emit stateChanged(state); +} + +void EventStream::start() +{ + if (m_state == State::Probing || m_state == State::Starting || m_state == State::Live) return; + setState(State::Probing); + QStringList help{QStringLiteral("--help")}; + if (!m_host.isEmpty()) help.prepend(QLatin1Char('@') + m_host); + auto request = HgsClient::commandRequest(m_hgs, help); request.timeoutMs = m_host.isEmpty() ? 5000 : 15000; + ProcessRunner::run(std::move(request), this, + [](ProcessRunner::Result result) { + return result.outcome == ProcessRunner::Result::Finished && result.exitCode == 0 + ? (result.standardOutput.contains(Marker) ? 1 : 0) : -1; + }, + [this](int supported) { + if (m_state != State::Probing) return; + if (supported == 1) return launch(); + if (supported == 0) { setState(State::Unsupported); m_restart.start(600000); return; } + failed(QStringLiteral("the computer did not answer")); + }); +} + +void EventStream::launch() +{ + setState(State::Starting); + QStringList args{QStringLiteral("watch"), QStringLiteral("--json")}; + if (!m_host.isEmpty()) args.prepend(QLatin1Char('@') + m_host); + const auto request = HgsClient::commandRequest(m_hgs, args); + QMetaObject::invokeMethod(m_worker, [worker = m_worker, request] { worker->begin(request); }, Qt::QueuedConnection); +} + +void EventStream::stop() +{ + m_restart.stop(); + QMetaObject::invokeMethod(m_worker, [worker = m_worker] { worker->finish(); }, Qt::QueuedConnection); + setState(State::Stopped); +} + +void EventStream::failed(const QString &reason) +{ + if (m_state == State::Stopped) return; + const auto now = QDateTime::currentMSecsSinceEpoch(); + // Failures more than ten minutes apart start the delays over. + m_failures = now - m_lastFailure > 600000 ? 1 : m_failures + 1; + m_lastFailure = now; + qWarning().noquote() << "hgs zerus: event stream" << (m_host.isEmpty() ? QStringLiteral("(local)") : m_host) << "stopped:" << reason; + setState(State::Failed); + m_restart.start(restartDelayMs(m_failures)); +} + +void EventStream::follow(const QString &name, const QString &runId, const QString &conversationId, qint64 after) +{ + if (name == m_followedName && runId == m_followedRun && conversationId == m_followedConversation) return; + m_followedName = name; m_followedRun = runId; m_followedConversation = conversationId; m_followedAfter = after; + if (live()) sendFollow(); +} + +void EventStream::unfollow() +{ + if (m_followedName.isEmpty()) return; + m_followedName.clear(); m_followedRun.clear(); m_followedConversation.clear(); m_followedAfter = 0; + if (live()) sendFollow(); +} + +void EventStream::sendFollow() +{ + QMetaObject::invokeMethod(m_worker, [worker = m_worker, name = m_followedName, run = m_followedRun, + conversation = m_followedConversation, after = m_followedAfter] { + worker->follow(name, run, conversation, after); + }, Qt::QueuedConnection); +} + +EventStreams::EventStreams(const QString &hgs, QObject *parent) : QObject(parent), m_hgs(hgs) +{ + Q_ASSERT(!s_instance); + s_instance = this; +} + +EventStreams::~EventStreams() +{ + if (s_instance == this) s_instance = nullptr; + qDeleteAll(std::exchange(m_streams, {})); +} + +EventStreams *EventStreams::instance() { return s_instance; } + +bool EventStreams::enabled() +{ + if (qEnvironmentVariable("ZERUS_EVENT_STREAM") == QLatin1String("0")) return false; + return QSettings().value(QStringLiteral("dataPlane/eventStream"), true).toBool(); +} + +void EventStreams::setHosts(const QStringList &hosts) +{ + for (auto it = m_streams.begin(); it != m_streams.end();) { + if (hosts.contains(it.key())) { ++it; continue; } + delete it.value(); it = m_streams.erase(it); + } + for (const auto &host : hosts) { + if (m_streams.contains(host)) continue; + auto *stream = new EventStream(m_hgs, host, this); + m_streams.insert(host, stream); + connect(stream, &EventStream::listingReady, this, [this, host](const BoxState &box) { emit listingReady(host, box); }); + connect(stream, &EventStream::activityReady, this, [this, host](const QString &name, const QJsonObject &inspection) { + emit activityReady(host, name, inspection); + }); + connect(stream, &EventStream::activityEnded, this, [this, host](const QString &name) { emit activityEnded(host, name); }); + connect(stream, &EventStream::stateChanged, this, [this, host](EventStream::State state) { emit stateChanged(host, state); }); + stream->start(); + } +} + +void EventStreams::follow(const QString &host, const QString &name, const QString &runId, const QString &conversationId, qint64 after) +{ + for (auto it = m_streams.cbegin(); it != m_streams.cend(); ++it) { + if (it.key() == host) it.value()->follow(name, runId, conversationId, after); + else it.value()->unfollow(); + } +} + +void EventStreams::unfollowAll() +{ + for (auto *stream : std::as_const(m_streams)) stream->unfollow(); +} diff --git a/tray/src/EventStream.h b/tray/src/EventStream.h new file mode 100644 index 000000000..280db1ef7 --- /dev/null +++ b/tray/src/EventStream.h @@ -0,0 +1,108 @@ +#pragma once + +#include "HgsClient.h" + +#include +#include +#include +#include +#include +#include + +#include + +class EventStreamWorker; + +// One computer's `hgs [@host] watch --json` (docs/reference.md, Event stream). +// The child process, NDJSON framing, JSON parsing and the computer's catalog +// and Activity state live on the stream's own thread. The GUI thread receives +// finished values: the same BoxState a listing poll delivers, and +// inspect-shaped Activity increments, at most one of each every 50 ms. +// +// A computer whose CLI lacks the watch-v1 marker stays Unsupported and keeps +// polling. A stream that fails restarts with a growing delay; while it is not +// Live, callers poll as before. +class EventStream : public QObject { + Q_OBJECT +public: + enum class State { Stopped, Probing, Starting, Live, Unsupported, Failed }; + Q_ENUM(State) + + EventStream(const QString &hgs, const QString &host, QObject *parent = nullptr); + ~EventStream() override; + + QString host() const { return m_host; } + State state() const { return m_state; } + bool live() const { return m_state == State::Live; } + void start(); + void stop(); + + // Follows one session's Activity; a later call for another session of this + // computer replaces it. after is the journal seq to resume after (0: tail). + void follow(const QString &name, const QString &runId, const QString &conversationId, qint64 after); + void unfollow(); + QString followed() const { return m_followedName; } + + // Tests: frames and the delay before a restart. + static constexpr int PublishMs = 50; + static constexpr int SilenceMs = 30000; + static int restartDelayMs(int failures); + // Tests: the least time between applied listings (2 s) and the time after + // which an unchanged listing is applied again (10 s). + static void setListingTimingForTests(int gapMs, int refreshMs); + +signals: + void stateChanged(EventStream::State state); + void listingReady(const BoxState &box); + void activityReady(const QString &name, const QJsonObject &inspection); + // The followed session left the catalog or changed run; poll it instead. + void activityEnded(const QString &name); + +private: + friend class EventStreamWorker; + void setState(State state); + void launch(); + void failed(const QString &reason); + void sendFollow(); + + QString m_hgs, m_host; + State m_state = State::Stopped; + QThread m_thread; + EventStreamWorker *m_worker = nullptr; + QTimer m_restart; + int m_failures = 0; + qint64 m_lastFailure = 0; + QString m_followedName, m_followedRun, m_followedConversation; + qint64 m_followedAfter = 0; +}; + +// The desktop's streams by computer alias ("" is this computer), owned by the +// tray agent. Windows read it to follow Activity and to skip polls. +class EventStreams : public QObject { + Q_OBJECT +public: + explicit EventStreams(const QString &hgs, QObject *parent = nullptr); + ~EventStreams() override; + static EventStreams *instance(); + // Off with ZERUS_EVENT_STREAM=0 or the setting dataPlane/eventStream=false. + static bool enabled(); + + EventStream *stream(const QString &host) const { return m_streams.value(host); } + bool live(const QString &host) const { const auto *s = stream(host); return s && s->live(); } + // Starts streams for these computers and stops the others. + void setHosts(const QStringList &hosts); + // Follows one session's Activity on its computer's stream and stops + // following on every other stream. + void follow(const QString &host, const QString &name, const QString &runId, const QString &conversationId, qint64 after); + void unfollowAll(); + +signals: + void listingReady(const QString &host, const BoxState &box); + void activityReady(const QString &host, const QString &name, const QJsonObject &inspection); + void activityEnded(const QString &host, const QString &name); + void stateChanged(const QString &host, EventStream::State state); + +private: + QString m_hgs; + QHash m_streams; +}; diff --git a/tray/src/HgsClient.cpp b/tray/src/HgsClient.cpp index 5e6e22b2c..2be034ef5 100644 --- a/tray/src/HgsClient.cpp +++ b/tray/src/HgsClient.cpp @@ -2,10 +2,12 @@ #include #include "DownloadCache.h" #include "ChildProcess.h" +#include "JsonParse.h" #include "ProcessRunner.h" #include "LocalFile.h" #include "MachinePlatform.h" #include "SessionNotifications.h" +#include "SettingsJson.h" #include #include @@ -26,10 +28,6 @@ #include #include #include -#include -#include -#include -#include #include #include @@ -40,6 +38,35 @@ namespace { +// A JSON object reply parsed on the preparation thread. error is empty when the +// reply was a JSON object within limit bytes (0: any size). +struct ObjectReply { + QJsonObject object; + QString error; + bool ok() const { return error.isEmpty(); } +}; +ObjectReply objectReply(const QByteArray &bytes, qsizetype limit = 0) +{ + ObjectReply reply; + if (limit > 0 && bytes.size() > limit) { reply.error = QStringLiteral("reply exceeds %1 bytes").arg(limit); return reply; } + reply.object = JsonParse::object(bytes, &reply.error); + return reply; +} +const auto parseObject = [](const QByteArray &bytes) { return objectReply(bytes); }; +// The reply as an object, or an empty object for anything else. +const auto parseJsonObject = [](const QByteArray &bytes) { return JsonParse::object(bytes); }; +// A finished child's result with its standard output parsed as an object. +struct ParsedResult { + ProcessRunner::Result result; + ObjectReply reply; +}; +ParsedResult parsedResult(ProcessRunner::Result result, qsizetype limit = 0) +{ + ParsedResult parsed; parsed.reply = objectReply(result.standardOutput, limit); parsed.result = std::move(result); + return parsed; +} + + // The executable a start would use: a path as given, otherwise a PATH lookup // (which honors PATHEXT on Windows, where C:\Zerus\hgs.exe is a path as well). QString resolvedProgram(const QString &program) @@ -273,7 +300,9 @@ quint64 HgsClient::requestRecovery(const QString &host, const QString &operation QStringList args{"recovery", operation}; if (!host.isEmpty()) args.prepend('@' + host); auto process = processRequest(m_hgs, args); process.timeoutMs = host.isEmpty() ? 10000 : 25000; process.input = operation != "get" ? QJsonDocument(data).toJson(QJsonDocument::Compact) : QByteArray(); - ProcessRunner::run(std::move(process), this, [this, request, host, operation](const ProcessRunner::Result &result) { + ProcessRunner::run(std::move(process), this, [](ProcessRunner::Result result) { return parsedResult(std::move(result)); }, + [this, request, host, operation](const ParsedResult &parsed) { + const auto &result = parsed.result; const auto fail = [this, request](const QString &error) { emit recoveryFinished(request, false, {}, error); }; if (result.outcome == ProcessRunner::Result::TimedOut) return fail(tr("Recovery request timed out. Reload its state before trying again.")); if (result.outcome == ProcessRunner::Result::FailedToStart) return fail(result.errorString); @@ -281,9 +310,8 @@ quint64 HgsClient::requestRecovery(const QString &host, const QString &operation const auto error = QString::fromUtf8(result.standardError).trimmed().left(2000); return fail(error.isEmpty() ? tr("Could not update recovery. Reload its state.") : error); } - QJsonParseError error; const auto doc = QJsonDocument::fromJson(result.standardOutput, &error); - if (error.error != QJsonParseError::NoError || !doc.isObject()) return fail(tr("Invalid recovery response")); - emit recoveryFinished(request, true, doc.object(), {}); if (operation != "get") emit sessionWriteDone(host); + if (!parsed.reply.ok()) return fail(tr("Invalid recovery response")); + emit recoveryFinished(request, true, parsed.reply.object, {}); if (operation != "get") emit sessionWriteDone(host); }); return request; } @@ -301,6 +329,9 @@ HgsClient::~HgsClient() } } +BoxState HgsClient::box(const QJsonObject &object) { return boxFromObject(object); } +ProcessRunner::Request HgsClient::commandRequest(const QString &hgs, const QStringList &arguments) { return processRequest(hgs, arguments); } + QList HgsClient::parseSessions(const QJsonArray &rows) { QJsonObject box{{"host","archive"},{"ok",true},{"sessions",rows}}; return boxFromObject(box).sessions; @@ -311,18 +342,14 @@ BoxState HgsClient::parseBox(const QByteArray &json, QString *error) if (error) error->clear(); - QJsonParseError perr; - const QJsonDocument doc = QJsonDocument::fromJson(json, &perr); - if (perr.error != QJsonParseError::NoError || !doc.isObject()) { - if (error) { - *error = perr.error != QJsonParseError::NoError - ? perr.errorString() - : QStringLiteral("expected a JSON object"); - } + QString parseError; + const QJsonObject object = JsonParse::object(json, &parseError); + if (!parseError.isEmpty()) { + if (error) *error = parseError; return BoxState{}; } - return boxFromObject(doc.object()); + return boxFromObject(object); } QList HgsClient::parseFleet(const QByteArray &json, QString *error) @@ -330,19 +357,14 @@ QList HgsClient::parseFleet(const QByteArray &json, QString *error) if (error) error->clear(); - QJsonParseError perr; - const QJsonDocument doc = QJsonDocument::fromJson(json, &perr); - if (perr.error != QJsonParseError::NoError || !doc.isArray()) { - if (error) { - *error = perr.error != QJsonParseError::NoError - ? perr.errorString() - : QStringLiteral("expected a JSON array"); - } + QString parseError; + const QJsonArray arr = JsonParse::array(json, &parseError); + if (!parseError.isEmpty()) { + if (error) *error = parseError; return {}; } QList boxes; - const QJsonArray arr = doc.array(); boxes.reserve(arr.size()); for (const QJsonValue &v : arr) boxes << boxFromObject(v.toObject()); @@ -354,19 +376,14 @@ QList HgsClient::parseProjects(const QByteArray &json, QString *err if (error) error->clear(); - QJsonParseError perr; - const QJsonDocument doc = QJsonDocument::fromJson(json, &perr); - if (perr.error != QJsonParseError::NoError || !doc.isArray()) { - if (error) { - *error = perr.error != QJsonParseError::NoError - ? perr.errorString() - : QStringLiteral("expected a JSON array"); - } + QString parseError; + const QJsonArray arr = JsonParse::array(json, &parseError); + if (!parseError.isEmpty()) { + if (error) *error = parseError; return {}; } QList projects; - const QJsonArray arr = doc.array(); projects.reserve(arr.size()); for (const QJsonValue &v : arr) projects << projectFromJson(v.toObject()); @@ -382,6 +399,22 @@ namespace { // instead of the GUI thread. ProcessRunner also reports exactly once: the finished() // that follows a timeout kill() must not reach onFailed() again (--selftest leaves // main() on the first failure). +// Why a call failed, or empty when it succeeded. +QString failureDetail(const QString &hgs, const ProcessRunner::Result &result) +{ + if (result.outcome == ProcessRunner::Result::TimedOut) + return QStringLiteral("timed out"); + if (result.outcome == ProcessRunner::Result::FailedToStart) + return QStringLiteral("cannot start '%1': %2").arg(hgs, result.errorString); + // hgs всегда возвращает 0 для "спящего пира" — ненулевой код или CrashExit + // означает, что сломан сам вызов, а не то, что застали пира офлайн. + if (result.exitStatus != QProcess::NormalExit || result.exitCode != 0) { + const QString detail = QString::fromLocal8Bit(result.standardError).trimmed().left(4000); + return detail.isEmpty() ? QStringLiteral("exit code %1").arg(result.exitCode) : detail; + } + return {}; +} + void runHgsDirect(const QString &hgs, const QStringList &args, int timeoutMs, QObject *parent, const std::function &onFinished, const std::function &onFailed, @@ -390,21 +423,34 @@ void runHgsDirect(const QString &hgs, const QStringList &args, int timeoutMs, QO auto request = processRequest(hgs, args); request.timeoutMs = timeoutMs; if (!input.isEmpty()) request.input = input; ProcessRunner::run(std::move(request), parent, [hgs, args, onFinished, onFailed](const ProcessRunner::Result &result) { - const QString what = hgs + QLatin1Char(' ') + args.join(QLatin1Char(' ')); - if (result.outcome == ProcessRunner::Result::TimedOut) - return onFailed(what, QStringLiteral("timed out")); - if (result.outcome == ProcessRunner::Result::FailedToStart) - return onFailed(what, QStringLiteral("cannot start '%1': %2").arg(hgs, result.errorString)); - // hgs всегда возвращает 0 для "спящего пира" — ненулевой код или CrashExit - // означает, что сломан сам вызов, а не то, что застали пира офлайн. - if (result.exitStatus != QProcess::NormalExit || result.exitCode != 0) { - const QString detail = QString::fromLocal8Bit(result.standardError).trimmed().left(4000); - return onFailed(what, detail.isEmpty() ? QStringLiteral("exit code %1").arg(result.exitCode) : detail); - } + if (const auto detail = failureDetail(hgs, result); !detail.isEmpty()) + return onFailed(hgs + QLatin1Char(' ') + args.join(QLatin1Char(' ')), detail); onFinished(result.standardOutput); }); } +// A successful call's output becomes parse(output) on the preparation thread +// (ProcessRunner.h); done(value) or onFailed then runs on parent's thread. +template +void runHgsParsedDirect(const QString &hgs, const QStringList &args, int timeoutMs, QObject *parent, Parse parse, Done done, + const std::function &onFailed, const QByteArray &input = {}) +{ + using Value = std::decay_t>; + struct Reply { QString detail; Value value{}; }; + auto request = processRequest(hgs, args); request.timeoutMs = timeoutMs; + if (!input.isEmpty()) request.input = input; + ProcessRunner::run(std::move(request), parent, + [hgs, parse = std::move(parse)](ProcessRunner::Result result) mutable { + Reply reply; reply.detail = failureDetail(hgs, result); + if (reply.detail.isEmpty()) reply.value = parse(result.standardOutput); + return reply; + }, + [hgs, args, done = std::move(done), onFailed](Reply reply) mutable { + if (!reply.detail.isEmpty()) return onFailed(hgs + QLatin1Char(' ') + args.join(QLatin1Char(' ')), reply.detail); + done(std::move(reply.value)); + }); +} + void guardMachine(QStringList &args,const QString &expected) { if(expected.isEmpty())return; @@ -424,6 +470,22 @@ void runHgs(const QString &hgs,const QStringList &args,int timeoutMs,QObject *pa },failed); } +// runHgs() for a JSON reply: parse runs on the preparation thread, done on +// parent's thread. Every hgs reply the client parses goes through here. +template +void runHgsParsed(const QString &hgs, const QStringList &args, int timeoutMs, QObject *parent, Parse parse, Done done, + const std::function &failed, const QByteArray &input = {}) +{ + const int at = !args.isEmpty() && args.first().startsWith('@') ? 1 : 0; + if (args.value(at) != "--expect-machine") { runHgsParsedDirect(hgs, args, timeoutMs, parent, std::move(parse), std::move(done), failed, input); return; } + QStringList help{"--help"}; if (at) help.prepend(args.first()); + runHgsDirect(hgs, help, 15000, parent, [=](const QByteArray &out) { + if (!out.contains("guarded-create-v1")) { failed({}, QObject::tr("The selected CLI cannot verify this creation's computer. Update hgs before continuing.")); return; } + runHgsParsedDirect(hgs, args, timeoutMs, parent, parse, done, failed, input); + }, failed); +} + + } // namespace void HgsClient::cancelArchivePages(){const auto jobs=std::exchange(m_archiveJobs,{});qDeleteAll(jobs);} @@ -433,18 +495,19 @@ quint64 HgsClient::requestArchivePage(const QString &host,const QJsonObject &que const quint64 request=++m_archiveRequest; QStringList args;if(!host.isEmpty())args<<'@'+host;args<<"archives"<<"--json"; const auto finish=[this,owner,request,host](const QJsonObject &data,const QString &error){m_archiveJobs.remove(host);owner->deleteLater();emit archivePageReady(request,host,data,error);}; - runHgs(m_hgs,args,30000,owner,[query,finish](const QByteArray &out){ - const auto doc=QJsonDocument::fromJson(out);const auto data=doc.object();QString error; + struct Page { QJsonObject data; QString error; }; + runHgsParsed(m_hgs,args,30000,owner,[query](const QByteArray &out){ + QString parseError;const auto data=JsonParse::object(out,&parseError);QString error; const auto rows=data["sessions"].toArray(); - if(!doc.isObject()||data["version"].toInt()!=1||!data["sessions"].isArray()||rows.size()>50||!data["next_cursor"].isString()||data["next_cursor"].toString().size()>1024||!data["total"].isDouble()||data["total"].toDouble()50||!data["next_cursor"].isString()||data["next_cursor"].toString().size()>1024||!data["total"].isDouble()||data["total"].toDouble()1024||!data["position"].isDouble()||position<0||position!=qFloor(position)||position+rows.size()>data["total"].toDouble())error=tr("Invalid archive window. Refresh the archive."); + if(!previous.isString()||previous.toString().size()>1024||!data["position"].isDouble()||position<0||position!=qFloor(position)||position+rows.size()>data["total"].toDouble())error=HgsClient::tr("Invalid archive window. Refresh the archive."); } - for(const auto &value:rows){const auto row=value.toObject();const auto id=row["archive_id"].toString();if(row["state"]!="archived"||row["name"].toString().isEmpty()||QUuid(id).toString(QUuid::WithoutBraces)!=id||!row["archived_at"].isDouble()||row["archived_at"].toDouble()=query["until"].toDouble())error=tr("Invalid archived session identity. Refresh the archive.");} - finish(data,error); - },[finish](const QString &,const QString &error){finish({},error);},QJsonDocument(query).toJson(QJsonDocument::Compact)); + for(const auto &value:rows){const auto row=value.toObject();const auto id=row["archive_id"].toString();if(row["state"]!="archived"||row["name"].toString().isEmpty()||QUuid(id).toString(QUuid::WithoutBraces)!=id||!row["archived_at"].isDouble()||row["archived_at"].toDouble()=query["until"].toDouble())error=HgsClient::tr("Invalid archived session identity. Refresh the archive.");} + return Page{data,error}; + },[finish](const Page &page){finish(page.data,page.error);},[finish](const QString &,const QString &error){finish({},error);},QJsonDocument(query).toJson(QJsonDocument::Compact)); return request; } @@ -468,10 +531,9 @@ void HgsClient::requestSessionReference(const QString &host, const QString &name if (!host.isEmpty()) args.prepend('@' + host); if (!archive.isEmpty()) args << "--archive" << archive; if (!agent.isEmpty()) args << "--agent" << agent; - runHgs(m_hgs,args,host.isEmpty()?15000:25000,this,[this](const QByteArray &out){ - QJsonParseError error;const auto data=QJsonDocument::fromJson(out,&error); - const auto uri=data.object().value("uri").toString(); - if(error.error!=QJsonParseError::NoError || !data.isObject() || !uri.startsWith("zerus://session?") || uri.size()>8192) + runHgsParsed(m_hgs,args,host.isEmpty()?15000:25000,this,parseObject,[this](const ObjectReply &data){ + const auto uri=data.object.value("uri").toString(); + if(!data.ok() || !uri.startsWith("zerus://session?") || uri.size()>8192) emit sessionReferenceFailed(tr("Could not read a session reference. Update hgs on this computer.")); else emit sessionReferenceReady(uri); },[this](const QString &,const QString &error){emit sessionReferenceFailed(error);}); @@ -484,11 +546,10 @@ quint64 HgsClient::requestAccounts(const QString &host, const QStringList &argum if (!host.isEmpty()) args << QLatin1Char('@') + host; args << QStringLiteral("account"); args.append(arguments); // Start waits up to 20 seconds for OpenCode's registration on that computer. - runHgs(m_hgs, args, arguments.value(0) == "set-key" ? 90000 : arguments.value(0) == "copy" || arguments.value(0) == "start" ? 60000 : 25000, this, - [this, request, host](const QByteArray &bytes) { - QJsonParseError error; const auto data = QJsonDocument::fromJson(bytes, &error); - if (error.error != QJsonParseError::NoError || !data.isObject()) emit accountsFailed(request, host, tr("Invalid account catalog response")); - else emit accountsReady(request, host, data.object()); + runHgsParsed(m_hgs, args, arguments.value(0) == "set-key" ? 90000 : arguments.value(0) == "copy" || arguments.value(0) == "start" ? 60000 : 25000, this, + parseObject, [this, request, host](const ObjectReply &data) { + if (!data.ok()) emit accountsFailed(request, host, tr("Invalid account catalog response")); + else emit accountsReady(request, host, data.object); }, [this, request, host](const QString &, const QString &error) { emit accountsFailed(request, host, error); }, input); return request; } @@ -500,16 +561,21 @@ quint64 HgsClient::requestOpenCodeModels(const QString &host, const QString &loc if (!host.isEmpty()) args << QLatin1Char('@') + host; args << QStringLiteral("account") << QStringLiteral("models") << QStringLiteral("native-opencode") << QStringLiteral("--location") << location << QStringLiteral("--json"); - runHgs(m_hgs, args, 30000, this, - [this, request, host, location](const QByteArray &bytes) { + struct Catalog { bool tooLarge = false; ObjectReply reply; }; + runHgsParsed(m_hgs, args, 30000, this, + [](const QByteArray &bytes) { // hgs keeps the catalog within 4 MiB; a larger answer is not one. - if (bytes.size() > 5 * 1024 * 1024) { + Catalog catalog; catalog.tooLarge = bytes.size() > 5 * 1024 * 1024; + if (!catalog.tooLarge) catalog.reply = objectReply(bytes); + return catalog; + }, + [this, request, host, location](const Catalog &catalog) { + if (catalog.tooLarge) { emit openCodeModelsFailed(request, host, location, tr("The OpenCode model catalog is too large")); return; } - QJsonParseError error; const auto data = QJsonDocument::fromJson(bytes, &error); - if (error.error != QJsonParseError::NoError || !data.isObject()) emit openCodeModelsFailed(request, host, location, tr("Invalid OpenCode model catalog response")); - else emit openCodeModelsReady(request, host, location, data.object()); + if (!catalog.reply.ok()) emit openCodeModelsFailed(request, host, location, tr("Invalid OpenCode model catalog response")); + else emit openCodeModelsReady(request, host, location, catalog.reply.object); }, [this, request, host, location](const QString &, const QString &error) { emit openCodeModelsFailed(request, host, location, error); }); return request; } @@ -524,11 +590,11 @@ quint64 HgsClient::requestOpenCodeFavorite(const QString &host, const QString &v args << QStringLiteral("account") << QStringLiteral("favorite") << QStringLiteral("native-opencode") << QStringLiteral("--model") << value << QStringLiteral("--favorite") << (favorite ? QStringLiteral("on") : QStringLiteral("off")) << QStringLiteral("--request-id") << identity << QStringLiteral("--json"); - runHgs(m_hgs, args, 25000, this, - [this, request, host, value, favorite, identity](const QByteArray &bytes) { - QJsonParseError error; const auto data = QJsonDocument::fromJson(bytes, &error); const auto receipt = data.object(); + runHgsParsed(m_hgs, args, 25000, this, parseObject, + [this, request, host, value, favorite, identity](const ObjectReply &data) { + const auto &receipt = data.object; // The receipt must name this exact change. - if (error.error != QJsonParseError::NoError || !data.isObject() || receipt["status"] != "applied" + if (!data.ok() || receipt["status"] != "applied" || receipt["request_id"] != identity || receipt["model"] != value || receipt["favorite"] != favorite) emit openCodeFavoriteFailed(request, host, tr("Invalid favorite response")); else emit openCodeFavoriteReady(request, host, receipt); @@ -552,10 +618,9 @@ quint64 HgsClient::requestMobile(const QString &host, const QStringList &argumen emit mobileFailed(request, host, tr("Update Zerus on %1 to manage phones from here.").arg(host.isEmpty() ? tr("this computer") : host)); return; } - runHgs(m_hgs, args, 40000, this, [this, request, host](const QByteArray &bytes) { - QJsonParseError error; const auto data = QJsonDocument::fromJson(bytes, &error); - if (error.error != QJsonParseError::NoError || !data.isObject()) emit mobileFailed(request, host, tr("Invalid phone management response")); - else emit mobileReady(request, host, data.object()); + runHgsParsed(m_hgs, args, 40000, this, parseObject, [this, request, host](const ObjectReply &data) { + if (!data.ok()) emit mobileFailed(request, host, tr("Invalid phone management response")); + else emit mobileReady(request, host, data.object); }, failed); }, failed); return request; @@ -565,11 +630,10 @@ quint64 HgsClient::requestMachines(const QStringList &arguments) { const quint64 request = ++m_machineRequest; QStringList args{QStringLiteral("machine")}; args.append(arguments); - runHgs(m_hgs, args, arguments.value(0) == "check" ? 25000 : 8000, this, - [this, request](const QByteArray &bytes) { - QJsonParseError error; const auto data = QJsonDocument::fromJson(bytes, &error); - if (error.error != QJsonParseError::NoError || !data.isObject()) emit machinesFailed(request, tr("Invalid machine settings response")); - else emit machinesReady(request, data.object()); + runHgsParsed(m_hgs, args, arguments.value(0) == "check" ? 25000 : 8000, this, parseObject, + [this, request](const ObjectReply &data) { + if (!data.ok()) emit machinesFailed(request, tr("Invalid machine settings response")); + else emit machinesReady(request, data.object); }, [this, request](const QString &, const QString &error) { emit machinesFailed(request, error); }); return request; } @@ -579,11 +643,10 @@ quint64 HgsClient::requestSwarm(const QStringList &arguments, const QJsonObject const quint64 request = ++m_swarmRequest; QStringList args{QStringLiteral("swarm")}; args.append(arguments); const bool network = QStringList{"preview", "join", "sync", "bind"}.contains(arguments.value(0)); - runHgs(m_hgs, args, network ? 180000 : 10000, this, - [this, request](const QByteArray &bytes) { - QJsonParseError error; const auto data = QJsonDocument::fromJson(bytes, &error); - if (error.error != QJsonParseError::NoError || !data.isObject()) emit swarmFailed(request, tr("Invalid project catalog response")); - else emit swarmReady(request, data.object()); + runHgsParsed(m_hgs, args, network ? 180000 : 10000, this, parseObject, + [this, request](const ObjectReply &data) { + if (!data.ok()) emit swarmFailed(request, tr("Invalid project catalog response")); + else emit swarmReady(request, data.object); }, [this, request](const QString &, const QString &error) { emit swarmFailed(request, error); }, input.isEmpty() ? QByteArray() : QJsonDocument(input).toJson(QJsonDocument::Compact)); return request; @@ -602,7 +665,14 @@ quint64 HgsClient::requestTask(const QStringList &arguments, const QString &inpu call.maximumOutputBytes = 36 * 1024 * 1024; // stdin always closes, even empty: a command that reads a text never waits on the GUI. call.input = input.toUtf8(); - ProcessRunner::run(std::move(call), this, [this, request](const ProcessRunner::Result &result) { + ProcessRunner::run(std::move(call), this, [](ProcessRunner::Result result) { + ParsedResult parsed; + if (!result.outputLimitExceeded && result.outcome == ProcessRunner::Result::Finished && result.exitStatus == QProcess::NormalExit + && result.exitCode == 0) parsed = parsedResult(std::move(result)); + else parsed.result = std::move(result); + return parsed; + }, [this, request](const ParsedResult &parsed) { + const auto &result = parsed.result; if (result.outputLimitExceeded) return emit taskFailed(request, tr("Task response exceeds 36 MiB")); if (result.outcome == ProcessRunner::Result::TimedOut) return emit taskFailed(request, tr("hgs task timed out")); if (result.outcome == ProcessRunner::Result::FailedToStart) @@ -611,19 +681,8 @@ quint64 HgsClient::requestTask(const QStringList &arguments, const QString &inpu const QString detail = QString::fromLocal8Bit(result.standardError).trimmed().left(4000); return emit taskFailed(request, detail.isEmpty() ? tr("exit code %1").arg(result.exitCode) : detail); } - auto promise = std::make_shared>(); promise->start(); - auto *watcher = new QFutureWatcher(this); - connect(watcher, &QFutureWatcher::finished, this, [this, request, watcher] { - const auto data = watcher->result(); watcher->deleteLater(); - if (!data.isObject()) emit taskFailed(request, tr("Invalid task response")); - else emit taskReady(request, data.object()); - }); - watcher->setFuture(promise->future()); - static QThreadPool parser; - static const bool configured = [] { parser.setMaxThreadCount(2); return true; }(); Q_UNUSED(configured); - parser.start(QRunnable::create([promise, bytes = result.standardOutput] { - promise->addResult(QJsonDocument::fromJson(bytes)); promise->finish(); - })); + if (!parsed.reply.ok()) emit taskFailed(request, tr("Invalid task response")); + else emit taskReady(request, parsed.reply.object); }); return request; } @@ -634,11 +693,10 @@ quint64 HgsClient::requestSessionHost(const QString &action) // install copies hgs.exe and a host image into the profile; status asks a // running host, which answers within five seconds or counts as unavailable. const int timeoutMs = action == QLatin1String("install") ? 60000 : action == QLatin1String("status") ? 15000 : 25000; - runHgs(m_hgs, {QStringLiteral("session-host"), action, QStringLiteral("--json")}, timeoutMs, this, - [this, request](const QByteArray &bytes) { - QJsonParseError error; const auto data = QJsonDocument::fromJson(bytes, &error); - if (error.error != QJsonParseError::NoError || !data.isObject()) emit sessionHostFailed(request, tr("Invalid session host response")); - else emit sessionHostReady(request, data.object()); + runHgsParsed(m_hgs, {QStringLiteral("session-host"), action, QStringLiteral("--json")}, timeoutMs, this, parseObject, + [this, request](const ObjectReply &data) { + if (!data.ok()) emit sessionHostFailed(request, tr("Invalid session host response")); + else emit sessionHostReady(request, data.object); }, [this, request](const QString &, const QString &error) { emit sessionHostFailed(request, error); }); return request; } @@ -689,12 +747,14 @@ void HgsClient::requestLocal() emit stateReadStarted({}); // Large native histories can take longer than five seconds. Keep one poll // in flight and allow it to finish instead of continually killing/restarting it. - runHgs(m_hgs, {QStringLiteral("ls"), QStringLiteral("--json"), QStringLiteral("--local"), QStringLiteral("--no-archives")}, + struct Listing { BoxState box; QString error; }; + runHgsParsed(m_hgs, {QStringLiteral("ls"), QStringLiteral("--json"), QStringLiteral("--local"), QStringLiteral("--no-archives")}, 15000, this, - [this](const QByteArray &out) { + [](const QByteArray &out) { Listing listing; listing.box = parseBox(out, &listing.error); return listing; }, + [this](const Listing &listing) { m_localInFlight = false; - QString err; - const BoxState box = parseBox(out, &err); + const QString &err = listing.error; + const BoxState &box = listing.box; if (!err.isEmpty()) { emit stateReadFailed(QString(), err); emit failed(m_hgs + QStringLiteral(" ls --json --local"), err); @@ -718,27 +778,32 @@ void HgsClient::requestPeer(const QString &peer) // ssh-рукопожатие может занять больше времени, чем локальный вызов — у самого hgs // уже есть ConnectTimeout=3, но обвязка (auth, motd) укладывается не всегда в 5с. - runHgs(m_hgs, {QLatin1Char('@') + peer, QStringLiteral("ls"), QStringLiteral("--json"), QStringLiteral("--no-archives")}, + struct Listing { BoxState box; QString error; }; + runHgsParsed(m_hgs, {QLatin1Char('@') + peer, QStringLiteral("ls"), QStringLiteral("--json"), QStringLiteral("--no-archives")}, 15000, this, - [this, peer](const QByteArray &out) { + [peer](const QByteArray &out) { + Listing listing; listing.box = parseBox(out, &listing.error); + // The CLI forwards the peer's own listing, whose host is the name + // that machine gives itself (HGS_SELF or its host name). A machine + // saved under another alias must still be found, polled and + // addressed by that alias. + if (listing.error.isEmpty()) listing.box.reportedHost = std::exchange(listing.box.host, peer); + return listing; + }, + [this, peer](const Listing &listing) { // Снятие метки здесь и в onFailed ниже -- ровно те же два колбэка, которыми // runHgs() гарантированно завершает вызов РОВНО один раз (см. reported в // runHgs). Без этого упавший или зависший по таймауту пир навсегда // застревал бы в m_peerInFlight, и его строка в меню замерла бы на // "polling…" до перезапуска трея. m_peerInFlight.remove(peer); - QString err; - BoxState box = parseBox(out, &err); + const QString &err = listing.error; + const BoxState &box = listing.box; if (!err.isEmpty()) { emit stateReadFailed(peer, err); emit failed(m_hgs + QStringLiteral(" ls --json"), err); return; } - // The CLI forwards the peer's own listing, whose host is the name - // that machine gives itself (HGS_SELF or its host name). A machine - // saved under another alias must still be found, polled and - // addressed by that alias. - box.reportedHost = std::exchange(box.host, peer); emit peerReady(box); }, [this, peer](const QString &what, const QString &detail) { @@ -760,12 +825,11 @@ quint64 HgsClient::requestSearch(const QString &host, const QString &query) const auto id = ++m_searchRequest; QStringList args{"search", "--query", query, "--limit", "100"}; if (!host.isEmpty()) args.prepend('@' + host); - runHgs(m_hgs, args, host.isEmpty() ? 12000 : 20000, this, - [this, id, host](const QByteArray &out) { - QJsonParseError error; const auto doc = QJsonDocument::fromJson(out, &error); - if (error.error != QJsonParseError::NoError || !doc.isObject() || !doc.object().value("results").isArray()) + runHgsParsed(m_hgs, args, host.isEmpty() ? 12000 : 20000, this, parseObject, + [this, id, host](const ObjectReply &doc) { + if (!doc.ok() || !doc.object.value("results").isArray()) emit searchFailed(id, host, tr("Invalid search response; update hgs on this machine.")); - else emit searchReady(id, host, doc.object()); + else emit searchReady(id, host, doc.object); }, [this, id, host](const QString &, const QString &error) { emit searchFailed(id, host, error); }); return id; } @@ -779,10 +843,9 @@ quint64 HgsClient::requestProcess(const QString &host, const QString &name, cons if(!archive.isEmpty())args<<"--archive"< 256 * 1024 || result.isEmpty()) error = tr("Invalid automatic approval journal response."); @@ -924,16 +987,20 @@ void HgsClient::transferFile(quint64 id, const QString &host, const QString &pat if (cacheKey.isEmpty()) emit fileDownloadFailed(id, error); else finishCachedDownload(cacheKey, {}, error); }; - runHgs(m_hgs, {'@' + host, "download", "--path", path, "--output", destination}, 135000, this, - [this, id, host, path, destination, pending, cacheKey, failed](const QByteArray &out) { - if (out.size() > 65536) { failed(tr("Invalid file transfer response.")); return; } - const auto object = QJsonDocument::fromJson(out).object(); + enum class Transfer { Saved, InvalidResponse, Unconfirmed }; + runHgsParsed(m_hgs, {'@' + host, "download", "--path", path, "--output", destination}, 135000, this, + [host, path, destination](const QByteArray &out) { + if (out.size() > 65536) return Transfer::InvalidResponse; + const auto object = JsonParse::object(out); const QFileInfo saved(destination); if (object.value("host").toString() != host || object.value("path").toString() != path || object.value("local_path").toString() != destination || !object.value("bytes").isDouble() - || !LocalFile::readable(saved) || object.value("bytes").toDouble() != LocalFile::size(saved)) { - failed(tr("The file transfer did not confirm the saved local file.")); return; - } + || !LocalFile::readable(saved) || object.value("bytes").toDouble() != LocalFile::size(saved)) return Transfer::Unconfirmed; + return Transfer::Saved; + }, + [this, id, host, path, destination, pending, cacheKey, failed](Transfer transfer) { + if (transfer == Transfer::InvalidResponse) { failed(tr("Invalid file transfer response.")); return; } + if (transfer == Transfer::Unconfirmed) { failed(tr("The file transfer did not confirm the saved local file.")); return; } if (!pending) { emit fileDownloaded(id, destination); return; } DownloadCache::async(this, [pending, host, path] { return DownloadCache::publish(pending, path, -1, {host, path}); }, @@ -976,16 +1043,22 @@ quint64 HgsClient::requestAttachment(const QString &host, const QString &name, c if(!archive.isEmpty())args<<"--archive"<MaximumAttachmentBytes*4/3+8192){emit attachmentFailed(id,tr("Attachment response is too large."));return;} - const auto object=QJsonDocument::fromJson(out).object(); + struct Attachment { bool tooLarge=false, valid=false; QJsonObject metadata; QByteArray bytes; }; + runHgsParsed(m_hgs,args,host.isEmpty()?10000:45000,this,[file](const QByteArray &out){ + Attachment attachment; + if(out.size()>MaximumAttachmentBytes*4/3+8192){attachment.tooLarge=true;return attachment;} + const auto object=JsonParse::object(out); const auto encoded=object.value("data_base64").toString().toLatin1(); const auto decoded=QByteArray::fromBase64Encoding(encoded,QByteArray::AbortOnBase64DecodingErrors); - if(out.size()>MaximumAttachmentBytes*4/3+8192 || object.value("request_id")!=file.value("request_id") - || object.value("index")!=file.value("index") || !decoded || decoded.decoded.isEmpty() || decoded.decoded.size()>MaximumAttachmentBytes) { - emit attachmentFailed(id,tr("Invalid attachment response. Try again."));return; - } - auto metadata=object;metadata.remove("data_base64");emit attachmentReady(id,metadata,decoded.decoded); + if(object.value("request_id")!=file.value("request_id") + || object.value("index")!=file.value("index") || !decoded || decoded.decoded.isEmpty() || decoded.decoded.size()>MaximumAttachmentBytes) + return attachment; + attachment.valid=true;attachment.metadata=object;attachment.metadata.remove("data_base64");attachment.bytes=decoded.decoded; + return attachment; + },[this,id](const Attachment &attachment){ + if(attachment.tooLarge){emit attachmentFailed(id,tr("Attachment response is too large."));return;} + if(!attachment.valid){emit attachmentFailed(id,tr("Invalid attachment response. Try again."));return;} + emit attachmentReady(id,attachment.metadata,attachment.bytes); },[this,id](const QString &,const QString &error){emit attachmentFailed(id,error);}); return id; } @@ -1019,14 +1092,13 @@ void HgsClient::requestSubagentInspection(const QString &host, const QString &na QStringList args{"inspect", name, "--agent", agentId, "--skip-processes"}; if (!archiveId.isEmpty()) args << "--archive" << archiveId; if (!host.isEmpty()) args.prepend('@' + host); - runHgs(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, - [this,host,name,agentId,archiveId,key](const QByteArray &out) { + runHgsParsed(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, parseObject, + [this,host,name,agentId,archiveId,key](const ObjectReply &doc) { m_inspectionsInFlight.remove(key); emit inspectionLoadingChanged(host, name, archiveId, agentId, false); - QJsonParseError error; const auto doc = QJsonDocument::fromJson(out,&error); - if (error.error != QJsonParseError::NoError || !doc.isObject()) + if (!doc.ok()) emit subagentInspectionFailed(host,name,agentId,archiveId,tr("Invalid subagent activity response")); - else emit subagentInspectionReady(host,name,agentId,archiveId,doc.object()); + else emit subagentInspectionReady(host,name,agentId,archiveId,doc.object); }, [this,host,name,agentId,archiveId,key](const QString &,const QString &error) { m_inspectionsInFlight.remove(key); emit inspectionLoadingChanged(host, name, archiveId, agentId, false); @@ -1038,7 +1110,16 @@ void HgsClient::runWrite(const QString &op, const QStringList &args, int timeout bool sessionWrite, const QString &host) { auto request = processRequest(m_hgs, args); request.timeoutMs = timeoutMs; - ProcessRunner::run(std::move(request), this, [this, op, sessionWrite, host](const ProcessRunner::Result &result) { + struct Write { ProcessRunner::Result result; QJsonObject cleanup; int cleanups = 0; }; + ProcessRunner::run(std::move(request), this, [](ProcessRunner::Result result) { + Write write; + for (const auto &line : result.standardError.split('\n')) if (line.startsWith("ZERUS_WORKTREE_CLEANUP ")) { + write.cleanup = JsonParse::object(line.mid(23)); ++write.cleanups; + } + write.result = std::move(result); + return write; + }, [this, op, sessionWrite, host](const Write &write) { + const auto &result = write.result; // Any QProcess error (failed start, crash) reports its own text, as the // errorOccurred() that precedes finished() did before. const bool timedOut = result.outcome == ProcessRunner::Result::TimedOut; @@ -1047,10 +1128,7 @@ void HgsClient::runWrite(const QString &op, const QStringList &args, int timeout emit writeDone(op, ok, timedOut ? QStringLiteral("timed out") : error ? result.errorString : QString::fromLocal8Bit(result.standardError).trimmed()); if (ok && (op.startsWith("archive ") || op.startsWith("terminate "))) { - QJsonObject cleanup; int count = 0; - for (const auto &line : result.standardError.split('\n')) if (line.startsWith("ZERUS_WORKTREE_CLEANUP ")) { - cleanup = QJsonDocument::fromJson(line.mid(23)).object(); ++count; - } + const auto &cleanup = write.cleanup; const int count = write.cleanups; if (count == 1 && !cleanup["path"].toString().isEmpty() && !cleanup["common_dir"].toString().isEmpty()) { if (cleanup["status"] == "removed") invalidateWorktreeCache(host, cleanup["common_dir"].toString()); emit worktreeCleanupFinished(host, op, cleanup); @@ -1106,13 +1184,12 @@ quint64 HgsClient::requestNativeUi(const QString &host, const QString &name) { const auto request = ++m_nativeUiRequest; QStringList args{"native-ui", name, "--json"}; if (!host.isEmpty()) args.prepend('@' + host); - runHgs(m_hgs, args, host.isEmpty() ? 45000 : 60000, this, - [this, request](const QByteArray &out) { - const auto doc = QJsonDocument::fromJson(out); - if (!doc.isObject() || !doc.object().value("url").isString()) { + runHgsParsed(m_hgs, args, host.isEmpty() ? 45000 : 60000, this, parseObject, + [this, request](const ObjectReply &doc) { + if (!doc.ok() || !doc.object.value("url").isString()) { emit nativeUiFailed(request, tr("Invalid native UI connection response")); return; } - emit nativeUiReady(request, doc.object()); + emit nativeUiReady(request, doc.object); }, [this, request](const QString &, const QString &error) { emit nativeUiFailed(request, error); }); return request; } @@ -1243,10 +1320,12 @@ void HgsClient::requestProjects(const QString &host) { QStringList args{QStringLiteral("project"), QStringLiteral("ls"), QStringLiteral("--json")}; if (!host.isEmpty()) args.prepend(QLatin1Char('@') + host); - runHgs(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, - [this, host](const QByteArray &out) { - QString err; - const auto projects = parseProjects(out, &err); + struct Projects { QList projects; QString error; }; + runHgsParsed(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, + [](const QByteArray &out) { Projects parsed; parsed.projects = parseProjects(out, &parsed.error); return parsed; }, + [this, host](const Projects &parsed) { + const QString &err = parsed.error; + const auto &projects = parsed.projects; if (!err.isEmpty()) { emit projectsFailed(host, err); return; } if (host.isEmpty()) emit projectsReady(projects); emit projectsForHostReady(host, projects); @@ -1254,23 +1333,36 @@ void HgsClient::requestProjects(const QString &host) } namespace { -void trimWorktreeCache(QHash &cache) { - qsizetype bytes=0;for(const auto &entry:cache)bytes+=QJsonDocument(entry).toJson(QJsonDocument::Compact).size(); +// sizes holds each entry's reply size, measured when its reply was parsed, +// so trimming never serializes the cache on the GUI thread. +void trimWorktreeCache(QHash &cache,QHash &sizes) { + qsizetype bytes=0;for(auto i=cache.cbegin();i!=cache.cend();++i)bytes+=sizes.value(i.key()); while(cache.size()>64||bytes>16*1024*1024) { auto oldest=cache.begin(); for(auto i=cache.begin();i!=cache.end();++i)if(i.value()["received_at"].toDouble()64||QJsonDocument(snapshots).toJson(QJsonDocument::Compact).size()>WorktreeSnapshotsLimit){QString oldest;double time=std::numeric_limits::max();for(auto i=snapshots.begin();i!=snapshots.end();++i)if(i.value().toObject()["sampled_at"].toDouble()8*1024*1024)return {}; - return QJsonDocument::fromJson(bytes).object().value(worktreeCacheKey(m_hgs,host,path)).toObject(); + return SettingsJson::object(WorktreeSnapshotsKey,WorktreeSnapshotsLimit).value(worktreeCacheKey(m_hgs,host,path)).toObject(); } QJsonObject HgsClient::gitStatusSnapshot(const QString &host, const QString &path) const { @@ -1311,8 +1403,9 @@ void HgsClient::pumpGitStatus() }; QStringList args{"git-status", "--path", path, "--json"}; if (!host.isEmpty()) args.prepend('@' + host); - runHgs(m_hgs, args, host.isEmpty() ? 5000 : 10000, this, - [finish](const QByteArray &out) { finish(out.size() <= 32768 ? QJsonDocument::fromJson(out).object() : QJsonObject()); }, + runHgsParsed(m_hgs, args, host.isEmpty() ? 5000 : 10000, this, + [](const QByteArray &out) { return out.size() <= 32768 ? JsonParse::object(out) : QJsonObject(); }, + [finish](const QJsonObject &data) { finish(data); }, [finish](const QString &, const QString &) { finish({}); }); } } @@ -1322,8 +1415,7 @@ quint64 HgsClient::requestWorktrees(const QString &host,const QString &path,bool if(!expectedMachine.isEmpty()){ const auto request=++m_worktreeRequest;QStringList args{"worktrees","--creation","--path",path,"--refresh","--json"}; if(!host.isEmpty())args.prepend('@'+host);guardMachine(args,expectedMachine); - runHgs(m_hgs,args,15000,this,[this,request,host,path](const QByteArray &out){ - const auto data=QJsonDocument::fromJson(out).object(); + runHgsParsed(m_hgs,args,15000,this,[](const QByteArray &out){return JsonParse::object(out);},[this,request,host,path](const QJsonObject &data){ if(!data["worktrees"].isArray()||!data["state"].isString())emit worktreesFailed(request,host,path,tr("Invalid verified worktree catalog.")); else emit worktreesReady(request,host,path,data); },[this,request,host,path](const QString &,const QString &error){emit worktreesFailed(request,host,path,error);}); @@ -1346,24 +1438,23 @@ quint64 HgsClient::requestWorktrees(const QString &host,const QString &path,bool if(m_worktreesInFlight.contains(key)){m_worktreesInFlight[key].append({request,path});return request;} m_worktreesInFlight[key].append({request,path}); QStringList args{"worktrees","--path",path,"--json"};if(refresh)args<<"--refresh";if(!host.isEmpty())args.prepend('@'+host); - runHgs(m_hgs,args,host.isEmpty()?6000:15000,this,[this,host,key,repository,generation,hostKey,hostGeneration](const QByteArray &out){ + runHgsParsed(m_hgs,args,host.isEmpty()?6000:15000,this,[](const QByteArray &out){return out.size()<=2*1024*1024?JsonParse::object(out):QJsonObject();}, + [this,host,key,repository,generation,hostKey,hostGeneration](QJsonObject data){ const auto requests=m_worktreesInFlight.take(key); if(m_worktreeCacheGenerations.value(repository)!=generation||m_worktreeCacheGenerations.value(hostKey)!=hostGeneration) { for(const auto &r:requests)emit worktreesFailed(r.first,host,r.second,tr("Folder changed. Refresh its catalog."));return; } - auto data=out.size()<=2*1024*1024?QJsonDocument::fromJson(out).object():QJsonObject(); data["received_at"] = QDateTime::currentSecsSinceEpoch(); if(!data.value("worktrees").isArray()||!data.value("state").isString()){ for(const auto &r:requests)emit worktreesFailed(r.first,host,r.second,tr("Invalid worktree catalog response"));return; } - QSettings settings;auto snapshots=QJsonDocument::fromJson(settings.value("workspace/worktreeSnapshots").toByteArray()).object(); + auto snapshots=SettingsJson::object(WorktreeSnapshotsKey); auto store=[&](const QString &path,QJsonObject value){snapshots[worktreeCacheKey(m_hgs,host,path)]=value;}; for(const auto &r:requests)store(r.second,data); if(data["state"]=="ok")for(const auto &v:data["worktrees"].toArray()){ const auto root=v.toObject()["path"].toString();auto sibling=data;sibling["path"]=root;sibling["selected_root"]=root;store(root,sibling); } - while(snapshots.size()>64||QJsonDocument(snapshots).toJson(QJsonDocument::Compact).size()>8*1024*1024){QString oldest;double time=std::numeric_limits::max();for(auto i=snapshots.begin();i!=snapshots.end();++i)if(i.value().toObject()["sampled_at"].toDouble()0)args<<"--offset"<0&&data["file"].toObject()["offset"].toInt(-1)!=offset)) { - finish({},tr("Worktree inspection could not be verified. Refresh and try again."));return; - } - finish(data,{}); - },[finish](const QString &,const QString &error){finish({},error.contains("usage: hgs worktrees")?tr("Update Zerus on this computer to inspect worktree changes."):error);}); + ||(offset>0&&data["file"].toObject()["offset"].toInt(-1)!=offset)); + return inspection; + },[finish](const Inspection &inspection){ + if(!inspection.verified){finish({},tr("Worktree inspection could not be verified. Refresh and try again."),0);return;} + finish(inspection.data,{},inspection.bytes); + },[finish](const QString &,const QString &error){finish({},error.contains("usage: hgs worktrees")?tr("Update Zerus on this computer to inspect worktree changes."):error,0);}); return request; } @@ -1549,9 +1648,11 @@ quint64 HgsClient::removeWorktree(const QString &host,const QString &path,const if(preserveArchiveRestore)args<<"--preserve-archive-restore"; const auto forget=[this,host,commonDir]{invalidateWorktreeCache(host,commonDir);}; auto process=processRequest(m_hgs,args); process.timeoutMs=330000; process.maximumOutputBytes=65536; - ProcessRunner::run(std::move(process),this,[this,request,token,path,commonDir,host,forget](const ProcessRunner::Result &reply){ + ProcessRunner::run(std::move(process),this,[](ProcessRunner::Result result){return parsedResult(std::move(result));}, + [this,request,token,path,commonDir,host,forget](const ParsedResult &parsed){ + const auto &reply=parsed.result; forget(); - const auto data=QJsonDocument::fromJson(reply.standardOutput).object(); + const auto &data=parsed.reply.object; const bool finished=reply.outcome==ProcessRunner::Result::Finished && reply.exitStatus==QProcess::NormalExit; const bool identity=data["request_id"]==token && data["path"]==path; if(finished && reply.exitCode==0 && identity && (data["status"]=="removed" || data["status"]=="forgotten")) { @@ -1574,8 +1675,7 @@ quint64 HgsClient::requestCreationDirectory(const QString &host,const QString &p if(!verified&&!expectedMachine.isEmpty()){emit directoriesFailed(request,host,tr("The original computer identity cannot be verified by this CLI."));return;} QStringList args{"dirs"};if(verified)args<<"--creation";args<<(path.isEmpty()?QStringLiteral("~"):path); if(!host.isEmpty())args.prepend('@'+host);guardMachine(args,expectedMachine); - runHgs(m_hgs,args,host.isEmpty()?10000:20000,this,[this,request,host,verified](const QByteArray &out){ - const auto data=QJsonDocument::fromJson(out).object(); + runHgsParsed(m_hgs,args,host.isEmpty()?10000:20000,this,parseJsonObject,[this,request,host,verified](const QJsonObject &data){ if(!data["path"].isString()||!data["directories"].isArray()||(verified&&(!data["creation_identity_v1"].toBool()||QUuid(data["computer_id"].toString()).isNull()))) emit directoriesFailed(request,host,tr("Invalid creation directory identity.")); else emit directoriesReady(request,host,data); @@ -1592,15 +1692,13 @@ quint64 HgsClient::requestDirectories(const QString &host, const QString &path, if (hidden) args << QStringLiteral("--hidden"); args << (path.isEmpty() ? QStringLiteral("~") : path); if (!host.isEmpty()) args.prepend(QLatin1Char('@') + host); - runHgs(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, - [this, host, request](const QByteArray &out) { - QJsonParseError error; - const auto doc = QJsonDocument::fromJson(out, &error); - if (!doc.isObject() || !doc.object().value("path").isString() - || !doc.object().value("directories").isArray()) { + runHgsParsed(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, parseObject, + [this, host, request](const ObjectReply &doc) { + if (!doc.ok() || !doc.object.value("path").isString() + || !doc.object.value("directories").isArray()) { emit directoriesFailed(request, host, tr("Invalid directory response")); return; } - emit directoriesReady(request, host, doc.object()); + emit directoriesReady(request, host, doc.object); }, [this, host, request](const QString &, const QString &detail) { emit directoriesFailed(request, host, detail); }); @@ -1621,9 +1719,8 @@ quint64 HgsClient::createDirectory(const QString &host, const QString &path) QStringList args{"dirs", "--create", path, "--request-id", token}; if (!host.isEmpty()) args.prepend('@' + host); const auto expected = MachinePlatform::clean(parent + '/' + leaf, style); - runHgs(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, - [this, host, path, request, token, style, expected, parent](const QByteArray &bytes) { - const auto data = QJsonDocument::fromJson(bytes).object(); + runHgsParsed(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, + parseJsonObject,[this, host, path, request, token, style, expected, parent](const QJsonObject &data){ const auto confirmed = data.value("path").toString(); if (data["status"] != "created" || data["request_id"] != token || data["requested_path"] != path || !MachinePlatform::absolute(confirmed, style) || !MachinePlatform::same(confirmed, expected, style) @@ -1652,9 +1749,8 @@ quint64 HgsClient::createDirectory(const QString &host, const QString &path) QStringList args{"dirs", parentPath}; if (!host.isEmpty()) args.prepend('@' + host); // Pin the canonical parent before mkdir, retaining symlink aliases. - runHgs(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, - [this, host, request, style, mutate](const QByteArray &bytes) { - const auto data = QJsonDocument::fromJson(bytes).object(); + runHgsParsed(m_hgs, args, host.isEmpty() ? 5000 : 15000, this, + parseJsonObject,[this, host, request, style, mutate](const QJsonObject &data){ const auto parent = data.value("path").toString(); if (!MachinePlatform::absolute(parent, style) || !data.value("directories").isArray()) { emit directoryCreationFailed(request, host, @@ -1731,9 +1827,10 @@ quint64 HgsClient::requestAnswerQuestion(const QString &host, const QString &nam if (!host.isEmpty()) args.prepend(QLatin1Char('@') + host); m_answersInFlight.insert(key); auto job = processRequest(program, args); job.input = payload; job.timeoutMs = host.isEmpty() ? 30000 : 45000; - ProcessRunner::run(std::move(job), this, + ProcessRunner::run(std::move(job), this, [](ProcessRunner::Result result) { return parsedResult(std::move(result), 65536); }, [this, request, host, name, key, requestId, questionId, questionHash, runId, conversationId, asyncQuestion, canSkip] - (const ProcessRunner::Result &result) { + (const ParsedResult &parsed) { + const auto &result = parsed.result; m_answersInFlight.remove(key); const auto fail = [this, request, host, name](const QString &detail, bool uncertain) { emit questionAnswerFailed(request, host, name, detail, uncertain); }; if (result.outcome == ProcessRunner::Result::FailedToStart) { fail(tr("Cannot start hgs: %1").arg(result.errorString), false); return; } @@ -1745,8 +1842,8 @@ quint64 HgsClient::requestAnswerQuestion(const QString &host, const QString &nam fail(errorText.isEmpty() ? tr("Answer transfer failed (exit %1).").arg(result.exitCode) : errorText, uncertain); return; } if (result.standardOutput.size() > 65536) { fail(tr("The answer acknowledgement is too large. Check Terminal before retrying."), true); return; } - QJsonParseError error; const auto doc = QJsonDocument::fromJson(result.standardOutput, &error); const auto receipt = doc.object(); - if (error.error != QJsonParseError::NoError || !doc.isObject() + const auto &receipt = parsed.reply.object; + if (!parsed.reply.ok() || (receipt.value("status") != "answered" && !(asyncQuestion && !canSkip && receipt.value("status") == "submitted" && !receipt.value("skipped").toBool())) || receipt.value("request_id").toString() != requestId || receipt.value("name").toString() != name || receipt.value("question_id").toString() != questionId || receipt.value("question_hash").toString() != questionHash @@ -1875,11 +1972,10 @@ quint64 HgsClient::requestSendMessage(const QString &host, const QString &name, const bool uncertain = status != QProcess::NormalExit || code != 1 || detail.contains(QStringLiteral("delivery uncertain"), Qt::CaseInsensitive); fail(detail.isEmpty() ? tr("Message transfer failed (exit %1).").arg(code) : detail, uncertain); - } else { - QJsonParseError error; - const auto document = QJsonDocument::fromJson(process->readAllStandardOutput(), &error); - const QJsonObject receipt = document.object(); - if (error.error != QJsonParseError::NoError || !document.isObject() + } else ProcessRunner::post(this, [bytes = process->readAllStandardOutput()] { return objectReply(bytes); }, + [this, fail, reported, request, host, name, key, messageId, expectedRunId, expectedConversationId, agentId](const ObjectReply &document) { + const QJsonObject &receipt = document.object; + if (!document.ok() || receipt.value("status").toString() != QLatin1String("submitted") || receipt.value("request_id").toString() != messageId || receipt.value("agent_id").toString() != agentId @@ -1895,7 +1991,7 @@ quint64 HgsClient::requestSendMessage(const QString &host, const QString &name, emit messageSent(request, host, name, receipt); emit sessionWriteDone(host); } - } + }); process->deleteLater(); }); m_messagesInFlight.insert(key); @@ -2010,8 +2106,8 @@ quint64 HgsClient::runSessionAction(const QString &host, const QString &name, co const auto error = QString::fromUtf8(process->readAllStandardError()).trimmed().left(2000); if (code || status != QProcess::NormalExit) fail(!error.isEmpty() ? error : scoped ? tr("Could not complete the request. Check the session before retrying.") : tr("Could not complete the request. Check Terminal.")); - else { - const auto result = QJsonDocument::fromJson(process->readAllStandardOutput()).object(); + else ProcessRunner::post(this, [bytes = process->readAllStandardOutput()] { return JsonParse::object(bytes); }, + [this, fail, reported, request, data, operation, name, host, scoped](const QJsonObject &result) { const auto state = result.value("status").toString(); const bool identity = result.value("request_id") == data.value("request_id") && result.value("name") == name && result.value("run_id") == data.value("expected_run_id") && result.value("conversation_id") == data.value("expected_conversation_id"); @@ -2027,7 +2123,7 @@ quint64 HgsClient::runSessionAction(const QString &host, const QString &name, co || (operation == "clear-context" && state == "confirmed"))) fail(scoped ? tr("Invalid acknowledgement. Check the session before retrying.") : tr("Invalid acknowledgement. Check Terminal before retrying.")); else { *reported = true; emit sessionActionFinished(request,true,result,{}); if (operation != "attachment") emit sessionWriteDone(host); } - } + }); } process->deleteLater(); }); @@ -2070,13 +2166,14 @@ quint64 HgsClient::requestQueueSendNow(const QString &host, const QString &name, if (code || status != QProcess::NormalExit) { const QString error = QString::fromUtf8(process->readAllStandardError()).trimmed().left(2000); fail(error.isEmpty() ? tr("Could not confirm sending the queue. Check Terminal before retrying.") : error); - } else { - QJsonParseError error; const auto doc = QJsonDocument::fromJson(process->readAllStandardOutput(), &error); const auto receipt = doc.object(); - if (error.error != QJsonParseError::NoError || receipt.value("status") != "submitted" || receipt.value("request_id") != requestId + } else ProcessRunner::post(this, [bytes = process->readAllStandardOutput()] { return objectReply(bytes); }, + [this, fail, reported, request, requestId, name, run, conversation, queue, host](const ObjectReply &doc) { + const auto &receipt = doc.object; + if (!doc.ok() || receipt.value("status") != "submitted" || receipt.value("request_id") != requestId || receipt.value("name") != name || receipt.value("run_id") != run || receipt.value("conversation_id") != conversation || receipt.value("queue_id") != queue) fail(tr("Invalid queue acknowledgement. Check Terminal before retrying.")); else { *reported = true; emit queueSendFinished(request, true, receipt, {}); emit sessionWriteDone(host); } - } + }); } process->deleteLater(); }); @@ -2119,13 +2216,14 @@ quint64 HgsClient::requestEffort(const QString &host, const QString &name, const if (code || status != QProcess::NormalExit) { const QString error = QString::fromUtf8(process->readAllStandardError()).trimmed().left(2000); fail(error.isEmpty() ? tr("Could not confirm effort. Check Terminal before retrying.") : error); - } else { - QJsonParseError error; const auto doc = QJsonDocument::fromJson(process->readAllStandardOutput(), &error); const auto receipt = doc.object(); - if (error.error != QJsonParseError::NoError || receipt.value("status") != "confirmed" || receipt.value("request_id") != requestId + } else ProcessRunner::post(this, [bytes = process->readAllStandardOutput()] { return objectReply(bytes); }, + [this, fail, reported, request, requestId, name, run, conversation, effort, host](const ObjectReply &doc) { + const auto &receipt = doc.object; + if (!doc.ok() || receipt.value("status") != "confirmed" || receipt.value("request_id") != requestId || receipt.value("name") != name || receipt.value("run_id") != run || receipt.value("conversation_id") != conversation || receipt.value("effort") != effort) fail(tr("Invalid effort acknowledgement. Check Terminal before retrying.")); else { *reported = true; emit effortFinished(request, true, receipt, {}); emit sessionWriteDone(host); } - } + }); } process->deleteLater(); }); @@ -2176,14 +2274,15 @@ quint64 HgsClient::requestSettings(const QString &host, const QString &name, con if (code || status != QProcess::NormalExit) { const QString error = QString::fromUtf8(process->readAllStandardError()).trimmed().left(2000); fail(error.isEmpty() ? unconfirmed : error); - } else { - QJsonParseError error; const auto doc = QJsonDocument::fromJson(process->readAllStandardOutput(), &error); const auto receipt = doc.object(); - if (error.error != QJsonParseError::NoError || (receipt.value("status") != "applied" && receipt.value("status") != "scheduled") || receipt.value("request_id") != requestId + } else ProcessRunner::post(this, [bytes = process->readAllStandardOutput()] { return objectReply(bytes); }, + [this, fail, reported, request, requestId, name, run, conversation, model, effort, host, invalid](const ObjectReply &doc) { + const auto &receipt = doc.object; + if (!doc.ok() || (receipt.value("status") != "applied" && receipt.value("status") != "scheduled") || receipt.value("request_id") != requestId || receipt.value("name") != name || receipt.value("run_id") != run || receipt.value("conversation_id") != conversation || (!model.isEmpty() && receipt.value("model") != model) || (!effort.isEmpty() && receipt.value("effort") != effort)) fail(invalid); else { *reported = true; emit settingsFinished(request, true, receipt, {}); emit sessionWriteDone(host); } - } + }); } process->deleteLater(); }); @@ -2232,14 +2331,14 @@ quint64 HgsClient::requestNotifications(const QString &host, const QString &name job.input = QJsonDocument(payload).toJson(QJsonDocument::Compact); job.timeoutMs = host.isEmpty() ? 15000 : 35000; job.maximumOutputBytes = 512 * 1024; // Includes the bounded 256-child suppression baseline. - ProcessRunner::run(std::move(job), this, [this, request, payload, expected](const ProcessRunner::Result &reply) { - QJsonParseError parse; - const auto document = QJsonDocument::fromJson(reply.standardOutput, &parse); - const auto result = document.object(), preferences = result.value("preferences").toObject(); + ProcessRunner::run(std::move(job), this, [](ProcessRunner::Result result) { return parsedResult(std::move(result)); }, + [this, request, payload, expected](const ParsedResult &parsed) { + const auto &reply = parsed.result; + const auto result = parsed.reply.object, preferences = result.value("preferences").toObject(); const auto revision = SessionNotifications::revision(preferences.value("revision")); const bool applied = result.value("applied").toBool(); const bool confirmed = reply.outcome == ProcessRunner::Result::Finished && reply.exitStatus == QProcess::NormalExit - && reply.exitCode == 0 && parse.error == QJsonParseError::NoError && document.isObject() + && reply.exitCode == 0 && parsed.reply.ok() && SessionNotifications::keys(result, {"ok", "applied", "run_id", "conversation_id", "preferences"}) && result.value("ok").isBool() && result.value("ok").toBool() && result.value("applied").isBool() && result.value("run_id") == payload.value("run_id") && result.value("conversation_id") == payload.value("conversation_id") @@ -2278,21 +2377,25 @@ quint64 HgsClient::requestRead(const QString &host, const QString &name, const Q }); connect(process, &QProcess::finished, process, [this, process, timer, fail, reported, request, payload](int code, QProcess::ExitStatus status) { timer->stop(); - if (!*reported) { - QJsonParseError parse; const auto result = QJsonDocument::fromJson(process->readAllStandardOutput(), &parse).object(); + if (!*reported) ProcessRunner::post(this, + [bytes = process->readAllStandardOutput(), errorText = QString::fromUtf8(process->readAllStandardError()).trimmed().left(2000)] { + ObjectReply reply = objectReply(bytes); return std::make_pair(reply, errorText); + }, + [fail, reported, request, payload, code, status, this](const std::pair &prepared) { + const auto &result = prepared.first.object; const auto applied = result.value("applied").toObject(); const bool notFound = code == 1 && result == QJsonObject{{"ok", false}, {"error", "not_found"}}; const bool answered = code == 0 && result.value("ok") == true && applied.value("reply_id").isBool() && applied.value("attention_signature").isBool() && (result.value("read").isNull() || result.value("read").toObject().value("conversation_id") == payload.value("conversation_id")); - if (status == QProcess::NormalExit && parse.error == QJsonParseError::NoError && (notFound || answered)) { + if (status == QProcess::NormalExit && prepared.first.ok() && (notFound || answered)) { *reported = true; emit readFinished(request, true, result, {}); } else { - const auto error = QString::fromUtf8(process->readAllStandardError()).trimmed().left(2000); + const auto &error = prepared.second; fail(error.isEmpty() ? tr("Could not save read state on that machine.") : error); } - } + }); process->deleteLater(); }); timer->start(host.isEmpty() ? 15000 : 35000); process->start(); return request; @@ -2315,13 +2418,15 @@ quint64 HgsClient::repositoryOperation(const QString &host,const QString &action guardMachine(args,expectedMachine); auto process=processRequest(m_hgs,args);process.input=QByteArray();process.maximumOutputBytes=65536; process.timeoutMs=action=="clone"?650000:30000; - ProcessRunner::run(std::move(process),this,[this,request,action,name,token,host](const ProcessRunner::Result &result){ + ProcessRunner::run(std::move(process),this,[](ProcessRunner::Result result){return parsedResult(std::move(result));}, + [this,request,action,name,token,host](const ParsedResult &parsed){ + const auto &result=parsed.result; if(result.outcome!=ProcessRunner::Result::Finished||result.exitStatus!=QProcess::NormalExit||result.exitCode!=0){ auto error=QString::fromUtf8(result.standardError).trimmed().left(4000); if(error.isEmpty())error=tr("The computer did not confirm the repository operation. Check its result before trying again."); emit repositoryFinished(request,false,{},error);return; } - const auto data=QJsonDocument::fromJson(result.standardOutput).object(); + const auto &data=parsed.reply.object; const auto style=host.isEmpty()&&MachinePlatform::windowsShaped(data["parent"].toString())?MachinePlatform::Style::Windows:MachinePlatform::Style::Posix; const auto status=data["status"].toString(); const auto expected=MachinePlatform::clean(data["parent"].toString()+'/'+name,style); diff --git a/tray/src/HgsClient.h b/tray/src/HgsClient.h index 9ae92a1d5..d1558d620 100644 --- a/tray/src/HgsClient.h +++ b/tray/src/HgsClient.h @@ -1,5 +1,7 @@ #pragma once +#include "ProcessRunner.h" + #include #include #include @@ -298,6 +300,11 @@ class HgsClient : public QObject { quint64 requestArchivePage(const QString &host, const QJsonObject &query); static QList parseSessions(const QJsonArray &rows); static BoxState parseBox(const QByteArray &json, QString *error); + // A listing that is already an object, as tests and prepared values hold it. + static BoxState box(const QJsonObject &object); + // How the client starts hgs with these arguments: its environment without + // an agent's attribution, and macOS argument handling. + static ProcessRunner::Request commandRequest(const QString &hgs, const QStringList &arguments); static QList parseFleet(const QByteArray &json, QString *error); static QList parseProjects(const QByteArray &json, QString *error); @@ -450,6 +457,10 @@ class HgsClient : public QObject { QHash> m_reviewsInFlight; QHash m_worktreeReviews; QHash m_worktreeInspections; + // Reply sizes of cached reviews and inspections, and each inspection's + // {host, path, common dir} scope, so trimming and invalidation never parse. + QHash m_worktreeReviewBytes, m_worktreeInspectionBytes; + QHash m_worktreeInspectionScopes; QHash> m_worktreeInspectionsInFlight; QHash m_worktreeCacheGenerations; quint64 m_nativeUiRequest = 0; diff --git a/tray/src/IdentityHash.h b/tray/src/IdentityHash.h new file mode 100644 index 000000000..93d2d615a --- /dev/null +++ b/tray/src/IdentityHash.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include +#include +#include + +// SHA-256 of a short identity, remembered: rows hash the same identities on +// every listing, and on macOS each QCryptographicHash call set up an OpenSSL +// digest (3.9 % of the GUI thread in a 2026-10-27 sample). Bounded; any thread. +namespace IdentityHash { +inline QByteArray sha256(const QByteArray &input) +{ + static QMutex mutex; + static QHash cache; + { + QMutexLocker lock(&mutex); + if (const auto found = cache.constFind(input); found != cache.cend()) return *found; + } + const QByteArray digest = QCryptographicHash::hash(input, QCryptographicHash::Sha256); + QMutexLocker lock(&mutex); + if (cache.size() >= 16384) cache.clear(); + cache.insert(input, digest); + return digest; +} +} diff --git a/tray/src/JsonParse.cpp b/tray/src/JsonParse.cpp new file mode 100644 index 000000000..eaeedb06a --- /dev/null +++ b/tray/src/JsonParse.cpp @@ -0,0 +1,93 @@ +#include "JsonParse.h" + +#include +#include +#include +#include +#include + +#include + +namespace JsonParse { +namespace { + +std::atomic s_violations{0}; +QMutex s_mutex; +std::function s_handler; +QSet s_reported; + +bool fatalChecks() +{ + static const bool fatal = qEnvironmentVariable("ZERUS_GUI_THREAD_CHECKS") == QLatin1String("fatal"); + return fatal; +} + +void violation(const char *file, int line) +{ + ++s_violations; + QMutexLocker lock(&s_mutex); + if (s_handler) { const auto handler = s_handler; lock.unlock(); handler(file, line); return; } + if (fatalChecks()) qFatal("JSON parsed on the GUI thread at %s:%d (JsonParse.h)", file, line); + const QByteArray site = QByteArray(file) + ':' + QByteArray::number(line); + if (s_reported.size() < 256 && !s_reported.contains(site)) { + s_reported.insert(site); + qWarning("JSON parsed on the GUI thread at %s", site.constData()); + } +} + +QJsonDocument decode(const QByteArray &bytes, QString *error) +{ + QJsonParseError parseError; + QJsonDocument document = QJsonDocument::fromJson(bytes, &parseError); + if (error) *error = parseError.error == QJsonParseError::NoError ? QString() : parseError.errorString(); + return document; +} + +} + +bool onGuiThread() +{ + const auto *application = QCoreApplication::instance(); + return application && QThread::currentThread() == application->thread() && application->inherits("QGuiApplication"); +} + +QJsonDocument parse(const QByteArray &bytes, QString *error, const char *file, int line) +{ + if (onGuiThread()) violation(file, line); + return decode(bytes, error); +} + +QJsonObject object(const QByteArray &bytes, QString *error, const char *file, int line) +{ + QString parseError; const auto document = parse(bytes, &parseError, file, line); + if (parseError.isEmpty() && !document.isObject()) parseError = QStringLiteral("expected a JSON object"); + if (error) *error = parseError; + return parseError.isEmpty() ? document.object() : QJsonObject(); +} + +QJsonArray array(const QByteArray &bytes, QString *error, const char *file, int line) +{ + QString parseError; const auto document = parse(bytes, &parseError, file, line); + if (parseError.isEmpty() && !document.isArray()) parseError = QStringLiteral("expected a JSON array"); + if (error) *error = parseError; + return parseError.isEmpty() ? document.array() : QJsonArray(); +} + +QJsonDocument local(const QByteArray &bytes, QString *error, const char *file, int line) +{ + if (bytes.size() > LocalLimit && onGuiThread()) violation(file, line); + return decode(bytes, error); +} + +QJsonObject localObject(const QByteArray &bytes, const char *file, int line) { return local(bytes, nullptr, file, line).object(); } +QJsonArray localArray(const QByteArray &bytes, const char *file, int line) { return local(bytes, nullptr, file, line).array(); } + +int violations() { return s_violations; } + +void setViolationHandler(std::function handler) +{ + QMutexLocker lock(&s_mutex); + s_handler = std::move(handler); +} + +} diff --git a/tray/src/JsonParse.h b/tray/src/JsonParse.h new file mode 100644 index 000000000..1ebb1aeb7 --- /dev/null +++ b/tray/src/JsonParse.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +// The tray's only JSON parser: scripts/ci/check-source.py rejects +// QJsonDocument::fromJson elsewhere in tray/src. Data from hgs, the event +// stream and files is parsed off the GUI thread, in a ProcessRunner prepare +// step or on another worker, and the GUI thread applies the finished value. +// +// A parse() on the application thread of a GUI application is a violation. +// It is counted and logged once per call site; with +// ZERUS_GUI_THREAD_CHECKS=fatal, which every Qt suite sets, it aborts with that +// call site. A command-line or test program without a GUI has no GUI thread. +// +// local() is the one exception, for small values the tray wrote itself +// (preferences, drag data) that a widget needs synchronously. It is limited to +// LocalLimit bytes, and check-source.py lists the files allowed to call it and +// how often; that list only shrinks. +namespace JsonParse { + +constexpr qsizetype LocalLimit = 64 * 1024; + +// error, which may be null, receives the parse error or is cleared. object() +// and array() also report a document of the other kind as an error. +QJsonDocument parse(const QByteArray &bytes, QString *error = nullptr, + const char *file = __builtin_FILE(), int line = __builtin_LINE()); +QJsonObject object(const QByteArray &bytes, QString *error = nullptr, + const char *file = __builtin_FILE(), int line = __builtin_LINE()); +QJsonArray array(const QByteArray &bytes, QString *error = nullptr, + const char *file = __builtin_FILE(), int line = __builtin_LINE()); + +QJsonDocument local(const QByteArray &bytes, QString *error = nullptr, + const char *file = __builtin_FILE(), int line = __builtin_LINE()); +QJsonObject localObject(const QByteArray &bytes, const char *file = __builtin_FILE(), int line = __builtin_LINE()); +QJsonArray localArray(const QByteArray &bytes, const char *file = __builtin_FILE(), int line = __builtin_LINE()); + +// True on the application thread of a GUI application. +bool onGuiThread(); + +// Tests: violations so far, and a handler that replaces logging and the fatal +// check (an empty handler restores them). +int violations(); +void setViolationHandler(std::function handler); + +} diff --git a/tray/src/LinuxAttentionNotifier.cpp b/tray/src/LinuxAttentionNotifier.cpp index fd786eddb..54e5d5fe1 100644 --- a/tray/src/LinuxAttentionNotifier.cpp +++ b/tray/src/LinuxAttentionNotifier.cpp @@ -1,3 +1,4 @@ +#include "SettingsJson.h" #include "LinuxAttentionNotifier.h" #include #include @@ -20,7 +21,7 @@ LinuxAttentionNotifier::LinuxAttentionNotifier(const QDBusConnection &bus) : m_b if (m_bus.interface()) m_owner = m_bus.interface()->serviceOwner(service).value(); QSettings settings; if (!m_owner.isEmpty() && settings.value("attention/linuxOwner").toString() == m_owner) { - const auto saved = QJsonDocument::fromJson(settings.value("attention/linuxTokens").toByteArray()).object(); + const auto saved = SettingsJson::object("attention/linuxTokens"); for (auto it = saved.begin(); it != saved.end(); ++it) m_tokens[it.key().toUInt()] = it.value().toString(); } m_bus.connect(service, "/org/freedesktop/Notifications", service, "ActionInvoked", this, SLOT(action(uint,QString))); diff --git a/tray/src/NewSessionDialog.cpp b/tray/src/NewSessionDialog.cpp index 25d6c9629..93cb6ea51 100644 --- a/tray/src/NewSessionDialog.cpp +++ b/tray/src/NewSessionDialog.cpp @@ -1,3 +1,4 @@ +#include "SettingsJson.h" #include "NewSessionDialog.h" #include "RepositoryClone.h" #include "WorkspaceIcons.h" @@ -470,7 +471,7 @@ NewSessionDialog::NewSessionDialog(const QString &hgsPath,const FleetState &flee m_creationCatalog["worktrees"]=entries;m_verifiedWorktrees[host()+'\n'+path]=m_creationCatalog; m_autoPreferred=false;selectPath(path);launch(path,m_creationFolder); }); - setGroups(SessionOrganization(QJsonDocument::fromJson(QSettings().value("workspace/organization").toByteArray()).object()),project); + setGroups(SessionOrganization(SettingsJson::object("workspace/organization")),project); } void NewSessionDialog::reject(){if(!m_creation&&(!m_repository||!m_repository->cloning())){m_cloneLaunchPending=false;if(m_repository)m_repository->cancelCheck();QDialog::reject();}} QString NewSessionDialog::host() const {return m_machine->currentData().toString();} @@ -845,7 +846,7 @@ void NewSessionDialog::launch(const QString &target,const QString &addFolder) // The next dialog starts with this agent and, per machine and agent, this account. settings.setValue("workspace/launchAgent",agent()); if(const auto selectedAccount=account();!selectedAccount.isEmpty()){ - auto accounts=QJsonDocument::fromJson(settings.value("workspace/launchAccounts").toByteArray()).object(); + auto accounts=SettingsJson::object("workspace/launchAccounts"); auto machine=accounts.value(host()).toObject();machine[agent()]=selectedAccount;accounts[host()]=machine; settings.setValue("workspace/launchAccounts",QJsonDocument(accounts).toJson(QJsonDocument::Compact)); } @@ -893,7 +894,7 @@ void NewSessionDialog::updateAccounts() } m_accountGroups=AccountCatalog::group(localProfiles); } - const auto remembered=QJsonDocument::fromJson(QSettings().value("workspace/launchAccounts").toByteArray()).object().value(host()).toObject(); + const auto remembered=SettingsJson::object("workspace/launchAccounts").value(host()).toObject(); const auto current=m_account->currentItem()?m_account->currentItem()->data(ProfileRole).toJsonObject():QJsonObject(); const auto currentId=current["host"].toString()==host()&¤t["provider"]==m_preferredAgent?current["id"].toString():QString(); QList profiles;QList members; @@ -918,7 +919,7 @@ void NewSessionDialog::updateAccounts() const bool dark=m_account->property("hgsDark").toBool(); const QColor iconColor(dark?"#a1adbb":"#647386"); int defaultIndex=-1,lastIndex=-1,explicitIndex=-1,currentIndex=-1; - const auto lastAccount=QJsonDocument::fromJson(QSettings().value("workspace/launchAccounts").toByteArray()).object() + const auto lastAccount=SettingsJson::object("workspace/launchAccounts") .value(host()).toObject().value(m_preferredAgent).toString(); for(int row=0;row prepare; }; struct Pending { QPointer owner; - std::function done; + std::function apply; QMetaObject::Connection destroyed; }; +std::atomic s_preparationThread{nullptr}; + class Runner { public: Runner() @@ -297,18 +300,38 @@ class Runner { m_worker = new QObject; m_worker->moveToThread(&m_thread); QObject::connect(&m_thread, &QThread::finished, m_worker, &QObject::deleteLater); m_thread.start(); + // One serial thread keeps results in the order their children finished. + m_prepareThread.setObjectName(QStringLiteral("hgs-prepare")); + m_preparer = new QObject; m_preparer->moveToThread(&m_prepareThread); + QObject::connect(&m_prepareThread, &QThread::finished, m_preparer, &QObject::deleteLater); + s_preparationThread = &m_prepareThread; + m_prepareThread.start(); } - void run(Request request, QObject *owner, std::function done) + void run(Request request, QObject *owner, std::function prepare, std::function apply) { const quint64 id = ++m_next; - Pending pending{owner, std::move(done), {}}; + Pending pending{owner, std::move(apply), {}}; // Owner destruction kills the child synchronously, as the old per-object // QProcess children did, so fixtures and sockets are not used afterwards. pending.destroyed = QObject::connect(owner, &QObject::destroyed, &m_front, [this, id] { cancel(id); }, Qt::DirectConnection); m_pending.insert(id, std::move(pending)); - QMetaObject::invokeMethod(m_worker, [this, id, request = std::move(request)]() mutable { start(id, std::move(request)); }, - Qt::QueuedConnection); + QMetaObject::invokeMethod(m_worker, [this, id, request = std::move(request), prepare = std::move(prepare)]() mutable { + start(id, std::move(request), std::move(prepare)); + }, Qt::QueuedConnection); + } + void post(QObject *owner, std::function prepare, std::function apply) + { + const quint64 id = ++m_next; + Pending pending{owner, std::move(apply), {}}; + pending.destroyed = QObject::connect(owner, &QObject::destroyed, &m_front, [this, id] { m_pending.remove(id); }, + Qt::DirectConnection); + m_pending.insert(id, std::move(pending)); + QMetaObject::invokeMethod(m_preparer, [this, id, prepare = std::move(prepare)]() mutable { + Prepared value = prepare(); + QMetaObject::invokeMethod(&m_front, [this, id, value = std::move(value)]() mutable { deliver(id, std::move(value)); }, + Qt::QueuedConnection); + }, Qt::QueuedConnection); } void stop() { @@ -321,14 +344,18 @@ class Runner { qDeleteAll(m_jobs); m_jobs.clear(); }, Qt::BlockingQueuedConnection); m_thread.quit(); m_thread.wait(); + // Preparations already queued only compute values; nothing applies them. + m_prepareThread.quit(); m_prepareThread.wait(); + s_preparationThread = nullptr; } QThread *thread() { return &m_thread; } + QThread *preparationThread() { return &m_prepareThread; } void setObserver(std::function observer) { QMutexLocker lock(&m_observerMutex); m_observer = std::move(observer); } private: - void start(quint64 id, Request request) // worker thread + void start(quint64 id, Request request, std::function prepare) // worker thread { - auto *job = new Job; m_jobs.insert(id, job); + auto *job = new Job; m_jobs.insert(id, job); job->prepare = std::move(prepare); auto *process = job->process = new Child(m_worker); process->setProgram(request.program); process->setArguments(request.arguments); if (request.environment) process->setProcessEnvironment(*request.environment); @@ -411,7 +438,11 @@ class Runner { if (job->reported) return; job->reported = true; Result result = outcome == Result::TimedOut ? Result{} : job->result; result.outcome = outcome; - QMetaObject::invokeMethod(&m_front, [this, id, result = std::move(result)] { deliver(id, result); }, Qt::QueuedConnection); + QMetaObject::invokeMethod(m_preparer, [this, id, prepare = std::move(job->prepare), result = std::move(result)]() mutable { + Prepared value = prepare(std::move(result)); + QMetaObject::invokeMethod(&m_front, [this, id, value = std::move(value)]() mutable { deliver(id, std::move(value)); }, + Qt::QueuedConnection); + }, Qt::QueuedConnection); } void finish(quint64 id, Job *job) // worker thread { @@ -428,13 +459,13 @@ class Runner { if (process->state() != QProcess::NotRunning) { process->kill(); process->waitForFinished(1000); } delete process; } - void deliver(quint64 id, const Result &result) // application thread + void deliver(quint64 id, Prepared value) // application thread { auto it = m_pending.find(id); if (it == m_pending.end()) return; Pending pending = std::move(*it); m_pending.erase(it); QObject::disconnect(pending.destroyed); - if (pending.owner) pending.done(result); + if (pending.owner) pending.apply(std::move(value)); } void cancel(quint64 id) // application thread, while the owner is destroyed { @@ -446,8 +477,8 @@ class Runner { }, Qt::BlockingQueuedConnection); } - QThread m_thread; - QObject *m_worker = nullptr; + QThread m_thread, m_prepareThread; + QObject *m_worker = nullptr, *m_preparer = nullptr; QObject m_front; // receives results on the application thread QHash m_pending; QHash m_jobs; @@ -479,13 +510,27 @@ Runner *runner() } -void run(Request request, QObject *owner, std::function done) +void runPrepared(Request request, QObject *owner, std::function prepare, std::function apply) { Q_ASSERT(owner && owner->thread() == QThread::currentThread()); // After the application's post routines nothing can receive a result. - if (auto *instance = runner()) instance->run(std::move(request), owner, std::move(done)); + if (auto *instance = runner()) instance->run(std::move(request), owner, std::move(prepare), std::move(apply)); +} + +void run(Request request, QObject *owner, std::function done) +{ + runPrepared(std::move(request), owner, [](Result result) -> Prepared { return std::make_shared(std::move(result)); }, + [done = std::move(done)](Prepared value) { done(*std::static_pointer_cast(value)); }); +} + +void postPrepared(QObject *owner, std::function prepare, std::function apply) +{ + Q_ASSERT(owner && owner->thread() == QThread::currentThread()); + if (auto *instance = runner()) instance->post(owner, std::move(prepare), std::move(apply)); } +bool isPreparationThread() { return QThread::currentThread() == s_preparationThread.load(); } + #ifdef Q_OS_MACOS const char *launchBackend() { return "posix_spawn"; } #else @@ -493,6 +538,7 @@ const char *launchBackend() { return "QProcess"; } #endif QThread *workerThread() { auto *instance = runner(); return instance ? instance->thread() : nullptr; } +QThread *preparationThread() { auto *instance = runner(); return instance ? instance->preparationThread() : nullptr; } void setLaunchObserver(std::function observer) { if (auto *instance = runner()) instance->setObserver(std::move(observer)); } diff --git a/tray/src/ProcessRunner.h b/tray/src/ProcessRunner.h index a3b11e33a..f46c2df7c 100644 --- a/tray/src/ProcessRunner.h +++ b/tray/src/ProcessRunner.h @@ -7,7 +7,10 @@ #include #include +#include #include +#include +#include class QObject; class QThread; @@ -16,6 +19,11 @@ class QThread; // the large GUI process, pipe I/O and child reaping then never stall painting: // the GUI thread only posts a request and later receives one result. Polls start // about one hgs child per second, and on macOS each fork() cost ~4 ms there. +// +// Every result then passes one shared preparation thread before it reaches its +// owner. A caller with a prepare step parses and shapes the output there, so the +// owner's thread only applies a finished value. Results keep the order in which +// their children finished, with or without a prepare step. namespace ProcessRunner { struct Request { @@ -49,8 +57,46 @@ struct Result { // kills and reaps its running children before returning and drops their results. void run(Request request, QObject *owner, std::function done); -// Tests: the worker thread, and a hook run there right before each start. +// A prepared value, type-erased between the two threads. +using Prepared = std::shared_ptr; +void runPrepared(Request request, QObject *owner, std::function prepare, + std::function apply); + +// As run(), but prepare(Result) runs on the preparation thread and its value goes +// to apply on owner's thread. prepare must not touch owner or any other +// application-thread object: it sees only the result and what it captured by +// value. A destroyed owner still drops the value instead of applying it. +template +void run(Request request, QObject *owner, Prepare prepare, Apply apply) +{ + using Value = std::decay_t>; + runPrepared(std::move(request), owner, + [prepare = std::move(prepare)](Result result) mutable -> Prepared { + return std::make_shared(prepare(std::move(result))); + }, + [apply = std::move(apply)](Prepared value) mutable { apply(std::move(*std::static_pointer_cast(value))); }); +} + +// Prepares data that did not come from a child of run(), such as a direct +// QProcess result or a file, the same way: prepare() on the preparation thread, +// then apply(value) on owner's thread unless owner is destroyed first. +void postPrepared(QObject *owner, std::function prepare, std::function apply); +template +void post(QObject *owner, Prepare prepare, Apply apply) +{ + using Value = std::decay_t>; + postPrepared(owner, + [prepare = std::move(prepare)]() mutable -> Prepared { return std::make_shared(prepare()); }, + [apply = std::move(apply)](Prepared value) mutable { apply(std::move(*std::static_pointer_cast(value))); }); +} + +// True on the shared preparation thread. +bool isPreparationThread(); + +// Tests: the worker and preparation threads, and a hook run on the worker +// right before each start. QThread *workerThread(); +QThread *preparationThread(); // Tests: how children start, "posix_spawn" on macOS and "QProcess" elsewhere. const char *launchBackend(); void setLaunchObserver(std::function observer); diff --git a/tray/src/ProjectsDialog.cpp b/tray/src/ProjectsDialog.cpp index 34268f5b8..ede032bc7 100644 --- a/tray/src/ProjectsDialog.cpp +++ b/tray/src/ProjectsDialog.cpp @@ -1,3 +1,4 @@ +#include "SettingsJson.h" #include "ProjectsDialog.h" #include "CloneRepositoryDialog.h" #include "WorktreePanel.h" @@ -90,7 +91,7 @@ class ProjectDelegate : public QStyledItemDelegate { ProjectsDialog::ProjectsDialog(HgsClient *client, QWidget *parent, bool embedded, SessionOrganization *projects) : QDialog(parent), m_client(client), - m_ownedProjects(QJsonDocument::fromJson(QSettings().value("workspace/organization").toByteArray()).object()), + m_ownedProjects(SettingsJson::object("workspace/organization")), m_projects(projects ? projects : &m_ownedProjects) { setObjectName("projectsPage"); setWindowTitle(tr("Projects")); resize(850, 560); @@ -309,7 +310,10 @@ void ProjectsDialog::showProject() } void ProjectsDialog::changed() { - if (m_projects == &m_ownedProjects) QSettings().setValue("workspace/organization",QJsonDocument(m_projects->toJson()).toJson(QJsonDocument::Compact)); + if (m_projects == &m_ownedProjects) { + const QJsonDocument document(m_projects->toJson()); const auto bytes = document.toJson(QJsonDocument::Compact); + QSettings().setValue("workspace/organization", bytes); SettingsJson::remember("workspace/organization", bytes, document); + } emit organizationChanged(); refresh(); m_status->setText(tr("Saved")); } void ProjectsDialog::selectProject(const QString &id) diff --git a/tray/src/QuestionReply.h b/tray/src/QuestionReply.h index 989559bf2..9cfeaec59 100644 --- a/tray/src/QuestionReply.h +++ b/tray/src/QuestionReply.h @@ -1,5 +1,6 @@ #pragma once +#include "JsonParse.h" #include #include #include @@ -19,9 +20,9 @@ inline QJsonArray parse(QString text) const QString start = ""; const QString end = ""; if (!text.startsWith(start) || !text.endsWith(end)) return {}; - QJsonParseError error; - const auto document = QJsonDocument::fromJson(text.mid(start.size(), text.size() - start.size() - end.size()).toUtf8(), &error); - if (error.error != QJsonParseError::NoError) return {}; + QString error; + const auto document = JsonParse::local(text.mid(start.size(), text.size() - start.size() - end.size()).toUtf8(), &error); + if (!error.isEmpty()) return {}; const auto replies = document.isObject() ? QJsonArray{document.object()} : document.array(); for (const auto &value : replies) { if (!value.isObject()) return {}; diff --git a/tray/src/ReadMarks.h b/tray/src/ReadMarks.h index 88fbe4199..960e6b479 100644 --- a/tray/src/ReadMarks.h +++ b/tray/src/ReadMarks.h @@ -1,5 +1,6 @@ #pragma once +#include "SettingsJson.h" #include #include #include @@ -51,7 +52,7 @@ struct ReadMark { // restart keeps retrying; the root agent and the workspace window share it. namespace ReadMarkStore { inline QJsonObject load() { - return QJsonDocument::fromJson(QSettings().value("attention/pendingReads").toByteArray()).object(); + return SettingsJson::object("attention/pendingReads"); } inline void save(const QJsonObject &marks) { QSettings().setValue("attention/pendingReads", QJsonDocument(marks).toJson(QJsonDocument::Compact)); @@ -73,7 +74,7 @@ inline void remove(const QList &marks) { save(stored); } inline QJsonArray migratedHosts() { - return QJsonDocument::fromJson(QSettings().value("attention/sharedReadMigrated").toByteArray()).array(); + return SettingsJson::array("attention/sharedReadMigrated"); } inline bool migrated(const QString &host) { return migratedHosts().contains(host); } inline void setMigrated(const QString &host) { diff --git a/tray/src/RecoverySettings.h b/tray/src/RecoverySettings.h index afcd8f61d..c04023316 100644 --- a/tray/src/RecoverySettings.h +++ b/tray/src/RecoverySettings.h @@ -1,4 +1,5 @@ #pragma once +#include "SettingsJson.h" #include "HgsClient.h" #include #include @@ -20,7 +21,7 @@ class RecoverySync : public QObject { public: std::function changed; explicit RecoverySync(const QString &executable,QObject *parent):QObject(parent),client(executable,this) { - best=QJsonDocument::fromJson(QSettings().value("recovery/sharedPolicy").toByteArray()).object(); + best=SettingsJson::object("recovery/sharedPolicy"); if(!best.value("policy").isObject()||best.value("order").toArray().size()!=2)best={}; connect(&client,&HgsClient::recoveryFinished,this,[this](quint64 id,bool ok,const QJsonObject &data,const QString &error){ if(!pending.contains(id))return;const auto request=pending.take(id); diff --git a/tray/src/RelayConfiguration.h b/tray/src/RelayConfiguration.h index e9a7e0a67..ca6262e6c 100644 --- a/tray/src/RelayConfiguration.h +++ b/tray/src/RelayConfiguration.h @@ -1,4 +1,5 @@ #pragma once +#include "JsonParse.h" #include #include #include @@ -210,8 +211,8 @@ class ConfigStore { struct stat after{};const bool stable=::fstat(fd,&after)==0&&after.st_size==st.st_size&&after.st_mtime==st.st_mtime; ::close(fd); if(n<0||!stable||bytes.size()>65536)return fail(QStringLiteral("Configuration changed while reading. Reload before saving.")); - QJsonParseError parse{};const auto doc=QJsonDocument::fromJson(bytes,&parse); - if(parse.error!=QJsonParseError::NoError||!doc.isObject())return fail(QStringLiteral("Connector configuration is not a JSON object.")); + QString parse;const auto doc=JsonParse::local(bytes,&parse); + if(!parse.isEmpty()||!doc.isObject())return fail(QStringLiteral("Connector configuration is not a JSON object.")); *out={true,doc.object(),bytes,quint64(st.st_dev),quint64(st.st_ino)};return true; } #endif diff --git a/tray/src/RepositoryClone.cpp b/tray/src/RepositoryClone.cpp index cec545030..277216325 100644 --- a/tray/src/RepositoryClone.cpp +++ b/tray/src/RepositoryClone.cpp @@ -1,3 +1,4 @@ +#include "SettingsJson.h" #include "RepositoryClone.h" #include "HgsClient.h" #include @@ -29,7 +30,7 @@ RepositoryClone::RepositoryClone(HgsClient *client,const QString &host,MachinePl { connect(m_client,&HgsClient::repositoryFinished,this,&RepositoryClone::finish); m_parent=QSettings().value(settingsKey()+"/parent","~").toString(); - const auto saved=QJsonDocument::fromJson(QSettings().value(settingsKey()+"/pending").toByteArray()).object(); + const auto saved=SettingsJson::object(settingsKey()+"/pending"); if(!saved.isEmpty()){ m_url=saved["url"].toString();m_parent=saved["parent"].toString();m_name=saved["name"].toString(); m_token=saved["request_id"].toString();m_expected=saved["path"].toString();m_uncertain=true; diff --git a/tray/src/SessionCreationController.cpp b/tray/src/SessionCreationController.cpp index c8bb0bec0..302ca565b 100644 --- a/tray/src/SessionCreationController.cpp +++ b/tray/src/SessionCreationController.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "SessionCreationController.h" #include "MachinePlatform.h" #include @@ -67,9 +68,8 @@ SessionCreationController::SessionCreationController(const QString &hgs,QObject if(file.exists()){ if(!file.open(QIODevice::ReadOnly)||file.size()>JournalLimit)error=tr("Could not read the creation journal. Existing requests are kept."); else{ - QJsonParseError parse;const auto document=QJsonDocument::fromJson(file.readAll(),&parse); - saved=document.object(); - if(parse.error!=QJsonParseError::NoError||saved.value("schema")!=1||!saved.value("operations").isObject() + QString parse;saved=JsonParse::object(file.readAll(),&parse); + if(!parse.isEmpty()||saved.value("schema")!=1||!saved.value("operations").isObject() ||saved.value("operations").toObject().size()>128)error=tr("Unsupported creation journal. Existing requests are kept."); } } @@ -243,7 +243,7 @@ void SessionCreationController::persist(const QString &id,std::function QSettings settings;settings.setValue("workspace/launchOpenTerminal",choices.value("terminal").toBool()); settings.setValue("workspace/launchAgent",choices.value("cmd").toString()); if(!choices.value("account").toString().isEmpty()){ - auto accounts=QJsonDocument::fromJson(settings.value("workspace/launchAccounts").toByteArray()).object(); + auto accounts=JsonParse::object(settings.value("workspace/launchAccounts").toByteArray()); auto machine=accounts[choices.value("host").toString()].toObject();machine[choices.value("cmd").toString()]=choices.value("account"); accounts[choices.value("host").toString()]=machine;settings.setValue("workspace/launchAccounts",QJsonDocument(accounts).toJson(QJsonDocument::Compact)); } diff --git a/tray/src/SessionList.cpp b/tray/src/SessionList.cpp index 3c593b08e..0c7b70705 100644 --- a/tray/src/SessionList.cpp +++ b/tray/src/SessionList.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "SessionList.h" #include "SessionCardDelegate.h" #include @@ -455,7 +456,7 @@ QMimeData *SessionList::mimeData(const QList &items) const bool SessionList::validDrop(const QMimeData *mime) const { if (!mime->hasFormat(Mime)) return false; - const auto data = QJsonDocument::fromJson(mime->data(Mime)).object(); + const auto data = JsonParse::localObject(mime->data(Mime)); if (data.value("token").toString() != m_token || data.value("id").toString().isEmpty()) return false; const int role = data.value("group").toBool() ? SessionRoles::Group : SessionRoles::Identity; for (int row = 0; row < count(); ++row) if (item(row)->data(role).toString() == data.value("id").toString()) return true; @@ -540,7 +541,7 @@ void SessionList::dragMoveEvent(QDragMoveEvent *e) if (!validDrop(e->mimeData())) { e->ignore(); return; } // Retain Qt's edge auto-scroll; the base drop mutation is deliberately unused. QListWidget::dragMoveEvent(e); - const auto target = placement(e->position().toPoint(), QJsonDocument::fromJson(e->mimeData()->data(Mime)).object().value("group").toBool()); + const auto target = placement(e->position().toPoint(), JsonParse::localObject(e->mimeData()->data(Mime)).value("group").toBool()); m_indicator = target.indicator; m_onGroup = target.onGroup; viewport()->update(); e->setDropAction(Qt::MoveAction); e->accept(); } @@ -562,7 +563,7 @@ void SessionList::dropEvent(QDropEvent *e) return; } if (!validDrop(e->mimeData())) { e->ignore(); return; } - const auto data = QJsonDocument::fromJson(e->mimeData()->data(Mime)).object(); + const auto data = JsonParse::localObject(e->mimeData()->data(Mime)); const auto target = placement(e->position().toPoint(), data.value("group").toBool()); m_indicator = {}; viewport()->update(); if (data.value("group").toBool()) emit groupMoved(data.value("id").toString(), target.before); diff --git a/tray/src/SessionRowPreferences.h b/tray/src/SessionRowPreferences.h index 1b629a704..351caeff5 100644 --- a/tray/src/SessionRowPreferences.h +++ b/tray/src/SessionRowPreferences.h @@ -1,7 +1,7 @@ #pragma once #include "HgsClient.h" -#include +#include "IdentityHash.h" #include #include #include @@ -21,7 +21,7 @@ class SessionRowPreferences { const QJsonArray identity = session.conversationId.isEmpty() || fallback ? QJsonArray{host, session.cmd, session.archiveId, session.name, session.created} : QJsonArray{host, session.cmd, session.archiveId, session.conversationId}; - return QCryptographicHash::hash(QJsonDocument(identity).toJson(QJsonDocument::Compact), QCryptographicHash::Sha256); + return IdentityHash::sha256(QJsonDocument(identity).toJson(QJsonDocument::Compact)); } bool collapsed(const QString &host, const SessionInfo &session) const { return m_collapsed.contains(key(host, session)); diff --git a/tray/src/SessionsWindow.cpp b/tray/src/SessionsWindow.cpp index 21410bb4e..cdd1e3ad9 100644 --- a/tray/src/SessionsWindow.cpp +++ b/tray/src/SessionsWindow.cpp @@ -1,6 +1,9 @@ +#include "SettingsJson.h" +#include "JsonParse.h" #include "ArchiveBrowser.h" #include "NativeSessionCapabilities.h" #include "SessionsWindow.h" +#include "EventStream.h" #include #include "WorktreeBatchCleanupDialog.h" #include "WorktreeCleanupStatus.h" @@ -306,11 +309,11 @@ SessionsWindow::SessionsWindow(const QString &hgsPath, QWidget *parent) m_expandedSessions = QSet(expanded.begin(), expanded.end()); m_sessionRowPreferences = SessionRowPreferences(settings.value("workspace/collapsedSessionRows").toByteArray()); m_savedOrganization = settings.value("workspace/organization").toByteArray(); - const auto savedProjects = QJsonDocument::fromJson(m_savedOrganization).object(); + const auto savedProjects = SettingsJson::object("workspace/organization"); if (!savedProjects.isEmpty() && !savedProjects.contains("projects") && !settings.contains("workspace/organizationBeforeProjects")) settings.setValue("workspace/organizationBeforeProjects", m_savedOrganization); m_organization = SessionOrganization(savedProjects); - m_pendingLaunches = QJsonDocument::fromJson(settings.value("workspace/pendingLaunches").toByteArray()).object(); + m_pendingLaunches = SettingsJson::object("workspace/pendingLaunches"); m_creations=new SessionCreationController(m_client.executable(),this); if (!settings.value("sessions/geometry").toByteArray().isEmpty()) restoreGeometry(settings.value("sessions/geometry").toByteArray()); auto *root = new QHBoxLayout(this); root->setContentsMargins(0, 0, 0, 0); root->setSpacing(0); @@ -1406,7 +1409,7 @@ SessionsWindow::SessionsWindow(const QString &hgsPath, QWidget *parent) m_inspector->addTab(m_children, tr("Plan && agents")); m_inspector->addTab(detailsPage, tr("Details")); m_inspector->addTab(m_worktrees, tr("Worktrees")); const auto taskLaunchBytes = settings.value("tasks/pendingExecutorLaunches").toByteArray(); - const auto taskLaunchDoc = taskLaunchBytes.size() <= 65536 ? QJsonDocument::fromJson(taskLaunchBytes) : QJsonDocument(); + const auto taskLaunchDoc = taskLaunchBytes.size() <= 65536 ? JsonParse::local(taskLaunchBytes) : QJsonDocument(); m_taskLaunches = taskLaunchDoc.object(); bool validLaunches = taskLaunchBytes.isEmpty() || taskLaunchDoc.isObject(); if (m_taskLaunches.size() > 16) validLaunches = false; @@ -1901,6 +1904,17 @@ SessionsWindow::SessionsWindow(const QString &hgsPath, QWidget *parent) emit refreshRequested(); }); connect(&m_client, &HgsClient::inspectionReady, this, &SessionsWindow::acceptInspection); + if (auto *streams = EventStreams::instance()) { + connect(streams, &EventStreams::activityReady, this, [this](const QString &host, const QString &name, const QJsonObject &inspection) { + acceptInspection(host, name, inspection, {}); + }); + // The stream stopped following this run (it ended or changed): poll it instead. + connect(streams, &EventStreams::activityEnded, this, [this](const QString &host, const QString &name) { + if (const auto *entry = selected(); entry && entry->host == host && entry->session.name == name) { + m_streamEnded.insert(host + '\n' + name + '\n' + entry->session.runId); inspect(); + } + }); + } connect(&m_client, &HgsClient::subagentInspectionReady, this, [this](const QString &host,const QString &name,const QString &id,const QString &archive,const QJsonObject &data) { const auto *entry = selected(); @@ -2498,6 +2512,7 @@ void SessionsWindow::saveOrganization() const auto data = QJsonDocument(m_organization.toJson()).toJson(QJsonDocument::Compact); if (data == m_savedOrganization) return; QSettings settings; settings.setValue("workspace/organization", data); settings.sync(); + SettingsJson::remember("workspace/organization", data, QJsonDocument(m_organization.toJson())); if (settings.status() == QSettings::NoError) m_savedOrganization = data; else showNotice(tr("Could not save the session layout. Check the settings file permissions."), true); } @@ -2687,7 +2702,7 @@ void SessionsWindow::setFleet(const FleetState &fleet) { if (auto *launch=qobject_cast(QApplication::activeModalWidget()); launch && launch->parentWidget()==this) launch->setFleet(fleet); m_settingsPage->setLocalSessions(fleet.local().localSessions, LocalSessionHost::canEnable(fleet.local())); m_settingsPage->setPeers(fleet.peerNames()); m_settingsPage->setMobileGateway(fleet.local().localSessions||MachinePlatform::of(fleet.local().metrics,true)==MachinePlatform::Style::Windows); - m_fleet = fleet; m_fleet.setAttentionMarks(QJsonDocument::fromJson(QSettings().value("attention/sessionMarks").toByteArray()).object()); m_fleet.setPendingReads(ReadMarkStore::load()); reconcileTerminations();m_usageStore->setFleet(fleet); m_accountsPage->setFleet(fleet); updateDashboard(); m_projectsPage->setFleet(fleet); m_machinesPage->setFleet(fleet); if (isVisible()) rebuild(); applyQueuedModelSettings(); + m_fleet = fleet; m_fleet.setAttentionMarks(SettingsJson::object("attention/sessionMarks")); m_fleet.setPendingReads(ReadMarkStore::load()); reconcileTerminations();m_usageStore->setFleet(fleet); m_accountsPage->setFleet(fleet); updateDashboard(); m_projectsPage->setFleet(fleet); m_machinesPage->setFleet(fleet); if (isVisible()) rebuild(); applyQueuedModelSettings(); updateConnectionStatus(); if (!m_nativeLaunchToOpen.isEmpty()) { for (const auto &entry : m_entries) if (entry.session.launchId==m_nativeLaunchToOpen) { @@ -3783,7 +3798,19 @@ void SessionsWindow::inspect() if (!isVisible() || m_pages->currentIndex() != 0 || !entry || !entry->online) return; const int processInterval=m_detailTabs->currentWidget()==m_processes?ProcessSettings::activeMs():ProcessSettings::backgroundMs(); const bool processes=ProcessSettings::enabled() && (!m_processPollAge.isValid() || m_processPollAge.elapsed()>=processInterval); - if(m_client.requestInspection(entry->host, entry->session.name, m_cursor, entry->session.archiveId,processes)) { + // A live event stream brings Activity as it happens (EventStream.h); only + // the process section is still polled, at its own interval. + auto *streams = EventStreams::instance(); + const bool streamed = streams && streams->live(entry->host) && entry->session.archiveId.isEmpty() + && !m_streamEnded.contains(entry->host + '\n' + entry->session.name + '\n' + entry->session.runId); + if (streams) { + if (streamed) streams->follow(entry->host, entry->session.name, entry->session.runId, entry->session.conversationId, m_cursor); + else streams->unfollowAll(); + } + if (streamed) { + m_inspectionAge.start(); + if (processes && m_client.requestInspection(entry->host, entry->session.name, m_cursor, entry->session.archiveId, true)) m_processPollAge.start(); + } else if(m_client.requestInspection(entry->host, entry->session.name, m_cursor, entry->session.archiveId,processes)) { m_inspectionAge.start();if(processes)m_processPollAge.start(); } refreshAccountUsage(); diff --git a/tray/src/SessionsWindow.h b/tray/src/SessionsWindow.h index 9cdbee1bd..30a11f5b1 100644 --- a/tray/src/SessionsWindow.h +++ b/tray/src/SessionsWindow.h @@ -461,6 +461,8 @@ class SessionsWindow : public QWidget { QHash m_remotePollAges; BusyIndicator *m_sessionLoading = nullptr; QElapsedTimer m_processPollAge,m_inspectionAge; + // Runs whose event-stream subscription ended; they are polled (EventStream.h). + QSet m_streamEnded; SessionList *m_sessions; SessionSearch *m_searchResults; QStackedWidget *m_listStack; diff --git a/tray/src/SettingsJson.cpp b/tray/src/SettingsJson.cpp new file mode 100644 index 000000000..5f1133d0a --- /dev/null +++ b/tray/src/SettingsJson.cpp @@ -0,0 +1,61 @@ +#include "SettingsJson.h" + +#include "JsonParse.h" +#include "ProcessRunner.h" + +#include +#include +#include +#include + +namespace SettingsJson { +namespace { + +struct Entry { + QByteArray bytes; + QJsonDocument document; +}; + +QMutex s_mutex; +QHash s_entries; + +QJsonDocument read(const QString &key, qsizetype maximumBytes = 0) +{ + const QByteArray bytes = QSettings().value(key).toByteArray(); + if (maximumBytes > 0 && bytes.size() > maximumBytes) return {}; + { + QMutexLocker lock(&s_mutex); + const auto found = s_entries.constFind(key); + if (found != s_entries.cend() && found->bytes == bytes) return found->document; + } + const QJsonDocument document = JsonParse::local(bytes); + QMutexLocker lock(&s_mutex); + s_entries.insert(key, {bytes, document}); + return document; +} + +} + +QJsonObject object(const QString &key, qsizetype maximumBytes) { return read(key, maximumBytes).object(); } +QJsonArray array(const QString &key) { return read(key).array(); } + +void remember(const QString &key, const QByteArray &bytes, const QJsonDocument &document) +{ + QMutexLocker lock(&s_mutex); + s_entries.insert(key, {bytes, document}); +} + +void warm(QObject *owner, const QStringList &keys) +{ + ProcessRunner::post(owner, [keys] { + for (const auto &key : keys) { + const QByteArray bytes = QSettings().value(key).toByteArray(); + const QJsonDocument document = JsonParse::parse(bytes); + QMutexLocker lock(&s_mutex); + s_entries.insert(key, {bytes, document}); + } + return true; + }, [](bool) {}); +} + +} diff --git a/tray/src/SettingsJson.h b/tray/src/SettingsJson.h new file mode 100644 index 000000000..1eb711bd7 --- /dev/null +++ b/tray/src/SettingsJson.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include +#include +#include + +class QObject; + +// JSON values the desktop keeps in QSettings, parsed once per change: a read +// whose stored bytes equal the previous read's returns the parsed value again +// without parsing. warm() parses keys on the preparation thread +// (ProcessRunner.h) ahead of their first read, so the GUI thread normally +// parses only values it has just changed. A parse that is still needed on the +// GUI thread goes through JsonParse::local() and its size limit. +namespace SettingsJson { + +// A stored value over maximumBytes (0: any size) reads as empty. +QJsonObject object(const QString &key, qsizetype maximumBytes = 0); +QJsonArray array(const QString &key); +void warm(QObject *owner, const QStringList &keys); +// A writer that has just stored bytes, serialized from document, under key +// records them, so the next read needs no parse. +void remember(const QString &key, const QByteArray &bytes, const QJsonDocument &document); + +} diff --git a/tray/src/SwarmController.cpp b/tray/src/SwarmController.cpp index 45e85e85e..167cc4781 100644 --- a/tray/src/SwarmController.cpp +++ b/tray/src/SwarmController.cpp @@ -1,3 +1,4 @@ +#include "SettingsJson.h" #include "SwarmController.h" #include #include @@ -77,8 +78,8 @@ void SwarmController::start(const QJsonObject &organization) if(m_started)return;m_started=true;m_desired=organization; QSettings settings; if(!settings.contains("workspace/organizationBeforeSwarm"))settings.setValue("workspace/organizationBeforeSwarm",QJsonDocument(organization).toJson(QJsonDocument::Compact)); - m_base=QJsonDocument::fromJson(settings.value("swarm/observedOrganization").toByteArray()).object(); - m_versions=QJsonDocument::fromJson(settings.value("swarm/observedVersions").toByteArray()).object(); + m_base=SettingsJson::object("swarm/observedOrganization"); + m_versions=SettingsJson::object("swarm/observedVersions"); m_ready=!m_base.isEmpty(); if(!m_ready)m_base=organization; m_poll.start(); @@ -119,6 +120,8 @@ void SwarmController::persist() QSettings settings; settings.setValue("swarm/observedOrganization", organization); settings.setValue("swarm/observedVersions", versions); + SettingsJson::remember("swarm/observedOrganization", organization, QJsonDocument(m_base)); + SettingsJson::remember("swarm/observedVersions", versions, QJsonDocument(m_versions)); settings.sync(); if (settings.status() == QSettings::NoError) { m_persistedOrganization = organization; diff --git a/tray/src/TaskDrag.h b/tray/src/TaskDrag.h index b0ca945be..4ce9873fe 100644 --- a/tray/src/TaskDrag.h +++ b/tray/src/TaskDrag.h @@ -1,4 +1,5 @@ #pragma once +#include "JsonParse.h" #include "TaskBoard.h" #include #include @@ -32,7 +33,7 @@ struct TaskDrag { } static std::optional from(const QMimeData *mime) { if (!mime || !mime->hasFormat(mimeType())) return std::nullopt; - const auto data = QJsonDocument::fromJson(mime->data(mimeType())).object(); + const auto data = JsonParse::localObject(mime->data(mimeType())); TaskDrag drag{data.value("project").toString(), data.value("uuid").toString(), data.value("id").toString(), data.value("stage").toString(), data.value("host").toString(), data.value("session").toString(), data.value("known").toBool(true)}; diff --git a/tray/src/TaskInspector.cpp b/tray/src/TaskInspector.cpp index 4d5b33fd4..6a68b2953 100644 --- a/tray/src/TaskInspector.cpp +++ b/tray/src/TaskInspector.cpp @@ -1,3 +1,4 @@ +#include "SettingsJson.h" #include "TaskInspector.h" #include #include "ActivityWidth.h" @@ -1026,7 +1027,7 @@ QString TaskInspector::inputKey(const QJsonObject &task) const { } void TaskInspector::restoreNewCard() { QSettings settings; const auto key = "tasks/newCard/" + m_project; - m_creation = QJsonDocument::fromJson(settings.value(key + "/request").toByteArray()).object(); + m_creation = SettingsJson::object(key + "/request"); m_newTitle->setText(m_creation.isEmpty() ? settings.value(key + "/title").toString() : m_creation.value("title").toString()); m_newTitle->setReadOnly(!m_creation.isEmpty()); m_creationHeader->setCurrentWidget(m_newTitle->text().isEmpty() ? static_cast(m_scope) : m_newTitle); m_add->setEnabled(!m_project.isEmpty()); m_createError->hide(); diff --git a/tray/src/TaskMessageOutbox.h b/tray/src/TaskMessageOutbox.h index 50e18ffac..073138501 100644 --- a/tray/src/TaskMessageOutbox.h +++ b/tray/src/TaskMessageOutbox.h @@ -1,4 +1,5 @@ #pragma once +#include "JsonParse.h" #include "ComposerDraftStore.h" #include #include @@ -23,7 +24,7 @@ inline Request read(const QString &file) { const QFileInfo info(file); QFile input(file); if (info.isSymLink() || !info.isFile() || info.size() > 32 * 1024 * 1024 || !input.open(QIODevice::ReadOnly)) return {{}, QObject::tr("The original message recovery snapshot is unavailable. This check sent nothing.")}; - const auto record = QJsonDocument::fromJson(input.readAll()).object(); + const auto record = JsonParse::parse(input.readAll()).object(); if (QUuid(record.value("id").toString()).isNull() || !record.value("intent").isObject()) return {{}, QObject::tr("The message recovery snapshot is invalid. Nothing was submitted.")}; return {record, {}}; diff --git a/tray/src/TrayAgent.cpp b/tray/src/TrayAgent.cpp index cb965eb4b..26481a9eb 100644 --- a/tray/src/TrayAgent.cpp +++ b/tray/src/TrayAgent.cpp @@ -1,4 +1,7 @@ #include "TrayAgent.h" +#include "EventStream.h" +#include "ProcessRunner.h" +#include "SettingsJson.h" #include "QuestionNotificationSettings.h" #include "AutomaticApprovals.h" @@ -58,10 +61,17 @@ std::unique_ptr makeTrayIcon() // совпал с открытием меню) -- дублирующий вызов requestPeer() для уже летящего запроса // не создаёт второй процесс: HgsClient сам игнорирует его (m_peerInFlight), эта функция // вызывающего кода не гейтит. +// A computer with a live event stream (EventStream.h) needs no periodic poll. void pollAllKnownPeers(HgsClient &client, const FleetState &state) { + const auto *streams = EventStreams::instance(); for (const QString &alias : state.peerNames()) - client.requestPeer(alias); + if (!streams || !streams->live(alias)) client.requestPeer(alias); +} +void pollLocal(HgsClient &client) +{ + const auto *streams = EventStreams::instance(); + if (!streams || !streams->live({})) client.requestLocal(); } // Терминал выбирается один раз на запуск. Явный шаблон в конфиге побеждает выбор @@ -144,7 +154,7 @@ TrayAgent::TrayAgent(const AppConfig &cfg, QObject *parent) , m_notifier(makeAttentionNotifier()) #endif , m_badge(makeApplicationBadge()) - , m_attention(QJsonDocument::fromJson(QSettings().value("attention/observed").toByteArray()).object()) + , m_attention(SettingsJson::object("attention/observed")) , m_localTimer(this) , m_peerTimer(this) { @@ -204,7 +214,7 @@ TrayAgent::TrayAgent(const AppConfig &cfg, QObject *parent) connect(&m_menu, &TrayMenu::openModeChosen, this, &TrayAgent::setOpenMode); connect(&m_menu, &TrayMenu::sessionsRequested, this, &TrayAgent::showSessions); connect(&m_menu, &TrayMenu::refreshRequested, this, [this]() { - m_client.requestLocal(); + pollLocal(m_client); pollAllKnownPeers(m_client, m_state); }); connect(&m_menu, &TrayMenu::quitRequested, qApp, &QCoreApplication::quit); @@ -243,9 +253,18 @@ TrayAgent::TrayAgent(const AppConfig &cfg, QObject *parent) // Локальный тик: опрашивает только этот бокс, никакого ssh, миллисекунды. Стартует // сразу же, не дожидаясь ни peerPollMs, ни обнаружения пиров ниже -- та часть, что // отвечает "тик и первая отрисовка не должны ждать пиров" из задания. + // One long-lived event stream per computer replaces its periodic polls + // while live; polling continues for a computer without one. + if (EventStreams::enabled()) { + m_streams = new EventStreams(m_client.executable(), this); + connect(m_streams, &EventStreams::listingReady, this, [this](const QString &host, const BoxState &box) { + if (host.isEmpty()) onLocalReady(box); else onPeerReady(box); + }); + m_streams->setHosts({QString()}); + } m_localTimer.setInterval(cfg.localPollMs); connect(&m_localTimer, &QTimer::timeout, this, [this] { - m_client.requestLocal(); + pollLocal(m_client); // Short-lived remote permission prompts must not wait for the ordinary // minute-long peer cadence. The client coalesces reads already in flight. const int interval = AutomaticApprovalPolicy::enabled() ? qMin(m_cfg.peerPollMs, 5000) : m_cfg.peerPollMs; @@ -284,6 +303,13 @@ TrayAgent::TrayAgent(const AppConfig &cfg, QObject *parent) pollAllKnownPeers(m_client, m_state); }); + // Parse the large saved values on the preparation thread before their first use. + SettingsJson::warm(this, {QStringLiteral("attention/readReplies"), QStringLiteral("attention/sessionMarks"), + QStringLiteral("attention/pendingReads"), QStringLiteral("attention/sharedReadMigrated"), + QStringLiteral("workspace/organization"), QStringLiteral("workspace/pendingLaunches"), + QStringLiteral("workspace/launchAccounts"), QStringLiteral("swarm/observedOrganization"), + QStringLiteral("swarm/observedVersions"), QStringLiteral("workspace/worktreeSnapshots")}); + auto *discover = new QProcess(this); discover->setProgram(cfg.hgsPath.isEmpty() ? QStringLiteral("hgs") : cfg.hgsPath); discover->setArguments({QStringLiteral("ls"), QStringLiteral("--json")}); @@ -328,10 +354,13 @@ TrayAgent::TrayAgent(const AppConfig &cfg, QObject *parent) *done = true; if (status == QProcess::NormalExit && exitCode == 0) { - QString err; - const QList fleet = - HgsClient::parseFleet(discover->readAllStandardOutput(), &err); - if (err.isEmpty()) { + // The whole fleet's listing is parsed off the GUI thread (ProcessRunner.h). + struct Fleet { QList boxes; QString error; }; + ProcessRunner::post(this, [bytes = discover->readAllStandardOutput()] { + Fleet parsed; parsed.boxes = HgsClient::parseFleet(bytes, &parsed.error); return parsed; + }, [this](const Fleet &parsed) { + const QList &fleet = parsed.boxes; + if (parsed.error.isEmpty()) { // The first box is this computer (onLocalReady brings its state) and // its "peers" are the aliases to poll; requestPeer() below brings their // sessions. A reachable peer's own entry carries the name it reports @@ -347,6 +376,7 @@ TrayAgent::TrayAgent(const AppConfig &cfg, QObject *parent) m_peerTimer.start(); pollAllKnownPeers(m_client, m_state); } + }); } discover->deleteLater(); }); @@ -369,8 +399,8 @@ TrayAgent::~TrayAgent() void TrayAgent::onLocalReady(const BoxState &box) { m_localReadError = box.ok ? QString() : (box.error.isEmpty() ? tr("Could not read local sessions") : box.error); - m_state.setReadReplies(QJsonDocument::fromJson(QSettings().value("attention/readReplies").toByteArray()).object()); - m_state.setAttentionMarks(QJsonDocument::fromJson(QSettings().value("attention/sessionMarks").toByteArray()).object()); + m_state.setReadReplies(SettingsJson::object("attention/readReplies")); + m_state.setAttentionMarks(SettingsJson::object("attention/sessionMarks")); m_state.setPendingReads(ReadMarkStore::load()); m_lastError.clear(); const auto previousPeers = m_state.peerNames(); @@ -382,6 +412,7 @@ void TrayAgent::onLocalReady(const BoxState &box) } for (const auto &peer : m_state.peerNames()) if (!previousPeers.contains(peer)) m_client.requestPeer(peer); + if (m_streams && previousPeers != m_state.peerNames()) m_streams->setHosts(QStringList{QString()} + m_state.peerNames()); if (box.peersKnown && !m_peerTimer.isActive()) m_peerTimer.start(); // A control-only computer reports no sessions; its answer only carries the // machine list and metrics. Every poll is a process start on Windows, so it @@ -404,8 +435,8 @@ void TrayAgent::onPeerReady(const BoxState &box) { // box.host is the alias this answer was polled under (see HgsClient::requestPeer). if (m_sessionsWindow) m_sessionsWindow->setPollingHosts(m_client.pollingHosts()); - m_state.setReadReplies(QJsonDocument::fromJson(QSettings().value("attention/readReplies").toByteArray()).object()); - m_state.setAttentionMarks(QJsonDocument::fromJson(QSettings().value("attention/sessionMarks").toByteArray()).object()); + m_state.setReadReplies(SettingsJson::object("attention/readReplies")); + m_state.setAttentionMarks(SettingsJson::object("attention/sessionMarks")); m_state.setPendingReads(ReadMarkStore::load()); if (m_state.local().peersKnown && !m_state.local().peers.contains(box.host)) return; m_lastError.clear(); @@ -689,7 +720,7 @@ void TrayAgent::showSessions() else m_client.requestPeer(host); }); connect(m_sessionsWindow, &SessionsWindow::refreshRequested, this, [this]() { - m_client.requestLocal(); pollAllKnownPeers(m_client, m_state); + pollLocal(m_client); pollAllKnownPeers(m_client, m_state); }); } m_sessionsWindow->setFleet(m_state); diff --git a/tray/src/TrayAgent.h b/tray/src/TrayAgent.h index 97127b78a..efb50eb33 100644 --- a/tray/src/TrayAgent.h +++ b/tray/src/TrayAgent.h @@ -113,6 +113,7 @@ private slots: SessionsWindow *m_sessionsWindow = nullptr; // Records desktop activity for phone alerts; see DesktopPresence.h. std::unique_ptr m_presence; + class EventStreams *m_streams = nullptr; QString m_presenceError; #ifdef Q_OS_WIN // The tray runs `hgs swarm worker` itself on Windows (SwarmWorkerSupervisor.h). diff --git a/tray/src/UpdateController.cpp b/tray/src/UpdateController.cpp index 9b4112d91..098ae367b 100644 --- a/tray/src/UpdateController.cpp +++ b/tray/src/UpdateController.cpp @@ -1,4 +1,6 @@ #include "UpdateController.h" +#include "JsonParse.h" +#include "ProcessRunner.h" #include "ChildProcess.h" #include #include @@ -85,8 +87,8 @@ QHash bundleValues(const QString &path){ DesktopUpdateFeed parseDev(const QByteArray &data){ DesktopUpdateFeed result; if(data.size()>256*1024)return result; - QJsonParseError error;auto doc=QJsonDocument::fromJson(data,&error); - if(error.error!=QJsonParseError::NoError||!doc.isObject())return result; + QString error;auto doc=JsonParse::local(data,&error); + if(!error.isEmpty()||!doc.isObject())return result; auto raw=doc.object(); if(!integer(raw["schemaVersion"],2,2)||raw["product"]!="zerus"||raw["platform"]!="desktop"||raw["channel"]!="dev"||raw["channelTag"]!="dev"||raw.contains("sourceDirty")||!integer(raw["buildId"],1,9007199254740991LL)||!integer(raw["buildNumber"],1,9007199254740991LL))return result; auto id=raw["buildId"].toInteger();auto commit=raw["sourceCommit"].toString();auto version=raw["versionName"].toString(); @@ -172,7 +174,7 @@ DesktopUpdateFeed UpdateController::parseFeed(const QByteArray &data,const QStri if(channel=="dev")return parseDev(data); DesktopUpdateFeed result; if(data.size()>256*1024||!(channel=="stable"||channel=="nightly"))return result; - QJsonParseError error;auto doc=QJsonDocument::fromJson(data,&error);if(error.error!=QJsonParseError::NoError||!doc.isObject())return result; + QString error;auto doc=JsonParse::local(data,&error);if(!error.isEmpty()||!doc.isObject())return result; auto raw=doc.object(); if(!integer(raw["schemaVersion"],1,1)||raw["product"]!="zerus"||raw["platform"]!="desktop"||raw["channel"]!=channel||raw.contains("sourceDirty")||!integer(raw["releaseId"],1,9007199254740991LL))return result; QString version=raw["versionName"].toString(),release=raw["releaseUrl"].toString(); @@ -308,8 +310,8 @@ QString UpdateController::wingetPackage(const QString &executable){ } bool UpdateController::installerRecord(const QByteArray &data){ if(data.isEmpty()||data.size()>installerRecordLimit)return false; - QJsonParseError error;const auto document=QJsonDocument::fromJson(data,&error); - if(error.error!=QJsonParseError::NoError||!document.isObject())return false; + QString error;const auto document=JsonParse::local(data,&error); + if(!error.isEmpty()||!document.isObject())return false; const auto record=document.object(); return integer(record["schema"],1,1)&&record["kind"]=="installer"&&record["product"]=="Zerus"; } @@ -591,12 +593,17 @@ void UpdateController::startCheck(bool automatic){ auto *manager=m_fetch?nullptr:new QNetworkAccessManager(this);QNetworkRequest request(QUrl("https://zerus.dev/updates/"+QString(selected=="dev"?"v2":"v1")+"/desktop/"+selected+".json"));request.setAttribute(QNetworkRequest::RedirectPolicyAttribute,QNetworkRequest::ManualRedirectPolicy);request.setTransferTimeout(10000); auto *reply=m_fetch?m_fetch(request):manager->get(request);m_activeReply=reply;auto data=std::make_shared();auto oversized=std::make_shared(false);auto *timer=new QTimer(reply);timer->setSingleShot(true);connect(timer,&QTimer::timeout,reply,&QNetworkReply::abort);timer->start(10000); connect(reply,&QNetworkReply::readyRead,this,[reply,data,oversized]{auto bytes=reply->readAll();if(data->size()+bytes.size()>256*1024){*oversized=true;reply->abort();}else *data+=bytes;}); - connect(reply,&QNetworkReply::finished,this,[this,reply,manager,timer,data,oversized,generation,selected]{timer->stop();if(generation==m_checkGeneration){ - m_busy=false;m_activeReply=nullptr;bool mayNotify=!m_automaticInFlight||automaticChecks();m_automaticInFlight=false;DesktopUpdateFeed verified;if(!*oversized&&reply->error()==QNetworkReply::NoError&&reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()==200)verified=parseFeed(*data,selected); + connect(reply,&QNetworkReply::finished,this,[this,reply,manager,timer,data,oversized,generation,selected]{timer->stop(); + // The downloaded catalog is parsed off the GUI thread (ProcessRunner.h). + const bool fetched=generation==m_checkGeneration&&!*oversized&&reply->error()==QNetworkReply::NoError&&reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()==200; + if(generation==m_checkGeneration)m_activeReply=nullptr; + reply->deleteLater();if(manager)manager->deleteLater(); + ProcessRunner::post(this,[fetched,bytes=*data,selected]{return fetched?parseFeed(bytes,selected):DesktopUpdateFeed{};},[this,data,generation,selected](const DesktopUpdateFeed &verified){if(generation==m_checkGeneration){ + m_busy=false;m_activeReply=nullptr;bool mayNotify=!m_automaticInFlight||automaticChecks();m_automaticInFlight=false; if(verified.valid){m_freshForNotification=mayNotify;m_cached=false;m_available=false;m_feed=verified;m_lastSuccess=QDateTime::currentSecsSinceEpoch();QSettings settings;settings.setValue("updates/catalog/"+selected,*data);settings.setValue("updates/lastSuccess/"+selected,m_lastSuccess);m_notice="Last successful check: "+QDateTime::fromSecsSinceEpoch(m_lastSuccess).toString(Qt::ISODate)+".";} else {m_error=tr("Check failed. Try again later.");m_cached=m_feed.valid;m_notice=m_feed.valid?"Update check failed. Showing the last verified catalog ("+QDateTime::fromSecsSinceEpoch(m_lastSuccess).toString(Qt::ISODate)+").":"Update check failed or the catalog was invalid. Try again later.";} present(m_feed.valid?comparisonMessage(m_installation,m_feed,QCoreApplication::applicationVersion(),0,false):QString());emit changed();compare(); - }reply->deleteLater();if(manager)manager->deleteLater();}); + }});}); } void UpdateController::completeComparison(bool newer,const QString &message){ m_available=newer;present(message);emit changed(); diff --git a/tray/src/WindowsMobileSettings.h b/tray/src/WindowsMobileSettings.h index c03b63a92..ed25ce21e 100644 --- a/tray/src/WindowsMobileSettings.h +++ b/tray/src/WindowsMobileSettings.h @@ -1,4 +1,6 @@ #pragma once +#include "ProcessRunner.h" +#include "JsonParse.h" #include "ChildProcess.h" #include "RelayConfiguration.h" #include @@ -105,11 +107,14 @@ class Panel : public QWidget { auto *deadline=new QTimer(p);deadline->setSingleShot(true); auto output=std::make_shared(); const auto previous=snapshot; - auto finish=[this,p,deadline,output,action,payload,previous](bool success){ - if(process!=p)return; - deadline->stop();process=nullptr; + auto finishing=std::make_shared(false); + auto finish=[this,p,deadline,output,action,payload,previous,finishing](bool success){ + if(process!=p||*finishing)return;*finishing=true; + deadline->stop(); const auto bytes=*output+p->readAllStandardOutput(); - const auto doc=QJsonDocument::fromJson(bytes);const auto result=doc.object(); + // The reply is parsed off the GUI thread; this request stays current until then. + ProcessRunner::post(this,[bytes]{return JsonParse::parse(bytes);},[this,p,action,payload,previous,success,bytes](const QJsonDocument &doc){ + process=nullptr;const auto result=doc.object(); if(!success||!doc.isObject()||bytes.size()>65536||!confirmed(action,result,payload,previous)){ notice->setText(tr("Connector operation was not confirmed. Reload status before retrying. Check that the portable ZIP includes zerus-mobile-connector.exe and private configuration is available.")); state->setText(tr("Connector: status unknown")); @@ -127,7 +132,7 @@ class Panel : public QWidget { state->setText(result.value("running").toBool()?tr("Connector: running. Relay connectivity is not verified here."):tr("Connector: stopped")); notice->setText(action=="save"?tr("Configuration saved privately. Start the connector, then refresh Phones below."):QString());controls(false); } - p->deleteLater(); + p->deleteLater();}); }; connect(deadline,&QTimer::timeout,p,[p]{p->kill();}); connect(p,&QProcess::readyReadStandardOutput,p,[p,output]{*output+=p->readAllStandardOutput();if(output->size()>65536)p->kill();}); diff --git a/tray/src/WorkspaceSettings.h b/tray/src/WorkspaceSettings.h index d4f80f5ce..2261253c3 100644 --- a/tray/src/WorkspaceSettings.h +++ b/tray/src/WorkspaceSettings.h @@ -1,4 +1,6 @@ #pragma once +#include "ProcessRunner.h" +#include "JsonParse.h" #include "ChildProcess.h" #include "RelayConfiguration.h" #include @@ -266,9 +268,13 @@ class Panel : public QWidget { for(const auto &key:{"HGS_SESSION","HGS_RUN_ID","HGS_EXPECTED_ID","HGS_AGENT","HGS_HOST_PANE","HGS_HOST_SESSION","TMUX","TMUX_PANE"})environment.remove(QLatin1String(key)); p->setProcessEnvironment(environment);p->setProcessChannelMode(QProcess::SeparateChannels); auto *deadline=new QTimer(p);deadline->setSingleShot(true);auto output=std::make_shared(); - auto finish=[this,p,deadline,output,action,payload,done](bool success){ - if(process!=p)return;deadline->stop();process=nullptr; - const auto bytes=*output+p->readAllStandardOutput();const auto doc=QJsonDocument::fromJson(bytes);const auto result=doc.object(); + auto finishing=std::make_shared(false); + auto finish=[this,p,deadline,output,action,payload,done,finishing](bool success){ + if(process!=p||*finishing)return;*finishing=true;deadline->stop(); + const auto bytes=*output+p->readAllStandardOutput(); + // The reply is parsed off the GUI thread; this request stays current until then. + ProcessRunner::post(this,[bytes]{return JsonParse::parse(bytes);},[this,p,action,payload,done,success,bytes](const QJsonDocument &doc){ + process=nullptr;const auto result=doc.object(); if(!success||!doc.isObject()||bytes.size()>65536||!confirmed(action,result,payload)){ enrollmentPoll->stop(); continueEnrollment=false; @@ -290,7 +296,7 @@ class Panel : public QWidget { } else if(done)done(result); } - p->deleteLater(); + p->deleteLater();}); }; connect(deadline,&QTimer::timeout,p,[p]{p->kill();}); connect(p,&QProcess::readyReadStandardOutput,p,[p,output]{*output+=p->readAllStandardOutput();if(output->size()>65536)p->kill();}); diff --git a/tray/src/WorktreeSettings.h b/tray/src/WorktreeSettings.h index 9de5b46a1..dc458e819 100644 --- a/tray/src/WorktreeSettings.h +++ b/tray/src/WorktreeSettings.h @@ -1,4 +1,5 @@ #pragma once +#include "JsonParse.h" #include "ProcessRunner.h" #include #include @@ -67,9 +68,10 @@ class WorktreeSettings : public QWidget { ProcessRunner::run(command({"--help"}),this,[this,action,policy](const ProcessRunner::Result &probe){ if(!success(probe)||!probe.standardOutput.contains("worktree-maintenance-v1")){fail(tr("This computer is unavailable or needs an updated hgs CLI for cleanup settings."));return;} auto r=command({"worktrees","policy",action,"--json"});if(action=="set")r.input=QJsonDocument(policy).toJson(QJsonDocument::Compact); - ProcessRunner::run(std::move(r),this,[this,action](const ProcessRunner::Result &result){ - if(!success(result)){fail(action=="set"?tr("Save was not confirmed. Refresh before changing settings again."):tr("Could not read cleanup settings."));return;} - const auto object=QJsonDocument::fromJson(result.standardOutput).object();const auto p=object.value("policy").toObject(); + ProcessRunner::run(std::move(r),this,[](ProcessRunner::Result result){return std::make_pair(success(result),success(result)?JsonParse::object(result.standardOutput):QJsonObject());}, + [this,action](const std::pair &reply){ + if(!reply.first){fail(action=="set"?tr("Save was not confirmed. Refresh before changing settings again."):tr("Could not read cleanup settings."));return;} + const auto &object=reply.second;const auto p=object.value("policy").toObject(); bool valid=object.value("schema").toInt()==1&&p.value("version").toInt()==1&&p.value("revision").isDouble()&&p.value("grace_hours").toInt()>=1&&p.value("grace_hours").toInt()<=8760; for(const auto *key:{"enabled","after_session","unused_integrated","missing"})valid=valid&&p.value(QLatin1String(key)).isBool(); if(!valid){fail(tr("Computer returned invalid cleanup settings."));return;} diff --git a/tray/src/WorktreesDialog.cpp b/tray/src/WorktreesDialog.cpp index 09b4585ad..750994e56 100644 --- a/tray/src/WorktreesDialog.cpp +++ b/tray/src/WorktreesDialog.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "WorktreesDialog.h" #include "DialogButtons.h" #include "WorktreeBatchCleanupDialog.h" @@ -599,7 +600,7 @@ void WorktreesDialog::invalidateReview(const QString &repository) { m_reviews.remove(repository); m_reviewErrors.remove(repository); m_reviewAttempted.remove(repository); for (auto it = m_reviewAttempted.begin(); it != m_reviewAttempted.end();) { - const auto parts = QJsonDocument::fromJson(it->toUtf8()).array(); + const auto parts = JsonParse::localArray(it->toUtf8()); if (!parts.isEmpty() && parts.first().toString() == repository) it = m_reviewAttempted.erase(it); else ++it; } } diff --git a/tray/src/speech/SpeechClient.cpp b/tray/src/speech/SpeechClient.cpp index 36366e0c7..523019c89 100644 --- a/tray/src/speech/SpeechClient.cpp +++ b/tray/src/speech/SpeechClient.cpp @@ -1,3 +1,4 @@ +#include "JsonParse.h" #include "SpeechClient.h" #include "ProcessRunner.h" #include @@ -82,14 +83,19 @@ void SpeechClient::dispatch(const QString &host, const QJsonObject &payload, Rep auto request = command(m_executable, host, session.isEmpty() ? QStringList{"speech", "--json"} : QStringList{"voice", session, "--json"}); auto start = [this, request, done, redact=management(payload)](QByteArray input) mutable { auto command=request;command.input=std::move(input); - ProcessRunner::run(std::move(command), this, [done, redact](const ProcessRunner::Result &result) { - const auto error = failure(result, redact); - if (!error.isEmpty()) { done({}, error); return; } - QJsonParseError parse; - const auto json = QJsonDocument::fromJson(result.standardOutput, &parse); - if (result.standardOutput.size() > 1024 * 1024 || parse.error != QJsonParseError::NoError || !json.isObject() - || json.object().value("schema").toInt() != 1) { done({}, tr("The speech computer returned an invalid response.")); return; } - auto reply=json.object();reply.remove("client_dispatch_started");done(reply, {}); + struct Prepared { QString error; bool valid = false; QJsonObject reply; }; + ProcessRunner::run(std::move(command), this, [redact](ProcessRunner::Result result) { + Prepared prepared; prepared.error = failure(result, redact); + if (!prepared.error.isEmpty() || result.standardOutput.size() > 1024 * 1024) return prepared; + QString parse; + const auto json = JsonParse::parse(result.standardOutput, &parse); + prepared.valid = parse.isEmpty() && json.isObject() && json.object().value("schema").toInt() == 1; + if (prepared.valid) { prepared.reply = json.object(); prepared.reply.remove("client_dispatch_started"); } + return prepared; + }, [done](const Prepared &prepared) { + if (!prepared.error.isEmpty()) { done({}, prepared.error); return; } + if (!prepared.valid) { done({}, tr("The speech computer returned an invalid response.")); return; } + done(prepared.reply, {}); }); }; if(!payload.contains("video")){start(QJsonDocument(payload).toJson(QJsonDocument::Compact));return;} diff --git a/tray/src/speech/SpeechSettings.cpp b/tray/src/speech/SpeechSettings.cpp index a57b4f379..1652a3d33 100644 --- a/tray/src/speech/SpeechSettings.cpp +++ b/tray/src/speech/SpeechSettings.cpp @@ -1,3 +1,4 @@ +#include "SettingsJson.h" #include "SpeechSettings.h" #include "SpeechClient.h" #include @@ -214,7 +215,7 @@ SpeechSettings::SpeechSettings(const QString &executable, QWidget *parent, std:: m_microphone = new QComboBox; m_microphone->setObjectName("speechMicrophone"); layout->addWidget(m_microphone); layout->addWidget(label(tr("Provider settings and credentials stay on their processing computer. Recordings are limited to two minutes. Original audio and recognized text remain in conversation history; your typed draft stays separate."))); m_statistics = label({}); m_statistics->setObjectName("speechStatistics"); layout->addWidget(m_statistics); layout->addStretch(); - m_pending = QJsonDocument::fromJson(QSettings().value(pendingKey).toByteArray()).object(); + m_pending = SettingsJson::object(pendingKey); if (!validMachine(m_pending.value("machine_id").toString()) || QUuid(m_pending.value("request_id").toString()).isNull()) m_pending = {}; setPeers({}); microphones(); controls(); connect(m_host, &QComboBox::currentIndexChanged, this, [this] { QSettings().setValue("speech/manage_host", m_host->currentData()); refresh(); }); diff --git a/tray/tests/CMakeLists.txt b/tray/tests/CMakeLists.txt index 004d3e8be..bc8fb045e 100644 --- a/tray/tests/CMakeLists.txt +++ b/tray/tests/CMakeLists.txt @@ -124,15 +124,24 @@ function(zerus_check_tests) endfunction() # Windows test programs find the Qt DLLs through PATH. The offscreen platform # reads fonts from QT_QPA_FONTDIR there; without it text has no glyphs or size. +# Every suite fails on JSON parsed on the GUI thread (JsonParse.h), whatever +# ENVIRONMENT it sets itself. function(zerus_test_environment) - if(NOT WIN32) + if(CMAKE_VERSION VERSION_LESS 3.22) + if(WIN32) + message(FATAL_ERROR "Windows tests need CMake 3.22 or later (ENVIRONMENT_MODIFICATION)") + endif() + message(WARNING "CMake 3.22 or later runs the Qt suites with ZERUS_GUI_THREAD_CHECKS=fatal; scripts/ci/gui.sh sets it too") return() endif() - if(CMAKE_VERSION VERSION_LESS 3.22) - message(FATAL_ERROR "Windows tests need CMake 3.22 or later (ENVIRONMENT_MODIFICATION)") + get_property(tests DIRECTORY PROPERTY TESTS) + foreach(test IN LISTS tests) + set_property(TEST ${test} APPEND PROPERTY ENVIRONMENT_MODIFICATION "ZERUS_GUI_THREAD_CHECKS=set:fatal") + endforeach() + if(NOT WIN32) + return() endif() file(TO_CMAKE_PATH "$ENV{SystemRoot}/Fonts" fonts) - get_property(tests DIRECTORY PROPERTY TESTS) foreach(test IN LISTS tests) set_property(TEST ${test} APPEND PROPERTY ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$" "QT_QPA_FONTDIR=set:${fonts}") @@ -245,6 +254,16 @@ target_include_directories(test_readmarksync PRIVATE ../src) target_link_libraries(test_readmarksync PRIVATE Qt6::Test Qt6::Core hgs-tray-core) zerus_add_test(readmarksync test_readmarksync) +add_executable(test_eventstream test_eventstream.cpp) +target_include_directories(test_eventstream PRIVATE ../src) +target_link_libraries(test_eventstream PRIVATE Qt6::Test Qt6::Gui hgs-tray-core) +zerus_add_test(eventstream test_eventstream) + +add_executable(test_jsonparse test_jsonparse.cpp) +target_include_directories(test_jsonparse PRIVATE ../src) +target_link_libraries(test_jsonparse PRIVATE Qt6::Test Qt6::Gui hgs-tray-core) +zerus_add_test(jsonparse test_jsonparse) + add_executable(test_processrunner test_processrunner.cpp) target_include_directories(test_processrunner PRIVATE ../src) target_link_libraries(test_processrunner PRIVATE Qt6::Test Qt6::Core hgs-tray-core) @@ -653,19 +672,19 @@ target_link_libraries(test_traymenu PRIVATE hgs-updates) add_executable(test_windowsmobilesettings test_windowsmobilesettings.cpp) target_include_directories(test_windowsmobilesettings PRIVATE ../src) -target_link_libraries(test_windowsmobilesettings PRIVATE Qt6::Test Qt6::Widgets) +target_link_libraries(test_windowsmobilesettings PRIVATE Qt6::Test Qt6::Widgets hgs-tray-core) zerus_add_test(windowsmobilesettings test_windowsmobilesettings) set_tests_properties(windowsmobilesettings PROPERTIES ENVIRONMENT "QT_QPA_PLATFORM=offscreen") add_executable(test_relaysettings test_relaysettings.cpp) target_include_directories(test_relaysettings PRIVATE ../src) -target_link_libraries(test_relaysettings PRIVATE Qt6::Test Qt6::Widgets) +target_link_libraries(test_relaysettings PRIVATE Qt6::Test Qt6::Widgets hgs-tray-core) zerus_add_test(relaysettings test_relaysettings) set_tests_properties(relaysettings PROPERTIES ENVIRONMENT "QT_QPA_PLATFORM=offscreen") add_executable(test_workspacesettings test_workspacesettings.cpp) target_include_directories(test_workspacesettings PRIVATE ../src) -target_link_libraries(test_workspacesettings PRIVATE Qt6::Test Qt6::Widgets) +target_link_libraries(test_workspacesettings PRIVATE Qt6::Test Qt6::Widgets hgs-tray-core) zerus_add_test(workspacesettings test_workspacesettings) set_tests_properties(workspacesettings PROPERTIES ENVIRONMENT "QT_QPA_PLATFORM=offscreen") diff --git a/tray/tests/support/OffGuiThread.h b/tray/tests/support/OffGuiThread.h new file mode 100644 index 000000000..905aa1d81 --- /dev/null +++ b/tray/tests/support/OffGuiThread.h @@ -0,0 +1,15 @@ +#pragma once + +#include +#include + +// Runs f on a separate thread and returns its result. GUI suites call parsers +// that production runs on workers through this: JsonParse.h fails a parse on +// the GUI thread (ZERUS_GUI_THREAD_CHECKS=fatal in every suite). +template +auto offGuiThread(F f) +{ + std::invoke_result_t result{}; + std::thread([&] { result = f(); }).join(); + return result; +} diff --git a/tray/tests/test_analytics.cpp b/tray/tests/test_analytics.cpp index c6c92c9d5..1fb3929d8 100644 --- a/tray/tests/test_analytics.cpp +++ b/tray/tests/test_analytics.cpp @@ -6,6 +6,7 @@ // native-preference and service panels in this isolated component suite. #include "SettingsPage.h" #include "FakeCli.h" +#include "OffGuiThread.h" #include #include #include @@ -22,6 +23,14 @@ #include #include #include + +namespace { +// The client parses analytics on a worker; these call the same parsers off the GUI thread. +std::optional offGuiSummary(const QByteArray &bytes, QDate from, QDate through) +{ return offGuiThread([&] { return AnalyticsData::summary(bytes, from, through); }); } +std::optional offGuiSpeech(const QByteArray &bytes, QDate from, QDate through) +{ return offGuiThread([&] { return AnalyticsData::speech(bytes, from, through); }); } +} #include namespace { @@ -140,21 +149,21 @@ void TestAnalytics::parserKeepsOnlyReviewedFieldsAndUnknowns() { auto rows=raw.value("buckets").toArray();auto row=rows[0].toObject();row.insert("path","private-path-sentinel"); auto counts=row.value("counters").toObject();counts.insert("arbitrary_counter_sentinel",123);row.insert("counters",counts);rows[0]=row;raw.insert("buckets",rows); auto coverage=raw.value("coverage").toObject();coverage.insert("gaps",QJsonArray{"private-error-sentinel"});raw.insert("coverage",coverage); - const auto parsed=AnalyticsData::summary(bytes(raw),first,last);QVERIFY(parsed); + const auto parsed=offGuiSummary(bytes(raw),first,last);QVERIFY(parsed); QVERIFY(!bytes(*parsed).contains("sentinel"));QCOMPARE(parsed->value("coverage").toObject().value("gaps").toArray(),QJsonArray{"unknown"}); const auto kept=parsed->value("buckets").toArray()[0].toObject().value("counters").toObject();QCOMPARE(kept.value("input_tokens").toInt(),17);QVERIFY(!kept.contains("output_tokens")); - raw.insert("source_epoch","00000000-0000-0000-0000-000000000000");QVERIFY(AnalyticsData::summary(bytes(raw),first,last)); + raw.insert("source_epoch","00000000-0000-0000-0000-000000000000");QVERIFY(offGuiSummary(bytes(raw),first,last)); } void TestAnalytics::parserRejectsInvalidAndUnboundedSummaries() { - QVERIFY(!AnalyticsData::summary(QByteArray(AnalyticsData::MaximumBytes+1,'x'),first,last)); - QVERIFY(!AnalyticsData::summary(bytes(summary()),first,first.addDays(90))); - for(const auto &key:{"machine_id","source_epoch","generated_at","revision"}){auto invalid=summary();invalid.insert(key,"private");QVERIFY2(!AnalyticsData::summary(bytes(invalid),first,last),key);} - auto duplicate=summary(QJsonArray{bucket(),bucket()});QVERIFY(!AnalyticsData::summary(bytes(duplicate),first,last)); - auto bad=bucket();auto c=bad.value("counters").toObject();c.insert("work_ms",-1);bad.insert("counters",c);QVERIFY(!AnalyticsData::summary(bytes(summary(QJsonArray{bad})),first,last)); - bad=bucket();bad.insert("provider","private-account-name");QVERIFY(!AnalyticsData::summary(bytes(summary(QJsonArray{bad})),first,last)); - auto histogram=bucket().value("histograms").toObject();auto work=histogram.value("work_ms").toObject();work.insert("count",2);histogram.insert("work_ms",work);bad=bucket();bad.insert("histograms",histogram);QVERIFY(!AnalyticsData::summary(bytes(summary(QJsonArray{bad})),first,last)); - QJsonArray excessive;for(int n=0;n<=AnalyticsData::MaximumBuckets;++n)excessive.append(bucket());QVERIFY(!AnalyticsData::summary(bytes(summary(excessive)),first,last)); + QVERIFY(!offGuiSummary(QByteArray(AnalyticsData::MaximumBytes+1,'x'),first,last)); + QVERIFY(!offGuiSummary(bytes(summary()),first,first.addDays(90))); + for(const auto &key:{"machine_id","source_epoch","generated_at","revision"}){auto invalid=summary();invalid.insert(key,"private");QVERIFY2(!offGuiSummary(bytes(invalid),first,last),key);} + auto duplicate=summary(QJsonArray{bucket(),bucket()});QVERIFY(!offGuiSummary(bytes(duplicate),first,last)); + auto bad=bucket();auto c=bad.value("counters").toObject();c.insert("work_ms",-1);bad.insert("counters",c);QVERIFY(!offGuiSummary(bytes(summary(QJsonArray{bad})),first,last)); + bad=bucket();bad.insert("provider","private-account-name");QVERIFY(!offGuiSummary(bytes(summary(QJsonArray{bad})),first,last)); + auto histogram=bucket().value("histograms").toObject();auto work=histogram.value("work_ms").toObject();work.insert("count",2);histogram.insert("work_ms",work);bad=bucket();bad.insert("histograms",histogram);QVERIFY(!offGuiSummary(bytes(summary(QJsonArray{bad})),first,last)); + QJsonArray excessive;for(int n=0;n<=AnalyticsData::MaximumBuckets;++n)excessive.append(bucket());QVERIFY(!offGuiSummary(bytes(summary(excessive)),first,last)); } void TestAnalytics::fleetReplacesSnapshotsAndEpochsInsteadOfSumming() { @@ -198,21 +207,21 @@ void TestAnalytics::invalidMissingAndOverflowingHistogramsStayUnavailable() { bad=validHistogram;bad.insert("counts",QJsonArray{0,0,0,0,0,0});bad.insert("count",0);malformed.append(bad); for(const auto &h:malformed) { auto invalid=valid;invalid.insert("histograms",QJsonObject{{"work_ms",h}}); - QVERIFY(!AnalyticsData::summary(bytes(summary({invalid})),first,last)); + QVERIFY(!offGuiSummary(bytes(summary({invalid})),first,last)); QVERIFY(!AnalyticsData::histogramTotals({valid,invalid,valid}).contains("work_ms")); } auto missing=valid;missing.insert("histograms",QJsonObject{}); - QVERIFY(AnalyticsData::summary(bytes(summary({missing})),first,last)); + QVERIFY(offGuiSummary(bytes(summary({missing})),first,last)); QVERIFY(!AnalyticsData::histogramTotals({valid,missing,valid}).contains("work_ms")); QVERIFY(!AnalyticsData::histogramTotals({missing,valid}).contains("work_ms")); constexpr qint64 maximum=9007199254740991LL; const auto countLimit=workBucket({maximum,0,0,0,0,0},maximum,0); - QVERIFY(AnalyticsData::summary(bytes(summary({countLimit})),first,last)); + QVERIFY(offGuiSummary(bytes(summary({countLimit})),first,last)); // Both same-bin overflow and total-count overflow across different bins fail closed. QVERIFY(!AnalyticsData::histogramTotals({countLimit,valid}).contains("work_ms")); QVERIFY(!AnalyticsData::histogramTotals({countLimit,workBucket({0,1,0,0,0,0},1,101)}).contains("work_ms")); const auto sumLimit=workBucket({0,0,0,0,0,1},1,maximum); - QVERIFY(AnalyticsData::summary(bytes(summary({sumLimit})),first,last)); + QVERIFY(offGuiSummary(bytes(summary({sumLimit})),first,last)); QVERIFY(!AnalyticsData::histogramTotals({sumLimit,valid}).contains("work_ms")); const auto justBelow=workBucket({0,0,0,0,0,1},1,maximum-100); QCOMPARE(AnalyticsData::histogramTotals({justBelow,valid}).value("work_ms").toObject().value("sum").toInteger(),maximum); @@ -220,9 +229,9 @@ void TestAnalytics::invalidMissingAndOverflowingHistogramsStayUnavailable() { void TestAnalytics::voiceCollapsesPrivateProfilesAndPreservesUnits() { auto raw=voice();auto days=raw.value("days").toArray();auto row=days[0].toObject();row.insert("profile_id","second-private-profile");row.insert("queue_ms",1.5);row.insert("error_codes",QJsonObject{{"private-provider-message",2}});days.append(row);raw.insert("days",days); - const auto parsed=AnalyticsData::speech(bytes(raw),first,last);QVERIFY(parsed);QVERIFY(!bytes(*parsed).contains("private"));QCOMPARE(parsed->value("days").toArray().size(),1); + const auto parsed=offGuiSpeech(bytes(raw),first,last);QVERIFY(parsed);QVERIFY(!bytes(*parsed).contains("private"));QCOMPARE(parsed->value("days").toArray().size(),1); const auto result=parsed->value("days").toArray()[0].toObject();QCOMPARE(result.value("submitted").toInt(),4);QCOMPARE(result.value("queue_ms").toDouble(),60001.5);QCOMPARE(result.value("error_codes").toObject().value("unknown").toInt(),2); - days.append(days[0]);raw.insert("days",days);QVERIFY(!AnalyticsData::speech(bytes(raw),first,last)); + days.append(days[0]);raw.insert("days",days);QVERIFY(!offGuiSpeech(bytes(raw),first,last)); } void TestAnalytics::retiredEpochNeverRevivesWithNewTimestamp() { @@ -236,7 +245,7 @@ void TestAnalytics::retiredEpochNeverRevivesWithNewTimestamp() { } void TestAnalytics::diagnosticsExcludeIdentityContentAndDetailedHistory() { - const auto agent=AnalyticsData::summary(bytes(summary()),first,last),v=AnalyticsData::speech(bytes(voice()),first,last);QVERIFY(agent&&v); + const auto agent=offGuiSummary(bytes(summary()),first,last),v=offGuiSpeech(bytes(voice()),first,last);QVERIFY(agent&&v); const auto exported=bytes(AnalyticsData::diagnostic({{"private-host-sentinel",*agent,*v,true,true}},first,last)); QVERIFY(!exported.contains(machine.toUtf8()));QVERIFY(!exported.contains(epoch.toUtf8()));QVERIFY(!exported.contains("private"));QVERIFY(!exported.contains("profile_id"));QVERIFY(!exported.contains("buckets"));QVERIFY(!exported.contains("\"days\":"));QVERIFY(exported.contains("reported_totals")); } diff --git a/tray/tests/test_dashboard.cpp b/tray/tests/test_dashboard.cpp index 3548d13f5..c7ad29785 100644 --- a/tray/tests/test_dashboard.cpp +++ b/tray/tests/test_dashboard.cpp @@ -67,10 +67,9 @@ private slots: for (const auto &value : contract.value("cases").toArray()) { const auto example = value.toObject(); auto raw = example.value("raw").toObject(); raw["name"] = "codex/contract/" + example.value("name").toString(); - QString error; - const auto box = HgsClient::parseBox(QJsonDocument(QJsonObject{{"host", "fixture"}, {"ok", true}, - {"sessions", QJsonArray{raw}}}).toJson(), &error); - QVERIFY2(error.isEmpty(), qPrintable(error)); QCOMPARE(box.sessions.size(), 1); + const auto box = HgsClient::box(QJsonObject{{"host", "fixture"}, {"ok", true}, + {"sessions", QJsonArray{raw}}}); + QCOMPARE(box.sessions.size(), 1); auto session = box.sessions.first(); const bool online = example.value("online").toBool(); // Desktop overlays these device/read marks after parsing CLI rows. session.reviewLater = raw.value("review_later").toBool(); @@ -92,15 +91,14 @@ private slots: const auto contract = QJsonDocument::fromJson(file.readAll()).object(); const auto parse = [](QJsonObject raw, const QString &name) { raw["name"] = "codex/contract/" + name; - QString error; - const auto box = HgsClient::parseBox(QJsonDocument(QJsonObject{{"host", "fixture"}, {"ok", true}, - {"sessions", QJsonArray{raw}}}).toJson(), &error); - return std::pair{box, error}; + const auto box = HgsClient::box(QJsonObject{{"host", "fixture"}, {"ok", true}, + {"sessions", QJsonArray{raw}}}); + return std::pair{box, QString()}; }; for (const auto &value : contract.value("cases").toArray()) { const auto example = value.toObject(); const auto name = example.value("name").toString(); const auto [box, error] = parse(example.value("raw").toObject(), name); - QVERIFY2(error.isEmpty(), qPrintable(error)); QCOMPARE(box.sessions.size(), 1); + QCOMPARE(box.sessions.size(), 1); const int count = SessionPresentation::optionalQuestionCount(box.sessions.first(), example.value("online").toBool()); QVERIFY2(count == example.value("count").toInt(), qPrintable(name)); const QString label = count > 0 ? OptionalQuestionBadge::countText(count) : QString(); diff --git a/tray/tests/test_eventstream.cpp b/tray/tests/test_eventstream.cpp new file mode 100644 index 000000000..29eb2fa01 --- /dev/null +++ b/tray/tests/test_eventstream.cpp @@ -0,0 +1,188 @@ +#include + +#include +#include + +#include "EventStream.h" +#include "FakeCli.h" +#include "JsonParse.h" + +// The desktop's consumer of `hgs watch --json` (EventStream.h), against a fake +// CLI that speaks the stream: listings and Activity arrive as the values polls +// deliver, parsed off the GUI thread, and a computer without the marker polls. +class TestEventStream : public QObject { + Q_OBJECT +private slots: + void init() { QVERIFY(m_dir.isValid()); } + void listingAndActivityArriveFromTheStream(); + void peerListingsKeepTheAlias(); + void computerWithoutTheMarkerIsUnsupported(); + void endedStreamFailsAndRestarts(); + void streamEndingBeforePublicationNeverGoesLive(); + void unchangedListingIsAppliedAgain(); + +private: + QString install(const QString &script); + QTemporaryDir m_dir; +}; + +namespace { +const char *Stream = R"PY(#!/usr/bin/env python3 +import json, sys +args = [a for a in sys.argv[1:] if not a.startswith('@')] +host = next((a[1:] for a in sys.argv[1:] if a.startswith('@')), '') +if args == ['--help']: + print('usage: hgs ...\n Event stream ABI: watch-v1') + sys.exit(0) +if args != ['watch', '--json']: + sys.exit(2) +seq = 0 +def send(t, **fields): + global seq + seq += 1 + print(json.dumps(dict(v=1, seq=seq, t=t, **fields)), flush=True) +row = lambda name, run, activity: {'name': name, 'cmd': 'codex', 'run_id': run, 'conversation_id': 'c-' + run, 'activity': activity} +send('hello', machine_id=None, host='fixture', hgs_version='1.57.0', stream='s', heartbeat_ms=10000, max_frame_bytes=1048576, + topics=['catalog', 'catalog.meta', 'activity'], limits={}) +send('snapshot', topic='catalog', items=[{'key': 'run:r2', 'row': row('codex/zeta', 'r2', 'idle')}, + {'key': 'run:r1', 'row': row('codex/Alpha', 'r1', 'idle')}], page=0, last=True, cursor=1) +send('snapshot', topic='catalog.meta', value={'host': 'fixture', 'ok': True, 'peers': [], 'hgs_version': '1.57.0'}, page=0, last=True, cursor=1) +for text in sys.stdin: + op = json.loads(text) + if op.get('op') == 'subscribe': + key = 'run:' + op['session']['run_id'] + send('events', topic='activity', key=key, events=[{'seq': 41, 'type': 'UserPromptSubmit', 'detail': 'go'}], cursor=41) + send('state', topic='activity', key=key, value={'run_id': op['session']['run_id'], 'conversation_id': 'c-r1', 'activity': 'busy'}, rev=1) + send('delta', topic='catalog', upsert=[{'key': 'run:r1', 'row': row('codex/Alpha', 'r1', 'busy')}], remove=[{'key': 'run:r2', 'reason': 'archived'}], cursor=2) + send('events', topic='activity', key=key, events=[{'seq': 42, 'type': 'Stop', 'detail': 'done'}], cursor=42) +)PY"; +} + +QString TestEventStream::install(const QString &script) +{ + const auto path = m_dir.filePath(QStringLiteral("hgs-%1").arg(QUuid::createUuid().toString(QUuid::Id128))); + if (!FakeCli::install(path, script.toUtf8())) return {}; + return FakeCli::program(path); +} + +void TestEventStream::listingAndActivityArriveFromTheStream() +{ + int guiThreadParses = 0; + JsonParse::setViolationHandler([&guiThreadParses](const char *, int) { ++guiThreadParses; }); + EventStream stream(install(QString::fromUtf8(Stream)), {}); + QSignalSpy listings(&stream, &EventStream::listingReady), activity(&stream, &EventStream::activityReady); + stream.start(); + QTRY_VERIFY(stream.live()); + QTRY_VERIFY(!listings.isEmpty()); + auto box = listings.first().first().value(); + QCOMPARE(box.host, QStringLiteral("fixture")); + QCOMPARE(box.sessions.size(), 2); + // Listing order, as ls --json sorts names. + QCOMPARE(box.sessions.at(0).name, QStringLiteral("codex/Alpha")); + QCOMPARE(box.sessions.at(1).name, QStringLiteral("codex/zeta")); + + stream.follow(QStringLiteral("codex/Alpha"), QStringLiteral("r1"), QStringLiteral("c-r1"), 0); + QTRY_VERIFY(activity.size() >= 1); + QTRY_COMPARE(listings.last().first().value().sessions.size(), 1); + box = listings.last().first().value(); + QCOMPARE(box.sessions.first().activity, QStringLiteral("busy")); + // Increments are inspect-shaped: the summary with the new events and cursor. + const auto received = [&activity] { + qsizetype count = 0; + for (const auto &arguments : activity) count += arguments.at(1).toJsonObject().value("events").toArray().size(); + return count; + }; + QTRY_COMPARE(received(), 2); + QJsonArray events; qint64 cursor = 0; + for (const auto &arguments : activity) { + const auto inspection = arguments.at(1).toJsonObject(); + QCOMPARE(arguments.at(0).toString(), QStringLiteral("codex/Alpha")); + QCOMPARE(inspection.value("run_id").toString(), QStringLiteral("r1")); + for (const auto &event : inspection.value("events").toArray()) events.append(event); + cursor = inspection.value("cursor").toInteger(); + } + QCOMPARE(events.at(0).toObject().value("seq").toInt(), 41); + QCOMPARE(events.at(1).toObject().value("seq").toInt(), 42); + QCOMPARE(cursor, 42); + QCOMPARE(guiThreadParses, 0); + JsonParse::setViolationHandler({}); + stream.stop(); + QCOMPARE(stream.state(), EventStream::State::Stopped); +} + +void TestEventStream::peerListingsKeepTheAlias() +{ + EventStream stream(install(QString::fromUtf8(Stream)), QStringLiteral("laptop")); + QSignalSpy listings(&stream, &EventStream::listingReady); + stream.start(); + QTRY_VERIFY(!listings.isEmpty()); + const auto box = listings.first().first().value(); + // A peer is found, polled and addressed by its saved alias. + QCOMPARE(box.host, QStringLiteral("laptop")); + QCOMPARE(box.reportedHost, QStringLiteral("fixture")); +} + +void TestEventStream::computerWithoutTheMarkerIsUnsupported() +{ + // Fake CLIs are Python scripts, which the Windows fake runs too. + EventStream stream(install(QStringLiteral("#!/usr/bin/env python3\nprint('usage: hgs ...')\n")), {}); + stream.start(); + QTRY_COMPARE(stream.state(), EventStream::State::Unsupported); + QVERIFY(!stream.live()); +} + +void TestEventStream::endedStreamFailsAndRestarts() +{ + const QString once = QStringLiteral(R"PY(#!/usr/bin/env python3 +import json, sys +if sys.argv[1:] == ['--help']: + print('Event stream ABI: watch-v1'); sys.exit(0) +print(json.dumps({'v': 1, 'seq': 1, 't': 'hello', 'host': 'fixture'}), flush=True) +print(json.dumps({'v': 1, 'seq': 2, 't': 'snapshot', 'topic': 'catalog', 'items': [], 'page': 0, 'last': True, 'cursor': 1}), flush=True) +print(json.dumps({'v': 1, 'seq': 3, 't': 'snapshot', 'topic': 'catalog.meta', 'value': {'host': 'fixture', 'ok': True}, 'page': 0, 'last': True, 'cursor': 1}), flush=True) +import time; time.sleep(1) +)PY"); + EventStream stream(install(once), {}); + QSignalSpy states(&stream, &EventStream::stateChanged); + stream.start(); + QTRY_VERIFY(states.contains(QVariantList{QVariant::fromValue(EventStream::State::Live)})); + QTRY_COMPARE(stream.state(), EventStream::State::Failed); + QCOMPARE(EventStream::restartDelayMs(1), 5000); + QCOMPARE(EventStream::restartDelayMs(9), 300000); +} + +void TestEventStream::streamEndingBeforePublicationNeverGoesLive() +{ + // Frames prepared just before the child exits are dropped with it: a failed + // stream never reports Live afterwards, so callers keep polling. + const QString brief = QStringLiteral(R"PY(#!/usr/bin/env python3 +import json, sys +if sys.argv[1:] == ['--help']: + print('Event stream ABI: watch-v1'); sys.exit(0) +print(json.dumps({'v': 1, 'seq': 1, 't': 'snapshot', 'topic': 'catalog', 'items': [], 'page': 0, 'last': True, 'cursor': 1}), flush=True) +print(json.dumps({'v': 1, 'seq': 2, 't': 'snapshot', 'topic': 'catalog.meta', 'value': {'ok': True}, 'page': 0, 'last': True, 'cursor': 1}), flush=True) +)PY"); + EventStream stream(install(brief), {}); + QSignalSpy states(&stream, &EventStream::stateChanged); + stream.start(); + QTRY_COMPARE(stream.state(), EventStream::State::Failed); + QTest::qWait(200); + QCOMPARE(stream.state(), EventStream::State::Failed); + QVERIFY(!states.contains(QVariantList{QVariant::fromValue(EventStream::State::Live)})); +} + +void TestEventStream::unchangedListingIsAppliedAgain() +{ + // Callers date a computer's data by its last listing (automatic approvals + // need a peer listed within 30 s), so a quiet stream re-applies it. + EventStream::setListingTimingForTests(100, 300); + EventStream stream(install(QString::fromUtf8(Stream)), QStringLiteral("laptop")); + QSignalSpy listings(&stream, &EventStream::listingReady); + stream.start(); + QTRY_VERIFY(!listings.isEmpty()); + QTRY_VERIFY_WITH_TIMEOUT(listings.size() >= 3, 3000); + EventStream::setListingTimingForTests(2000, 10000); +} + +QTEST_MAIN(TestEventStream) +#include "test_eventstream.moc" diff --git a/tray/tests/test_externalfile.cpp b/tray/tests/test_externalfile.cpp index f53f1a766..69552a532 100644 --- a/tray/tests/test_externalfile.cpp +++ b/tray/tests/test_externalfile.cpp @@ -3,6 +3,7 @@ #include #include "ExternalFile.h" #include "DownloadCache.h" +#include "OffGuiThread.h" namespace { QString write(const QTemporaryDir &dir, const QString &name, const QByteArray &bytes) @@ -114,7 +115,7 @@ private slots: #if defined(Q_OS_WIN) || defined(Q_OS_MACOS) QVERIFY(!marker(result.path).isEmpty()); #endif - const auto usage = DownloadCache::trim(dir.path(), 1, DownloadCache::OpeningGraceMs + 1001); + const auto usage = offGuiThread([&] { return DownloadCache::trim(dir.path(), 1, DownloadCache::OpeningGraceMs + 1001); }); QVERIFY2(usage.error.isEmpty(), qPrintable(usage.error)); QCOMPARE(usage.files, 0); QVERIFY(!QFileInfo::exists(result.path)); } diff --git a/tray/tests/test_jsonparse.cpp b/tray/tests/test_jsonparse.cpp new file mode 100644 index 000000000..7cd3600ca --- /dev/null +++ b/tray/tests/test_jsonparse.cpp @@ -0,0 +1,96 @@ +#include + +#include +#include + +#include + +#include "FakeCli.h" +#include "HgsClient.h" +#include "JsonParse.h" +#include "ProcessRunner.h" + +// The GUI thread applies values that workers prepared (JsonParse.h). Every +// suite fails on a parse there; this one checks the rule itself and that the +// client's polls parse off it. +class TestJsonParse : public QObject { + Q_OBJECT +private slots: + void init() + { + m_sites.clear(); + JsonParse::setViolationHandler([this](const char *file, int line) { + m_sites << QStringLiteral("%1:%2").arg(QFileInfo(QString::fromUtf8(file)).fileName()).arg(line); + }); + } + void cleanup() { JsonParse::setViolationHandler({}); } + void parsingOnTheGuiThreadIsAViolation(); + void workersParseWithoutViolation(); + void localValuesAreLimited(); + void clientPollsParseOffTheGuiThread(); + +private: + QStringList m_sites; +}; + +void TestJsonParse::parsingOnTheGuiThreadIsAViolation() +{ + QVERIFY(JsonParse::onGuiThread()); + const int before = JsonParse::violations(); + QString error; + const int line = __LINE__ + 1; + const auto object = JsonParse::object(R"({"a":1})", &error); + QVERIFY(error.isEmpty()); QCOMPARE(object.value("a").toInt(), 1); + QCOMPARE(JsonParse::violations(), before + 1); + QCOMPARE(m_sites, QStringList{QStringLiteral("test_jsonparse.cpp:%1").arg(line)}); + JsonParse::array(R"({"a":1})", &error); + QCOMPARE(error, QStringLiteral("expected a JSON array")); +} + +void TestJsonParse::workersParseWithoutViolation() +{ + QJsonObject object; bool guiThread = true; + std::thread([&] { guiThread = JsonParse::onGuiThread(); object = JsonParse::object(R"({"b":2})"); }).join(); + QVERIFY(!guiThread); QCOMPARE(object.value("b").toInt(), 2); + bool applied = false; QObject owner; + ProcessRunner::post(&owner, [] { return JsonParse::object(R"({"c":3})"); }, + [&applied](const QJsonObject &value) { applied = value.value("c").toInt() == 3; }); + QTRY_VERIFY(applied); + QVERIFY(m_sites.isEmpty()); +} + +void TestJsonParse::localValuesAreLimited() +{ + QCOMPARE(JsonParse::localObject(R"({"small":true})").value("small").toBool(), true); + QVERIFY(m_sites.isEmpty()); + const QByteArray large = "[\"" + QByteArray(JsonParse::LocalLimit, 'x') + "\"]"; + QCOMPARE(JsonParse::localArray(large).size(), 1); + QCOMPARE(m_sites.size(), 1); +} + +void TestJsonParse::clientPollsParseOffTheGuiThread() +{ + QTemporaryDir dir; QVERIFY(dir.isValid()); + const auto fixture = dir.filePath("hgs"); + QVERIFY(FakeCli::install(fixture, R"PY(#!/usr/bin/env python3 +import json, sys +args = sys.argv[1:] +if args[:1] == ["ls"]: + print(json.dumps({"host": "fixture", "ok": True, "sessions": [{"name": "codex/demo", "cmd": "codex", "activity": "idle"}]})) +elif args[:1] == ["inspect"]: + print(json.dumps({"cursor": 7, "events": [{"seq": 7, "kind": "Stop"}]})) +else: + sys.exit(2) +)PY")); + HgsClient client(FakeCli::program(fixture)); + QSignalSpy local(&client, &HgsClient::localReady), inspected(&client, &HgsClient::inspectionReady); + client.requestLocal(); + QVERIFY(client.requestInspection({}, QStringLiteral("codex/demo"))); + QTRY_COMPARE(local.size(), 1); QTRY_COMPARE(inspected.size(), 1); + QCOMPARE(local.first().first().value().sessions.first().name, QStringLiteral("codex/demo")); + QCOMPARE(inspected.first().at(2).toJsonObject().value("cursor").toInt(), 7); + QVERIFY2(m_sites.isEmpty(), qPrintable(m_sites.join(", "))); +} + +QTEST_MAIN(TestJsonParse) +#include "test_jsonparse.moc" diff --git a/tray/tests/test_processrunner.cpp b/tray/tests/test_processrunner.cpp index c3aa0aeb0..4150810e8 100644 --- a/tray/tests/test_processrunner.cpp +++ b/tray/tests/test_processrunner.cpp @@ -31,6 +31,9 @@ class TestProcessRunner : public QObject { private slots: void cleanup() { ProcessRunner::setLaunchObserver({}); } void launchesOnWorkerAndReportsOnOwnerThread(); + void preparesOnPreparationThreadAndAppliesOnOwnerThread(); + void preparedAndPlainResultsKeepCompletionOrder(); + void destroyedOwnerDropsPreparedValue(); void capturesOutputErrorAndExitCode(); void capsCombinedOutputWhileDraining(); void outputCapKillsOnlyTheDirectChild(); @@ -185,6 +188,57 @@ void TestProcessRunner::launchesOnWorkerAndReportsOnOwnerThread() QCOMPARE(result.standardOutput, QByteArray("hello")); } +void TestProcessRunner::preparesOnPreparationThreadAndAppliesOnOwnerThread() +{ + QObject owner; std::atomic prepareThread{nullptr}; QThread *applyThread = nullptr; + int applied = 0; QByteArray value; + ProcessRunner::run(child(QStringLiteral("printf hello"), {"out", "hello"}), &owner, + [&prepareThread](ProcessRunner::Result result) { + prepareThread = QThread::currentThread(); + return ProcessRunner::isPreparationThread() ? result.standardOutput.toUpper() : QByteArray(); + }, + [&](QByteArray prepared) { ++applied; applyThread = QThread::currentThread(); value = prepared; }); + QTRY_COMPARE(applied, 1); + QCOMPARE(prepareThread.load(), ProcessRunner::preparationThread()); + QVERIFY(prepareThread.load() != QThread::currentThread()); + QVERIFY(prepareThread.load() != ProcessRunner::workerThread()); + QVERIFY(!ProcessRunner::isPreparationThread()); + QCOMPARE(applyThread, QThread::currentThread()); + QCOMPARE(value, QByteArray("HELLO")); +} + +void TestProcessRunner::preparedAndPlainResultsKeepCompletionOrder() +{ + // The first child finishes first but prepares slowly. The second, plain + // result still reaches its owner after it: one serial thread passes both. + QObject owner; QStringList order; + ProcessRunner::run(child(QStringLiteral("printf first"), {"out", "first"}), &owner, + [](ProcessRunner::Result result) { QThread::msleep(400); return result.standardOutput; }, + [&order](QByteArray prepared) { order << QString::fromLatin1(prepared); }); + ProcessRunner::run(child(QStringLiteral("sleep 0.1; printf second"), {"sleep", "0.1", "out", "second"}), &owner, + [&order](const ProcessRunner::Result &result) { order << QString::fromLatin1(result.standardOutput); }); + QTRY_COMPARE(order.size(), 2); + QCOMPARE(order, QStringList({QStringLiteral("first"), QStringLiteral("second")})); +} + +void TestProcessRunner::destroyedOwnerDropsPreparedValue() +{ + auto owner = std::make_unique(); std::atomic preparing{false}, release{false}, prepared{false}; + int applied = 0; + ProcessRunner::run(child(QStringLiteral("printf done"), {"out", "done"}), owner.get(), + [&](ProcessRunner::Result result) { + preparing = true; + while (!release) QThread::msleep(5); + prepared = true; return result.standardOutput; + }, + [&applied](QByteArray) { ++applied; }); + QTRY_VERIFY(preparing.load()); + owner.reset(); release = true; + QTRY_VERIFY(prepared.load()); + QTest::qWait(200); + QCOMPARE(applied, 0); +} + void TestProcessRunner::capturesOutputErrorAndExitCode() { QObject owner; int results = 0; ProcessRunner::Result result; diff --git a/tray/tests/test_sessionswindow.cpp b/tray/tests/test_sessionswindow.cpp index 9b43f23e3..2b877dd8b 100644 --- a/tray/tests/test_sessionswindow.cpp +++ b/tray/tests/test_sessionswindow.cpp @@ -108,6 +108,7 @@ #include "LaunchWarnings.h" #include "LocalSessionHost.h" #include "ProcessRunner.h" +#include "SettingsJson.h" #include "SessionCreationController.h" #include "ProcessSettings.h" #include "QuestionNotificationSettings.h" @@ -1894,6 +1895,11 @@ void TestSessionsWindow::rebuildKeepsProjectOrderAtScale() {"sessions", storedOrder ? QJsonArray::fromStringList(ordered) : QJsonArray{}}}}}}); organization.markImported(box.host); QSettings().setValue("workspace/organization", QJsonDocument(organization.toJson()).toJson(QJsonDocument::Compact)); + // As at desktop startup (TrayAgent), the large saved layout is parsed off the GUI thread first. + QObject warming; bool warmed = false; + SettingsJson::warm(&warming, {QStringLiteral("workspace/organization")}); + ProcessRunner::post(&warming, [] { return true; }, [&warmed](bool) { warmed = true; }); + QTRY_VERIFY(warmed); FleetState state; state.setLocal(box, QDateTime::currentMSecsSinceEpoch()); SessionsWindow window(program()); keepDetailsPollsOut(window); window.show(); QElapsedTimer elapsed; elapsed.start(); window.setFleet(state);