From 066b6424ae3de154cd3f67a081e4b246f65602b3 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 17 Sep 2026 06:36:01 -0600 Subject: [PATCH 01/18] fix: enforce crash-only persistence coordination --- docs/contributing/oracle-coverage.md | 19 +- .../browser-db-sqlite-persistence/README.md | 51 +- .../src/browser-coordinator.ts | 1346 +++++- .../src/browser-persistence.ts | 30 +- .../src/index.ts | 9 + .../tests/browser-coordinator.test.ts | 3474 +++++++++++++++- .../per-collection-coordinator-oracle.test.ts | 3666 +++++++++++++++++ packages/db-sqlite-persistence-core/README.md | 133 +- .../db-sqlite-persistence-core/src/errors.ts | 101 + .../db-sqlite-persistence-core/src/index.ts | 1 + .../src/persisted.ts | 456 +- .../src/remote-subset-wire.ts | 899 ++++ .../src/sqlite-core-adapter.ts | 61 +- .../tests/persisted.test-d.ts | 225 +- .../tests/persisted.test.ts | 845 +++- .../db/skills/db-core/persistence/SKILL.md | 27 +- .../tests/electric-recovery-oracle.test.ts | 24 + .../electron-db-sqlite-persistence/README.md | 65 + .../src/electron-coordinator.ts | 1290 +++++- .../src/index.ts | 13 +- .../src/renderer.ts | 26 +- .../tests/electron-ipc.test.ts | 2988 +++++++++++++- 22 files changed, 15445 insertions(+), 304 deletions(-) create mode 100644 packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts create mode 100644 packages/db-sqlite-persistence-core/src/remote-subset-wire.ts diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 1b2aa2f157..d95cf11712 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -35,15 +35,32 @@ comment and the current API/architecture contract before extending its model. | Drafts and native values | [proxy](../../packages/db/tests/proxy.test.ts), [detachment](../../packages/db/tests/proxy-detachment-contract.test.ts), [iteration](../../packages/db/tests/proxy-iteration-contract.test.ts) | Native-operation controls, exact patches and actual stored rows; alias/cycle/adversarial-key histories. General native-mutator and symbol-write support is not established by a plain-object oracle. | | Query DB and observer | [ownership](../../packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts), [load lifecycle](../../packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts), [observer histories](../../packages/db/tests/live-query-observer-history.property.test.ts) | Real QueryClient boundary and a per-listener eligibility ledger, not a duplicate dispatch queue. Check reentry, peer survival, FIFO and disposal independently of final rows. | | Ordered acquisition | [pagination](../../packages/db/tests/query/pagination-oracle.property.test.ts), [ordered work](../../packages/db/tests/query/ordered-work-oracle.property.test.ts), [ordered lifecycle](../../packages/db/tests/query/ordered-lifecycle-oracle.property.test.ts) | Complete finite provider results, pending windows, ties/nulls, ownership and documented repair timing. Request completion is not proof of unrequested source extent. | +| Opaque backend pagination | [window oracle](../../packages/query-db-collection/tests/cursor-pagination.oracle.test.ts), [cache histories](../../packages/query-db-collection/tests/cursor-pagination.cache-oracle.test.ts), [cache publication](../../packages/query-db-collection/tests/cursor-pagination.publication-oracle.test.ts), [browser acquisition boundaries](../../packages/query-db-collection/tests/cursor-pagination.boundary-oracle.test.ts), [QueryCollection integration](../../packages/query-db-collection/tests/cursor-pagination.integration.test.ts) | Full filter/sort/slice reference, opaque token transport, actual Query cache expiry/invalidation/GC, forced refresh during growth, protocol failure publication/recovery, bounded slice work, nested cancellation/replacement, reader abort, browser retry defaults, manual-write cache isolation, and production window publications. Stable backend sequences; not snapshot guarantees for changing endpoints. Peek-ahead remains enabled. | | Electric and TrailBase | [Electric histories](../../packages/electric-db-collection/tests/electric-oracle.property.test.ts), [PostgreSQL semantics](../../packages/electric-db-collection/e2e/sql-predicate-semantics.e2e.test.ts), [TrailBase contract](../../packages/trailbase-db-collection/tests/ORACLE.md) | Installed SDK delivery/framing, independent predicates, exact subscription arguments and late errors. SDK fixtures and a real service test earn different credit. | | PowerSync | [tests](../../packages/powersync-db-collection/tests) | Applied receipt positions crossed with held peers, native SQLite/SDK and cleanup evidence. A timeout mutant proves a progress failure, not every value assertion. | -| SQLite persistence and native hosts | [persisted histories](../../packages/db-sqlite-persistence-core/tests/persisted.test.ts), [driver contracts](../../packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [113-law manifest](../../packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Cache/remote rejection/peer/reopen histories and exact driver results. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | +| SQLite persistence and native hosts | [persisted histories](../../packages/db-sqlite-persistence-core/tests/persisted.test.ts), [Browser composed-owner histories](../../packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts), [Browser coordinator RPC](../../packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts), [Electron IPC and composed owner](../../packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts), [driver contracts](../../packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [113-law manifest](../../packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Core cache/remote rejection/peer/reopen histories and exact driver results. Browser composes public source commits with per-collection elected-owner routing and covers the complete committed-transaction wire partition through deterministic Node transport seams. Electron composes source commits with a per-collection renderer owner, IPC adapter, real SQLite, and reopen checks. The Browser seams are not real multi-context/OPFS-worker execution; the Electron harness is not an actual Electron process unless its explicit runtime-bridge mode runs. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | | Offline execution | [scheduler](../../packages/offline-transactions/tests/KeyScheduler.property.test.ts), [leadership](../../packages/offline-transactions/tests/leadership-replay.property.test.ts), [settlement](../../packages/offline-transactions/tests/transaction-settlement.property.test.ts), [serialization](../../packages/offline-transactions/tests/transaction-serializer.property.test.ts) | Declarative FIFO eligibility, per-transaction outcomes, durable state and typed wire trees. Issued work may finish after ownership loss, but new work must not start. Exactly-once network execution is not promised. | | Frameworks | [React conformance](../../packages/react-db/tests/conformance.test.tsx), [React pagination](../../packages/react-db/tests/infinite-query-conformance.test.tsx), [shared suites](../../packages/db-collection-e2e/src/suites) | Exact exposed rows/pages and each framework's own lifecycle cuts. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. | | Small structures and test mechanics | [SortedMap](../../packages/db/tests/SortedMap.test.ts), [cleanup queue](../../packages/db/tests/cleanup-queue.property.test.ts), [guarded replay](../../packages/db/tests/oracle-replay.test.ts) | Map/full-sort and appointment-list models; executed target/seed/path checks. Callback-reentrant scheduling is outside the initial cleanup-queue domain. | ## Acceptance map +The post-merge review added three missing domains to existing owners: + +- [Top-K batch contracts](../../packages/db-ivm/tests/operators/topk-batch-contract.test.ts) + cross sparse-array length/holes and RegExp source/flags/position with equal + controls, replacement order, hash consolidation, and actual retained graph + output. Ordinary replacements also run without the global `File` constructor. +- [Leadership replay](../../packages/offline-transactions/tests/leadership-replay.property.test.ts) + holds real storage-read delivery across successful and permanently rejected + durable removals, with bounded scans, concurrent loads, and unfinished peers. + This is distinct from exactly-once execution across independent owners. +- [Accepted-snapshot retention](../../packages/db/tests/collection-state-retention-oracle.property.test.ts) + varies truncate before/during/after an optimistic delete, rejection versus + rollback, post-capture direct insertion, and later ordinary sync/key reuse. A + hidden accepted insert returns after rollback; an uncaptured insert retires, + and neither snapshot is rebased onto synced fields. + | Issue obligation | Implemented evidence | Limit | | --- | --- | --- | | Metamorphic laws | Includes cross-formulation/partition, D2 independent-key commutation, DBSP incremental/full recomputation, pagination provider/UI boundaries, optimistic snapshot stability | Equivalence premises are explicit; not arbitrary query rewrites. | diff --git a/packages/browser-db-sqlite-persistence/README.md b/packages/browser-db-sqlite-persistence/README.md index 84175bf7cd..08a5222935 100644 --- a/packages/browser-db-sqlite-persistence/README.md +++ b/packages/browser-db-sqlite-persistence/README.md @@ -9,6 +9,7 @@ is opt-in by passing a `BrowserCollectionCoordinator`. - `createBrowserWASQLitePersistence(...)` - `openBrowserWASQLiteOPFSDatabase(...)` +- `BrowserCollectionCoordinator` - `persistedCollectionOptions(...)` (re-exported from core) ## Quick start (single-tab) @@ -97,13 +98,61 @@ export const todosCollection = createCollection( See [`examples/react/offline-transactions`](../../examples/react/offline-transactions/src/db/persisted-todos.ts) for a full multi-tab example. +### Committed transaction ownership + +The persisted sync wrapper sends every source transaction with durable effects +through the coordinator's required +`requestApplyCommittedTx(collectionId, tx)` method. +`BrowserCollectionCoordinator` routes the complete transaction to the current +leader for that collection. The leader applies it with the adapter registered +for the same collection id, including that collection's resolved mode and +`schemaVersion`. + +The route preserves truncation, row changes, row metadata, collection metadata, +and stream position as one `PersistedTx`. It does not feature-detect a partial +route or fall back to row-only mutation RPC. A custom coordinator that omits +`requestApplyCommittedTx` is rejected while the collection is configured, +before its sync source can publish rows. + +Single-tab mode uses the same complete transaction contract. Its +`SingleProcessCoordinator` skips election and channel traffic but still routes +the transaction to the resolved adapter for that collection. + +If a mutating RPC loses its response, Browser coordination replays it only +while the requester still knows the same non-null leader id and term. An +unknown initial route or any leader/term change rejects with +`IndeterminateCommitError`; the application must reconcile the outcome. The +coordinator does not retry that mutation against an unknown or replacement +leader. + +### Remote subset requests + +`BrowserCollectionCoordinator.requestEnsureRemoteSubset(...)` validates and +projects the request before it chooses the local leader or `BroadcastChannel` +route. Registered owners receive the exported +`TransportedLoadSubsetOptions` type. It contains the supported +structured-clone wire data and excludes live `signal` and `subscription` +fields. Unsupported nested values fail immediately with +`RemoteSubsetWireValueError` and the exact value path; no owner callback or +channel post occurs. See the core package's remote subset wire contract for the +complete supported domain. + +Each accepted request is an explicit lease. Retries of the same request object +reuse its acquisition identity, while distinct equal request objects remain +independent. Release is routed to the elected collection owner and unloads the +exact acquired options once. Leadership loss unloads the retiring owner's live +leases, and requesters replay still-live acquisitions against the next leader. +Registering a second owner for one collection throws +`DuplicateRemoteSubsetOwnerError`; no adapter fallback replaces the owner. + ## Notes - `openBrowserWASQLiteOPFSDatabase(...)` starts a dedicated Web Worker and routes SQL operations through it. OPFS sync access handle APIs are used in that worker context. - Single-tab mode does not require `BroadcastChannel` or Web Locks for - correctness. + election, but committed transactions still go through the collection's + registered persistence owner. - Multi-tab mode requires `BroadcastChannel` and the Web Locks API; both are available in all modern browsers. - OPFS capability failures are surfaced as `PersistenceUnavailableError`. diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index 1babddc5a7..e5c8c419cb 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -1,12 +1,30 @@ -import { safeRandomUUID } from '@tanstack/db-sqlite-persistence-core' +import { + DuplicateRemoteSubsetOwnerError, + IndeterminateCommitError, + PersistedCollectionDurabilityError, + safeRandomUUID, + toPersistedCollectionDurabilityError, + toTransportedLoadSubsetOptions, +} from '@tanstack/db-sqlite-persistence-core' import type { + ApplyCommittedTxResponse, ApplyLocalMutationsResponse, + EnsureRemoteSubsetRequest, + EnsureRemoteSubsetResponse, + IndeterminateCommitRequestType, PersistedCollectionCoordinator, PersistedIndexSpec, PersistedMutationEnvelope, + PersistedRowMetadataMutation, + PersistedTx, PersistenceAdapter, ProtocolEnvelope, PullSinceResponse, + ReleaseRemoteSubsetRequest, + ReleaseRemoteSubsetResponse, + RemoteSubsetOwner, + TransportedLoadSubsetOptions, + TxCommitted, } from '@tanstack/db-sqlite-persistence-core' import type { LoadSubsetOptions } from '@tanstack/db' @@ -26,11 +44,8 @@ const WRITER_LOCK_MAX_RETRIES = 20 // --------------------------------------------------------------------------- type RPCRequest = - | { - type: `rpc:ensureRemoteSubset:req` - rpcId: string - options: LoadSubsetOptions - } + | EnsureRemoteSubsetRequest + | ReleaseRemoteSubsetRequest | { type: `rpc:ensurePersistedIndex:req` rpcId: string @@ -43,6 +58,12 @@ type RPCRequest = envelopeId: string mutations: Array } + | { + type: `rpc:applyCommittedTx:req` + rpcId: string + envelopeId: string + tx: PersistedTx + } | { type: `rpc:pullSince:req` rpcId: string @@ -50,12 +71,8 @@ type RPCRequest = } type RPCResponse = - | { - type: `rpc:ensureRemoteSubset:res` - rpcId: string - ok: boolean - error?: string - } + | EnsureRemoteSubsetResponse + | ReleaseRemoteSubsetResponse | { type: `rpc:ensurePersistedIndex:res` rpcId: string @@ -63,6 +80,7 @@ type RPCResponse = error?: string } | ApplyLocalMutationsResponse + | ApplyCommittedTxResponse | PullSinceResponse type PendingRPC = { @@ -73,6 +91,7 @@ type PendingRPC = { type CollectionState = { isLeader: boolean + leaderId: string | null lockAbortController: AbortController | null heartbeatTimer: ReturnType | null latestTerm: number @@ -81,8 +100,8 @@ type CollectionState = { subscribers: Set<(message: ProtocolEnvelope) => void> } -// Adapter with pullSince support -type AdapterWithPullSince = PersistenceAdapter & { +// Adapter capabilities used by coordinator-side operations +type CoordinatorAdapter = PersistenceAdapter & { pullSince?: ( collectionId: string, fromRowVersion: number, @@ -105,13 +124,76 @@ type AdapterWithPullSince = PersistenceAdapter & { }> } +type ActiveRemoteSubsetAcquisition = { + collectionId: string + requesterId: string + acquisitionId: string + owner: RemoteSubsetOwner + options: TransportedLoadSubsetOptions + load: Promise + transferred: boolean + released: boolean + terminalRelease: boolean + release: Promise | null +} + +type AwaitingRemoteSubsetOwnerAcquisition = { + collectionId: string + requesterId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + released: true + awaitingOwner: true +} + +type RemoteSubsetAcquisition = + | ActiveRemoteSubsetAcquisition + | AwaitingRemoteSubsetOwnerAcquisition + | { + collectionId: string + requesterId: string + acquisitionId: string + released: true + } + +type OutboundRemoteSubsetAcquisition = { + collectionId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + acquiredLeaderId: string | null + inFlight: Promise | null + forceReplay: boolean +} + +type AppliedEnvelope = + | { + appliedAt: number + requestType: `rpc:applyLocalMutations:req` + response: ApplyLocalMutationsResponse + } + | { + appliedAt: number + requestType: `rpc:applyCommittedTx:req` + response: ApplyCommittedTxResponse + } + +type InFlightEnvelope = + | { + requestType: `rpc:applyLocalMutations:req` + response: Promise + } + | { + requestType: `rpc:applyCommittedTx:req` + response: Promise + } + // --------------------------------------------------------------------------- // Options // --------------------------------------------------------------------------- export type BrowserCollectionCoordinatorOptions = { dbName: string - adapter?: AdapterWithPullSince + adapter?: CoordinatorAdapter } // --------------------------------------------------------------------------- @@ -121,11 +203,26 @@ export type BrowserCollectionCoordinatorOptions = { export class BrowserCollectionCoordinator implements PersistedCollectionCoordinator { private readonly nodeId = safeRandomUUID() private readonly dbName: string - private adapter: AdapterWithPullSince | null + private defaultAdapter: CoordinatorAdapter | null + private readonly collectionAdapters = new Map() + private readonly remoteSubsetOwners = new Map() + private readonly remoteSubsetIds = new Map< + string, + WeakMap + >() + private readonly outboundRemoteSubsetAcquisitions = new Map< + string, + OutboundRemoteSubsetAcquisition + >() + private readonly inboundRemoteSubsetAcquisitions = new Map< + string, + RemoteSubsetAcquisition + >() private readonly channel: BroadcastChannel private readonly collections = new Map() private readonly pendingRPCs = new Map() - private readonly appliedEnvelopeIds = new Map() + private readonly appliedEnvelopes = new Map() + private readonly inFlightEnvelopes = new Map() private disposed = false /** Method indirection to prevent TypeScript from narrowing `disposed` across awaits */ @@ -133,18 +230,20 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina return this.disposed } - private requireAdapter(): AdapterWithPullSince { - if (!this.adapter) { + private requireAdapter(collectionId: string): CoordinatorAdapter { + const adapter = + this.collectionAdapters.get(collectionId) ?? this.defaultAdapter + if (!adapter) { throw new Error( - `BrowserCollectionCoordinator: adapter not set. Call setAdapter() before using leader-side operations.`, + `BrowserCollectionCoordinator: adapter not set for collection "${collectionId}". Call setAdapterForCollection() before using leader-side operations.`, ) } - return this.adapter + return adapter } constructor(options: BrowserCollectionCoordinatorOptions) { this.dbName = options.dbName - this.adapter = options.adapter ?? null + this.defaultAdapter = options.adapter ?? null this.channel = new BroadcastChannel(`tsdb:coord:${this.dbName}`) this.channel.onmessage = (event: MessageEvent) => { this.onChannelMessage(event.data) @@ -156,8 +255,38 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina * Called by `createBrowserWASQLitePersistence` to wire the internally-created * adapter into the coordinator. */ - setAdapter(adapter: AdapterWithPullSince): void { - this.adapter = adapter + setAdapter(adapter: CoordinatorAdapter): void { + this.defaultAdapter = adapter + } + + /** Register the persistence adapter that owns one collection. */ + setAdapterForCollection( + collectionId: string, + adapter: CoordinatorAdapter, + ): void { + this.collectionAdapters.set(collectionId, adapter) + } + + registerRemoteSubsetOwner( + collectionId: string, + owner: RemoteSubsetOwner, + ): () => void { + if (this.remoteSubsetOwners.has(collectionId)) { + throw new DuplicateRemoteSubsetOwnerError(collectionId) + } + this.remoteSubsetOwners.set(collectionId, owner) + for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + if (acquisition.collectionId !== collectionId) continue + acquisition.acquiredLeaderId = null + acquisition.forceReplay = true + } + void this.replayRemoteSubsetAcquisitions(collectionId) + this.rebindRemoteInboundSubsetAcquisitions(collectionId, owner) + return () => { + if (this.remoteSubsetOwners.get(collectionId) !== owner) return + this.remoteSubsetOwners.delete(collectionId) + this.releaseInboundRemoteSubsetAcquisitions(collectionId, owner) + } } // ----------------------------------------------------------------------- @@ -197,23 +326,139 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina collectionId: string, options: LoadSubsetOptions, ): Promise { - if (this.isLeader(collectionId)) return + const transportedOptions = toTransportedLoadSubsetOptions(options) + let collectionIds = this.remoteSubsetIds.get(collectionId) + if (!collectionIds) { + collectionIds = new WeakMap() + this.remoteSubsetIds.set(collectionId, collectionIds) + } + let acquisitionId = collectionIds.get(options) + let acquisition = acquisitionId + ? this.outboundRemoteSubsetAcquisitions.get( + remoteSubsetAcquisitionKey(collectionId, acquisitionId), + ) + : undefined + if (!acquisition) { + acquisitionId = safeRandomUUID() + collectionIds.set(options, acquisitionId) + acquisition = { + collectionId, + acquisitionId, + options: transportedOptions, + acquiredLeaderId: null, + inFlight: null, + forceReplay: false, + } + this.outboundRemoteSubsetAcquisitions.set( + remoteSubsetAcquisitionKey(collectionId, acquisitionId), + acquisition, + ) + } - const response = await this.sendRPC<{ - type: `rpc:ensureRemoteSubset:res` - rpcId: string - ok: boolean - error?: string - }>(collectionId, { - type: `rpc:ensureRemoteSubset:req`, + await this.acquireRemoteSubset(acquisition) + } + + async requestReleaseRemoteSubset( + collectionId: string, + options: LoadSubsetOptions, + ): Promise { + const collectionIds = this.remoteSubsetIds.get(collectionId) + const acquisitionId = collectionIds?.get(options) + if (!acquisitionId) return + const key = remoteSubsetAcquisitionKey(collectionId, acquisitionId) + if (!this.outboundRemoteSubsetAcquisitions.delete(key)) return + collectionIds!.delete(options) + + const request: Extract< + RPCRequest, + { type: `rpc:releaseRemoteSubset:req` } + > = { + type: `rpc:releaseRemoteSubset:req`, rpcId: safeRandomUUID(), - options, - }) + acquisitionId, + } + const response = this.isLeader(collectionId) + ? await this.handleReleaseRemoteSubset(collectionId, request, this.nodeId) + : await this.sendRPC(collectionId, request) if (!response.ok) { - throw new Error( - `ensureRemoteSubset failed: ${response.error ?? `unknown error`}`, + throw new Error(`releaseRemoteSubset failed: ${response.error}`) + } + } + + private async acquireRemoteSubset( + acquisition: OutboundRemoteSubsetAcquisition, + ): Promise { + if (acquisition.inFlight) return acquisition.inFlight + + const route = { localOwner: false } + let resolveWork!: () => void + let rejectWork!: (error: unknown) => void + const work = new Promise((resolve, reject) => { + resolveWork = resolve + rejectWork = reject + }) + acquisition.inFlight = work + const run = async (): Promise => { + const request: Extract< + RPCRequest, + { type: `rpc:ensureRemoteSubset:req` } + > = { + type: `rpc:ensureRemoteSubset:req`, + rpcId: safeRandomUUID(), + acquisitionId: acquisition.acquisitionId, + options: acquisition.options, + } + route.localOwner = this.isLeader(acquisition.collectionId) + const response = route.localOwner + ? await this.handleEnsureRemoteSubset( + acquisition.collectionId, + request, + this.nodeId, + ) + : await this.sendRPC( + acquisition.collectionId, + request, + ) + + if (!response.ok) { + throw new Error(`ensureRemoteSubset failed: ${response.error}`) + } + acquisition.acquiredLeaderId = response.leaderId + } + void run().then(resolveWork, rejectWork) + let acquired = false + try { + await work + acquired = true + } catch (error) { + if (!route.localOwner) { + const owner = this.remoteSubsetOwners.get(acquisition.collectionId) + if (owner) reportRemoteSubsetOwnerError(owner, error) + } + throw error + } finally { + if (acquisition.inFlight === work) acquisition.inFlight = null + const current = this.collections.get(acquisition.collectionId) + const currentLeaderId = current?.isLeader + ? this.nodeId + : (current?.leaderId ?? null) + const key = remoteSubsetAcquisitionKey( + acquisition.collectionId, + acquisition.acquisitionId, ) + if ( + acquired && + this.outboundRemoteSubsetAcquisitions.get(key) === acquisition && + (acquisition.forceReplay || + (currentLeaderId !== null && + acquisition.acquiredLeaderId !== currentLeaderId)) + ) { + acquisition.forceReplay = false + void this.acquireRemoteSubset(acquisition).catch(() => { + // Failure is already reported; only new demand or ownership change retries. + }) + } } } @@ -223,7 +468,11 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina spec: PersistedIndexSpec, ): Promise { if (this.isLeader(collectionId)) { - await this.requireAdapter().ensureIndex(collectionId, signature, spec) + await this.requireAdapter(collectionId).ensureIndex( + collectionId, + signature, + spec, + ) return } @@ -267,6 +516,23 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina }) } + async requestApplyCommittedTx( + collectionId: string, + tx: PersistedTx, + ): Promise { + const request: Extract = { + type: `rpc:applyCommittedTx:req`, + rpcId: safeRandomUUID(), + envelopeId: safeRandomUUID(), + tx, + } + if (this.isLeader(collectionId)) { + return this.handleApplyCommittedTx(collectionId, request) + } + + return this.sendRPC(collectionId, request) + } + async pullSince( collectionId: string, fromRowVersion: number, @@ -293,6 +559,12 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina dispose(): void { this.disposed = true + for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + this.postRemoteSubsetRelease(acquisition) + } + this.outboundRemoteSubsetAcquisitions.clear() + this.remoteSubsetIds.clear() + for (const [collectionId, state] of this.collections) { this.releaseLeadership(collectionId, state) } @@ -305,6 +577,14 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina this.channel.close() this.collections.clear() + this.collectionAdapters.clear() + for (const collectionId of this.remoteSubsetOwners.keys()) { + this.releaseInboundRemoteSubsetAcquisitions(collectionId) + } + this.remoteSubsetOwners.clear() + this.inboundRemoteSubsetAcquisitions.clear() + this.appliedEnvelopes.clear() + this.inFlightEnvelopes.clear() } // ----------------------------------------------------------------------- @@ -316,6 +596,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina if (!state) { state = { isLeader: false, + leaderId: null, lockAbortController: null, heartbeatTimer: null, latestTerm: 0, @@ -348,7 +629,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina try { // Restore stream position from DB before claiming leadership - const adapter = this.requireAdapter() + const adapter = this.requireAdapter(collectionId) if (adapter.getStreamPosition) { const pos = await adapter.getStreamPosition(collectionId) state.latestTerm = pos.latestTerm @@ -358,8 +639,10 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina state.latestTerm++ state.isLeader = true + state.leaderId = this.nodeId this.emitHeartbeat(collectionId, state) + void this.replayRemoteSubsetAcquisitions(collectionId) state.heartbeatTimer = setInterval(() => { this.emitHeartbeat(collectionId, state) }, HEARTBEAT_INTERVAL_MS) @@ -377,7 +660,9 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina abortController.signal.addEventListener(`abort`, onAbort) }) } finally { + this.releaseInboundRemoteSubsetAcquisitions(collectionId) state.isLeader = false + state.leaderId = null if (state.heartbeatTimer) { clearInterval(state.heartbeatTimer) state.heartbeatTimer = null @@ -399,9 +684,10 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina } private releaseLeadership( - _collectionId: string, + collectionId: string, state: CollectionState, ): void { + this.releaseInboundRemoteSubsetAcquisitions(collectionId) if (state.lockAbortController) { state.lockAbortController.abort() state.lockAbortController = null @@ -411,6 +697,68 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina state.heartbeatTimer = null } state.isLeader = false + state.leaderId = null + } + + private postRemoteSubsetRelease( + acquisition: OutboundRemoteSubsetAcquisition, + ): void { + const request: Extract< + RPCRequest, + { type: `rpc:releaseRemoteSubset:req` } + > = { + type: `rpc:releaseRemoteSubset:req`, + rpcId: safeRandomUUID(), + acquisitionId: acquisition.acquisitionId, + } + if (this.isLeader(acquisition.collectionId)) { + void this.handleReleaseRemoteSubset( + acquisition.collectionId, + request, + this.nodeId, + ).catch(() => { + // The owner already received the exact unload failure through onError. + }) + return + } + this.channel.postMessage({ + v: 1, + dbName: this.dbName, + collectionId: acquisition.collectionId, + senderId: this.nodeId, + ts: Date.now(), + payload: request, + } satisfies ProtocolEnvelope) + } + + private async replayRemoteSubsetAcquisitions( + collectionId: string, + ): Promise { + if (this.isDisposed()) return + const state = this.collections.get(collectionId) + const leaderId = state?.isLeader ? this.nodeId : state?.leaderId + if (!leaderId) return + + const replays: Array> = [] + for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + if ( + acquisition.collectionId !== collectionId || + (!acquisition.forceReplay && acquisition.acquiredLeaderId === leaderId) + ) { + continue + } + if (acquisition.inFlight) { + acquisition.forceReplay = true + continue + } + acquisition.forceReplay = false + replays.push( + this.acquireRemoteSubset(acquisition).catch(() => { + // Failure is already reported; only new demand or ownership change retries. + }), + ) + } + await Promise.all(replays) } private emitHeartbeat(collectionId: string, state: CollectionState): void { @@ -448,6 +796,35 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina const type = (payload as Record).type as string | undefined + if (type === `leader:heartbeat`) { + const heartbeat = payload as { + leaderId?: unknown + term?: unknown + latestSeq?: unknown + latestRowVersion?: unknown + } + if ( + typeof heartbeat.leaderId === `string` && + typeof heartbeat.term === `number` && + typeof heartbeat.latestSeq === `number` && + typeof heartbeat.latestRowVersion === `number` + ) { + const state = this.ensureCollectionState(envelope.collectionId) + if (heartbeat.term < state.latestTerm) return + const changedLeader = state.leaderId !== heartbeat.leaderId + state.leaderId = heartbeat.leaderId + state.latestTerm = Math.max(state.latestTerm, heartbeat.term) + state.latestSeq = Math.max(state.latestSeq, heartbeat.latestSeq) + state.latestRowVersion = Math.max( + state.latestRowVersion, + heartbeat.latestRowVersion, + ) + if (changedLeader) { + void this.replayRemoteSubsetAcquisitions(envelope.collectionId) + } + } + } + // Handle RPC responses (for pending outbound RPCs) if (type && type.endsWith(`:res`)) { const rpcId = (payload as { rpcId?: string }).rpcId @@ -464,7 +841,11 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina if (type && type.endsWith(`:req`)) { const collectionId = envelope.collectionId if (this.isLeader(collectionId)) { - void this.handleRPCRequest(collectionId, payload as RPCRequest) + void this.handleRPCRequest( + collectionId, + payload as RPCRequest, + envelope.senderId, + ) } return } @@ -487,15 +868,53 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina request: RPCRequest, ): Promise { let lastError: Error | undefined + let firstTransportCause: unknown + const mutationRequestType = isMutatingRPCRequest(request) + ? request.type + : undefined + const mutationRoute = mutationRequestType + ? this.captureMutationRoute(collectionId) + : undefined for (let attempt = 0; attempt <= RPC_RETRY_ATTEMPTS; attempt++) { if (attempt > 0) { await sleep(RPC_RETRY_DELAY_MS * attempt) } + if ( + mutationRoute && + mutationRequestType && + firstTransportCause !== undefined + ) { + this.assertMutationRouteUnchanged( + collectionId, + mutationRequestType, + mutationRoute, + firstTransportCause, + ) + } + + if (this.isLeader(collectionId)) { + return (await this.dispatchRPCRequest( + collectionId, + request, + this.nodeId, + )) as T + } + try { return await this.sendRPCOnce(collectionId, request) } catch (error) { + if (this.isDisposed()) throw error + firstTransportCause ??= error + if (mutationRoute && mutationRequestType) { + this.assertMutationRouteUnchanged( + collectionId, + mutationRequestType, + mutationRoute, + firstTransportCause, + ) + } lastError = error instanceof Error ? error : new Error(String(error)) } } @@ -503,6 +922,43 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina throw lastError ?? new Error(`RPC failed after retries`) } + private captureMutationRoute(collectionId: string): { + leaderId: string | null + term: number | null + } { + const state = this.collections.get(collectionId) + return { + leaderId: state?.isLeader ? this.nodeId : (state?.leaderId ?? null), + term: state?.latestTerm ?? null, + } + } + + private assertMutationRouteUnchanged( + collectionId: string, + requestType: IndeterminateCommitRequestType, + previous: { leaderId: string | null; term: number | null }, + cause: unknown, + ): void { + const current = this.captureMutationRoute(collectionId) + if ( + previous.leaderId !== null && + previous.term !== null && + current.leaderId === previous.leaderId && + current.term === previous.term + ) { + return + } + throw new IndeterminateCommitError({ + collectionId, + requestType, + previousLeaderId: previous.leaderId, + previousTerm: previous.term, + currentLeaderId: current.leaderId, + currentTerm: current.term, + cause, + }) + } + private sendRPCOnce( collectionId: string, request: RPCRequest, @@ -542,38 +998,22 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina private async handleRPCRequest( collectionId: string, request: RPCRequest, + requesterId: string, ): Promise { let response: RPCResponse try { - switch (request.type) { - case `rpc:ensureRemoteSubset:req`: - response = await this.handleEnsureRemoteSubset(collectionId, request) - break - case `rpc:ensurePersistedIndex:req`: - response = await this.handleEnsurePersistedIndex( - collectionId, - request, - ) - break - case `rpc:applyLocalMutations:req`: - response = await this.handleApplyLocalMutations(collectionId, request) - break - case `rpc:pullSince:req`: - response = await this.handlePullSince(collectionId, request) - break - default: - return - } + response = await this.dispatchRPCRequest( + collectionId, + request, + requesterId, + ) } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error) - response = { - type: request.type.replace(`:req`, `:res`) as RPCResponse[`type`], - rpcId: request.rpcId, - ok: false, - error: errorMessage, - } as RPCResponse + response = createRPCErrorResponse(request, error) + } + + if (this.isDisposed()) { + return } const envelope: ProtocolEnvelope = { @@ -587,16 +1027,357 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina this.channel.postMessage(envelope) } - private handleEnsureRemoteSubset( - _collectionId: string, - request: { type: `rpc:ensureRemoteSubset:req`; rpcId: string }, - ): RPCResponse { - // Leader doesn't need to do anything special — the remote subset - // is ensured by the leader's own sync connection + private dispatchRPCRequest( + collectionId: string, + request: RPCRequest, + requesterId: string, + ): Promise { + switch (request.type) { + case `rpc:ensureRemoteSubset:req`: + return this.handleEnsureRemoteSubset(collectionId, request, requesterId) + case `rpc:releaseRemoteSubset:req`: + return this.handleReleaseRemoteSubset( + collectionId, + request, + requesterId, + ) + case `rpc:ensurePersistedIndex:req`: + return this.handleEnsurePersistedIndex(collectionId, request) + case `rpc:applyLocalMutations:req`: + return this.handleApplyLocalMutations(collectionId, request) + case `rpc:applyCommittedTx:req`: + return this.handleApplyCommittedTx(collectionId, request) + case `rpc:pullSince:req`: + return this.handlePullSince(collectionId, request) + } + } + + private async handleEnsureRemoteSubset( + collectionId: string, + request: Extract, + requesterId: string, + ): Promise { + const key = inboundRemoteSubsetAcquisitionKey( + collectionId, + requesterId, + request.acquisitionId, + ) + const existing = this.inboundRemoteSubsetAcquisitions.get(key) + const awaitingOwner = + existing && `awaitingOwner` in existing ? existing : undefined + if (existing) { + if (`owner` in existing) { + if (!existing.released) { + await existing.load + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + await existing.release + if (existing.terminalRelease) { + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + if (this.inboundRemoteSubsetAcquisitions.get(key) === existing) { + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } else if (!(`awaitingOwner` in existing)) { + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + } + + const owner = this.remoteSubsetOwners.get(collectionId) + if (!owner) { + throw new Error( + `BrowserCollectionCoordinator: no remote subset owner registered for collection "${collectionId}"`, + ) + } + + const acquisition: ActiveRemoteSubsetAcquisition = { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + owner, + options: awaitingOwner?.options ?? request.options, + load: Promise.resolve(), + transferred: false, + released: false, + terminalRelease: false, + release: null, + } + this.inboundRemoteSubsetAcquisitions.set(key, acquisition) + let resolveLoad!: () => void + let rejectLoad!: (error: unknown) => void + acquisition.load = new Promise((resolve, reject) => { + resolveLoad = resolve + rejectLoad = reject + }) + try { + const load = owner(acquisition.options) + acquisition.transferred = true + void Promise.resolve(load).then(resolveLoad, rejectLoad) + } catch (error) { + rejectLoad(error) + } + try { + await acquisition.load + } catch (error) { + if ( + !acquisition.transferred && + this.inboundRemoteSubsetAcquisitions.get(key) === acquisition + ) { + if (awaitingOwner) { + this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + } else { + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } + reportRemoteSubsetOwnerError(owner, error) + throw error + } return { type: `rpc:ensureRemoteSubset:res`, rpcId: request.rpcId, ok: true, + leaderId: this.nodeId, + } + } + + private async handleReleaseRemoteSubset( + collectionId: string, + request: Extract, + requesterId: string, + ): Promise { + const key = inboundRemoteSubsetAcquisitionKey( + collectionId, + requesterId, + request.acquisitionId, + ) + const acquisition = this.inboundRemoteSubsetAcquisitions.get(key) + if (!acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } else if (`owner` in acquisition) { + acquisition.terminalRelease = true + await this.releaseRemoteSubsetAcquisition(acquisition) + if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } + } else if (`awaitingOwner` in acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } + return { + type: `rpc:releaseRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + } + } + + private releaseRemoteSubsetAcquisition( + acquisition: ActiveRemoteSubsetAcquisition, + ): Promise { + if (acquisition.release) return acquisition.release + acquisition.released = true + acquisition.release = (async () => { + try { + await acquisition.load + } catch { + // A returned promise transfers the lease even when initial loading fails. + } + try { + if (acquisition.transferred) { + await unloadRemoteSubsetOwner(acquisition.owner, acquisition.options) + } + } finally { + if (!acquisition.terminalRelease) { + const key = inboundRemoteSubsetAcquisitionKey( + acquisition.collectionId, + acquisition.requesterId, + acquisition.acquisitionId, + ) + if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId: acquisition.collectionId, + requesterId: acquisition.requesterId, + acquisitionId: acquisition.acquisitionId, + options: acquisition.options, + released: true, + awaitingOwner: true, + }) + } + } + } + })() + return acquisition.release + } + + private releaseInboundRemoteSubsetAcquisitions( + collectionId: string, + owner?: RemoteSubsetOwner, + ): void { + for (const acquisition of this.inboundRemoteSubsetAcquisitions.values()) { + if ( + !(`owner` in acquisition) || + acquisition.collectionId !== collectionId || + (owner && acquisition.owner !== owner) + ) { + continue + } + void this.releaseRemoteSubsetAcquisition(acquisition).catch( + () => undefined, + ) + } + } + + private rebindRemoteInboundSubsetAcquisitions( + collectionId: string, + owner: RemoteSubsetOwner, + ): void { + for (const acquisition of this.inboundRemoteSubsetAcquisitions.values()) { + if (`awaitingOwner` in acquisition) { + if ( + acquisition.collectionId === collectionId && + acquisition.requesterId !== this.nodeId + ) { + void this.bindAwaitingRemoteSubsetAcquisition( + acquisition, + owner, + ).catch(() => { + // The owner receives the exact load failure through onError. + }) + } + continue + } + if ( + !(`owner` in acquisition) || + acquisition.collectionId !== collectionId || + acquisition.requesterId === this.nodeId || + !acquisition.released || + acquisition.terminalRelease + ) { + continue + } + + void this.rebindRemoteInboundSubsetAcquisition(acquisition, owner).catch( + () => undefined, + ) + } + } + + private async rebindRemoteInboundSubsetAcquisition( + previous: ActiveRemoteSubsetAcquisition, + owner: RemoteSubsetOwner, + ): Promise { + await previous.release + if ( + previous.terminalRelease || + this.remoteSubsetOwners.get(previous.collectionId) !== owner + ) { + return + } + + const key = inboundRemoteSubsetAcquisitionKey( + previous.collectionId, + previous.requesterId, + previous.acquisitionId, + ) + const current = this.inboundRemoteSubsetAcquisitions.get(key) + if (current && current !== previous) { + if (`awaitingOwner` in current) { + await this.bindAwaitingRemoteSubsetAcquisition(current, owner) + } + return + } + + const awaitingOwner: AwaitingRemoteSubsetOwnerAcquisition = { + collectionId: previous.collectionId, + requesterId: previous.requesterId, + acquisitionId: previous.acquisitionId, + options: previous.options, + released: true, + awaitingOwner: true, + } + this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + await this.bindAwaitingRemoteSubsetAcquisition(awaitingOwner, owner) + } + + private async bindAwaitingRemoteSubsetAcquisition( + awaitingOwner: AwaitingRemoteSubsetOwnerAcquisition, + owner: RemoteSubsetOwner, + ): Promise { + if (this.remoteSubsetOwners.get(awaitingOwner.collectionId) !== owner) { + return + } + const key = inboundRemoteSubsetAcquisitionKey( + awaitingOwner.collectionId, + awaitingOwner.requesterId, + awaitingOwner.acquisitionId, + ) + if (this.inboundRemoteSubsetAcquisitions.get(key) !== awaitingOwner) return + + const acquisition: ActiveRemoteSubsetAcquisition = { + collectionId: awaitingOwner.collectionId, + requesterId: awaitingOwner.requesterId, + acquisitionId: awaitingOwner.acquisitionId, + owner, + options: awaitingOwner.options, + load: Promise.resolve(), + transferred: false, + released: false, + terminalRelease: false, + release: null, + } + this.inboundRemoteSubsetAcquisitions.set(key, acquisition) + let resolveLoad!: () => void + let rejectLoad!: (error: unknown) => void + acquisition.load = new Promise((resolve, reject) => { + resolveLoad = resolve + rejectLoad = reject + }) + try { + const load = owner(acquisition.options) + acquisition.transferred = true + void Promise.resolve(load).then(resolveLoad, rejectLoad) + } catch (error) { + rejectLoad(error) + } + try { + await acquisition.load + } catch (error) { + if ( + !acquisition.transferred && + this.inboundRemoteSubsetAcquisitions.get(key) === acquisition + ) { + this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + } + reportRemoteSubsetOwnerError(owner, error) + throw error } } @@ -610,7 +1391,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina }, ): Promise { await this.withWriterLock(() => - this.requireAdapter().ensureIndex( + this.requireAdapter(collectionId).ensureIndex( collectionId, request.signature, request.spec, @@ -632,8 +1413,12 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina mutations: Array }, ): Promise { - // Dedupe by envelopeId - if (this.appliedEnvelopeIds.has(request.envelopeId)) { + const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) + const appliedEnvelope = this.appliedEnvelopes.get(envelopeKey) + if (appliedEnvelope?.requestType === `rpc:applyLocalMutations:req`) { + return { ...appliedEnvelope.response, rpcId: request.rpcId } + } + if (appliedEnvelope) { return { type: `rpc:applyLocalMutations:res`, rpcId: request.rpcId, @@ -643,6 +1428,40 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina } } + const inFlightEnvelope = this.inFlightEnvelopes.get(envelopeKey) + if (inFlightEnvelope?.requestType === `rpc:applyLocalMutations:req`) { + const response = await inFlightEnvelope.response + return { ...response, rpcId: request.rpcId } + } + if (inFlightEnvelope) { + return { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error: `envelope ${request.envelopeId} is already in flight`, + } + } + + const response = this.applyLocalMutationsOnce(collectionId, request) + const pendingEnvelope: InFlightEnvelope = { + requestType: request.type, + response, + } + this.inFlightEnvelopes.set(envelopeKey, pendingEnvelope) + try { + return await response + } finally { + if (this.inFlightEnvelopes.get(envelopeKey) === pendingEnvelope) { + this.inFlightEnvelopes.delete(envelopeKey) + } + } + } + + private async applyLocalMutationsOnce( + collectionId: string, + request: Extract, + ): Promise { const state = this.collections.get(collectionId) if (!state || !state.isLeader) { return { @@ -663,6 +1482,21 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina const rowVersion = state.latestRowVersion // Build and apply the persisted transaction + const rowMetadataMutations: Array = [] + for (const mutation of request.mutations) { + if (!(`metadataChanged` in mutation) || !mutation.metadataChanged) { + continue + } + rowMetadataMutations.push( + mutation.metadata === undefined + ? { type: `delete`, key: mutation.key } + : { + type: `set`, + key: mutation.key, + value: mutation.metadata, + }, + ) + } const tx = { txId: safeRandomUUID(), term, @@ -672,16 +1506,42 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina type: m.type, key: m.key, value: m.value, + ...(`metadataChanged` in m + ? { metadata: m.metadata, metadataChanged: m.metadataChanged } + : {}), })), + rowMetadataMutations, } - await this.withWriterLock(() => - this.requireAdapter().applyCommittedTx(collectionId, tx), - ) + try { + await this.withWriterLock(() => + this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), + ) + } catch (error) { + throw toPersistedCollectionDurabilityError(collectionId, error) + } - // Track envelope for dedup - this.appliedEnvelopeIds.set(request.envelopeId, Date.now()) - this.pruneAppliedEnvelopeIds() + const response: ApplyLocalMutationsResponse = { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: true, + term, + seq, + latestRowVersion: rowVersion, + acceptedMutationIds: request.mutations.map((m) => m.mutationId), + } + if (this.isDisposed()) { + return response + } + this.appliedEnvelopes.set( + appliedEnvelopeKey(collectionId, request.envelopeId), + { + appliedAt: Date.now(), + requestType: request.type, + response, + }, + ) + this.pruneAppliedEnvelopes() // Broadcast tx:committed to all tabs const changedRows = request.mutations @@ -706,6 +1566,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina requiresFullReload: false, changedRows, deletedKeys, + rowMetadataMutations, }, } this.channel.postMessage(txCommitted) @@ -715,15 +1576,151 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina subscriber(txCommitted) } - return { - type: `rpc:applyLocalMutations:res`, + return response + } + + private async handleApplyCommittedTx( + collectionId: string, + request: Extract, + ): Promise { + const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) + const appliedEnvelope = this.appliedEnvelopes.get(envelopeKey) + if (appliedEnvelope?.requestType === `rpc:applyCommittedTx:req`) { + return { ...appliedEnvelope.response, rpcId: request.rpcId } + } + if (appliedEnvelope) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error: `envelope ${request.envelopeId} already applied`, + } + } + + const inFlightEnvelope = this.inFlightEnvelopes.get(envelopeKey) + if (inFlightEnvelope?.requestType === `rpc:applyCommittedTx:req`) { + const response = await inFlightEnvelope.response + return { ...response, rpcId: request.rpcId } + } + if (inFlightEnvelope) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error: `envelope ${request.envelopeId} is already in flight`, + } + } + + const response = this.applyCommittedTxOnce(collectionId, request) + const pendingEnvelope: InFlightEnvelope = { + requestType: request.type, + response, + } + this.inFlightEnvelopes.set(envelopeKey, pendingEnvelope) + try { + return await response + } finally { + if (this.inFlightEnvelopes.get(envelopeKey) === pendingEnvelope) { + this.inFlightEnvelopes.delete(envelopeKey) + } + } + } + + private async applyCommittedTxOnce( + collectionId: string, + request: Extract, + ): Promise { + const state = this.collections.get(collectionId) + if (!state || !state.isLeader) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `NOT_LEADER`, + error: `not the leader for ${collectionId}`, + } + } + + state.latestSeq++ + state.latestRowVersion++ + const tx: PersistedTx = { + ...request.tx, + term: state.latestTerm, + seq: state.latestSeq, + rowVersion: state.latestRowVersion, + } + + try { + await this.withWriterLock(() => + this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), + ) + } catch (error) { + throw toPersistedCollectionDurabilityError(collectionId, error) + } + const response: ApplyCommittedTxResponse = { + type: `rpc:applyCommittedTx:res`, rpcId: request.rpcId, ok: true, - term, - seq, - latestRowVersion: rowVersion, - acceptedMutationIds: request.mutations.map((m) => m.mutationId), + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + } + if (this.isDisposed()) { + return response + } + this.appliedEnvelopes.set( + appliedEnvelopeKey(collectionId, request.envelopeId), + { + appliedAt: Date.now(), + requestType: request.type, + response, + }, + ) + this.pruneAppliedEnvelopes() + + const committedBase = { + type: `tx:committed` as const, + term: tx.term, + seq: tx.seq, + txId: tx.txId, + latestRowVersion: tx.rowVersion, + } + const committedPayload: TxCommitted = tx.truncate + ? { + ...committedBase, + requiresFullReload: true, + } + : { + ...committedBase, + requiresFullReload: false, + changedRows: tx.mutations + .filter((mutation) => mutation.type !== `delete`) + .map((mutation) => ({ + key: mutation.key, + value: mutation.value, + })), + deletedKeys: tx.mutations + .filter((mutation) => mutation.type === `delete`) + .map((mutation) => mutation.key), + rowMetadataMutations: tx.rowMetadataMutations, + collectionMetadataMutations: tx.collectionMetadataMutations, + } + const committed: ProtocolEnvelope = { + v: 1, + dbName: this.dbName, + collectionId, + senderId: this.nodeId, + ts: Date.now(), + payload: committedPayload, + } + this.channel.postMessage(committed) + for (const subscriber of state.subscribers) { + subscriber(committed) } + + return response } private async handlePullSince( @@ -736,7 +1733,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina ): Promise { const state = this.collections.get(collectionId) - const adapter = this.requireAdapter() + const adapter = this.requireAdapter(collectionId) if (!adapter.pullSince) { return { type: `rpc:pullSince:res`, @@ -784,9 +1781,16 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina const lockName = `tsdb:writer:${this.dbName}` for (let attempt = 0; attempt <= WRITER_LOCK_MAX_RETRIES; attempt++) { + const callbackState = { entered: false } try { - return await navigator.locks.request(lockName, async () => fn()) + return await navigator.locks.request(lockName, async () => { + callbackState.entered = true + return fn() + }) } catch (error) { + if (callbackState.entered) { + throw error + } if (error instanceof DOMException && error.name === `AbortError`) { throw error } @@ -808,12 +1812,12 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina // Helpers // ----------------------------------------------------------------------- - private pruneAppliedEnvelopeIds(): void { + private pruneAppliedEnvelopes(): void { // Keep envelopes for 60 seconds for dedup const cutoff = Date.now() - 60_000 - for (const [id, ts] of this.appliedEnvelopeIds) { - if (ts < cutoff) { - this.appliedEnvelopeIds.delete(id) + for (const [key, envelope] of this.appliedEnvelopes) { + if (envelope.appliedAt < cutoff) { + this.appliedEnvelopes.delete(key) } } } @@ -838,3 +1842,157 @@ function isProtocolEnvelope(data: unknown): data is ProtocolEnvelope { function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)) } + +async function unloadRemoteSubsetOwner( + owner: RemoteSubsetOwner, + options: TransportedLoadSubsetOptions, +): Promise { + try { + const result = ( + owner.unloadSubset as unknown as ( + options: TransportedLoadSubsetOptions, + ) => unknown + )(options) + await Promise.resolve(result) + } catch (error) { + reportRemoteSubsetOwnerError(owner, error) + throw error + } +} + +function reportRemoteSubsetOwnerError( + owner: RemoteSubsetOwner, + error: unknown, +): void { + try { + owner.onError(error) + } catch { + // Reporting must not replace the original owner failure. + } +} + +function appliedEnvelopeKey(collectionId: string, envelopeId: string): string { + return JSON.stringify([collectionId, envelopeId]) +} + +function remoteSubsetAcquisitionKey( + collectionId: string, + acquisitionId: string, +): string { + return JSON.stringify([collectionId, acquisitionId]) +} + +function inboundRemoteSubsetAcquisitionKey( + collectionId: string, + requesterId: string, + acquisitionId: string, +): string { + return JSON.stringify([collectionId, requesterId, acquisitionId]) +} + +function createRPCErrorResponse( + request: RPCRequest, + cause: unknown, +): RPCResponse { + const error = cause instanceof Error ? cause.message : String(cause) + switch (request.type) { + case `rpc:ensureRemoteSubset:req`: + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: false, + error, + } + case `rpc:releaseRemoteSubset:req`: + return { + type: `rpc:releaseRemoteSubset:res`, + rpcId: request.rpcId, + ok: false, + error, + } + case `rpc:ensurePersistedIndex:req`: + return { + type: `rpc:ensurePersistedIndex:res`, + rpcId: request.rpcId, + ok: false, + error, + } + case `rpc:applyLocalMutations:req`: + if (cause instanceof PersistedCollectionDurabilityError) { + return { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: false, + code: `PERSISTENCE_ERROR`, + error, + ...toSafeDurabilityDetails(cause), + } + } + return { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error, + } + case `rpc:applyCommittedTx:req`: + if (cause instanceof PersistedCollectionDurabilityError) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `PERSISTENCE_ERROR`, + error, + ...toSafeDurabilityDetails(cause), + } + } + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error, + } + case `rpc:pullSince:req`: + return { + type: `rpc:pullSince:res`, + rpcId: request.rpcId, + ok: false, + error, + } + } +} + +function isMutatingRPCRequest(request: RPCRequest): request is Extract< + RPCRequest, + { + type: IndeterminateCommitRequestType + } +> { + return ( + request.type === `rpc:applyLocalMutations:req` || + request.type === `rpc:applyCommittedTx:req` + ) +} + +function toSafeDurabilityDetails(error: PersistedCollectionDurabilityError): { + sourceCode?: string | number + path?: string | ReadonlyArray +} { + const sourceCode = + typeof error.code === `string` || typeof error.code === `number` + ? error.code + : undefined + const path = + typeof error.path === `string` || + (Array.isArray(error.path) && + error.path.every( + (part) => typeof part === `string` || typeof part === `number`, + )) + ? (error.path as string | ReadonlyArray) + : undefined + return { + ...(sourceCode === undefined ? {} : { sourceCode }), + ...(path === undefined ? {} : { path }), + } +} diff --git a/packages/browser-db-sqlite-persistence/src/browser-persistence.ts b/packages/browser-db-sqlite-persistence/src/browser-persistence.ts index b36e564900..8e76967a58 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-persistence.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-persistence.ts @@ -131,22 +131,28 @@ export function createBrowserWASQLitePersistence( }) adapterCache.set(cacheKey, adapter) - // Wire the adapter into the multi-tab coordinator so it can handle - // leader-side RPCs (applyCommittedTx, pullSince, ensureIndex, etc.) - if (resolvedCoordinator instanceof BrowserCollectionCoordinator) { - resolvedCoordinator.setAdapter(adapter) - } - return adapter } const createCollectionPersistence = ( mode: PersistedCollectionMode, schemaVersion: number | undefined, - ): PersistedCollectionPersistence => ({ - adapter: getAdapterForCollection(mode, schemaVersion), - coordinator: resolvedCoordinator, - }) + collectionId?: string, + ): PersistedCollectionPersistence => { + const adapter = getAdapterForCollection(mode, schemaVersion) + if (resolvedCoordinator instanceof BrowserCollectionCoordinator) { + if (collectionId === undefined) { + resolvedCoordinator.setAdapter(adapter) + } else { + resolvedCoordinator.setAdapterForCollection(collectionId, adapter) + } + } + + return { + adapter, + coordinator: resolvedCoordinator, + } + } const defaultPersistence = createCollectionPersistence( `sync-absent`, @@ -155,8 +161,8 @@ export function createBrowserWASQLitePersistence( return { ...defaultPersistence, - resolvePersistenceForCollection: ({ mode, schemaVersion }) => - createCollectionPersistence(mode, schemaVersion), + resolvePersistenceForCollection: ({ collectionId, mode, schemaVersion }) => + createCollectionPersistence(mode, schemaVersion, collectionId), resolvePersistenceForMode: (mode) => createCollectionPersistence(mode, undefined), } diff --git a/packages/browser-db-sqlite-persistence/src/index.ts b/packages/browser-db-sqlite-persistence/src/index.ts index 992e8c092f..c6ab075fd8 100644 --- a/packages/browser-db-sqlite-persistence/src/index.ts +++ b/packages/browser-db-sqlite-persistence/src/index.ts @@ -11,9 +11,18 @@ export type { BrowserCollectionCoordinatorOptions } from './browser-coordinator' export { DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + DuplicateRemoteSubsetOwnerError, + IndeterminateCommitError, + PersistedCollectionDurabilityError, + RemoteSubsetWireValueError, persistedCollectionOptions, } from '@tanstack/db-sqlite-persistence-core' export type { PersistedCollectionCoordinator, PersistedCollectionPersistence, + IndeterminateCommitRequestType, + RemoteSubsetOwner, + RemoteSubsetWireExpression, + RemoteSubsetWireValue, + TransportedLoadSubsetOptions, } from '@tanstack/db-sqlite-persistence-core' diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 5f554006e2..98fbcd751b 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -1,7 +1,18 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { IR } from '@tanstack/db' import { BrowserCollectionCoordinator } from '../src/browser-coordinator' +import type { LoadSubsetOptions, Subscription } from '@tanstack/db' +import type { + ApplyCommittedTxRequest, + ApplyLocalMutationsRequest, + IndeterminateCommitError, + PersistedCollectionDurabilityError, + PersistedTx, + PersistenceAdapter, + RemoteSubsetOwner, + TransportedLoadSubsetOptions, +} from '@tanstack/db-sqlite-persistence-core' import type { BrowserCollectionCoordinatorOptions } from '../src/browser-coordinator' -import type { PersistenceAdapter } from '@tanstack/db-sqlite-persistence-core' // --------------------------------------------------------------------------- // BroadcastChannel mock @@ -12,6 +23,11 @@ const channels: Map< string, Set<{ onmessage: MessageHandler | null }> > = new Map() +let dropNextBroadcastMessage: ((data: unknown) => boolean) | undefined +let duplicateNextBroadcastMessage: + | ((data: unknown) => unknown | undefined) + | undefined +let observeBroadcastMessage: ((data: unknown) => void) | undefined class MockBroadcastChannel { readonly name: string @@ -26,6 +42,21 @@ class MockBroadcastChannel { } postMessage(data: unknown): void { + observeBroadcastMessage?.(data) + if (dropNextBroadcastMessage?.(data)) { + dropNextBroadcastMessage = undefined + return + } + + const duplicate = duplicateNextBroadcastMessage?.(data) + if (duplicate !== undefined) { + duplicateNextBroadcastMessage = undefined + } + // BroadcastChannel performs serialization during postMessage, so clone + // failures are synchronous rather than deferred to delivery. + const deliveries = ( + duplicate === undefined ? [data] : [data, duplicate] + ).map((delivery) => structuredClone(delivery)) const peers = channels.get(this.name) if (!peers) return // Deliver to all other instances on same channel (simulating cross-tab) @@ -33,7 +64,9 @@ class MockBroadcastChannel { if (peer !== this && peer.onmessage) { // Use microtask to simulate async delivery const handler = peer.onmessage - queueMicrotask(() => handler({ data: structuredClone(data) })) + for (const delivery of deliveries) { + queueMicrotask(() => handler({ data: structuredClone(delivery) })) + } } } } @@ -43,6 +76,15 @@ class MockBroadcastChannel { } } +function injectBroadcastMessage(channelName: string, data: unknown): void { + for (const endpoint of channels.get(channelName) ?? []) { + const handler = endpoint.onmessage + if (handler) { + queueMicrotask(() => handler({ data: structuredClone(data) })) + } + } +} + // --------------------------------------------------------------------------- // Web Locks mock // --------------------------------------------------------------------------- @@ -162,6 +204,9 @@ function installGlobals(): void { } function cleanupGlobals(): void { + dropNextBroadcastMessage = undefined + duplicateNextBroadcastMessage = undefined + observeBroadcastMessage = undefined channels.clear() heldLocks.clear() lockQueues.clear() @@ -186,15 +231,15 @@ function createStubAdapter(): PersistenceAdapter & { latestSeq: number latestRowVersion: number }> - appliedTxs: Array<{ collectionId: string; txId: string }> + appliedTxs: Array<{ collectionId: string; tx: PersistedTx }> } { - const appliedTxs: Array<{ collectionId: string; txId: string }> = [] + const appliedTxs: Array<{ collectionId: string; tx: PersistedTx }> = [] return { appliedTxs, loadSubset: () => Promise.resolve([]), applyCommittedTx: (collectionId, tx) => { - appliedTxs.push({ collectionId, txId: tx.txId }) + appliedTxs.push({ collectionId, tx }) return Promise.resolve() }, ensureIndex: () => Promise.resolve(), @@ -221,9 +266,248 @@ function createCoordinator( dbName: `test-db`, adapter: adapter ?? createStubAdapter(), } - return new BrowserCollectionCoordinator(opts) + const coordinator = new BrowserCollectionCoordinator(opts) + liveCoordinators.add(coordinator) + return coordinator +} + +const liveCoordinators = new Set() + +function subsetWithNestedValue(value: unknown): LoadSubsetOptions { + return { + where: new IR.Func(`in`, [ + new IR.PropRef([`todos`, `status`]), + new IR.Value([`kept`, value]), + ]), + } +} + +function withUnusedUnloadSubset< + T extends (options: TransportedLoadSubsetOptions) => Promise | void, +>(owner: T): T & RemoteSubsetOwner { + // These fixtures exercise admission and delivery only. The required no-op + // release hook keeps that unrelated scope explicit without weakening the + // production coordinator contract. + return Object.assign(owner, { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) +} + +class UnsupportedSubsetValue { + constructor(readonly label: string) {} +} + +function withEnumerableExpando(value: T): T { + Object.defineProperty(value, `extra`, { + value: `would be lost`, + enumerable: true, + }) + return value +} + +function detachBufferWithView( + kind: `buffer` | `data-view` | `typed-array`, +): ArrayBuffer | DataView | Uint8Array { + const buffer = new ArrayBuffer(4) + const value = + kind === `buffer` + ? buffer + : kind === `data-view` + ? new DataView(buffer) + : new Uint8Array(buffer) + structuredClone(buffer, { transfer: [buffer] }) + return value +} + +type ResizableArrayBufferConstructor = { + new (byteLength: number, options: { maxByteLength: number }): ArrayBuffer +} + +type Float16ArrayConstructor = { + new (values: ArrayLike): ArrayBufferView +} + +function createResizableArrayBuffer(): ArrayBuffer | undefined { + try { + const buffer = new (ArrayBuffer as ResizableArrayBufferConstructor)(4, { + maxByteLength: 8, + }) + return (buffer as ArrayBuffer & { readonly resizable?: boolean }).resizable + ? buffer + : undefined + } catch { + return undefined + } } +const unsupportedValuePath = `options.where.args[1].value[1]` +const resizableArrayBufferValueCases = createResizableArrayBuffer() + ? ([ + [ + `resizable ArrayBuffer`, + () => createResizableArrayBuffer()!, + unsupportedValuePath, + ], + [ + `length-tracking DataView`, + () => new DataView(createResizableArrayBuffer()!), + unsupportedValuePath, + ], + [ + `length-tracking typed array`, + () => new Uint8Array(createResizableArrayBuffer()!), + unsupportedValuePath, + ], + ] as const) + : ([] as const) +const Float16ArrayValue = Reflect.get(globalThis, `Float16Array`) as + | Float16ArrayConstructor + | undefined +const float16ArrayValueCases = Float16ArrayValue + ? ([ + [ + `Float16Array outside the exported typed-array union`, + () => new Float16ArrayValue([1, 2]), + unsupportedValuePath, + ], + ] as const) + : ([] as const) + +const unsupportedSubsetValueCases = [ + [`function`, () => () => {}, unsupportedValuePath], + [`symbol`, () => Symbol(`unsupported`), unsupportedValuePath], + [`WeakMap`, () => new WeakMap(), unsupportedValuePath], + [`WeakSet`, () => new WeakSet(), unsupportedValuePath], + [`Promise`, () => Promise.resolve(`unsupported`), unsupportedValuePath], + [ + `custom prototype`, + () => new UnsupportedSubsetValue(`unsupported`), + unsupportedValuePath, + ], + [ + `custom prototype with a throwing constructor getter`, + () => { + const prototype = Object.create(null) as object + Object.defineProperty(prototype, `constructor`, { + get: () => { + throw new Error(`constructor getter must not be invoked`) + }, + }) + return Object.create(prototype) as object + }, + unsupportedValuePath, + ], + [ + `nonzero RegExp lastIndex`, + () => { + const regexp = /wire/g + regexp.lastIndex = 2 + return regexp + }, + `${unsupportedValuePath}.lastIndex`, + ], + [`SharedArrayBuffer`, () => new SharedArrayBuffer(4), unsupportedValuePath], + [ + `function-valued Map key`, + () => new Map([[() => {}, `value`]]), + `${unsupportedValuePath}.entries[0].key`, + ], + [ + `symbol-valued Map value`, + () => new Map([[`key`, Symbol(`unsupported`)]]), + `${unsupportedValuePath}.entries[0].value`, + ], + [ + `function-valued Set entry`, + () => new Set([() => {}]), + `${unsupportedValuePath}.values[0]`, + ], + [ + `detached view nested in a Map`, + () => new Map([[`key`, detachBufferWithView(`typed-array`)]]), + `${unsupportedValuePath}.entries[0].value`, + ], + [ + `detached ArrayBuffer`, + () => detachBufferWithView(`buffer`), + unsupportedValuePath, + ], + [ + `detached DataView`, + () => detachBufferWithView(`data-view`), + unsupportedValuePath, + ], + [ + `detached typed array`, + () => detachBufferWithView(`typed-array`), + unsupportedValuePath, + ], + [ + `symbol-keyed record`, + () => ({ [Symbol(`unsupported`)]: true }), + `${unsupportedValuePath}[Symbol(unsupported)]`, + ], + [ + `accessor property`, + () => + Object.defineProperty({}, `computed`, { + enumerable: true, + get: () => { + throw new Error(`accessor must not be invoked`) + }, + }), + `${unsupportedValuePath}.computed`, + ], + [ + `non-enumerable property`, + () => Object.defineProperty({}, `hidden`, { value: true }), + `${unsupportedValuePath}.hidden`, + ], + [ + `Date expando`, + () => withEnumerableExpando(new Date(0)), + `${unsupportedValuePath}.extra`, + ], + [ + `RegExp expando`, + () => withEnumerableExpando(/wire/), + `${unsupportedValuePath}.extra`, + ], + [ + `Map expando`, + () => withEnumerableExpando(new Map()), + `${unsupportedValuePath}.extra`, + ], + [ + `Set expando`, + () => withEnumerableExpando(new Set()), + `${unsupportedValuePath}.extra`, + ], + [ + `ArrayBuffer expando`, + () => withEnumerableExpando(new ArrayBuffer(4)), + `${unsupportedValuePath}.extra`, + ], + [ + `DataView expando`, + () => withEnumerableExpando(new DataView(new ArrayBuffer(4))), + `${unsupportedValuePath}.extra`, + ], + [ + `typed array expando`, + () => withEnumerableExpando(new Uint8Array(4)), + `${unsupportedValuePath}.extra`, + ], + [ + `array expando`, + () => withEnumerableExpando([]), + `${unsupportedValuePath}.extra`, + ], + ...resizableArrayBufferValueCases, + ...float16ArrayValueCases, +] as const + async function flush(ms: number = 10): Promise { await new Promise((resolve) => setTimeout(resolve, ms)) } @@ -237,7 +521,10 @@ describe(`BrowserCollectionCoordinator`, () => { installGlobals() }) - afterEach(() => { + afterEach(async () => { + for (const coordinator of liveCoordinators) coordinator.dispose() + liveCoordinators.clear() + await flush(0) cleanupGlobals() }) @@ -460,6 +747,3121 @@ describe(`BrowserCollectionCoordinator`, () => { coord.dispose() }) + + it(`coalesces a local-mutation envelope while application is in flight`, async () => { + const leaderAdapter = createStubAdapter() + const originalApply = leaderAdapter.applyCommittedTx + let releaseApplication = (): void => {} + const applicationGate = new Promise((resolve) => { + releaseApplication = resolve + }) + leaderAdapter.applyCommittedTx = vi.fn(async (collectionId, tx) => { + await applicationGate + return originalApply(collectionId, tx) + }) + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator() + const observedResponses: Array<{ + rpcId: string + term: number + seq: number + latestRowVersion: number + acceptedMutationIds: Array + }> = [] + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + try { + duplicateNextBroadcastMessage = (data) => { + const envelope = data as { payload?: { type?: string } } + if (envelope.payload?.type !== `rpc:applyLocalMutations:req`) { + return undefined + } + const duplicate = structuredClone(data) as { + payload: { rpcId: string } + } + duplicate.payload.rpcId = `coalesced-local-rpc` + return duplicate + } + observeBroadcastMessage = (data) => { + const payload = ( + data as { + payload?: { + type?: string + rpcId?: string + ok?: boolean + term?: number + seq?: number + latestRowVersion?: number + acceptedMutationIds?: Array + } + } + ).payload + if ( + payload?.type === `rpc:applyLocalMutations:res` && + payload.ok === true + ) { + observedResponses.push({ + rpcId: payload.rpcId!, + term: payload.term!, + seq: payload.seq!, + latestRowVersion: payload.latestRowVersion!, + acceptedMutationIds: payload.acceptedMutationIds!, + }) + } + } + + const responsePromise = follower.requestApplyLocalMutations(`todos`, [ + { + mutationId: `mut-in-flight`, + type: `insert`, + key: `todo-in-flight-local`, + value: { id: `todo-in-flight-local` }, + }, + ]) + await flush() + releaseApplication() + const response = await responsePromise + await flush() + + const responsesByRpcId = Object.fromEntries( + observedResponses.map(({ rpcId, ...result }) => [rpcId, result]), + ) + const expectedResult = { + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: [`mut-in-flight`], + } + expect(leaderAdapter.applyCommittedTx).toHaveBeenCalledTimes(1) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + expect(responsesByRpcId).toEqual({ + [response.rpcId]: expectedResult, + 'coalesced-local-rpc': expectedResult, + }) + } finally { + releaseApplication() + observeBroadcastMessage = undefined + duplicateNextBroadcastMessage = undefined + leader.dispose() + follower.dispose() + } + }) + + it(`replays the exact local-mutation success after same-leader response loss`, async () => { + const leaderAdapter = createStubAdapter() + const followerAdapter = createStubAdapter() + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator(followerAdapter) + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + vi.useFakeTimers() + try { + let droppedResponse: unknown + dropNextBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type !== `rpc:applyLocalMutations:res`) return false + droppedResponse = structuredClone(payload) + return true + } + + const responsePromise = follower.requestApplyLocalMutations(`todos`, [ + { + mutationId: `local-response-loss`, + type: `insert`, + key: `todo-local-response-loss`, + value: { id: `todo-local-response-loss` }, + }, + ]) + + await vi.advanceTimersByTimeAsync(0) + expect(droppedResponse).toMatchObject({ + type: `rpc:applyLocalMutations:res`, + ok: true, + }) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + + await vi.advanceTimersByTimeAsync(10_200) + const response = await responsePromise + + expect(response).toEqual(droppedResponse) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + expect(followerAdapter.appliedTxs).toEqual([]) + } finally { + leader.dispose() + follower.dispose() + vi.useRealTimers() + } + }) + + it(`fails indeterminate when a local-mutation success is lost across leader change`, async () => { + const retiredLeaderAdapter = createStubAdapter() + const requesterAdapter = createStubAdapter() + const retiredLeader = createCoordinator(retiredLeaderAdapter) + const requester = createCoordinator(requesterAdapter) + + retiredLeader.subscribe(`todos`, () => {}) + requester.subscribe(`todos`, () => {}) + await flush(50) + + vi.useFakeTimers() + try { + let publications = 0 + observeBroadcastMessage = (data) => { + if ( + (data as { payload?: { type?: string } }).payload?.type === + `tx:committed` + ) { + publications++ + } + } + dropNextBroadcastMessage = (data) => + (data as { payload?: { type?: string } }).payload?.type === + `rpc:applyLocalMutations:res` + const outcomePromise = requester + .requestApplyLocalMutations(`todos`, [ + { + mutationId: `local-requester-takeover`, + type: `insert`, + key: `local-requester-takeover`, + value: { id: `local-requester-takeover` }, + }, + ]) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + await vi.advanceTimersByTimeAsync(0) + expect(retiredLeaderAdapter.appliedTxs).toHaveLength(1) + retiredLeader.dispose() + await vi.advanceTimersByTimeAsync(0) + expect(requester.isLeader(`todos`)).toBe(true) + + await vi.advanceTimersByTimeAsync(31_000) + const outcome = await outcomePromise + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyLocalMutations:req`, + previousLeaderId: (retiredLeader as unknown as { nodeId: string }) + .nodeId, + previousTerm: 1, + currentLeaderId: (requester as unknown as { nodeId: string }) + .nodeId, + currentTerm: 1, + cause: expect.objectContaining({ + message: expect.stringContaining(`timed out`), + }), + } satisfies Partial), + }) + expect(retiredLeaderAdapter.appliedTxs).toHaveLength(1) + expect(requesterAdapter.appliedTxs).toEqual([]) + expect(publications).toBe(1) + } finally { + observeBroadcastMessage = undefined + retiredLeader.dispose() + requester.dispose() + vi.useRealTimers() + } + }) + + it(`fails indeterminate instead of retrying local mutations without an initial leader route`, async () => { + const coordinator = createCoordinator() + const transportError = new Error(`unknown leader response was lost`) + const attemptedMutations: Array = + [] + const failedTransport = vi.fn( + (_collectionId: string, request: ApplyLocalMutationsRequest) => { + attemptedMutations.push(structuredClone(request.mutations)) + if (attemptedMutations.length === 1) { + return Promise.reject(transportError) + } + return Promise.resolve({ + type: `rpc:applyLocalMutations:res` as const, + rpcId: request.rpcId, + ok: true as const, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: request.mutations.map( + (mutation) => mutation.mutationId, + ), + }) + }, + ) + Object.defineProperty(coordinator, `sendRPCOnce`, { + value: failedTransport, + configurable: true, + }) + + try { + const outcome = await coordinator + .requestApplyLocalMutations(`todos`, [ + { + mutationId: `browser-unknown-leader-local`, + type: `insert`, + key: `browser-unknown-leader-local`, + value: { id: `browser-unknown-leader-local` }, + }, + ]) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyLocalMutations:req`, + previousLeaderId: null, + previousTerm: null, + currentLeaderId: null, + currentTerm: null, + cause: transportError, + } satisfies Partial), + }) + expect(failedTransport).toHaveBeenCalledTimes(1) + expect(attemptedMutations).toEqual([ + [ + { + mutationId: `browser-unknown-leader-local`, + type: `insert`, + key: `browser-unknown-leader-local`, + value: { id: `browser-unknown-leader-local` }, + }, + ], + ]) + } finally { + coordinator.dispose() + } + }) + + it(`classifies local-mutation durability failures on local and follower routes`, async () => { + const failure = Object.assign( + new Error(`local mutation persistence failed`), + { + code: `SQLITE_IOERR`, + path: [`todos`, `rows`], + }, + ) + const adapter = createStubAdapter() + adapter.applyCommittedTx = vi.fn().mockRejectedValue(failure) + const leader = createCoordinator(adapter) + const follower = createCoordinator(createStubAdapter()) + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + try { + const followerResponse = await follower.requestApplyLocalMutations( + `todos`, + [ + { + mutationId: `remote-durability-failure`, + type: `insert`, + key: `remote-durability-failure`, + value: { id: `remote-durability-failure` }, + }, + ], + ) + + expect(followerResponse).toEqual({ + type: `rpc:applyLocalMutations:res`, + rpcId: expect.any(String), + ok: false, + code: `PERSISTENCE_ERROR`, + error: expect.stringContaining(failure.message), + sourceCode: `SQLITE_IOERR`, + path: [`todos`, `rows`], + }) + expect(adapter.applyCommittedTx).toHaveBeenCalledTimes(1) + + await expect( + leader.requestApplyLocalMutations(`todos`, [ + { + mutationId: `local-durability-failure`, + type: `insert`, + key: `local-durability-failure`, + value: { id: `local-durability-failure` }, + }, + ]), + ).rejects.toMatchObject({ + name: `PersistedCollectionDurabilityError`, + code: `SQLITE_IOERR`, + path: [`todos`, `rows`], + cause: failure, + } satisfies Partial) + expect(adapter.applyCommittedTx).toHaveBeenCalledTimes(2) + } finally { + leader.dispose() + follower.dispose() + } + }) + }) + + describe(`RPC - applyCommittedTx`, () => { + it(`routes a complete source transaction to the leader-owned adapter`, async () => { + const leaderAdapter = createStubAdapter() + const followerAdapter = createStubAdapter() + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator(followerAdapter) + const followerMessages: Array = [] + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, (message) => { + if ((message.payload as { type?: string }).type === `tx:committed`) { + followerMessages.push(message.payload) + } + }) + await flush(50) + + const response = await follower.requestApplyCommittedTx(`todos`, { + txId: `source-tx`, + term: 91, + seq: 92, + rowVersion: 93, + truncate: true, + mutations: [ + { + type: `insert`, + key: `todo-1`, + value: { id: `todo-1`, title: `Owned source row` }, + }, + ], + rowMetadataMutations: [ + { type: `set`, key: `todo-1`, value: { source: `remote` } }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `resume`, value: { offset: `next` } }, + ], + }) + + expect(response.ok).toBe(true) + expect(followerAdapter.appliedTxs).toEqual([]) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + expect(leaderAdapter.appliedTxs[0]).toMatchObject({ + collectionId: `todos`, + tx: { + txId: `source-tx`, + truncate: true, + mutations: [ + { + type: `insert`, + key: `todo-1`, + value: { id: `todo-1`, title: `Owned source row` }, + }, + ], + rowMetadataMutations: [ + { type: `set`, key: `todo-1`, value: { source: `remote` } }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `resume`, value: { offset: `next` } }, + ], + }, + }) + expect(leaderAdapter.appliedTxs[0]!.tx.term).not.toBe(91) + expect(leaderAdapter.appliedTxs[0]!.tx.seq).not.toBe(92) + expect(leaderAdapter.appliedTxs[0]!.tx.rowVersion).not.toBe(93) + + await flush() + expect(followerMessages).toHaveLength(1) + expect(followerMessages[0]).toEqual({ + type: `tx:committed`, + term: expect.any(Number), + seq: expect.any(Number), + txId: `source-tx`, + latestRowVersion: expect.any(Number), + requiresFullReload: true, + }) + + leader.dispose() + follower.dispose() + }) + + it(`classifies committed durability failures on local and follower routes`, async () => { + const leaderAdapter = createStubAdapter() + const failure = Object.assign(new Error(`persistence failed`), { + code: `SQLITE_FULL`, + path: `/tmp/browser.sqlite`, + }) + leaderAdapter.applyCommittedTx = vi.fn().mockRejectedValue(failure) + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator(createStubAdapter()) + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + const response = await follower.requestApplyCommittedTx(`todos`, { + txId: `source-tx-failure`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }) + + expect(response).toEqual({ + type: `rpc:applyCommittedTx:res`, + rpcId: expect.any(String), + ok: false, + code: `PERSISTENCE_ERROR`, + error: expect.stringContaining(`persistence failed`), + sourceCode: `SQLITE_FULL`, + path: `/tmp/browser.sqlite`, + }) + expect(leaderAdapter.applyCommittedTx).toHaveBeenCalledTimes(1) + + await expect( + leader.requestApplyCommittedTx(`todos`, { + txId: `source-tx-local-failure`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }), + ).rejects.toMatchObject({ + name: `PersistedCollectionDurabilityError`, + code: `SQLITE_FULL`, + path: `/tmp/browser.sqlite`, + cause: failure, + } satisfies Partial) + expect(leaderAdapter.applyCommittedTx).toHaveBeenCalledTimes(2) + + leader.dispose() + follower.dispose() + }) + + it(`does not retry application failures after entering the writer lock`, async () => { + const applicationError = new Error(`persistence failed once`) + const adapter = createStubAdapter() + adapter.applyCommittedTx = vi.fn().mockRejectedValue(applicationError) + const coordinator = createCoordinator(adapter) + + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + + vi.useFakeTimers() + try { + let settled = false + const outcomePromise = coordinator + .requestApplyCommittedTx(`todos`, { + txId: `source-tx-application-failure`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }) + .then( + (response) => { + settled = true + return { response } + }, + (error: unknown) => { + settled = true + return { error } + }, + ) + + await vi.advanceTimersByTimeAsync(0) + const firstTurn = { + applicationCalls: vi.mocked(adapter.applyCommittedTx).mock.calls + .length, + settled, + } + + coordinator.dispose() + await vi.advanceTimersByTimeAsync(5_000) + const outcome = await outcomePromise + + expect(firstTurn).toEqual({ applicationCalls: 1, settled: true }) + expect(adapter.applyCommittedTx).toHaveBeenCalledTimes(1) + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `PersistedCollectionDurabilityError`, + cause: applicationError, + }), + }) + } finally { + coordinator.dispose() + vi.useRealTimers() + } + }) + + it(`replays the successful response when only that response is lost`, async () => { + const leaderAdapter = createStubAdapter() + const followerAdapter = createStubAdapter() + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator(followerAdapter) + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + vi.useFakeTimers() + try { + let droppedResponse: unknown + dropNextBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type !== `rpc:applyCommittedTx:res`) return false + droppedResponse = structuredClone(payload) + return true + } + + const responsePromise = follower.requestApplyCommittedTx(`todos`, { + txId: `source-tx-response-loss`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `todo-response-loss`, + value: { id: `todo-response-loss` }, + }, + ], + }) + + await vi.advanceTimersByTimeAsync(0) + expect(droppedResponse).toMatchObject({ + type: `rpc:applyCommittedTx:res`, + ok: true, + }) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + + await vi.advanceTimersByTimeAsync(10_200) + const response = await responsePromise + + expect(response).toEqual(droppedResponse) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + expect(followerAdapter.appliedTxs).toEqual([]) + } finally { + leader.dispose() + follower.dispose() + vi.useRealTimers() + } + }) + + it(`coalesces the same envelope while its first application is in flight`, async () => { + const leaderAdapter = createStubAdapter() + const followerAdapter = createStubAdapter() + const originalApply = leaderAdapter.applyCommittedTx + let releaseApplication = (): void => {} + const applicationGate = new Promise((resolve) => { + releaseApplication = resolve + }) + leaderAdapter.applyCommittedTx = vi.fn(async (collectionId, tx) => { + await applicationGate + return originalApply(collectionId, tx) + }) + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator(followerAdapter) + const observedResponses: Array<{ + rpcId: string + term: number + seq: number + latestRowVersion: number + }> = [] + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + try { + duplicateNextBroadcastMessage = (data) => { + const envelope = data as { + payload?: { type?: string; rpcId?: string } + } + if (envelope.payload?.type !== `rpc:applyCommittedTx:req`) { + return undefined + } + const duplicate = structuredClone(data) as { + payload: { rpcId: string } + } + duplicate.payload.rpcId = `coalesced-rpc` + return duplicate + } + observeBroadcastMessage = (data) => { + const payload = ( + data as { + payload?: { + type?: string + rpcId?: string + ok?: boolean + term?: number + seq?: number + latestRowVersion?: number + } + } + ).payload + if ( + payload?.type === `rpc:applyCommittedTx:res` && + payload.ok === true + ) { + observedResponses.push({ + rpcId: payload.rpcId!, + term: payload.term!, + seq: payload.seq!, + latestRowVersion: payload.latestRowVersion!, + }) + } + } + + const responsePromise = follower.requestApplyCommittedTx(`todos`, { + txId: `source-tx-in-flight-envelope`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `todo-in-flight-envelope`, + value: { id: `todo-in-flight-envelope` }, + }, + ], + }) + await flush() + const callsBeforeRelease = vi.mocked(leaderAdapter.applyCommittedTx) + .mock.calls.length + + releaseApplication() + const response = await responsePromise + await flush() + + const responsesByRpcId = Object.fromEntries( + observedResponses.map(({ rpcId, ...position }) => [rpcId, position]), + ) + const expectedPosition = { + term: 1, + seq: 1, + latestRowVersion: 1, + } + expect({ + callsBeforeRelease, + appliedTxs: leaderAdapter.appliedTxs.length, + responsesByRpcId, + }).toEqual({ + callsBeforeRelease: 1, + appliedTxs: 1, + responsesByRpcId: { + [response.rpcId]: expectedPosition, + 'coalesced-rpc': expectedPosition, + }, + }) + expect(followerAdapter.appliedTxs).toEqual([]) + } finally { + releaseApplication() + observeBroadcastMessage = undefined + duplicateNextBroadcastMessage = undefined + leader.dispose() + follower.dispose() + } + }) + + it(`allows the same envelope to retry after its application fails`, async () => { + const applicationError = new Error(`first application failed`) + const leaderAdapter = createStubAdapter() + const originalApply = leaderAdapter.applyCommittedTx + leaderAdapter.applyCommittedTx = vi + .fn() + .mockRejectedValueOnce(applicationError) + .mockImplementation((collectionId, tx) => + originalApply(collectionId, tx), + ) + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator() + let requestEnvelope: unknown + const observedResponses: Array<{ ok?: boolean; error?: string }> = [] + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + try { + observeBroadcastMessage = (data) => { + const payload = ( + data as { + payload?: { type?: string; ok?: boolean; error?: string } + } + ).payload + if ( + payload?.type === `rpc:applyCommittedTx:req` && + requestEnvelope === undefined + ) { + requestEnvelope = structuredClone(data) + } + if (payload?.type === `rpc:applyCommittedTx:res`) { + observedResponses.push({ + ok: payload.ok, + ...(payload.error === undefined ? {} : { error: payload.error }), + }) + } + } + + const firstResponse = await follower.requestApplyCommittedTx(`todos`, { + txId: `source-tx-retry-after-failure`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }) + expect(firstResponse).toMatchObject({ + ok: false, + code: `PERSISTENCE_ERROR`, + error: expect.stringContaining(applicationError.message), + }) + expect(leaderAdapter.applyCommittedTx).toHaveBeenCalledTimes(1) + expect(requestEnvelope).toBeDefined() + + injectBroadcastMessage(`tsdb:coord:test-db`, requestEnvelope) + await flush() + + expect(leaderAdapter.applyCommittedTx).toHaveBeenCalledTimes(2) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + expect(observedResponses).toEqual([ + { + ok: false, + error: expect.stringContaining(applicationError.message), + }, + { ok: true }, + ]) + } finally { + observeBroadcastMessage = undefined + leader.dispose() + follower.dispose() + } + }) + + it(`rejects a different mutation operation that reuses an in-flight envelope`, async () => { + const leaderAdapter = createStubAdapter() + const originalApply = leaderAdapter.applyCommittedTx + let releaseApplication = (): void => {} + const applicationGate = new Promise((resolve) => { + releaseApplication = resolve + }) + leaderAdapter.applyCommittedTx = vi.fn(async (collectionId, tx) => { + await applicationGate + return originalApply(collectionId, tx) + }) + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator() + let crossOperationResponse: unknown + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + try { + duplicateNextBroadcastMessage = (data) => { + const envelope = data as { + payload?: { type?: string; envelopeId?: string } + } + if (envelope.payload?.type !== `rpc:applyCommittedTx:req`) { + return undefined + } + const duplicate = structuredClone(data) as { + payload: Record + } + duplicate.payload = { + type: `rpc:applyLocalMutations:req`, + rpcId: `cross-operation-rpc`, + envelopeId: envelope.payload.envelopeId, + mutations: [], + } + return duplicate + } + observeBroadcastMessage = (data) => { + const payload = (data as { payload?: { rpcId?: string } }).payload + if (payload?.rpcId === `cross-operation-rpc`) { + crossOperationResponse = structuredClone(payload) + } + } + + const responsePromise = follower.requestApplyCommittedTx(`todos`, { + txId: `source-tx-cross-operation-envelope`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }) + await flush() + + expect(crossOperationResponse).toMatchObject({ + type: `rpc:applyLocalMutations:res`, + rpcId: `cross-operation-rpc`, + ok: false, + code: `CONFLICT`, + error: expect.stringContaining(`is already in flight`), + }) + expect(leaderAdapter.applyCommittedTx).toHaveBeenCalledTimes(1) + + releaseApplication() + const response = await responsePromise + expect(response).toMatchObject({ ok: true, seq: 1 }) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + } finally { + releaseApplication() + observeBroadcastMessage = undefined + duplicateNextBroadcastMessage = undefined + leader.dispose() + follower.dispose() + } + }) + + it(`fails indeterminate when a committed success is lost across leader change`, async () => { + const retiredLeaderAdapter = createStubAdapter() + const requesterAdapter = createStubAdapter() + const retiredLeader = createCoordinator(retiredLeaderAdapter) + const requester = createCoordinator(requesterAdapter) + + retiredLeader.subscribe(`todos`, () => {}) + requester.subscribe(`todos`, () => {}) + await flush(50) + expect(retiredLeader.isLeader(`todos`)).toBe(true) + expect(requester.isLeader(`todos`)).toBe(false) + + vi.useFakeTimers() + try { + let publications = 0 + observeBroadcastMessage = (data) => { + if ( + (data as { payload?: { type?: string } }).payload?.type === + `tx:committed` + ) { + publications++ + } + } + let droppedResponse: unknown + dropNextBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type !== `rpc:applyCommittedTx:res`) return false + droppedResponse = structuredClone(payload) + return true + } + + const outcomePromise = requester + .requestApplyCommittedTx(`todos`, { + txId: `source-tx-requester-takeover`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `todo-requester-takeover`, + value: { id: `todo-requester-takeover` }, + }, + ], + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + await vi.advanceTimersByTimeAsync(0) + expect(droppedResponse).toMatchObject({ ok: true }) + expect(retiredLeaderAdapter.appliedTxs).toHaveLength(1) + + retiredLeader.dispose() + await vi.advanceTimersByTimeAsync(0) + expect(requester.isLeader(`todos`)).toBe(true) + + await vi.advanceTimersByTimeAsync(31_000) + const outcome = await outcomePromise + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyCommittedTx:req`, + previousLeaderId: (retiredLeader as unknown as { nodeId: string }) + .nodeId, + previousTerm: 1, + currentLeaderId: (requester as unknown as { nodeId: string }) + .nodeId, + currentTerm: 1, + cause: expect.objectContaining({ + message: expect.stringContaining(`timed out`), + }), + } satisfies Partial), + }) + expect(retiredLeaderAdapter.appliedTxs).toHaveLength(1) + expect(requesterAdapter.appliedTxs).toEqual([]) + expect(publications).toBe(1) + } finally { + observeBroadcastMessage = undefined + retiredLeader.dispose() + requester.dispose() + vi.useRealTimers() + } + }) + + it(`fails indeterminate instead of retrying a committed transaction without an initial leader route`, async () => { + const coordinator = createCoordinator() + const transportError = new Error(`unknown leader response was lost`) + const attemptedTransactions: Array = [] + const failedTransport = vi.fn( + (_collectionId: string, request: ApplyCommittedTxRequest) => { + attemptedTransactions.push(structuredClone(request.tx)) + if (attemptedTransactions.length === 1) { + return Promise.reject(transportError) + } + return Promise.resolve({ + type: `rpc:applyCommittedTx:res` as const, + rpcId: request.rpcId, + ok: true as const, + term: 1, + seq: 1, + latestRowVersion: 1, + }) + }, + ) + Object.defineProperty(coordinator, `sendRPCOnce`, { + value: failedTransport, + configurable: true, + }) + + try { + const outcome = await coordinator + .requestApplyCommittedTx(`todos`, { + txId: `browser-unknown-leader-committed`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `browser-unknown-leader-committed`, + value: { id: `browser-unknown-leader-committed` }, + }, + ], + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyCommittedTx:req`, + previousLeaderId: null, + previousTerm: null, + currentLeaderId: null, + currentTerm: null, + cause: transportError, + } satisfies Partial), + }) + expect(failedTransport).toHaveBeenCalledTimes(1) + expect(attemptedTransactions).toEqual([ + { + txId: `browser-unknown-leader-committed`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `browser-unknown-leader-committed`, + value: { id: `browser-unknown-leader-committed` }, + }, + ], + }, + ]) + } finally { + coordinator.dispose() + } + }) + + it(`does not retain completed or in-flight committed transactions after disposal`, async () => { + const adapter = createStubAdapter() + const originalApply = adapter.applyCommittedTx + let heldApplyEntered = false + let releaseHeldApply = (): void => {} + const heldApplyGate = new Promise((resolve) => { + releaseHeldApply = resolve + }) + adapter.applyCommittedTx = vi.fn(async (collectionId, tx) => { + if (tx.txId === `held-during-disposal`) { + heldApplyEntered = true + await heldApplyGate + } + await originalApply(collectionId, tx) + }) + const coordinator = createCoordinator(adapter) + const internals = coordinator as unknown as { + handleApplyCommittedTx: ( + collectionId: string, + request: ApplyCommittedTxRequest, + ) => Promise<{ + ok: boolean + code?: string + }> + appliedEnvelopes: Map + inFlightEnvelopes: Map + } + const completedRequest: ApplyCommittedTxRequest = { + type: `rpc:applyCommittedTx:req`, + rpcId: `completed-rpc`, + envelopeId: `completed-envelope`, + tx: { + txId: `completed-before-disposal`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }, + } + const heldRequest: ApplyCommittedTxRequest = { + type: `rpc:applyCommittedTx:req`, + rpcId: `held-rpc`, + envelopeId: `held-envelope`, + tx: { + txId: `held-during-disposal`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }, + } + + coordinator.subscribe(`todos`, () => {}) + await flush(50) + + try { + expect( + await internals.handleApplyCommittedTx(`todos`, completedRequest), + ).toMatchObject({ ok: true }) + const heldResponse = internals.handleApplyCommittedTx( + `todos`, + heldRequest, + ) + await vi.waitFor(() => expect(heldApplyEntered).toBe(true)) + expect({ + completed: internals.appliedEnvelopes.size, + inFlight: internals.inFlightEnvelopes.size, + }).toEqual({ completed: 1, inFlight: 1 }) + + coordinator.dispose() + expect({ + completed: internals.appliedEnvelopes.size, + inFlight: internals.inFlightEnvelopes.size, + }).toEqual({ completed: 0, inFlight: 0 }) + + releaseHeldApply() + await expect(heldResponse).resolves.toMatchObject({ ok: true }) + expect({ + completed: internals.appliedEnvelopes.size, + inFlight: internals.inFlightEnvelopes.size, + }).toEqual({ completed: 0, inFlight: 0 }) + await expect( + internals.handleApplyCommittedTx(`todos`, completedRequest), + ).resolves.toMatchObject({ ok: false, code: `NOT_LEADER` }) + } finally { + releaseHeldApply() + coordinator.dispose() + } + }) + + it(`does not answer a held follower request after its leader is disposed`, async () => { + const leaderAdapter = createStubAdapter() + const originalApply = leaderAdapter.applyCommittedTx + let applyEntered = false + let releaseApply = (): void => {} + const applyGate = new Promise((resolve) => { + releaseApply = resolve + }) + leaderAdapter.applyCommittedTx = vi.fn(async (collectionId, tx) => { + applyEntered = true + await applyGate + await originalApply(collectionId, tx) + }) + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator(createStubAdapter()) + const leaderInternals = leader as unknown as { + appliedEnvelopes: Map + inFlightEnvelopes: Map + } + const followerInternals = follower as unknown as { + sendRPCOnce: ( + collectionId: string, + request: ApplyCommittedTxRequest, + ) => Promise + } + let responsePosts = 0 + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + try { + observeBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type === `rpc:applyCommittedTx:res`) { + responsePosts++ + } + } + const outcomePromise = followerInternals + .sendRPCOnce(`todos`, { + type: `rpc:applyCommittedTx:req`, + rpcId: `held-follower-rpc`, + envelopeId: `held-follower-envelope`, + tx: { + txId: `held-follower-tx`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }, + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + await vi.waitFor(() => expect(applyEntered).toBe(true)) + leader.dispose() + releaseApply() + await vi.waitFor(() => expect(leaderAdapter.appliedTxs).toHaveLength(1)) + await flush() + + expect({ + responsePosts, + completed: leaderInternals.appliedEnvelopes.size, + inFlight: leaderInternals.inFlightEnvelopes.size, + }).toEqual({ responsePosts: 0, completed: 0, inFlight: 0 }) + + follower.dispose() + const outcome = await outcomePromise + expect(outcome).toEqual({ error: new Error(`coordinator disposed`) }) + } finally { + observeBroadcastMessage = undefined + releaseApply() + leader.dispose() + follower.dispose() + } + }) + }) + + describe(`RPC - ensureRemoteSubset`, () => { + it(`rejects a second live remote subset owner instead of replacing the first`, () => { + const coordinator = createCoordinator() + const first = Object.assign( + vi.fn((_options: TransportedLoadSubsetOptions) => Promise.resolve()), + { + loadSubset: vi.fn((_options: TransportedLoadSubsetOptions) => + Promise.resolve(), + ), + unloadSubset: vi.fn((_options: TransportedLoadSubsetOptions) => {}), + onError: vi.fn(), + }, + ) + const second = Object.assign( + vi.fn((_options: TransportedLoadSubsetOptions) => Promise.resolve()), + { + loadSubset: vi.fn((_options: TransportedLoadSubsetOptions) => + Promise.resolve(), + ), + unloadSubset: vi.fn((_options: TransportedLoadSubsetOptions) => {}), + onError: vi.fn(), + }, + ) + const unregisterFirst = coordinator.registerRemoteSubsetOwner( + `todos`, + first, + ) + + try { + expect(() => + coordinator.registerRemoteSubsetOwner(`todos`, second), + ).toThrowError( + expect.objectContaining({ + name: `DuplicateRemoteSubsetOwnerError`, + collectionId: `todos`, + }), + ) + } finally { + unregisterFirst() + coordinator.dispose() + } + }) + + it(`rejects an unsupported nested membership value before local owner work`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + + const owner = withUnusedUnloadSubset(vi.fn(() => Promise.resolve())) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + + try { + await expect( + coordinator.requestEnsureRemoteSubset( + `todos`, + subsetWithNestedValue(() => {}), + ), + ).rejects.toMatchObject({ + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1].value[1]`, + message: `Unsupported remote subset wire value at options.where.args[1].value[1]: function`, + }) + expect(owner).not.toHaveBeenCalled() + } finally { + unregisterOwner() + coordinator.dispose() + } + }) + + it(`rejects a sparse function argument at its exact Browser wire path`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + const owner = withUnusedUnloadSubset(vi.fn()) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const args = [new IR.PropRef([`todos`, `status`])] + args.length = 2 + + try { + await expect( + coordinator.requestEnsureRemoteSubset(`todos`, { + where: new IR.Func(`eq`, args), + }), + ).rejects.toMatchObject({ + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1]`, + }) + expect(owner).not.toHaveBeenCalled() + } finally { + unregisterOwner() + coordinator.dispose() + } + }) + + it(`holds same-stack Browser subset reentry behind the original owner load`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + const options: LoadSubsetOptions = { limit: 1 } + let releaseLoad = (): void => {} + const loadGate = new Promise((resolve) => { + releaseLoad = resolve + }) + let duplicate: Promise | undefined + let didReenter = false + const owner = Object.assign( + vi.fn(() => { + if (!didReenter) { + didReenter = true + duplicate = coordinator.requestEnsureRemoteSubset(`todos`, options) + } + return loadGate + }), + { unloadSubset: vi.fn(), onError: vi.fn() }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + + try { + let firstSettled = false + let duplicateSettled = false + const first = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then(() => { + firstSettled = true + }) + await vi.waitFor(() => expect(duplicate).toBeDefined()) + const duplicateResult = duplicate!.then(() => { + duplicateSettled = true + }) + await flush(0) + + expect({ + ownerCalls: owner.mock.calls.length, + firstSettled, + duplicateSettled, + }).toEqual({ + ownerCalls: 1, + firstSettled: false, + duplicateSettled: false, + }) + + releaseLoad() + await Promise.all([first, duplicateResult]) + } finally { + releaseLoad() + unregisterOwner() + coordinator.dispose() + } + }) + + it(`compacts a terminal same-stack Browser release after the real owner load finishes`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + const options: LoadSubsetOptions = { limit: 1 } + let releaseLoad = (): void => {} + const loadGate = new Promise((resolve) => { + releaseLoad = resolve + }) + let release: Promise | undefined + const events: Array = [] + const owner = Object.assign( + vi.fn(() => { + events.push(`load`) + release = coordinator.requestReleaseRemoteSubset(`todos`, options) + return loadGate + }), + { + unloadSubset: vi.fn(() => { + events.push(`unload`) + }), + onError: vi.fn(), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const internals = coordinator as unknown as { + outboundRemoteSubsetAcquisitions: Map + inboundRemoteSubsetAcquisitions: Map> + } + + try { + let releaseSettled = false + const load = coordinator.requestEnsureRemoteSubset(`todos`, options) + await vi.waitFor(() => expect(release).toBeDefined()) + const terminalRelease = release!.then(() => { + releaseSettled = true + }) + await flush(0) + expect({ events: [...events], releaseSettled }).toEqual({ + events: [`load`], + releaseSettled: false, + }) + + releaseLoad() + await Promise.all([load, terminalRelease]) + const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() + expect({ + events, + outbound: internals.outboundRemoteSubsetAcquisitions.size, + inbound: internals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + events: [`load`, `unload`], + outbound: 0, + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + releaseLoad() + unregisterOwner() + coordinator.dispose() + } + }) + + it(`releases a transferred Browser lease whose initial load rejected`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + const loadError = new Error(`browser transferred load failed`) + const ownerErrors: Array = [] + const owner = Object.assign( + vi.fn((_options: TransportedLoadSubsetOptions) => + Promise.reject(loadError), + ), + { + unloadSubset: vi.fn( + (_options: TransportedLoadSubsetOptions) => undefined, + ), + onError: (error: unknown) => ownerErrors.push(error), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { offset: 20 } + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + inboundRemoteSubsetAcquisitions: Map> + } + + try { + const ensureError = await coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then( + () => undefined, + (error: unknown) => error, + ) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + await flush(0) + const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() + + expect(ensureError).toBe(loadError) + expect(owner).toHaveBeenCalledTimes(1) + expect(owner.unloadSubset).toHaveBeenCalledTimes(1) + expect(owner.unloadSubset.mock.calls[0]?.[0]).toBe( + owner.mock.calls[0]?.[0], + ) + expect(ownerErrors).toEqual([loadError]) + expect(unhandled).toEqual([]) + expect({ + inbound: internals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`keeps a Browser release tombstone when a transferred load rejects concurrently`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + const loadError = new Error(`browser concurrent transferred load failed`) + let rejectLoad = (_error: unknown): void => {} + const loadGate = new Promise((_resolve, reject) => { + rejectLoad = reject + }) + const ownerErrors: Array = [] + const owner = Object.assign( + vi.fn((_options: TransportedLoadSubsetOptions) => loadGate), + { + unloadSubset: vi.fn( + (_options: TransportedLoadSubsetOptions) => undefined, + ), + onError: (error: unknown) => ownerErrors.push(error), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { offset: 21 } + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + nodeId: string + outboundRemoteSubsetAcquisitions: Map + inboundRemoteSubsetAcquisitions: Map> + handleEnsureRemoteSubset: ( + collectionId: string, + request: { + type: `rpc:ensureRemoteSubset:req` + rpcId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + }, + requesterId: string, + ) => Promise<{ ok: boolean }> + } + + try { + const ensure = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then( + () => ({ status: `fulfilled` as const }), + (error: unknown) => ({ status: `rejected` as const, error }), + ) + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(1)) + const [outbound] = internals.outboundRemoteSubsetAcquisitions.values() + const release = coordinator + .requestReleaseRemoteSubset(`todos`, options) + .then( + () => ({ status: `fulfilled` as const }), + (error: unknown) => ({ status: `rejected` as const, error }), + ) + rejectLoad(loadError) + const outcomes = await Promise.all([ensure, release]) + const duplicate = await internals + .handleEnsureRemoteSubset( + `todos`, + { + type: `rpc:ensureRemoteSubset:req`, + rpcId: `delayed-browser-duplicate`, + acquisitionId: outbound!.acquisitionId, + options: owner.mock.calls[0]![0], + }, + internals.nodeId, + ) + .then( + (response) => ({ status: `fulfilled` as const, response }), + (error: unknown) => ({ status: `rejected` as const, error }), + ) + await flush(0) + const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() + + expect(outcomes).toEqual([ + { status: `rejected`, error: loadError }, + { status: `fulfilled` }, + ]) + expect(duplicate).toEqual({ + status: `fulfilled`, + response: expect.objectContaining({ ok: true }), + }) + expect(owner).toHaveBeenCalledTimes(1) + expect(owner.unloadSubset).toHaveBeenCalledTimes(1) + expect(ownerErrors).toEqual([loadError]) + expect(unhandled).toEqual([]) + expect({ + inbound: internals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + rejectLoad(loadError) + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`rejects unsupported nested values before follower transport or owner work`, async () => { + const leader = createCoordinator() + const follower = createCoordinator() + const owner = withUnusedUnloadSubset(vi.fn(() => Promise.resolve())) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + let subsetPosts = 0 + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + expect(leader.isLeader(`todos`)).toBe(true) + expect(follower.isLeader(`todos`)).toBe(false) + observeBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type === `rpc:ensureRemoteSubset:req`) subsetPosts++ + } + + try { + await expect( + follower.requestEnsureRemoteSubset( + `todos`, + subsetWithNestedValue(() => {}), + ), + ).rejects.toMatchObject({ + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1].value[1]`, + message: `Unsupported remote subset wire value at options.where.args[1].value[1]: function`, + }) + expect({ subsetPosts, ownerCalls: owner.mock.calls.length }).toEqual({ + subsetPosts: 0, + ownerCalls: 0, + }) + } finally { + observeBroadcastMessage = undefined + unregisterOwner() + leader.dispose() + follower.dispose() + } + }) + + it.each(unsupportedSubsetValueCases)( + `rejects a nested %s through both local and follower admission`, + async (_label, createUnsupported, path) => { + const leader = createCoordinator() + const follower = createCoordinator() + const owner = withUnusedUnloadSubset(vi.fn(() => Promise.resolve())) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + let subsetPosts = 0 + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + observeBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type === `rpc:ensureRemoteSubset:req`) subsetPosts++ + } + + try { + const outcomes = await Promise.all( + [leader, follower].map((coordinator) => + coordinator + .requestEnsureRemoteSubset( + `todos`, + subsetWithNestedValue(createUnsupported()), + ) + .then( + () => ({ ok: true as const }), + (error: unknown) => ({ ok: false as const, error }), + ), + ), + ) + for (const outcome of outcomes) { + expect(outcome).toMatchObject({ + ok: false, + error: { name: `RemoteSubsetWireValueError`, path }, + }) + } + expect({ + subsetPosts, + ownerCalls: owner.mock.calls.length, + }).toEqual({ subsetPosts: 0, ownerCalls: 0 }) + } finally { + observeBroadcastMessage = undefined + unregisterOwner() + leader.dispose() + follower.dispose() + } + }, + ) + + it.each([ + [ + `where expression`, + { + where: new IR.Value({ nested: () => {} }), + }, + `options.where.value.nested`, + ], + [ + `non-identifier object key`, + { + where: new IR.Value({ [`dotted.key`]: () => {} }), + }, + `options.where.value["dotted.key"]`, + ], + [ + `property-reference path`, + { + where: new IR.PropRef([`todos`, (() => {}) as unknown as string]), + }, + `options.where.path[1]`, + ], + [ + `order expression`, + { + orderBy: [ + { + expression: new IR.Value({ nested: () => {} }), + compareOptions: { direction: `asc`, nulls: `last` }, + }, + ], + }, + `options.orderBy[0].expression.value.nested`, + ], + [ + `comparison options`, + { + orderBy: [ + { + expression: new IR.PropRef([`todos`, `title`]), + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + localeOptions: { matcher: () => {} }, + }, + }, + ], + }, + `options.orderBy[0].compareOptions.localeOptions.matcher`, + ], + [ + `cursor from expression`, + { + cursor: { + whereFrom: new IR.Value({ nested: () => {} }), + whereCurrent: new IR.Value(`current`), + }, + }, + `options.cursor.whereFrom.value.nested`, + ], + [ + `cursor current expression`, + { + cursor: { + whereFrom: new IR.Value(`from`), + whereCurrent: new IR.Value({ nested: () => {} }), + }, + }, + `options.cursor.whereCurrent.value.nested`, + ], + [ + `cursor key`, + { + cursor: { + whereFrom: new IR.Value(`from`), + whereCurrent: new IR.Value(`current`), + lastKey: () => {}, + }, + }, + `options.cursor.lastKey`, + ], + [`limit`, { limit: () => {} }, `options.limit`], + [`offset`, { offset: () => {} }, `options.offset`], + ] as const)( + `reports the exact path for an unsupported %s value`, + async (_label, options, path) => { + const leader = createCoordinator() + const follower = createCoordinator() + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + const owner = withUnusedUnloadSubset(vi.fn(() => Promise.resolve())) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + let subsetPosts = 0 + observeBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type === `rpc:ensureRemoteSubset:req`) subsetPosts++ + } + + try { + const outcomes = await Promise.all( + [leader, follower].map((coordinator) => + coordinator + .requestEnsureRemoteSubset( + `todos`, + options as unknown as LoadSubsetOptions, + ) + .then( + () => ({ ok: true as const }), + (error: unknown) => ({ ok: false as const, error }), + ), + ), + ) + for (const outcome of outcomes) { + expect(outcome).toMatchObject({ + ok: false, + error: { name: `RemoteSubsetWireValueError`, path }, + }) + } + expect({ + subsetPosts, + ownerCalls: owner.mock.calls.length, + }).toEqual({ subsetPosts: 0, ownerCalls: 0 }) + } finally { + observeBroadcastMessage = undefined + unregisterOwner() + leader.dispose() + follower.dispose() + } + }, + ) + + it.each([ + [ + `wire record reused as an expression`, + () => { + const shared = { bad: 1 } + return { + where: new IR.Func(`and`, [ + new IR.Value(shared), + shared as unknown as IR.BasicExpression, + ]), + } + }, + `options.where.args[1].type`, + ], + [ + `wire array reused as a property-reference path`, + () => { + const shared = [`todos`, 1] + return { + where: new IR.Func(`and`, [ + new IR.Value(shared), + new IR.PropRef(shared as unknown as Array), + ]), + } + }, + `options.where.args[1].path[1]`, + ], + [ + `wire array reused as an expression array`, + () => { + const shared = [1] + return { + where: new IR.Func(`and`, [ + new IR.Value(shared), + new IR.Func( + `nested`, + shared as unknown as Array, + ), + ]), + } + }, + `options.where.args[1].args[0]`, + ], + [ + `wire record reused as comparison options`, + () => { + const shared = { bad: 1 } + return { + where: new IR.Value(shared), + orderBy: [ + { + expression: new IR.PropRef([`todos`, `title`]), + compareOptions: shared, + }, + ], + } + }, + `options.orderBy[0].compareOptions.bad`, + ], + ] as const)( + `does not reuse a projected %s across incompatible schema roles`, + async (_label, createOptions, path) => { + const leader = createCoordinator() + const follower = createCoordinator() + const owner = withUnusedUnloadSubset(vi.fn(() => Promise.resolve())) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + let subsetPosts = 0 + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + observeBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type === `rpc:ensureRemoteSubset:req`) subsetPosts++ + } + + try { + const outcomes = await Promise.all( + [leader, follower].map((coordinator) => + coordinator + .requestEnsureRemoteSubset( + `todos`, + createOptions() as unknown as LoadSubsetOptions, + ) + .then( + () => ({ ok: true as const }), + (error: unknown) => ({ ok: false as const, error }), + ), + ), + ) + for (const outcome of outcomes) { + expect(outcome).toMatchObject({ + ok: false, + error: { name: `RemoteSubsetWireValueError`, path }, + }) + } + expect({ + subsetPosts, + ownerCalls: owner.mock.calls.length, + }).toEqual({ subsetPosts: 0, ownerCalls: 0 }) + } finally { + observeBroadcastMessage = undefined + unregisterOwner() + leader.dispose() + follower.dispose() + } + }, + ) + + it(`preserves supported native values, aliases, and cycles for both routes`, async () => { + const leader = createCoordinator() + const follower = createCoordinator() + const received: Array = [] + const owner = withUnusedUnloadSubset( + vi.fn((options: TransportedLoadSubsetOptions) => { + received.push(options) + return Promise.resolve() + }), + ) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + const shared = { label: `shared` } + const cycle: Record = { label: `cycle` } + cycle.self = cycle + const buffer = new ArrayBuffer(8) + const view = new DataView(buffer, 2, 4) + const typed = new Uint16Array(buffer, 0, 2) + typed.set([17, 42]) + const typedArrays = [ + new Int8Array([-1, 2]), + new Uint8Array([0, 255]), + new Uint8ClampedArray([0, 255]), + new Int16Array([-2, 3]), + new Uint16Array([2, 65_535]), + new Int32Array([-3, 4]), + new Uint32Array([3, 4_000_000_000]), + new Float32Array([-0, Number.NaN, Number.POSITIVE_INFINITY]), + new Float64Array([Number.NEGATIVE_INFINITY, Math.PI]), + new BigInt64Array([-4n, 5n]), + new BigUint64Array([4n, 5n]), + ] + const sparse = new Array(3) + sparse[1] = undefined + const reservedKeys = { constructor: `own constructor value` } + Object.defineProperty(reservedKeys, `__proto__`, { + value: { label: `own __proto__ value` }, + enumerable: true, + writable: true, + configurable: true, + }) + const richValue = { + undefinedValue: undefined, + nullValue: null, + booleanValue: true, + stringValue: `wire`, + bigintValue: 9_007_199_254_740_993n, + nanValue: Number.NaN, + positiveInfinity: Number.POSITIVE_INFINITY, + negativeInfinity: Number.NEGATIVE_INFINITY, + negativeZero: -0, + date: new Date(`2026-09-16T12:34:56.000Z`), + invalidDate: new Date(Number.NaN), + regexp: /wire/giu, + buffer, + view, + typed, + typedArrays, + map: new Map([[shared, cycle]]), + set: new Set([shared, cycle]), + sparse, + reservedKeys, + first: shared, + second: shared, + cycle, + } + const options = { + where: new IR.Func(`eq`, [ + new IR.PropRef([`todos`, `payload`]), + new IR.Value(richValue), + ]), + signal: new AbortController().signal, + subscription: { + on: () => () => {}, + } as unknown as Subscription, + } + + try { + await leader.requestEnsureRemoteSubset(`todos`, options) + await follower.requestEnsureRemoteSubset(`todos`, options) + expect(received).toHaveLength(2) + + for (const decoded of received) { + const value = ( + decoded.where as unknown as { + args: Array<{ value?: typeof richValue }> + } + ).args[1]!.value! + expect(value.undefinedValue).toBeUndefined() + expect(value.nullValue).toBeNull() + expect(value.booleanValue).toBe(true) + expect(value.stringValue).toBe(`wire`) + expect(value.bigintValue).toBe(9_007_199_254_740_993n) + expect(Object.is(value.nanValue, Number.NaN)).toBe(true) + expect( + Object.is(value.positiveInfinity, Number.POSITIVE_INFINITY), + ).toBe(true) + expect( + Object.is(value.negativeInfinity, Number.NEGATIVE_INFINITY), + ).toBe(true) + expect(Object.is(value.negativeZero, -0)).toBe(true) + expect(value.date.getTime()).toBe(richValue.date.getTime()) + expect(Number.isNaN(value.invalidDate.getTime())).toBe(true) + expect({ + source: value.regexp.source, + flags: value.regexp.flags, + lastIndex: value.regexp.lastIndex, + }).toEqual({ source: `wire`, flags: `giu`, lastIndex: 0 }) + expect(value.buffer).toBeInstanceOf(ArrayBuffer) + expect(value.view).toBeInstanceOf(DataView) + expect(value.typed).toBeInstanceOf(Uint16Array) + expect(Array.from(new Uint8Array(value.buffer))).toEqual( + Array.from(new Uint8Array(richValue.buffer)), + ) + expect({ + byteOffset: value.view.byteOffset, + byteLength: value.view.byteLength, + }).toEqual({ byteOffset: 2, byteLength: 4 }) + expect(Array.from(value.typed)).toEqual([17, 42]) + expect(value.view.buffer).toBe(value.buffer) + expect(value.typed.buffer).toBe(value.buffer) + expect(value.typedArrays).toHaveLength(typedArrays.length) + for (let index = 0; index < typedArrays.length; index++) { + const actual = value.typedArrays[index]! + const expected = typedArrays[index]! + expect(actual.constructor).toBe(expected.constructor) + expect(actual.byteOffset).toBe(expected.byteOffset) + expect(actual.byteLength).toBe(expected.byteLength) + expect( + Array.from( + new Uint8Array( + actual.buffer, + actual.byteOffset, + actual.byteLength, + ), + ), + ).toEqual( + Array.from( + new Uint8Array( + expected.buffer, + expected.byteOffset, + expected.byteLength, + ), + ), + ) + } + expect(value.map).toBeInstanceOf(Map) + expect(value.set).toBeInstanceOf(Set) + expect(value.first).toBe(value.second) + expect([...value.map.keys()]).toEqual([value.first]) + expect(value.map.get(value.first)).toBe(value.cycle) + expect(value.set.has(value.first)).toBe(true) + expect(value.set.has(value.cycle)).toBe(true) + expect(value.cycle.self).toBe(value.cycle) + expect(0 in value.sparse).toBe(false) + expect(1 in value.sparse).toBe(true) + expect(2 in value.sparse).toBe(false) + expect(Object.getPrototypeOf(value.reservedKeys)).toBe( + Object.prototype, + ) + expect( + Object.getOwnPropertyDescriptor(value.reservedKeys, `__proto__`), + ).toMatchObject({ + value: { label: `own __proto__ value` }, + enumerable: true, + writable: true, + configurable: true, + }) + expect(value.reservedKeys.constructor).toBe(`own constructor value`) + expect(decoded).not.toHaveProperty(`signal`) + expect(decoded).not.toHaveProperty(`subscription`) + } + // Local delivery need not clone identity, but it must receive the same + // validated, live-field-free wire domain as follower delivery. + expect(received[1]).not.toBe(options) + } finally { + unregisterOwner() + leader.dispose() + follower.dispose() + } + }) + + it(`direct leader requests await the registered collection owner`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + + let releaseOwner = (): void => {} + const owner = withUnusedUnloadSubset( + vi.fn( + () => + new Promise((resolve) => { + releaseOwner = resolve + }), + ), + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + + try { + let settled = false + const request = coordinator + .requestEnsureRemoteSubset(`todos`, { limit: 2, offset: 1 }) + .then(() => { + settled = true + }) + await Promise.resolve() + const beforeRelease = { ownerCalls: owner.mock.calls.length, settled } + + releaseOwner() + await request + + expect(beforeRelease).toEqual({ ownerCalls: 1, settled: false }) + expect(owner).toHaveBeenCalledWith({ limit: 2, offset: 1 }) + } finally { + releaseOwner() + unregisterOwner() + coordinator.dispose() + } + }) + + it(`preserves the receiver of an explicitly registered adapter owner`, async () => { + const receivers: Array = [] + const leaderAdapter = createStubAdapter() as ReturnType< + typeof createStubAdapter + > & { + ensureRemoteSubset: ( + collectionId: string, + options: { limit?: number }, + ) => Promise + } + leaderAdapter.ensureRemoteSubset = function ( + this: typeof leaderAdapter, + _collectionId, + _options, + ) { + receivers.push(this) + return Promise.resolve() + } + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator() + const owner = withUnusedUnloadSubset( + vi.fn((options: TransportedLoadSubsetOptions) => + leaderAdapter.ensureRemoteSubset(`todos`, options), + ), + ) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + try { + await follower.requestEnsureRemoteSubset(`todos`, { limit: 1 }) + expect(receivers).toEqual([leaderAdapter]) + } finally { + unregisterOwner() + leader.dispose() + follower.dispose() + } + }) + + it(`scopes a reused subset options object to each Browser collection`, async () => { + const coordinator = createCoordinator() + const alpha = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const beta = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterAlpha = coordinator.registerRemoteSubsetOwner( + `alpha`, + alpha, + ) + const unregisterBeta = coordinator.registerRemoteSubsetOwner(`beta`, beta) + coordinator.subscribe(`alpha`, () => {}) + coordinator.subscribe(`beta`, () => {}) + await flush(50) + expect({ + alpha: coordinator.isLeader(`alpha`), + beta: coordinator.isLeader(`beta`), + }).toEqual({ alpha: true, beta: true }) + + const shared: LoadSubsetOptions = { limit: 2, offset: 1 } + try { + await coordinator.requestEnsureRemoteSubset(`alpha`, shared) + await coordinator.requestEnsureRemoteSubset(`beta`, shared) + await coordinator.requestEnsureRemoteSubset(`alpha`, shared) + + const afterAcquire = { + alphaLoads: alpha.mock.calls.length, + betaLoads: beta.mock.calls.length, + } + await coordinator.requestReleaseRemoteSubset(`alpha`, shared) + await coordinator.requestReleaseRemoteSubset(`beta`, shared) + + expect({ + afterAcquire, + alphaUnloads: alpha.unloadSubset.mock.calls.length, + betaUnloads: beta.unloadSubset.mock.calls.length, + }).toEqual({ + afterAcquire: { alphaLoads: 1, betaLoads: 1 }, + alphaUnloads: 1, + betaUnloads: 1, + }) + } finally { + unregisterAlpha() + unregisterBeta() + coordinator.dispose() + } + }) + + it(`replays after an earlier Browser leader responds behind a new heartbeat`, async () => { + const leader = createCoordinator() + const follower = createCoordinator() + let releaseOwner = (): void => {} + const owner = withUnusedUnloadSubset( + vi.fn( + () => + new Promise((resolve) => { + releaseOwner = resolve + }), + ), + ) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + expect({ + leader: leader.isLeader(`todos`), + follower: follower.isLeader(`todos`), + }).toEqual({ leader: true, follower: false }) + + const leaderInternals = leader as unknown as { + nodeId: string + channel: { postMessage: (message: unknown) => void } + } + const followerInternals = follower as unknown as { + onChannelMessage: (message: unknown) => void + outboundRemoteSubsetAcquisitions: Map< + string, + { acquiredLeaderId: string | null } + > + } + const originalLeaderPost = leaderInternals.channel.postMessage.bind( + leaderInternals.channel, + ) + let heldResponse: unknown + leaderInternals.channel.postMessage = (message) => { + const type = (message as { payload?: { type?: string } }).payload?.type + if (type === `rpc:ensureRemoteSubset:res`) { + heldResponse = structuredClone(message) + return + } + originalLeaderPost(message) + } + let requestPosts = 0 + observeBroadcastMessage = (message) => { + if ( + (message as { payload?: { type?: string } }).payload?.type === + `rpc:ensureRemoteSubset:req` + ) { + requestPosts++ + } + } + + try { + const request = follower.requestEnsureRemoteSubset(`todos`, { + limit: 1, + }) + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(1)) + + followerInternals.onChannelMessage({ + v: 1, + dbName: `test-db`, + collectionId: `todos`, + senderId: `replacement-browser-leader`, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term: 2, + leaderId: `replacement-browser-leader`, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + releaseOwner() + await vi.waitFor(() => expect(heldResponse).toBeDefined()) + leader.isLeader = () => false + followerInternals.onChannelMessage(heldResponse) + await request + await flush() + + const [acquisition] = + followerInternals.outboundRemoteSubsetAcquisitions.values() + expect({ + requestPosts, + acquiredLeaderId: acquisition?.acquiredLeaderId, + }).toEqual({ + requestPosts: 2, + acquiredLeaderId: leaderInternals.nodeId, + }) + } finally { + observeBroadcastMessage = undefined + releaseOwner() + unregisterOwner() + leader.dispose() + follower.dispose() + } + }) + + it(`replays a held Browser lease after an A to B to A leader cycle`, async () => { + const leader = createCoordinator() + const follower = createCoordinator() + let releaseFirstLoad = (): void => {} + const firstLoadGate = new Promise((resolve) => { + releaseFirstLoad = resolve + }) + const owner = Object.assign( + vi.fn().mockImplementationOnce(() => firstLoadGate), + { unloadSubset: vi.fn(), onError: vi.fn() }, + ) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + const leaderInternals = leader as unknown as { + nodeId: string + channel: { postMessage: (message: unknown) => void } + inboundRemoteSubsetAcquisitions: Map + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + } + const followerInternals = follower as unknown as { + onChannelMessage: (message: unknown) => void + outboundRemoteSubsetAcquisitions: Map< + string, + { acquiredLeaderId: string | null } + > + } + const originalLeaderPost = leaderInternals.channel.postMessage.bind( + leaderInternals.channel, + ) + let heldResponse: unknown + leaderInternals.channel.postMessage = (message) => { + const type = (message as { payload?: { type?: string } }).payload?.type + if ( + type === `rpc:ensureRemoteSubset:res` && + heldResponse === undefined + ) { + heldResponse = structuredClone(message) + return + } + originalLeaderPost(message) + } + let requestPosts = 0 + const wireRequests: Array<{ + acquisitionId: string + rpcId: string + }> = [] + observeBroadcastMessage = (message) => { + const payload = ( + message as { + payload?: { + type?: string + acquisitionId?: string + rpcId?: string + } + } + ).payload + if (payload?.type === `rpc:ensureRemoteSubset:req`) { + requestPosts++ + wireRequests.push({ + acquisitionId: payload.acquisitionId!, + rpcId: payload.rpcId!, + }) + } + } + const heartbeat = (leaderId: string, term: number) => ({ + v: 1, + dbName: `test-db`, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + const options: LoadSubsetOptions = { limit: 1 } + + try { + const request = follower.requestEnsureRemoteSubset(`todos`, options) + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(1)) + followerInternals.onChannelMessage( + heartbeat(`replacement-browser-leader`, 2), + ) + leaderInternals.releaseInboundRemoteSubsetAcquisitions(`todos`) + followerInternals.onChannelMessage(heartbeat(leaderInternals.nodeId, 3)) + releaseFirstLoad() + await vi.waitFor(() => expect(heldResponse).toBeDefined()) + followerInternals.onChannelMessage(heldResponse) + await request + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(2)) + + const [acquisition] = + followerInternals.outboundRemoteSubsetAcquisitions.values() + expect({ + requestPosts, + loads: owner.mock.calls.length, + unloads: owner.unloadSubset.mock.calls.length, + inbound: leaderInternals.inboundRemoteSubsetAcquisitions.size, + acquiredLeaderId: acquisition?.acquiredLeaderId, + }).toEqual({ + requestPosts: 2, + loads: 2, + unloads: 1, + inbound: 1, + acquiredLeaderId: leaderInternals.nodeId, + }) + expect(wireRequests).toHaveLength(2) + expect(wireRequests[0]!.acquisitionId).not.toBe(``) + expect(wireRequests[1]!.acquisitionId).toBe( + wireRequests[0]!.acquisitionId, + ) + expect(wireRequests[1]!.rpcId).not.toBe(wireRequests[0]!.rpcId) + + await follower.requestReleaseRemoteSubset(`todos`, options) + expect(owner.unloadSubset).toHaveBeenCalledTimes(2) + } finally { + observeBroadcastMessage = undefined + releaseFirstLoad() + unregisterOwner() + leader.dispose() + follower.dispose() + } + }) + + it(`reports a failed Browser replay once without self-retrying`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + const replayError = new Error(`replacement owner is not ready`) + const ownerErrors: Array = [] + const owner = Object.assign( + vi + .fn() + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(replayError), + { + unloadSubset: vi.fn(), + onError: (error: unknown) => ownerErrors.push(error), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { limit: 1 } + await coordinator.requestEnsureRemoteSubset(`todos`, options) + type Acquisition = { + acquiredLeaderId: string | null + inFlight: Promise | null + } + const internals = coordinator as unknown as { + outboundRemoteSubsetAcquisitions: Map + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + const [acquisition] = internals.outboundRemoteSubsetAcquisitions.values() + internals.releaseInboundRemoteSubsetAcquisitions(`todos`) + await vi.waitFor(() => + expect(owner.unloadSubset).toHaveBeenCalledTimes(1), + ) + acquisition!.acquiredLeaderId = `retired-browser-leader` + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + + try { + await internals.replayRemoteSubsetAcquisitions(`todos`) + await flush() + expect({ + attempts: owner.mock.calls.length - 1, + ownerErrors, + acquiredLeaderId: acquisition!.acquiredLeaderId, + inFlight: acquisition!.inFlight, + unhandled, + }).toEqual({ + attempts: 1, + ownerErrors: [replayError], + acquiredLeaderId: `retired-browser-leader`, + inFlight: null, + unhandled: [], + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`reports a failed remote Browser follower replay to its local owner once`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + const internals = coordinator as unknown as { + onChannelMessage: (message: unknown) => void + sendRPC: (collectionId: string, request: unknown) => Promise + outboundRemoteSubsetAcquisitions: Map< + string, + { acquiredLeaderId: string | null; inFlight: Promise | null } + > + } + coordinator.isLeader = () => false + const heartbeat = (leaderId: string, term: number) => + internals.onChannelMessage({ + v: 1, + dbName: `test-db`, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + heartbeat(`remote-browser-a`, 1) + const ownerErrors: Array = [] + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + internals.sendRPC = vi.fn().mockResolvedValue({ + type: `rpc:ensureRemoteSubset:res`, + rpcId: `initial-browser`, + ok: true, + leaderId: `remote-browser-a`, + }) + const options: LoadSubsetOptions = { limit: 1 } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, options) + const replayError = new Error(`remote Browser replay transport failed`) + const replayRPC = vi.fn().mockRejectedValue(replayError) + internals.sendRPC = replayRPC + heartbeat(`remote-browser-b`, 2) + await vi.waitFor(() => expect(replayRPC).toHaveBeenCalledTimes(1)) + await vi.waitFor(() => { + const [acquisition] = + internals.outboundRemoteSubsetAcquisitions.values() + expect(acquisition?.inFlight).toBeNull() + }) + + const [acquisition] = + internals.outboundRemoteSubsetAcquisitions.values() + expect({ + ownerErrors, + attempts: replayRPC.mock.calls.length, + acquiredLeaderId: acquisition?.acquiredLeaderId, + retained: internals.outboundRemoteSubsetAcquisitions.size, + }).toEqual({ + ownerErrors: [replayError], + attempts: 1, + acquiredLeaderId: `remote-browser-a`, + retained: 1, + }) + } finally { + unregisterOwner() + coordinator.dispose() + } + }) + + it(`ignores a lower-term Browser heartbeat without replaying to its stale leader`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + coordinator.isLeader = () => false + const internals = coordinator as unknown as { + onChannelMessage: (message: unknown) => void + collections: Map< + string, + { leaderId: string | null; latestTerm: number } + > + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + const replay = vi.fn(() => Promise.resolve()) + internals.replayRemoteSubsetAcquisitions = replay + const heartbeat = (leaderId: string, term: number) => + internals.onChannelMessage({ + v: 1, + dbName: `test-db`, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + + try { + heartbeat(`remote-browser-b`, 2) + replay.mockClear() + heartbeat(`retired-browser-a`, 1) + const state = internals.collections.get(`todos`) + + expect({ + leaderId: state?.leaderId, + latestTerm: state?.latestTerm, + replayCalls: replay.mock.calls.length, + }).toEqual({ + leaderId: `remote-browser-b`, + latestTerm: 2, + replayCalls: 0, + }) + } finally { + coordinator.dispose() + } + }) + + it(`rebinds a live Browser acquisition when its owner is replaced`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + let releaseFirstLoad = (): void => {} + const firstLoadGate = new Promise((resolve) => { + releaseFirstLoad = resolve + }) + const firstOwner = Object.assign( + vi.fn(() => firstLoadGate), + { + unloadSubset: vi.fn(), + onError: vi.fn(), + }, + ) + const secondOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn( + (_options: TransportedLoadSubsetOptions) => undefined, + ), + onError: vi.fn(), + }) + const unregisterFirst = coordinator.registerRemoteSubsetOwner( + `todos`, + firstOwner, + ) + let unregisterSecond: (() => void) | undefined + const options: LoadSubsetOptions = { limit: 1 } + + try { + const initialAcquire = coordinator.requestEnsureRemoteSubset( + `todos`, + options, + ) + await vi.waitFor(() => expect(firstOwner).toHaveBeenCalledTimes(1)) + unregisterFirst() + unregisterSecond = coordinator.registerRemoteSubsetOwner( + `todos`, + secondOwner, + ) + releaseFirstLoad() + await initialAcquire + await vi.waitFor(() => expect(secondOwner).toHaveBeenCalledTimes(1)) + + expect(firstOwner.unloadSubset).toHaveBeenCalledTimes(1) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + expect(secondOwner.unloadSubset).toHaveBeenCalledTimes(1) + } finally { + releaseFirstLoad() + unregisterFirst() + unregisterSecond?.() + coordinator.dispose() + } + }) + + it(`rebinds a live remote Browser follower lease when the same leader replaces its owner`, async () => { + const leader = createCoordinator() + const follower = createCoordinator() + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + expect({ + leader: leader.isLeader(`todos`), + follower: follower.isLeader(`todos`), + }).toEqual({ leader: true, follower: false }) + let releaseFirstLoad = (): void => {} + const firstLoadGate = new Promise((resolve) => { + releaseFirstLoad = resolve + }) + const firstOwner = Object.assign( + vi.fn(() => firstLoadGate), + { + unloadSubset: vi.fn(), + onError: vi.fn(), + }, + ) + const secondOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterFirst = leader.registerRemoteSubsetOwner( + `todos`, + firstOwner, + ) + let unregisterSecond: (() => void) | undefined + const options: LoadSubsetOptions = { limit: 1 } + let requestPosts = 0 + observeBroadcastMessage = (message) => { + if ( + (message as { payload?: { type?: string } }).payload?.type === + `rpc:ensureRemoteSubset:req` + ) { + requestPosts++ + } + } + + try { + const initialAcquire = follower.requestEnsureRemoteSubset( + `todos`, + options, + ) + await vi.waitFor(() => expect(firstOwner).toHaveBeenCalledTimes(1)) + unregisterFirst() + unregisterSecond = leader.registerRemoteSubsetOwner( + `todos`, + secondOwner, + ) + releaseFirstLoad() + await initialAcquire + await vi.waitFor(() => expect(secondOwner).toHaveBeenCalledTimes(1)) + + expect({ + requestPosts, + firstUnloads: firstOwner.unloadSubset.mock.calls.length, + }).toEqual({ requestPosts: 1, firstUnloads: 1 }) + await follower.requestReleaseRemoteSubset(`todos`, options) + expect(secondOwner.unloadSubset).toHaveBeenCalledTimes(1) + } finally { + observeBroadcastMessage = undefined + releaseFirstLoad() + unregisterFirst() + unregisterSecond?.() + leader.dispose() + follower.dispose() + } + }) + + it(`rebinds remote Browser demand when its replacement owner registers after release settlement`, async () => { + const leader = createCoordinator() + const follower = createCoordinator() + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + expect({ + leader: leader.isLeader(`todos`), + follower: follower.isLeader(`todos`), + }).toEqual({ leader: true, follower: false }) + const firstOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const secondOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterFirst = leader.registerRemoteSubsetOwner( + `todos`, + firstOwner, + ) + let unregisterSecond: (() => void) | undefined + const options: LoadSubsetOptions = { offset: 22 } + const leaderInternals = leader as unknown as { + inboundRemoteSubsetAcquisitions: Map> + } + + try { + await follower.requestEnsureRemoteSubset(`todos`, options) + unregisterFirst() + await vi.waitFor(() => + expect(firstOwner.unloadSubset).toHaveBeenCalledTimes(1), + ) + await flush(0) + + unregisterSecond = leader.registerRemoteSubsetOwner( + `todos`, + secondOwner, + ) + await flush(0) + + expect({ + firstLoads: firstOwner.mock.calls.length, + firstUnloads: firstOwner.unloadSubset.mock.calls.length, + secondLoads: secondOwner.mock.calls.length, + inbound: leaderInternals.inboundRemoteSubsetAcquisitions.size, + }).toEqual({ + firstLoads: 1, + firstUnloads: 1, + secondLoads: 1, + inbound: 1, + }) + + await follower.requestReleaseRemoteSubset(`todos`, options) + const [terminal] = + leaderInternals.inboundRemoteSubsetAcquisitions.values() + expect(secondOwner.unloadSubset).toHaveBeenCalledTimes(1) + expect(secondOwner.unloadSubset.mock.calls[0]?.[0]).toBe( + secondOwner.mock.calls[0]?.[0], + ) + expect({ + inbound: leaderInternals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + unregisterFirst() + unregisterSecond?.() + leader.dispose() + follower.dispose() + } + }) + + it(`reacquires the same Browser lease after leadership retirement`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const internals = coordinator as unknown as { + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + } + const options: LoadSubsetOptions = { limit: 1 } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, options) + internals.releaseInboundRemoteSubsetAcquisitions(`todos`) + await vi.waitFor(() => + expect(owner.unloadSubset).toHaveBeenCalledTimes(1), + ) + + await coordinator.requestEnsureRemoteSubset(`todos`, options) + expect(owner).toHaveBeenCalledTimes(2) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + expect(owner.unloadSubset).toHaveBeenCalledTimes(2) + } finally { + unregisterOwner() + coordinator.dispose() + } + }) + + it(`starts independent Browser lease replays without sibling head-of-line blocking`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 1 }) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 2 }) + + type Acquisition = { + collectionId: string + options: TransportedLoadSubsetOptions + acquiredLeaderId: string | null + } + const internals = coordinator as unknown as { + outboundRemoteSubsetAcquisitions: Map + acquireRemoteSubset: (acquisition: Acquisition) => Promise + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + const acquisitions = [ + ...internals.outboundRemoteSubsetAcquisitions.values(), + ] + for (const acquisition of acquisitions) { + acquisition.acquiredLeaderId = `retired-browser-leader` + } + let releaseFirstReplay = (): void => {} + const firstReplayGate = new Promise((resolve) => { + releaseFirstReplay = resolve + }) + const starts: Array = [] + internals.acquireRemoteSubset = async (acquisition) => { + starts.push(acquisition.options.offset) + if (acquisition.options.offset === 1) await firstReplayGate + } + + try { + const replay = internals.replayRemoteSubsetAcquisitions(`todos`) + await Promise.resolve() + const startsBeforeFirstFinished = [...starts] + expect(startsBeforeFirstFinished).toEqual([1, 2]) + releaseFirstReplay() + await replay + } finally { + releaseFirstReplay() + unregisterOwner() + coordinator.dispose() + } + }) + + it(`reports owner unload rejection while completing sibling Browser cleanup`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + const unloadError = new Error(`browser owner unload failed`) + const ownerErrors: Array = [] + const unloadSubset = vi.fn((options: TransportedLoadSubsetOptions) => + options.offset === 1 ? Promise.reject(unloadError) : undefined, + ) + const owner = Object.assign(vi.fn(), { + unloadSubset, + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 1 }) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 2 }) + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + inboundRemoteSubsetAcquisitions: Map< + string, + { release: Promise | null } + > + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + } + + try { + internals.releaseInboundRemoteSubsetAcquisitions(`todos`) + const releases = [ + ...internals.inboundRemoteSubsetAcquisitions.values(), + ].map((acquisition) => acquisition.release!) + const outcomes = await Promise.allSettled(releases) + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect({ + outcomes: outcomes.map((outcome) => + outcome.status === `rejected` ? outcome.reason : outcome.status, + ), + unloadOffsets: unloadSubset.mock.calls.map( + ([options]) => options.offset, + ), + ownerErrors, + unhandled, + }).toEqual({ + outcomes: [unloadError, `fulfilled`], + unloadOffsets: [1, 2], + ownerErrors: [unloadError], + unhandled: [], + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`reports local Browser disposal unload rejection once and completes cleanup`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + const unloadError = new Error(`browser disposal unload failed`) + const ownerErrors: Array = [] + const unloadSubset = vi.fn((options: TransportedLoadSubsetOptions) => + options.offset === 23 ? Promise.reject(unloadError) : undefined, + ) + const owner = Object.assign(vi.fn(), { + unloadSubset, + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 23 }) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 24 }) + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + collections: Map + pendingRPCs: Map + remoteSubsetOwners: Map + outboundRemoteSubsetAcquisitions: Map + inboundRemoteSubsetAcquisitions: Map + } + + try { + coordinator.dispose() + await flush(0) + + expect({ + unloadOffsets: unloadSubset.mock.calls.map( + ([options]) => options.offset, + ), + ownerErrors, + unhandled, + collections: internals.collections.size, + pendingRPCs: internals.pendingRPCs.size, + owners: internals.remoteSubsetOwners.size, + outbound: internals.outboundRemoteSubsetAcquisitions.size, + inbound: internals.inboundRemoteSubsetAcquisitions.size, + channelEndpoints: channels.get(`tsdb:coord:test-db`)?.size ?? 0, + }).toEqual({ + unloadOffsets: [23, 24], + ownerErrors: [unloadError], + unhandled: [], + collections: 0, + pendingRPCs: 0, + owners: 0, + outbound: 0, + inbound: 0, + channelEndpoints: 0, + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) }) describe(`RPC - pullSince`, () => { @@ -551,6 +3953,64 @@ describe(`BrowserCollectionCoordinator`, () => { }) describe(`dispose`, () => { + it(`stops Browser RPC retry after disposal and preserves the first failure`, async () => { + vi.useFakeTimers() + const coordinator = createCoordinator() + const firstError = new Error(`browser transport failed during disposal`) + let firstAttempt = true + const internals = coordinator as unknown as { + sendRPC: ( + collectionId: string, + request: { + type: `rpc:pullSince:req` + rpcId: string + fromRowVersion: number + }, + ) => Promise + sendRPCOnce: () => Promise + } + internals.sendRPCOnce = vi.fn(() => { + if (firstAttempt) { + firstAttempt = false + coordinator.dispose() + } + return Promise.reject(firstError) + }) + let outcome: { error: unknown } | undefined + + try { + const pending = internals + .sendRPC(`todos`, { + type: `rpc:pullSince:req`, + rpcId: `browser-disposal-rpc`, + fromRowVersion: 0, + }) + .then( + () => ({ error: undefined }), + (error: unknown) => ({ error }), + ) + .then((result) => { + outcome = result + return result + }) + await vi.advanceTimersByTimeAsync(0) + const immediate = { + calls: vi.mocked(internals.sendRPCOnce).mock.calls.length, + settled: outcome !== undefined, + } + await vi.runAllTimersAsync() + const finalOutcome = await pending + + expect(immediate).toEqual({ calls: 1, settled: true }) + expect(finalOutcome).toEqual({ error: firstError }) + expect(internals.sendRPCOnce).toHaveBeenCalledTimes(1) + } finally { + await vi.runAllTimersAsync() + vi.useRealTimers() + coordinator.dispose() + } + }) + it(`cleans up on dispose`, async () => { const coord = createCoordinator() coord.subscribe(`todos`, () => {}) diff --git a/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts new file mode 100644 index 0000000000..f2ff086f13 --- /dev/null +++ b/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts @@ -0,0 +1,3666 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { setImmediate as nextTurn } from 'node:timers/promises' +import { fc } from '@fast-check/vitest' +import { IR, createCollection } from '@tanstack/db' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { BrowserCollectionCoordinator } from '../src/browser-coordinator' +import { + createBrowserWASQLitePersistence, + persistedCollectionOptions, +} from '../src' +import { createWASQLiteTestDatabase } from './helpers/wa-sqlite-test-db' +import type { + PersistedCollectionCoordinator, + PersistedMutationEnvelope, + PersistedTx, + PersistenceAdapter, + TransportedLoadSubsetOptions, +} from '@tanstack/db-sqlite-persistence-core' +import type { LoadSubsetOptions, Subscription, SyncConfig } from '@tanstack/db' +import type { BrowserWASQLiteDatabase } from '../src' + +const seedText = process.env.TANSTACK_DB_COORDINATOR_ORACLE_SEED ?? `165902` +const runText = process.env.TANSTACK_DB_COORDINATOR_ORACLE_RUNS ?? `12` +const SEED = Number(seedText) +const RUNS = Number(runText) +const PATH = process.env.TANSTACK_DB_COORDINATOR_ORACLE_PATH + +if (!/^-?\d+$/.test(seedText) || !Number.isSafeInteger(SEED)) { + throw new Error(`Invalid TANSTACK_DB_COORDINATOR_ORACLE_SEED`) +} +if (!/^\d+$/.test(runText) || !Number.isSafeInteger(RUNS) || RUNS < 1) { + throw new Error(`Invalid TANSTACK_DB_COORDINATOR_ORACLE_RUNS`) +} +if (PATH !== undefined && !/^\d+(?::\d+)*$/.test(PATH)) { + throw new Error( + `TANSTACK_DB_COORDINATOR_ORACLE_PATH must be a numeric shrink path`, + ) +} + +/* +Coverage owner and test cards +============================= + +This suite owns the RFC #1659 Workstream 2 cut across +BrowserCollectionCoordinator, browser persistence resolution, and the +persisted sync wrapper. browser-coordinator.test.ts is still the focused RPC +unit owner, and db-sqlite-persistence-core/tests/persisted.test.ts is still the +single-runtime owner. Putting this history in either file would replace the +other production boundary with a stub and lose the collection x tab x adapter +identity question. The real SQLite witness below extends the existing browser +runtime helper instead of claiming a new driver model. + +Routing law and source: +- RFC #1659 release invariants 3, 4, and 9; BrowserCollectionCoordinator's + collectionId-bearing PersistedCollectionCoordinator API; issues #1589 and + #1753. Configuration and adapter effects for collection A cannot be handled + by collection B's adapter, before or after leadership transfer. +- Domain: two distinct collections, distinct schema versions/policies, two + tabs, reordered request delivery, local writes, and leader -> follower + takeover. The reference is a Map keyed by (tab, collection), not the + coordinator's mutable adapter slot. +- Production/checkpoint: real coordinator RPC plus Web Locks seam; compare the + adapter identity that applies each collection's transaction after both the + initial and takeover phases. The fixed witness additionally checks real + SQLite row count, registry schema, and reset epoch after merely subscribing. + +Wire/ack law and source: +- RFC invariant 4, LoadSubsetOptions' immutable-request contract, and + LoadSubsetFn's applied-receipt contract; issue #1498. The emitted heartbeat, + RPC, and tx variants exercised by the partition below must survive + structuredClone. Subset expression/cursor/order/window semantics cross the + wire; live signal/subscription ownership stays local. A successful follower + response is permitted only after the leader's exact upstream subset load has + completed. +- Domain: a callback-bearing Subscription witness plus clone-safe expression, + limit, and offset requests. Requests and responses may be held separately. +- Production/checkpoint: BrowserCollectionCoordinator.sendRPC and the public + persisted on-demand leader collection. Record postMessage clone failures, + upstream load entry/completion, and follower settlement at response delivery. + +Write-ownership law and source: +- RFC invariants 3 and 9 and issue #1753. A sync-ingested persistent write has + the same supported-owner requirement as a local write. +- Domain: public source begin/write/commit after a modeled leadership change. +- Production/checkpoint: persistedCollectionOptions' wrapped source commit; + record collection-visible rows, adapter calls, and the owner active at the + exact applyCommittedTx call. + +Reach, challenge, replay, cleanup, and limits: +- Green calibration tests prove structuredClone rejects a function payload, + the leader upstream fixture can really enter/complete, and the ownership + checker rejects an unowned apply. Every RED test records a reached production + checkpoint before comparing the independent ledger. +- FastCheck prints seed/path. Replay with TANSTACK_DB_COORDINATOR_ORACLE_{SEED, + PATH,RUNS}; the thrown report retains the first failing (original) and final + shrunk candidate traces. +- Coordinators/collections/databases are disposed by failure-preserving cleanup. A final + lifecycle test proves no channel, held lock, queued lock, or delayed delivery + remains and that a fresh db name elects normally. +- The BroadcastChannel and Web Locks seams below perform real structuredClone + and real coordinator code but are deterministic Node controls. They do not + earn real-browser, multi-context, OPFS exclusive-handle, worker, Electric, + PowerSync, or service credit. Those cells remain required for GREEN closeout. +*/ + +type MessageHandler = (event: { data: unknown }) => void + +type DelayedDelivery = { + channel: string + source: ControlledBroadcastChannel + target: ControlledBroadcastChannel + data: unknown +} + +type CloneAttempt = { + data: unknown + errorName?: string +} + +class ControlledBroadcastChannel { + static readonly endpoints = new Map>() + static readonly delayed: Array = [] + static readonly cloneAttempts: Array = [] + static retiredDeliveries = 0 + + readonly name: string + onmessage: MessageHandler | null = null + + constructor(name: string) { + this.name = name + const peers = ControlledBroadcastChannel.endpoints.get(name) ?? new Set() + peers.add(this) + ControlledBroadcastChannel.endpoints.set(name, peers) + } + + postMessage(data: unknown): void { + let copy: unknown + try { + copy = structuredClone(data) + ControlledBroadcastChannel.cloneAttempts.push({ data }) + } catch (error) { + ControlledBroadcastChannel.cloneAttempts.push({ + data, + errorName: error instanceof Error ? error.name : String(error), + }) + throw error + } + + for (const target of ControlledBroadcastChannel.endpoints.get(this.name) ?? + []) { + if (target === this || !target.onmessage) continue + ControlledBroadcastChannel.delayed.push({ + channel: this.name, + source: this, + target, + data: copy, + }) + } + } + + close(): void { + const peers = ControlledBroadcastChannel.endpoints.get(this.name) + peers?.delete(this) + if (peers?.size === 0) { + ControlledBroadcastChannel.endpoints.delete(this.name) + } + for ( + let index = ControlledBroadcastChannel.delayed.length - 1; + index >= 0; + index-- + ) { + const delivery = ControlledBroadcastChannel.delayed[index] + if (delivery?.source === this || delivery?.target === this) { + ControlledBroadcastChannel.delayed.splice(index, 1) + ControlledBroadcastChannel.retiredDeliveries++ + } + } + this.onmessage = null + } + + static deliverAt(index: number): boolean { + const [delivery] = this.delayed.splice(index, 1) + if (!delivery) return false + delivery.target.onmessage?.({ data: delivery.data }) + return true + } + + static deliverWhere(predicate: (data: unknown) => boolean): boolean { + const index = this.delayed.findIndex((delivery) => predicate(delivery.data)) + return index >= 0 ? this.deliverAt(index) : false + } + + static reset(): void { + this.endpoints.clear() + this.delayed.length = 0 + this.cloneAttempts.length = 0 + this.retiredDeliveries = 0 + } +} + +type LockCallback = (lock: { name: string }) => Promise +type LockQueueEntry = { + callback: LockCallback + signal?: AbortSignal + resolve: (value: unknown) => void + reject: (error: unknown) => void +} + +const heldLocks = new Set() +const queuedLocks = new Map>() + +function grantNextLock(name: string): void { + if (heldLocks.has(name)) return + const queue = queuedLocks.get(name) + const next = queue?.shift() + if (!next) return + if (queue?.length === 0) queuedLocks.delete(name) + if (next.signal?.aborted) { + grantNextLock(name) + return + } + + heldLocks.add(name) + void Promise.resolve(next.callback({ name })).then( + (value) => { + heldLocks.delete(name) + next.resolve(value) + grantNextLock(name) + }, + (error) => { + heldLocks.delete(name) + next.reject(error) + grantNextLock(name) + }, + ) +} + +const controlledLocks = { + request: ( + name: string, + optionsOrCallback: { signal?: AbortSignal } | LockCallback, + maybeCallback?: LockCallback, + ): Promise => { + const callback = + typeof optionsOrCallback === `function` + ? optionsOrCallback + : maybeCallback! + const signal = + typeof optionsOrCallback === `object` + ? optionsOrCallback.signal + : undefined + + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(new DOMException(`Lock request aborted`, `AbortError`)) + return + } + const entry = { callback, signal, resolve, reject } + const queue = queuedLocks.get(name) ?? [] + queue.push(entry) + queuedLocks.set(name, queue) + signal?.addEventListener( + `abort`, + () => { + const current = queuedLocks.get(name) + const index = current?.indexOf(entry) ?? -1 + if (index >= 0) { + current!.splice(index, 1) + if (current!.length === 0) queuedLocks.delete(name) + reject(new DOMException(`Lock request aborted`, `AbortError`)) + } + }, + { once: true }, + ) + grantNextLock(name) + }) + }, +} + +let originalBroadcastChannelDescriptor: PropertyDescriptor | undefined +let originalNavigatorDescriptor: PropertyDescriptor | undefined + +function installControlledBrowserSeams(): void { + originalBroadcastChannelDescriptor = Object.getOwnPropertyDescriptor( + globalThis, + `BroadcastChannel`, + ) + originalNavigatorDescriptor = Object.getOwnPropertyDescriptor( + globalThis, + `navigator`, + ) + Object.defineProperty(globalThis, `BroadcastChannel`, { + value: ControlledBroadcastChannel, + writable: true, + configurable: true, + }) + Object.defineProperty(globalThis, `navigator`, { + value: { locks: controlledLocks }, + writable: true, + configurable: true, + }) +} + +function restoreControlledBrowserSeams(): void { + if (originalBroadcastChannelDescriptor) { + Object.defineProperty( + globalThis, + `BroadcastChannel`, + originalBroadcastChannelDescriptor, + ) + } else { + Reflect.deleteProperty(globalThis, `BroadcastChannel`) + } + if (originalNavigatorDescriptor) { + Object.defineProperty(globalThis, `navigator`, originalNavigatorDescriptor) + } else { + Reflect.deleteProperty(globalThis, `navigator`) + } + originalBroadcastChannelDescriptor = undefined + originalNavigatorDescriptor = undefined +} + +function payloadType(data: unknown): string | undefined { + if (!data || typeof data !== `object` || !(`payload` in data)) + return undefined + const payload = (data as { payload?: unknown }).payload + if (!payload || typeof payload !== `object` || !(`type` in payload)) { + return undefined + } + return String((payload as { type?: unknown }).type) +} + +function envelopeCollection(data: unknown): string | undefined { + if (!data || typeof data !== `object` || !(`collectionId` in data)) { + return undefined + } + return String((data as { collectionId?: unknown }).collectionId) +} + +async function pumpNetwork(): Promise { + for (let pass = 0; pass < 100; pass++) { + while (ControlledBroadcastChannel.deliverAt(0)) { + await Promise.resolve() + } + await nextTurn() + if (ControlledBroadcastChannel.delayed.length === 0) return + } + throw new Error(`controlled network failed to become idle`) +} + +async function waitFor( + predicate: () => boolean, + checkpoint: string, +): Promise { + for (let pass = 0; pass < 100; pass++) { + if (predicate()) return + await pumpNetwork() + await nextTurn() + } + throw new Error(`did not reach ${checkpoint}`) +} + +async function waitForWithoutDelivery( + predicate: () => boolean, + checkpoint: string, +): Promise { + for (let pass = 0; pass < 100; pass++) { + if (predicate()) return + await nextTurn() + } + throw new Error(`did not reach ${checkpoint}`) +} + +type CleanupDiagnostic = { + resource: string + error: string +} + +const NO_PRIMARY_FAILURE = Symbol(`no-primary-failure`) + +async function captureCleanup( + diagnostics: Array, + resource: string, + cleanup: () => Promise | void, +): Promise { + try { + await cleanup() + } catch (error) { + diagnostics.push({ + resource, + error: + error instanceof Error + ? `${error.name}: ${error.message}` + : String(error), + }) + } +} + +async function withFailurePreservingCleanup( + body: () => Promise, + cleanups: ReadonlyArray Promise | void]>, +): Promise { + let primaryFailure: unknown | typeof NO_PRIMARY_FAILURE = NO_PRIMARY_FAILURE + const cleanupDiagnostics: Array = [] + try { + await body() + } catch (error) { + primaryFailure = error + } + for (const [resource, cleanup] of cleanups) { + await captureCleanup(cleanupDiagnostics, resource, cleanup) + } + expect.soft(cleanupDiagnostics).toEqual([]) + if (primaryFailure !== NO_PRIMARY_FAILURE) throw primaryFailure +} + +type AdapterCall = { + operation: `load` | `apply` | `index` | `position` | `remote-subset` | `pull` + adapterId: string + collectionId: string + ownerAtCall?: string + argument?: SemanticValue +} + +type PullFixtureResult = { + latestRowVersion: number + requiresFullReload: false + changedKeys: Array + deletedKeys: Array +} + +type RecordedSubsetOptions = LoadSubsetOptions | TransportedLoadSubsetOptions + +type RecordingAdapter = PersistenceAdapter & { + id: string + schemaVersion: number + policy: `sync-present-reset` | `sync-absent-error` + calls: Array + getStreamPosition: (collectionId: string) => Promise<{ + latestTerm: number + latestSeq: number + latestRowVersion: number + }> + ensureRemoteSubset: ( + collectionId: string, + options: RecordedSubsetOptions, + ) => Promise + pullSince: ( + collectionId: string, + fromRowVersion: number, + ) => Promise +} + +function createRecordingAdapter(options: { + id: string + schemaVersion?: number + policy?: RecordingAdapter[`policy`] + owner?: () => string | undefined + remoteSubsetOwner?: ( + collectionId: string, + options: RecordedSubsetOptions, + ) => Promise + pullResult?: PullFixtureResult +}): RecordingAdapter { + const calls: Array = [] + const record = ( + operation: AdapterCall[`operation`], + collectionId: string, + argument?: SemanticValue, + ) => { + calls.push({ + operation, + adapterId: options.id, + collectionId, + ownerAtCall: options.owner?.(), + ...(argument === undefined ? {} : { argument }), + }) + } + + return { + id: options.id, + schemaVersion: options.schemaVersion ?? 1, + policy: options.policy ?? `sync-present-reset`, + calls, + loadSubset: (collectionId) => { + record(`load`, collectionId) + return Promise.resolve([]) + }, + applyCommittedTx: (collectionId) => { + record(`apply`, collectionId) + return Promise.resolve() + }, + ensureIndex: (collectionId, signature, spec) => { + record(`index`, collectionId, semanticValue({ signature, spec })) + return Promise.resolve() + }, + getStreamPosition: (collectionId) => { + record(`position`, collectionId) + return Promise.resolve({ + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + }) + }, + ensureRemoteSubset: (collectionId, subsetOptions) => { + record(`remote-subset`, collectionId, subsetSemantics(subsetOptions)) + if (!options.remoteSubsetOwner) { + throw new Error(`no remote subset owner for ${collectionId}`) + } + return options.remoteSubsetOwner(collectionId, subsetOptions) + }, + pullSince: (collectionId, fromRowVersion) => { + record(`pull`, collectionId, semanticValue({ fromRowVersion })) + const result = options.pullResult ?? { + latestRowVersion: 0, + requiresFullReload: false, + changedKeys: [], + deletedKeys: [], + } + return Promise.resolve(structuredClone(result)) + }, + } +} + +type SemanticValue = + | null + | boolean + | number + | string + | Array + | { [key: string]: SemanticValue } + +function semanticValue(value: unknown): SemanticValue { + if ( + value === null || + typeof value === `boolean` || + typeof value === `number` || + typeof value === `string` + ) { + return value + } + if (value instanceof Date) return { date: value.toISOString() } + if (value instanceof Uint8Array) return { bytes: Array.from(value) } + if (Array.isArray(value)) return value.map(semanticValue) + if (typeof value === `object`) { + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, semanticValue(entry)]), + ) + } + throw new Error(`unsupported semantic wire value: ${typeof value}`) +} + +function subsetSemantics( + options: LoadSubsetOptions | TransportedLoadSubsetOptions, +): SemanticValue { + return semanticValue({ + ...(options.where === undefined ? {} : { where: options.where }), + ...(options.orderBy === undefined ? {} : { orderBy: options.orderBy }), + ...(options.limit === undefined ? {} : { limit: options.limit }), + ...(options.cursor === undefined ? {} : { cursor: options.cursor }), + ...(options.offset === undefined ? {} : { offset: options.offset }), + }) +} + +function complexSubsetDemand(includeSubscription: boolean): LoadSubsetOptions { + return { + where: new IR.Func(`eq`, [ + new IR.PropRef([`alpha`, `createdAt`]), + new IR.Value(new Date(`2026-09-16T12:34:56.000Z`)), + ]), + orderBy: [ + { + expression: new IR.PropRef([`alpha`, `score`]), + compareOptions: { direction: `desc`, nulls: `last` }, + }, + ], + limit: 3, + cursor: { + whereFrom: new IR.Func(`gt`, [ + new IR.PropRef([`alpha`, `token`]), + new IR.Value(new Uint8Array([1, 2, 255])), + ]), + whereCurrent: new IR.Func(`eq`, [ + new IR.PropRef([`alpha`, `score`]), + new IR.Value(42), + ]), + lastKey: `a-42`, + }, + offset: 1, + ...(includeSubscription + ? { + subscription: { + on: () => () => {}, + } as unknown as Subscription, + } + : {}), + } +} + +const liveCoordinators = new Set() + +function createCoordinator( + dbName: string, + adapter: PersistenceAdapter, +): BrowserCollectionCoordinator { + const coordinator = new BrowserCollectionCoordinator({ dbName, adapter }) + liveCoordinators.add(coordinator) + return coordinator +} + +function registerCollectionAdapter( + coordinator: BrowserCollectionCoordinator, + collectionId: string, + adapter: RecordingAdapter, + registerOwner: boolean = true, +): void { + coordinator.setAdapterForCollection(collectionId, adapter) + if (!registerOwner) return + coordinator.registerRemoteSubsetOwner( + collectionId, + Object.assign( + (options: TransportedLoadSubsetOptions) => + adapter.ensureRemoteSubset(collectionId, options), + { + // These route/wire laws do not exercise lease release semantics; the + // dedicated remote-ownership oracle below owns exact unload behavior. + unloadSubset: () => {}, + onError: () => {}, + }, + ), + ) +} + +async function disposeCoordinator( + coordinator?: BrowserCollectionCoordinator, +): Promise { + if (!coordinator || !liveCoordinators.has(coordinator)) return + coordinator.dispose() + liveCoordinators.delete(coordinator) + await nextTurn() +} + +function coordinatorFixtureSnapshot() { + return { + delayed: ControlledBroadcastChannel.delayed.length, + endpoints: ControlledBroadcastChannel.endpoints.size, + heldLocks: heldLocks.size, + queuedLocks: Array.from(queuedLocks.values()).reduce( + (count, queue) => count + queue.length, + 0, + ), + } +} + +beforeEach(() => { + installControlledBrowserSeams() +}) + +afterEach(async () => { + const cleanupDiagnostics: Array = [] + for (const coordinator of liveCoordinators) { + await captureCleanup(cleanupDiagnostics, `afterEach live coordinator`, () => + coordinator.dispose(), + ) + } + liveCoordinators.clear() + await captureCleanup( + cleanupDiagnostics, + `afterEach lock release`, + async () => { + await waitForWithoutDelivery( + () => heldLocks.size === 0 && queuedLocks.size === 0, + `afterEach lock release`, + ) + }, + ) + const cleanupSnapshot = coordinatorFixtureSnapshot() + expect.soft(cleanupSnapshot).toEqual({ + delayed: 0, + endpoints: 0, + heldLocks: 0, + queuedLocks: 0, + }) + await captureCleanup( + cleanupDiagnostics, + `global browser descriptors`, + restoreControlledBrowserSeams, + ) + await captureCleanup(cleanupDiagnostics, `vitest timers and mocks`, () => { + vi.useRealTimers() + vi.restoreAllMocks() + }) + ControlledBroadcastChannel.reset() + heldLocks.clear() + queuedLocks.clear() + expect.soft(cleanupDiagnostics).toEqual([]) +}) + +type SubsetAckTrace = { + ownerInvocation: `coordinator` | `fixture-fallback` + decoded: SemanticValue | undefined + expected: SemanticValue + prematureResponses: number + acknowledgedBeforeResponseDelivery: boolean + ownerCompletedBeforeResponse: boolean + acknowledgedAfterResponseDelivery: boolean +} + +type SubsetWireTrace = { + outcome: { ok: true } | { ok: false; errorName: string } | undefined + decoded: ReadonlyArray + expected: SemanticValue +} + +function subsetWireViolations(trace: SubsetWireTrace): Array { + const violations: Array = [] + if (trace.outcome?.ok !== true) violations.push(`subset-transport-failed`) + if ( + trace.decoded.length !== 1 || + JSON.stringify(trace.decoded[0]) !== JSON.stringify(trace.expected) + ) { + violations.push(`subset-semantics-mismatch`) + } + return violations +} + +function subsetAckViolations(trace: SubsetAckTrace): Array { + const violations: Array = [] + if (trace.ownerInvocation !== `coordinator`) { + violations.push(`leader-owner-not-invoked-by-coordinator`) + } + if (JSON.stringify(trace.decoded) !== JSON.stringify(trace.expected)) { + violations.push(`decoded-subset-mismatch`) + } + if (trace.prematureResponses !== 0) { + violations.push(`response-before-owner-completion`) + } + if (trace.acknowledgedBeforeResponseDelivery) { + violations.push(`ack-before-response-delivery`) + } + if (!trace.ownerCompletedBeforeResponse) { + violations.push(`response-before-owner-completion-checkpoint`) + } + if (!trace.acknowledgedAfterResponseDelivery) { + violations.push(`missing-ack-after-response`) + } + return violations +} + +async function observeOwnerEntryOrResponse( + ownerEntered: () => boolean, +): Promise<`owner-entered` | `response-queued` | `no-progress`> { + for (let turn = 0; turn < 20; turn++) { + if (ownerEntered()) return `owner-entered` + if ( + ControlledBroadcastChannel.delayed.some( + ({ data }) => payloadType(data) === `rpc:ensureRemoteSubset:res`, + ) + ) { + return `response-queued` + } + await nextTurn() + } + return `no-progress` +} + +describe(`per-collection coordinator wire and acknowledgement oracle`, () => { + it(`posts the callback-bearing subset witness as clone-safe wire data`, async () => { + const decoded: Array = [] + const leaderAdapter = createRecordingAdapter({ + id: `leader`, + remoteSubsetOwner: (_collectionId, options) => { + decoded.push(subsetSemantics(options)) + return Promise.resolve() + }, + }) + const followerAdapter = createRecordingAdapter({ id: `follower` }) + let leader: BrowserCollectionCoordinator | undefined + let follower: BrowserCollectionCoordinator | undefined + + await withFailurePreservingCleanup(async () => { + leader = createCoordinator(`clone-witness`, leaderAdapter) + follower = createCoordinator(`clone-witness`, followerAdapter) + registerCollectionAdapter(leader, `alpha`, leaderAdapter) + registerCollectionAdapter(follower, `alpha`, followerAdapter) + leader.subscribe(`alpha`, () => {}) + follower.subscribe(`alpha`, () => {}) + await waitFor( + () => leader!.isLeader(`alpha`) && !follower!.isLeader(`alpha`), + `clone-witness leader/follower topology`, + ) + const safeTransportBeforeRequest = + ControlledBroadcastChannel.cloneAttempts.filter( + (attempt) => attempt.errorName === undefined, + ).length + expect(safeTransportBeforeRequest).toBeGreaterThan(0) + + const options = complexSubsetDemand(true) + vi.useFakeTimers() + let outcome: { ok: true } | { ok: false; errorName: string } | undefined + void follower.requestEnsureRemoteSubset(`alpha`, options).then( + () => { + outcome = { ok: true } + }, + (error: unknown) => { + outcome = { + ok: false, + errorName: error instanceof Error ? error.name : String(error), + } + }, + ) + for (let step = 0; step < 12 && outcome === undefined; step++) { + while (ControlledBroadcastChannel.deliverAt(0)) { + await Promise.resolve() + } + await vi.advanceTimersByTimeAsync(100) + } + + expect(outcome).toBeDefined() + // Preserve the current-main DataCloneError as an observation without + // making the defect a positive reach requirement for the GREEN suite. + const observedCloneFault = ControlledBroadcastChannel.cloneAttempts.find( + (attempt) => attempt.errorName !== undefined, + ) + if (outcome?.ok === false) { + expect(outcome.errorName).toBe(observedCloneFault?.errorName) + } + // RED D3: after clone-safe projection, the registered leader owner must + // receive the exact supported semantics and the follower call succeeds. + expect( + subsetWireViolations({ + outcome, + decoded, + expected: subsetSemantics(options), + }), + ).toEqual([]) + }, [ + [ + `clone witness leader`, + async () => { + await disposeCoordinator(leader) + }, + ], + [ + `clone witness follower`, + async () => { + await disposeCoordinator(follower) + }, + ], + [ + `clone witness timers`, + async () => { + vi.clearAllTimers() + vi.useRealTimers() + await nextTurn() + }, + ], + ]) + }) + + it(`calibrates the clone checker against safe and function-valued payloads`, () => { + const safe = { + collectionId: `alpha`, + where: { type: `eq`, field: `group`, value: `kept` }, + limit: 2, + } + const unsafe = { ...safe, callback: () => {} } + + expect(structuredClone(safe)).toEqual(safe) + expect(() => structuredClone(unsafe)).toThrowError( + expect.objectContaining({ name: `DataCloneError` }), + ) + + const expected = subsetSemantics(complexSubsetDemand(false)) + const allowed: SubsetWireTrace = { + outcome: { ok: true }, + decoded: [expected], + expected, + } + expect(subsetWireViolations(allowed)).toEqual([]) + expect( + subsetWireViolations({ + ...allowed, + outcome: { ok: false, errorName: `DataCloneError` }, + }), + ).toContain(`subset-transport-failed`) + expect( + subsetWireViolations({ ...allowed, decoded: [semanticValue({})] }), + ).toContain(`subset-semantics-mismatch`) + }) + + it(`round-trips supported subset semantics across the emitted message partition`, async () => { + const leaderAdapter = createRecordingAdapter({ + id: `partition-leader`, + remoteSubsetOwner: () => Promise.resolve(), + }) + const followerAdapter = createRecordingAdapter({ id: `partition-follower` }) + let leader: BrowserCollectionCoordinator | undefined + let follower: BrowserCollectionCoordinator | undefined + + await withFailurePreservingCleanup(async () => { + leader = createCoordinator(`wire-partition`, leaderAdapter) + follower = createCoordinator(`wire-partition`, followerAdapter) + registerCollectionAdapter(leader, `alpha`, leaderAdapter) + registerCollectionAdapter(follower, `alpha`, followerAdapter) + leader.subscribe(`alpha`, () => {}) + follower.subscribe(`alpha`, () => {}) + await waitFor( + () => leader!.isLeader(`alpha`) && !follower!.isLeader(`alpha`), + `wire partition leader/follower topology`, + ) + + const demand = complexSubsetDemand(false) + const subsetRequest = follower.requestEnsureRemoteSubset(`alpha`, demand) + const queuedSubset = ControlledBroadcastChannel.delayed.find( + ({ data }) => payloadType(data) === `rpc:ensureRemoteSubset:req`, + )?.data as { payload?: { options?: LoadSubsetOptions } } | undefined + expect(subsetSemantics(queuedSubset?.payload?.options ?? {})).toEqual( + subsetSemantics(demand), + ) + await pumpNetwork() + await subsetRequest + + const mutationValue = { + id: `native-values`, + at: new Date(`2026-09-16T12:34:56.000Z`), + bytes: new Uint8Array([0, 127, 255]), + } + const mutationRequest = follower.requestApplyLocalMutations(`alpha`, [ + { + mutationId: `native-values`, + type: `insert`, + key: `native-values`, + value: mutationValue, + }, + ]) + const queuedMutation = ControlledBroadcastChannel.delayed.find( + ({ data }) => payloadType(data) === `rpc:applyLocalMutations:req`, + )?.data as + | { + payload?: { + mutations?: Array<{ value?: unknown }> + } + } + | undefined + expect( + semanticValue(queuedMutation?.payload?.mutations?.[0]?.value), + ).toEqual(semanticValue(mutationValue)) + await pumpNetwork() + await mutationRequest + + const committedTx: PersistedTx = { + txId: `committed-native-values`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `update`, + key: `native-values`, + value: mutationValue, + }, + ], + rowMetadataMutations: [ + { + type: `set`, + key: `native-values`, + value: { source: `wire-partition` }, + }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `resume`, value: { offset: 4 } }, + ], + } + const committedRequest = follower.requestApplyCommittedTx( + `alpha`, + committedTx, + ) + const queuedCommitted = ControlledBroadcastChannel.delayed.find( + ({ data }) => payloadType(data) === `rpc:applyCommittedTx:req`, + )?.data as { payload?: { tx?: PersistedTx } } | undefined + expect(semanticValue(queuedCommitted?.payload?.tx)).toEqual( + semanticValue(committedTx), + ) + await pumpNetwork() + const committedResponse = await committedRequest + expect(committedResponse).toEqual({ + type: `rpc:applyCommittedTx:res`, + rpcId: expect.any(String), + ok: true, + term: 1, + seq: 2, + latestRowVersion: 2, + }) + + const successfulPayloads = ControlledBroadcastChannel.cloneAttempts + .filter(({ errorName }) => errorName === undefined) + .map(({ data }) => + data && typeof data === `object` && `payload` in data + ? (data as { payload?: unknown }).payload + : undefined, + ) + const matchingCommittedResponses = successfulPayloads.filter( + (payload) => + payload !== undefined && + payload !== null && + typeof payload === `object` && + `type` in payload && + payload.type === `rpc:applyCommittedTx:res` && + `rpcId` in payload && + payload.rpcId === committedResponse.rpcId, + ) + const matchingCommittedEvents = successfulPayloads.filter( + (payload) => + payload !== undefined && + payload !== null && + typeof payload === `object` && + `type` in payload && + payload.type === `tx:committed` && + `txId` in payload && + payload.txId === committedTx.txId, + ) + expect(matchingCommittedResponses).toEqual([committedResponse]) + expect(matchingCommittedEvents).toEqual([ + { + type: `tx:committed`, + term: 1, + seq: 2, + txId: `committed-native-values`, + latestRowVersion: 2, + requiresFullReload: false, + changedRows: [{ key: `native-values`, value: mutationValue }], + deletedKeys: [], + rowMetadataMutations: [ + { + type: `set`, + key: `native-values`, + value: { source: `wire-partition` }, + }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `resume`, value: { offset: 4 } }, + ], + }, + ]) + + const indexRequest = follower.requestEnsurePersistedIndex( + `alpha`, + `score-index`, + { expressionSql: [`score`] }, + ) + await pumpNetwork() + await indexRequest + const pullRequest = follower.pullSince(`alpha`, 0) + await pumpNetwork() + await pullRequest + + const successfulTypes = new Set( + ControlledBroadcastChannel.cloneAttempts + .filter(({ errorName }) => errorName === undefined) + .map(({ data }) => payloadType(data)) + .filter((type): type is string => type !== undefined), + ) + expect(successfulTypes).toEqual( + new Set([ + `leader:heartbeat`, + `rpc:ensureRemoteSubset:req`, + `rpc:ensureRemoteSubset:res`, + `rpc:applyLocalMutations:req`, + `rpc:applyLocalMutations:res`, + `rpc:applyCommittedTx:req`, + `rpc:applyCommittedTx:res`, + `tx:committed`, + `rpc:ensurePersistedIndex:req`, + `rpc:ensurePersistedIndex:res`, + `rpc:pullSince:req`, + `rpc:pullSince:res`, + ]), + ) + }, [ + [ + `wire partition leader`, + async () => { + await disposeCoordinator(leader) + }, + ], + [ + `wire partition follower`, + async () => { + await disposeCoordinator(follower) + }, + ], + ]) + }) + + it(`calibrates acknowledgement checks against entry-only and false-success faults`, () => { + const expected = subsetSemantics(complexSubsetDemand(false)) + const allowed: SubsetAckTrace = { + ownerInvocation: `coordinator`, + decoded: expected, + expected, + prematureResponses: 0, + acknowledgedBeforeResponseDelivery: false, + ownerCompletedBeforeResponse: true, + acknowledgedAfterResponseDelivery: true, + } + expect(subsetAckViolations(allowed)).toEqual([]) + expect( + subsetAckViolations({ + ...allowed, + ownerCompletedBeforeResponse: false, + }), + ).toContain(`response-before-owner-completion-checkpoint`) + expect( + subsetAckViolations({ + ...allowed, + ownerInvocation: `fixture-fallback`, + prematureResponses: 1, + }), + ).toEqual([ + `leader-owner-not-invoked-by-coordinator`, + `response-before-owner-completion`, + ]) + }) + + it(`acknowledges a follower subset only after the leader source completed it`, async () => { + const upstreamEvents: Array<{ + phase: `entered` | `completed` + semantics: SemanticValue + }> = [] + const releases: Array<() => void> = [] + const ownerDelegate: { + invoke?: ( + collectionId: string, + options: RecordedSubsetOptions, + ) => Promise + } = {} + let leader: BrowserCollectionCoordinator | undefined + let follower: BrowserCollectionCoordinator | undefined + let cleanupLeaderCollection: (() => Promise) | undefined + + await withFailurePreservingCleanup(async () => { + const leaderAdapter = createRecordingAdapter({ + id: `leader-alpha`, + remoteSubsetOwner: (collectionId, options) => { + if (!ownerDelegate.invoke) { + throw new Error(`leader owner not attached`) + } + return ownerDelegate.invoke(collectionId, options) + }, + }) + const followerAdapter = createRecordingAdapter({ + id: `follower-alpha`, + }) + leader = createCoordinator(`subset-ack`, leaderAdapter) + follower = createCoordinator(`subset-ack`, followerAdapter) + registerCollectionAdapter(leader, `alpha`, leaderAdapter, false) + registerCollectionAdapter(follower, `alpha`, followerAdapter, false) + const source: SyncConfig<{ id: string; group: string }, string> = { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: (options) => { + upstreamEvents.push({ + phase: `entered`, + semantics: subsetSemantics(options), + }) + return new Promise((resolve) => { + let released = false + releases.push(() => { + if (released) return + released = true + upstreamEvents.push({ + phase: `completed`, + semantics: subsetSemantics(options), + }) + resolve() + }) + }) + }, + } + }, + } + const leaderCollection = createCollection( + persistedCollectionOptions({ + id: `alpha`, + getKey: (row: { id: string; group: string }) => row.id, + syncMode: `on-demand`, + sync: source, + persistence: { adapter: leaderAdapter, coordinator: leader }, + }), + ) + cleanupLeaderCollection = () => leaderCollection.cleanup() + ownerDelegate.invoke = async (collectionId, options) => { + if (collectionId !== `alpha`) { + throw new Error(`wrong leader subset collection ${collectionId}`) + } + await Promise.resolve( + leaderCollection._sync.loadSubset( + options as unknown as LoadSubsetOptions, + ), + ) + } + + leaderCollection.startSyncImmediate() + follower.subscribe(`alpha`, () => {}) + await waitFor( + () => leader!.isLeader(`alpha`) && !follower!.isLeader(`alpha`), + `subset-ack leader/follower topology`, + ) + + const demand = complexSubsetDemand(false) + const expectedSemantics = subsetSemantics(demand) + let acknowledged = false + const acknowledgement = follower + .requestEnsureRemoteSubset(`alpha`, demand) + .then(() => { + acknowledged = true + }) + + expect( + ControlledBroadcastChannel.deliverWhere( + (data) => + payloadType(data) === `rpc:ensureRemoteSubset:req` && + envelopeCollection(data) === `alpha`, + ), + ).toBe(true) + const firstProgress = await observeOwnerEntryOrResponse(() => + upstreamEvents.some(({ phase }) => phase === `entered`), + ) + expect(firstProgress).not.toBe(`no-progress`) + const prematureResponses = ControlledBroadcastChannel.delayed.filter( + ({ data }) => + payloadType(data) === `rpc:ensureRemoteSubset:res` && + envelopeCollection(data) === `alpha`, + ) + let ownerInvocation: SubsetAckTrace[`ownerInvocation`] = `coordinator` + let ownerWork: Promise | undefined + if (!upstreamEvents.some(({ phase }) => phase === `entered`)) { + ownerInvocation = `fixture-fallback` + for (const response of prematureResponses) { + const index = ControlledBroadcastChannel.delayed.indexOf(response) + if (index >= 0) ControlledBroadcastChannel.delayed.splice(index, 1) + } + ownerWork = leaderAdapter.ensureRemoteSubset(`alpha`, demand) + await waitForWithoutDelivery( + () => upstreamEvents.some(({ phase }) => phase === `entered`), + `fallback leader owner entry`, + ) + } + const entered = upstreamEvents.find(({ phase }) => phase === `entered`) + expect( + ControlledBroadcastChannel.delayed.some( + ({ data }) => payloadType(data) === `rpc:ensureRemoteSubset:res`, + ), + ).toBe(false) + expect(acknowledged).toBe(false) + releases.at(-1)?.() + await ownerWork + await waitForWithoutDelivery( + () => upstreamEvents.some(({ phase }) => phase === `completed`), + `leader subset completion`, + ) + if (ownerInvocation === `fixture-fallback`) { + ControlledBroadcastChannel.delayed.push(...prematureResponses) + } else { + await waitForWithoutDelivery( + () => + ControlledBroadcastChannel.delayed.some( + ({ data }) => payloadType(data) === `rpc:ensureRemoteSubset:res`, + ), + `response after leader subset completion`, + ) + } + const acknowledgedBeforeResponseDelivery = acknowledged + expect(acknowledged).toBe(false) + expect( + ControlledBroadcastChannel.deliverWhere( + (data) => + payloadType(data) === `rpc:ensureRemoteSubset:res` && + envelopeCollection(data) === `alpha`, + ), + ).toBe(true) + await acknowledgement + const trace: SubsetAckTrace = { + ownerInvocation, + decoded: entered?.semantics, + expected: expectedSemantics, + prematureResponses: prematureResponses.length, + acknowledgedBeforeResponseDelivery, + ownerCompletedBeforeResponse: upstreamEvents.some( + ({ phase }) => phase === `completed`, + ), + acknowledgedAfterResponseDelivery: acknowledged, + } + // RED D13: one comparator owns both the production verdict and hostile + // entry-only/false-success calibrations above. + expect(subsetAckViolations(trace)).toEqual([]) + }, [ + [ + `pending subset source releases`, + () => { + for (const release of releases) release() + }, + ], + [ + `leader collection`, + async () => { + await cleanupLeaderCollection?.() + }, + ], + [ + `leader coordinator`, + async () => { + await disposeCoordinator(leader) + }, + ], + [ + `follower coordinator`, + async () => { + await disposeCoordinator(follower) + }, + ], + ]) + }) +}) + +type RemoteLeaseEvent = { + phase: `load` | `unload` + token: number + semantics: SemanticValue + exactLoadedObject?: boolean +} + +class RemoteLeaseOwnerLedger { + private nextToken = 0 + private readonly tokens = new WeakMap() + private readonly loaded = new WeakSet() + readonly active = new Set() + readonly events: Array = [] + + private token(options: TransportedLoadSubsetOptions): number { + let token = this.tokens.get(options) + if (token === undefined) { + token = ++this.nextToken + this.tokens.set(options, token) + } + return token + } + + readonly loadSubset = async ( + options: TransportedLoadSubsetOptions, + ): Promise => { + const token = this.token(options) + this.loaded.add(options) + this.active.add(options) + this.events.push({ + phase: `load`, + token, + semantics: subsetSemantics(options as unknown as LoadSubsetOptions), + }) + } + + readonly unloadSubset = (options: TransportedLoadSubsetOptions): void => { + const token = this.token(options) + const exactLoadedObject = this.loaded.has(options) + this.active.delete(options) + this.events.push({ + phase: `unload`, + token, + semantics: subsetSemantics(options as unknown as LoadSubsetOptions), + exactLoadedObject, + }) + } + + /** Callable for today's owner boundary; properties are the crash-only owner. */ + readonly boundary = Object.assign( + (options: TransportedLoadSubsetOptions) => this.loadSubset(options), + { + loadSubset: this.loadSubset, + unloadSubset: this.unloadSubset, + onError: () => {}, + }, + ) + + snapshot() { + return { + loads: this.events.filter(({ phase }) => phase === `load`).length, + unloads: this.events.filter(({ phase }) => phase === `unload`).length, + active: this.active.size, + exactUnloads: this.events + .filter( + (event): event is RemoteLeaseEvent & { phase: `unload` } => + event.phase === `unload`, + ) + .every(({ exactLoadedObject }) => exactLoadedObject === true), + } + } +} + +type RemoteLeaseTakeoverTrace = { + initialOwnerAfterAcquire: ReturnType + retiredOwnerAfterTakeover: ReturnType + newOwnerAfterTakeover: ReturnType + newOwnerAfterRelease: ReturnType +} + +type RemoteLeaseTakeoverViolation = { + checkpoint: string + field: keyof ReturnType + expected: number | boolean + actual: number | boolean +} + +const expectedRemoteLeaseTakeoverTrace: RemoteLeaseTakeoverTrace = { + initialOwnerAfterAcquire: { + loads: 1, + unloads: 0, + active: 1, + exactUnloads: true, + }, + retiredOwnerAfterTakeover: { + loads: 1, + unloads: 1, + active: 0, + exactUnloads: true, + }, + newOwnerAfterTakeover: { + loads: 1, + unloads: 0, + active: 1, + exactUnloads: true, + }, + newOwnerAfterRelease: { + loads: 1, + unloads: 1, + active: 0, + exactUnloads: true, + }, +} + +function remoteLeaseTakeoverViolations( + actual: RemoteLeaseTakeoverTrace, +): Array { + const violations: Array = [] + for (const checkpoint of Object.keys( + expectedRemoteLeaseTakeoverTrace, + ) as Array) { + const expected = expectedRemoteLeaseTakeoverTrace[checkpoint] + for (const field of Object.keys(expected) as Array) { + if (actual[checkpoint][field] !== expected[field]) { + violations.push({ + checkpoint, + field, + expected: expected[field], + actual: actual[checkpoint][field], + }) + } + } + } + return violations +} + +type CollectionScopedLeaseSnapshot = Record< + `alpha` | `beta`, + { loads: number; unloads: number; active: number } +> + +function collectionScopedLeaseViolations( + actual: CollectionScopedLeaseSnapshot, + expected: CollectionScopedLeaseSnapshot, +): Array { + const violations: Array = [] + for (const collectionId of [`alpha`, `beta`] as const) { + for (const field of [`loads`, `unloads`, `active`] as const) { + if (actual[collectionId][field] !== expected[collectionId][field]) { + violations.push(`${collectionId}:${field}`) + } + } + } + return violations +} + +type RemoteLeaseHistory = { + siblingCount: number + releasePrefix: number + identical: boolean + duplicateRelease: boolean + reverseFinalRelease: boolean +} + +type RemoteLeaseHistoryTrace = { + firstAfterAcquire: ReturnType + firstAfterPrefixRelease: ReturnType + firstAfterTakeover: ReturnType + secondAfterTakeover: ReturnType + secondAfterFinalRelease: ReturnType +} + +const remoteLeaseHistoryArbitrary = fc.record({ + siblingCount: fc.integer({ min: 1, max: 3 }), + releasePrefix: fc.integer({ min: 0, max: 2 }), + identical: fc.boolean(), + duplicateRelease: fc.boolean(), + reverseFinalRelease: fc.boolean(), +}) + +function remoteLeaseHistoryViolations( + history: RemoteLeaseHistory, + actual: RemoteLeaseHistoryTrace, +): Array { + const releaseCount = Math.min(history.releasePrefix, history.siblingCount - 1) + const remaining = history.siblingCount - releaseCount + const expected: RemoteLeaseHistoryTrace = { + firstAfterAcquire: { + loads: history.siblingCount, + unloads: 0, + active: history.siblingCount, + exactUnloads: true, + }, + firstAfterPrefixRelease: { + loads: history.siblingCount, + unloads: releaseCount, + active: remaining, + exactUnloads: true, + }, + firstAfterTakeover: { + loads: history.siblingCount, + unloads: history.siblingCount, + active: 0, + exactUnloads: true, + }, + secondAfterTakeover: { + loads: remaining, + unloads: 0, + active: remaining, + exactUnloads: true, + }, + secondAfterFinalRelease: { + loads: remaining, + unloads: remaining, + active: 0, + exactUnloads: true, + }, + } + const violations: Array = [] + for (const checkpoint of Object.keys(expected) as Array< + keyof RemoteLeaseHistoryTrace + >) { + for (const field of Object.keys(expected[checkpoint]) as Array< + keyof ReturnType + >) { + if (actual[checkpoint][field] !== expected[checkpoint][field]) { + violations.push({ + checkpoint, + field, + expected: expected[checkpoint][field], + actual: actual[checkpoint][field], + }) + } + } + } + return violations +} + +describe(`remote subset ownership lease oracle`, () => { + it(`calibrates collection-scoped acquisition identity against a global-id mutant`, () => { + const expected: CollectionScopedLeaseSnapshot = { + alpha: { loads: 1, unloads: 0, active: 1 }, + beta: { loads: 1, unloads: 0, active: 1 }, + } + expect(collectionScopedLeaseViolations(expected, expected)).toEqual([]) + expect( + collectionScopedLeaseViolations( + { + alpha: { loads: 2, unloads: 0, active: 2 }, + beta: { loads: 1, unloads: 0, active: 1 }, + }, + expected, + ), + ).toEqual([`alpha:loads`, `alpha:active`]) + }) + + it(`keeps reused option identities independent per collection across bounded histories`, async () => { + type Action = { + kind: `ensure` | `release` + collectionId: `alpha` | `beta` + objectId: 0 | 1 + } + const histories: ReadonlyArray<{ + name: string + actions: ReadonlyArray + }> = [ + { + name: `shared object A-B-A`, + actions: [ + { kind: `ensure`, collectionId: `alpha`, objectId: 0 }, + { kind: `ensure`, collectionId: `beta`, objectId: 0 }, + { kind: `ensure`, collectionId: `alpha`, objectId: 0 }, + { kind: `release`, collectionId: `alpha`, objectId: 0 }, + { kind: `release`, collectionId: `beta`, objectId: 0 }, + ], + }, + { + name: `released object reacquired beside sibling collection`, + actions: [ + { kind: `ensure`, collectionId: `alpha`, objectId: 0 }, + { kind: `release`, collectionId: `alpha`, objectId: 0 }, + { kind: `ensure`, collectionId: `beta`, objectId: 0 }, + { kind: `ensure`, collectionId: `alpha`, objectId: 0 }, + { kind: `release`, collectionId: `alpha`, objectId: 0 }, + { kind: `release`, collectionId: `beta`, objectId: 0 }, + ], + }, + { + name: `two identities with duplicate releases`, + actions: [ + { kind: `ensure`, collectionId: `alpha`, objectId: 0 }, + { kind: `ensure`, collectionId: `alpha`, objectId: 1 }, + { kind: `ensure`, collectionId: `beta`, objectId: 0 }, + { kind: `ensure`, collectionId: `beta`, objectId: 1 }, + { kind: `ensure`, collectionId: `alpha`, objectId: 0 }, + { kind: `release`, collectionId: `beta`, objectId: 1 }, + { kind: `release`, collectionId: `beta`, objectId: 1 }, + { kind: `release`, collectionId: `alpha`, objectId: 1 }, + { kind: `release`, collectionId: `alpha`, objectId: 0 }, + { kind: `release`, collectionId: `beta`, objectId: 0 }, + ], + }, + ] + + for (const [historyIndex, history] of histories.entries()) { + const owners = { + alpha: new RemoteLeaseOwnerLedger(), + beta: new RemoteLeaseOwnerLedger(), + } + const live = { + alpha: new Set(), + beta: new Set(), + } + const expected: CollectionScopedLeaseSnapshot = { + alpha: { loads: 0, unloads: 0, active: 0 }, + beta: { loads: 0, unloads: 0, active: 0 }, + } + const coordinator = createCoordinator( + `collection-scoped-lease-${historyIndex}`, + createRecordingAdapter({ id: `collection-scoped-${historyIndex}` }), + ) + const unregisterAlpha = coordinator.registerRemoteSubsetOwner( + `alpha`, + owners.alpha.boundary, + ) + const unregisterBeta = coordinator.registerRemoteSubsetOwner( + `beta`, + owners.beta.boundary, + ) + coordinator.subscribe(`alpha`, () => {}) + coordinator.subscribe(`beta`, () => {}) + const objects: [LoadSubsetOptions, LoadSubsetOptions] = [ + { limit: 1, offset: 0 }, + { limit: 2, offset: 1 }, + ] + + await withFailurePreservingCleanup(async () => { + await waitFor( + () => coordinator.isLeader(`alpha`) && coordinator.isLeader(`beta`), + `${history.name} leadership`, + ) + for (const [actionIndex, action] of history.actions.entries()) { + const liveObjects = live[action.collectionId] + const expectedCollection = expected[action.collectionId] + if (action.kind === `ensure`) { + await coordinator.requestEnsureRemoteSubset( + action.collectionId, + objects[action.objectId], + ) + if (!liveObjects.has(action.objectId)) { + liveObjects.add(action.objectId) + expectedCollection.loads++ + expectedCollection.active++ + } + } else { + await coordinator.requestReleaseRemoteSubset( + action.collectionId, + objects[action.objectId], + ) + if (liveObjects.delete(action.objectId)) { + expectedCollection.unloads++ + expectedCollection.active-- + } + } + + const alpha = owners.alpha.snapshot() + const beta = owners.beta.snapshot() + const actual: CollectionScopedLeaseSnapshot = { + alpha: { + loads: alpha.loads, + unloads: alpha.unloads, + active: alpha.active, + }, + beta: { + loads: beta.loads, + unloads: beta.unloads, + active: beta.active, + }, + } + expect( + collectionScopedLeaseViolations(actual, expected), + `${history.name} action ${actionIndex}: ${JSON.stringify(action)}; actual=${JSON.stringify(actual)} expected=${JSON.stringify(expected)}`, + ).toEqual([]) + } + }, [ + [`${history.name} alpha owner`, unregisterAlpha], + [`${history.name} beta owner`, unregisterBeta], + [ + `${history.name} coordinator`, + async () => { + await disposeCoordinator(coordinator) + }, + ], + ]) + } + }) + + it(`keeps one public acquisition live across leader takeover and releases both owners exactly`, async () => { + type Todo = { id: string; title: string } + const firstOwner = new RemoteLeaseOwnerLedger() + const secondOwner = new RemoteLeaseOwnerLedger() + const dbName = `remote-lease-takeover-fixed` + const collectionId = `lease-todos` + let firstLeader: BrowserCollectionCoordinator | undefined + let secondLeader: BrowserCollectionCoordinator | undefined + let requester: BrowserCollectionCoordinator | undefined + let unregisterFirst: (() => void) | undefined + let unregisterSecond: (() => void) | undefined + let cleanupCollection: (() => Promise) | undefined + + await withFailurePreservingCleanup(async () => { + firstLeader = createCoordinator( + dbName, + createRecordingAdapter({ id: `lease-first` }), + ) + secondLeader = createCoordinator( + dbName, + createRecordingAdapter({ id: `lease-second` }), + ) + requester = createCoordinator( + dbName, + createRecordingAdapter({ id: `lease-requester` }), + ) + unregisterFirst = firstLeader.registerRemoteSubsetOwner( + collectionId, + firstOwner.boundary, + ) + unregisterSecond = secondLeader.registerRemoteSubsetOwner( + collectionId, + secondOwner.boundary, + ) + firstLeader.subscribe(collectionId, () => {}) + secondLeader.subscribe(collectionId, () => {}) + await waitFor( + () => + firstLeader!.isLeader(collectionId) && + !secondLeader!.isLeader(collectionId), + `initial remote subset owner`, + ) + + const collection = createCollection( + persistedCollectionOptions({ + id: collectionId, + getKey: (todo) => todo.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return {} + }, + }, + persistence: { + adapter: createRecordingAdapter({ id: `lease-requester-runtime` }), + coordinator: requester, + }, + }), + ) + cleanupCollection = () => collection.cleanup() + collection.startSyncImmediate() + + const demand: LoadSubsetOptions = { + where: new IR.Func(`eq`, [ + new IR.PropRef([collectionId, `group`]), + new IR.Value(`owned`), + ]), + limit: 2, + } + const acquisition = Promise.resolve(collection._sync.loadSubset(demand)) + await pumpNetwork() + await acquisition + const initialOwnerAfterAcquire = firstOwner.snapshot() + + await disposeCoordinator(firstLeader) + await waitFor( + () => secondLeader!.isLeader(collectionId), + `remote subset takeover leader`, + ) + // Deliver the new leader heartbeat and every acquisition replay it + // causes. The public acquisition remains live throughout this cut. + await pumpNetwork() + await nextTurn() + await pumpNetwork() + const retiredOwnerAfterTakeover = firstOwner.snapshot() + const newOwnerAfterTakeover = secondOwner.snapshot() + + collection._sync.unloadSubset(demand) + await pumpNetwork() + await nextTurn() + await pumpNetwork() + const newOwnerAfterRelease = secondOwner.snapshot() + + const trace: RemoteLeaseTakeoverTrace = { + initialOwnerAfterAcquire, + retiredOwnerAfterTakeover, + newOwnerAfterTakeover, + newOwnerAfterRelease, + } + expect(remoteLeaseTakeoverViolations(trace)).toEqual([]) + }, [ + [ + `remote lease requester collection`, + async () => { + await cleanupCollection?.() + }, + ], + [`first remote subset owner`, () => unregisterFirst?.()], + [`second remote subset owner`, () => unregisterSecond?.()], + [ + `first remote subset leader`, + async () => { + await disposeCoordinator(firstLeader) + }, + ], + [ + `second remote subset leader`, + async () => { + await disposeCoordinator(secondLeader) + }, + ], + [ + `remote subset requester`, + async () => { + await disposeCoordinator(requester) + }, + ], + ]) + }) + + it(`calibrates the lease checker against leak, missing replay, and inexact release mutants`, () => { + expect( + remoteLeaseTakeoverViolations(expectedRemoteLeaseTakeoverTrace), + ).toEqual([]) + const hostile: RemoteLeaseTakeoverTrace = structuredClone( + expectedRemoteLeaseTakeoverTrace, + ) + hostile.retiredOwnerAfterTakeover = { + loads: 1, + unloads: 0, + active: 1, + exactUnloads: true, + } + hostile.newOwnerAfterTakeover = { + loads: 0, + unloads: 0, + active: 0, + exactUnloads: true, + } + hostile.newOwnerAfterRelease.exactUnloads = false + expect( + remoteLeaseTakeoverViolations(hostile).map( + ({ checkpoint, field }) => `${checkpoint}:${field}`, + ), + ).toEqual([ + `retiredOwnerAfterTakeover:unloads`, + `retiredOwnerAfterTakeover:active`, + `newOwnerAfterTakeover:loads`, + `newOwnerAfterTakeover:active`, + `newOwnerAfterRelease:exactUnloads`, + ]) + }) + + it(`preserves independent sibling leases through release, retry-safe takeover, and cleanup histories`, async () => { + let originalHistory: RemoteLeaseHistory | undefined + let originalViolation: RemoteLeaseTakeoverViolation | undefined + let targetDiscriminant: string | undefined + const cleanupDiagnostics: Array< + CleanupDiagnostic & { history: RemoteLeaseHistory } + > = [] + const executionDiagnostics: Array< + CleanupDiagnostic & { history: RemoteLeaseHistory; checkpoint: string } + > = [] + let propertyFailure: unknown | typeof NO_PRIMARY_FAILURE = + NO_PRIMARY_FAILURE + let run = 0 + + try { + await fc.assert( + fc.asyncProperty(remoteLeaseHistoryArbitrary, async (history) => { + const firstOwner = new RemoteLeaseOwnerLedger() + const secondOwner = new RemoteLeaseOwnerLedger() + const suffix = `${++run}-${history.siblingCount}-${history.releasePrefix}` + const dbName = `remote-lease-history-${suffix}` + const collectionId = `lease-history-${suffix}` + let firstLeader: BrowserCollectionCoordinator | undefined + let secondLeader: BrowserCollectionCoordinator | undefined + let requester: BrowserCollectionCoordinator | undefined + let unregisterFirst: (() => void) | undefined + let unregisterSecond: (() => void) | undefined + let cleanupCollection: (() => Promise) | undefined + let semanticFailure: unknown | typeof NO_PRIMARY_FAILURE = + NO_PRIMARY_FAILURE + let checkpoint = `coordinator allocation` + + try { + firstLeader = createCoordinator( + dbName, + createRecordingAdapter({ id: `history-first` }), + ) + secondLeader = createCoordinator( + dbName, + createRecordingAdapter({ id: `history-second` }), + ) + requester = createCoordinator( + dbName, + createRecordingAdapter({ id: `history-requester` }), + ) + unregisterFirst = firstLeader.registerRemoteSubsetOwner( + collectionId, + firstOwner.boundary, + ) + unregisterSecond = secondLeader.registerRemoteSubsetOwner( + collectionId, + secondOwner.boundary, + ) + firstLeader.subscribe(collectionId, () => {}) + secondLeader.subscribe(collectionId, () => {}) + checkpoint = `initial leadership` + await waitFor( + () => + firstLeader!.isLeader(collectionId) && + !secondLeader!.isLeader(collectionId), + `generated remote subset owner`, + ) + + const collection = createCollection( + persistedCollectionOptions<{ id: string }, string>({ + id: collectionId, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return {} + }, + }, + persistence: { + adapter: createRecordingAdapter({ id: `history-runtime` }), + coordinator: requester, + }, + }), + ) + cleanupCollection = () => collection.cleanup() + collection.startSyncImmediate() + const demands = Array.from( + { length: history.siblingCount }, + (_, index): LoadSubsetOptions => ({ + limit: 1, + ...(history.identical ? {} : { offset: index }), + }), + ) + + checkpoint = `initial acquisitions` + for (const demand of demands) { + const acquisition = Promise.resolve( + collection._sync.loadSubset(demand), + ) + await pumpNetwork() + await acquisition + } + const firstAfterAcquire = firstOwner.snapshot() + + checkpoint = `prefix releases` + const releaseCount = Math.min( + history.releasePrefix, + history.siblingCount - 1, + ) + for (const demand of demands.slice(0, releaseCount)) { + collection._sync.unloadSubset(demand) + if (history.duplicateRelease) { + collection._sync.unloadSubset(demand) + } + } + await pumpNetwork() + const firstAfterPrefixRelease = firstOwner.snapshot() + + checkpoint = `leadership takeover` + await disposeCoordinator(firstLeader) + await waitFor( + () => secondLeader!.isLeader(collectionId), + `generated remote subset takeover`, + ) + await pumpNetwork() + await nextTurn() + await pumpNetwork() + const firstAfterTakeover = firstOwner.snapshot() + const secondAfterTakeover = secondOwner.snapshot() + + checkpoint = `final releases` + const finalDemands = demands.slice(releaseCount) + if (history.reverseFinalRelease) finalDemands.reverse() + for (const demand of finalDemands) { + collection._sync.unloadSubset(demand) + if (history.duplicateRelease) { + collection._sync.unloadSubset(demand) + } + } + await pumpNetwork() + await nextTurn() + await pumpNetwork() + const secondAfterFinalRelease = secondOwner.snapshot() + + checkpoint = `semantic comparison` + const trace: RemoteLeaseHistoryTrace = { + firstAfterAcquire, + firstAfterPrefixRelease, + firstAfterTakeover, + secondAfterTakeover, + secondAfterFinalRelease, + } + const [violation] = remoteLeaseHistoryViolations(history, trace) + if (violation) { + const discriminant = `${violation.checkpoint}:${violation.field}` + if (targetDiscriminant === undefined) { + targetDiscriminant = discriminant + originalHistory = structuredClone(history) + originalViolation = structuredClone(violation) + } + if (discriminant === targetDiscriminant) { + semanticFailure = new Error( + `remote lease mismatch; discriminant=${discriminant}; originalViolation=${JSON.stringify(originalViolation)}; reducedViolation=${JSON.stringify(violation)}; original=${JSON.stringify(originalHistory)}; reduced=${JSON.stringify(history)}; trace=${JSON.stringify(trace)}`, + ) + } + } + } catch (error) { + executionDiagnostics.push({ + resource: `generated remote lease execution`, + error: + error instanceof Error + ? `${error.name}: ${error.message}` + : String(error), + history: structuredClone(history), + checkpoint, + }) + } + + const historyCleanupDiagnostics: Array = [] + await captureCleanup( + historyCleanupDiagnostics, + `generated remote lease collection`, + async () => { + await cleanupCollection?.() + }, + ) + await captureCleanup( + historyCleanupDiagnostics, + `generated first owner registration`, + () => unregisterFirst?.(), + ) + await captureCleanup( + historyCleanupDiagnostics, + `generated second owner registration`, + () => unregisterSecond?.(), + ) + for (const [resource, coordinator] of [ + [`generated first leader`, firstLeader], + [`generated second leader`, secondLeader], + [`generated requester`, requester], + ] as const) { + await captureCleanup( + historyCleanupDiagnostics, + resource, + async () => { + await disposeCoordinator(coordinator) + }, + ) + } + await captureCleanup( + historyCleanupDiagnostics, + `generated remote lease transport and locks`, + async () => { + await waitForWithoutDelivery( + () => + heldLocks.size === 0 && + queuedLocks.size === 0 && + ControlledBroadcastChannel.delayed.length === 0, + `generated remote lease cleanup`, + ) + }, + ) + cleanupDiagnostics.push( + ...historyCleanupDiagnostics.map((diagnostic) => ({ + ...diagnostic, + history: structuredClone(history), + })), + ) + if (semanticFailure !== NO_PRIMARY_FAILURE) throw semanticFailure + }), + { + seed: SEED, + numRuns: RUNS, + ...(PATH === undefined ? {} : { path: PATH }), + examples: [ + [ + { + siblingCount: 2, + releasePrefix: 1, + identical: true, + duplicateRelease: true, + reverseFinalRelease: true, + }, + ], + ], + }, + ) + } catch (error) { + propertyFailure = error + } + + expect.soft(cleanupDiagnostics).toEqual([]) + expect.soft(executionDiagnostics).toEqual([]) + if (propertyFailure !== NO_PRIMARY_FAILURE) throw propertyFailure + }) +}) + +type RawCollectionSnapshot = { + registryRowCount: number + schemaVersion: number + resetEpoch: number + rowCount: number +} + +type RawSnapshotViolation = { + field: keyof RawCollectionSnapshot + expected: number + actual: number +} + +function rawSnapshotViolations( + actual: RawCollectionSnapshot, + expected: RawCollectionSnapshot, +): Array { + return ( + [`registryRowCount`, `schemaVersion`, `resetEpoch`, `rowCount`] as const + ).flatMap((field) => + actual[field] === expected[field] + ? [] + : [{ field, expected: expected[field], actual: actual[field] }], + ) +} + +type FreshInitOutcome = + | { status: `fulfilled` } + | { status: `rejected`; error: string } + +type FreshInitViolation = + | { kind: `missing-outcome`; index: number } + | { kind: `unexpected-outcome`; index: number } + | { kind: `rejected`; index: number; error: string } + +function freshInitViolations( + outcomes: ReadonlyArray, +): Array { + const violations: Array = [] + for (let index = 0; index < 2; index++) { + const outcome = outcomes[index] + if (!outcome) { + violations.push({ kind: `missing-outcome`, index }) + } else if (outcome.status === `rejected`) { + violations.push({ kind: `rejected`, index, error: outcome.error }) + } + } + for (let index = 2; index < outcomes.length; index++) { + violations.push({ kind: `unexpected-outcome`, index }) + } + return violations +} + +async function readRawCollectionSnapshot( + database: BrowserWASQLiteDatabase, + collectionId: string, +): Promise { + const registry = await database.execute<{ + table_name: string + schema_version: number + }>( + `SELECT table_name, schema_version FROM collection_registry WHERE collection_id = ?`, + [collectionId], + ) + const row = registry[0] + if (!row) throw new Error(`missing registry row for ${collectionId}`) + const reset = await database.execute<{ reset_epoch: number }>( + `SELECT reset_epoch FROM collection_reset_epoch WHERE collection_id = ?`, + [collectionId], + ) + const tableName = `"${row.table_name.replaceAll(`"`, `""`)}"` + const count = await database.execute<{ count: number }>( + `SELECT COUNT(*) AS count FROM ${tableName}`, + ) + return { + registryRowCount: registry.length, + schemaVersion: row.schema_version, + resetEpoch: reset[0]?.reset_epoch ?? -1, + rowCount: count[0]?.count ?? -1, + } +} + +describe(`per-collection adapter and SQLite-state oracle`, () => { + it(`preserves alpha rows, schema, and reset epoch when beta registers a different adapter`, async () => { + let directory: string | undefined + let database: BrowserWASQLiteDatabase | undefined + let coordinator: BrowserCollectionCoordinator | undefined + + await withFailurePreservingCleanup(async () => { + directory = mkdtempSync(join(tmpdir(), `db-coordinator-routing-`)) + const createdDatabase = createWASQLiteTestDatabase({ + filename: join(directory, `state.sqlite`), + }) + database = createdDatabase + const createdCoordinator = createCoordinator( + `real-sqlite-routing`, + createRecordingAdapter({ id: `bootstrap` }), + ) + coordinator = createdCoordinator + const persistence = createBrowserWASQLitePersistence({ + database: createdDatabase, + coordinator: createdCoordinator, + }) + const resolve = ( + collectionId: string, + schemaVersion: number, + ): PersistenceAdapter => { + const resolved = persistence.resolvePersistenceForCollection?.({ + collectionId, + mode: `sync-present`, + schemaVersion, + }) + if (!resolved) throw new Error(`missing per-collection resolver`) + return resolved.adapter + } + + const alphaAdapter = resolve(`alpha`, 11) + await alphaAdapter.applyCommittedTx(`alpha`, { + txId: `alpha-seed`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `a-1`, + value: { id: `a-1`, group: `kept` }, + }, + ], + }) + const expected: RawCollectionSnapshot = { + registryRowCount: 1, + schemaVersion: 11, + resetEpoch: 0, + rowCount: 1, + } + const before = await readRawCollectionSnapshot(createdDatabase, `alpha`) + expect(rawSnapshotViolations(before, expected)).toEqual([]) + + // Merely resolving beta replaces the coordinator's one adapter slot. + // The later alpha leadership read therefore runs through beta's schema. + const betaAdapter = resolve(`beta`, 22) + await betaAdapter.loadSubset(`beta`, {}) + createdCoordinator.subscribe(`alpha`, () => {}) + await waitFor( + () => createdCoordinator.isLeader(`alpha`), + `alpha leadership after beta registration`, + ) + const after = await readRawCollectionSnapshot(createdDatabase, `alpha`) + + // RED D2 fixed cross-collection contamination witness. The shared + // comparator reports schema, reset, and row losses independently. + expect(rawSnapshotViolations(after, expected)).toEqual([]) + }, [ + [ + `real SQLite coordinator`, + async () => { + await disposeCoordinator(coordinator) + }, + ], + [ + `real SQLite database`, + async () => { + await Promise.resolve(database?.close?.()) + }, + ], + [ + `real SQLite directory`, + () => { + if (directory) rmSync(directory, { recursive: true, force: true }) + }, + ], + ]) + }) + + it(`initializes one fresh collection safely across matching adapter instances`, async () => { + let directory: string | undefined + let database: BrowserWASQLiteDatabase | undefined + + await withFailurePreservingCleanup(async () => { + directory = mkdtempSync(join(tmpdir(), `db-adapter-init-race-`)) + const createdDatabase = createWASQLiteTestDatabase({ + filename: join(directory, `state.sqlite`), + }) + database = createdDatabase + const createAdapter = (): PersistenceAdapter => { + const persistence = createBrowserWASQLitePersistence({ + database: createdDatabase, + }) + const resolved = persistence.resolvePersistenceForCollection?.({ + collectionId: `shared`, + mode: `sync-present`, + schemaVersion: 7, + }) + if (!resolved) throw new Error(`missing per-collection resolver`) + return resolved.adapter + } + const first = createAdapter() + const second = createAdapter() + + const outcomes = await Promise.allSettled([ + first.loadSubset(`shared`, {}), + second.loadSubset(`shared`, {}), + ]) + const observed: Array = outcomes.map((outcome) => + outcome.status === `fulfilled` + ? { status: outcome.status } + : { + status: outcome.status, + error: + outcome.reason instanceof Error + ? outcome.reason.message + : String(outcome.reason), + }, + ) + + const initialized = await readRawCollectionSnapshot( + createdDatabase, + `shared`, + ) + expect( + rawSnapshotViolations(initialized, { + registryRowCount: 1, + schemaVersion: 7, + resetEpoch: 0, + rowCount: 0, + }), + ).toEqual([]) + + // Separate #1753 checkpoint: same collection and same version/policy is + // legal in two tabs. Both production adapter initializations must finish; + // this does not rely on the cross-version reset assertion above. + expect(freshInitViolations(observed)).toEqual([]) + }, [ + [ + `race SQLite database`, + async () => { + await Promise.resolve(database?.close?.()) + }, + ], + [ + `race SQLite directory`, + () => { + if (directory) rmSync(directory, { recursive: true, force: true }) + }, + ], + ]) + }) + + it(`calibrates SQLite-state and fresh-registry checks against hostile results`, () => { + const expected: RawCollectionSnapshot = { + registryRowCount: 1, + schemaVersion: 11, + resetEpoch: 0, + rowCount: 1, + } + expect(rawSnapshotViolations(expected, expected)).toEqual([]) + expect( + rawSnapshotViolations( + { + registryRowCount: 0, + schemaVersion: 22, + resetEpoch: 1, + rowCount: 0, + }, + expected, + ).map(({ field }) => field), + ).toEqual([`registryRowCount`, `schemaVersion`, `resetEpoch`, `rowCount`]) + + const allowed: Array = [ + { status: `fulfilled` }, + { status: `fulfilled` }, + ] + expect(freshInitViolations(allowed)).toEqual([]) + expect( + freshInitViolations([ + allowed[0]!, + { + status: `rejected`, + error: `UNIQUE constraint failed: collection_registry`, + }, + ]), + ).toEqual([ + { + kind: `rejected`, + index: 1, + error: `UNIQUE constraint failed: collection_registry`, + }, + ]) + expect(freshInitViolations([allowed[0]!])).toEqual([ + { kind: `missing-outcome`, index: 1 }, + ]) + }) +}) + +type CollectionName = `alpha` | `beta` +type TabName = `leader` | `follower` + +type RoutingHistory = { + alphaVersion: number + betaVersion: number + registrationOrder: [CollectionName, CollectionName] + firstDeliveryOrder: [CollectionName, CollectionName] + secondDeliveryOrder: [CollectionName, CollectionName] +} + +type RoutedApply = { + phase: `initial-owner` | `after-takeover` + collectionId: CollectionName + adapterId: string +} + +type RouteViolation = { + kind: `missing-apply` | `duplicate-apply` | `misrouted-apply` + checkpoint: RoutedApply[`phase`] + collectionId: CollectionName + expectedAdapterId: string + actualAdapterIds: Array +} + +type CoordinatorRouteObservation = { + phase: RoutedApply[`phase`] + operation: `remote-subset` | `index` | `pull` + collectionId: CollectionName + calls: Array<{ + adapterId: string + collectionId: string + argument?: SemanticValue + }> + result: SemanticValue +} + +type CoordinatorRouteViolation = { + operation: CoordinatorRouteObservation[`operation`] + checkpoint: CoordinatorRouteObservation[`phase`] + collectionId: CollectionName + field: `calls` | `result` + expected: SemanticValue + actual: SemanticValue +} + +function coordinatorRouteViolations( + observed: ReadonlyArray, + expected: ReadonlyArray, +): Array { + const violations: Array = [] + for (const expectedCell of expected) { + const actualCell = observed.find( + ({ phase, operation, collectionId }) => + phase === expectedCell.phase && + operation === expectedCell.operation && + collectionId === expectedCell.collectionId, + ) + for (const field of [`calls`, `result`] as const) { + const expectedValue = semanticValue(expectedCell[field]) + const actualValue = semanticValue(actualCell?.[field] ?? []) + if (JSON.stringify(actualValue) !== JSON.stringify(expectedValue)) { + violations.push({ + operation: expectedCell.operation, + checkpoint: expectedCell.phase, + collectionId: expectedCell.collectionId, + field, + expected: expectedValue, + actual: actualValue, + }) + } + } + } + return violations +} + +function routeViolationDiscriminant(violation: RouteViolation): string { + return `${violation.kind}:${violation.checkpoint}:${violation.collectionId}` +} + +function routeViolations( + observed: ReadonlyArray, + expected: ReadonlyArray, +): Array { + const violations: Array = [] + for (const expectedRoute of expected) { + const actualAdapterIds = observed + .filter( + ({ phase, collectionId }) => + phase === expectedRoute.phase && + collectionId === expectedRoute.collectionId, + ) + .map(({ adapterId }) => adapterId) + if (actualAdapterIds.length === 0) { + violations.push({ + kind: `missing-apply`, + checkpoint: expectedRoute.phase, + collectionId: expectedRoute.collectionId, + expectedAdapterId: expectedRoute.adapterId, + actualAdapterIds, + }) + continue + } + if (actualAdapterIds.length !== 1) { + violations.push({ + kind: `duplicate-apply`, + checkpoint: expectedRoute.phase, + collectionId: expectedRoute.collectionId, + expectedAdapterId: expectedRoute.adapterId, + actualAdapterIds, + }) + continue + } + if (actualAdapterIds[0] !== expectedRoute.adapterId) { + violations.push({ + kind: `misrouted-apply`, + checkpoint: expectedRoute.phase, + collectionId: expectedRoute.collectionId, + expectedAdapterId: expectedRoute.adapterId, + actualAdapterIds, + }) + } + } + return violations +} + +class PerCollectionRouteOracle { + private readonly routes = new Map() + private readonly owners = new Map() + + register(tab: TabName, collection: CollectionName, adapterId: string): void { + this.routes.set(`${tab}:${collection}`, adapterId) + } + + setOwner(collection: CollectionName, tab: TabName): void { + this.owners.set(collection, tab) + } + + expectedApply( + phase: RoutedApply[`phase`], + collectionId: CollectionName, + ): RoutedApply { + const owner = this.owners.get(collectionId) + const adapterId = owner + ? this.routes.get(`${owner}:${collectionId}`) + : undefined + if (!owner || !adapterId) { + throw new Error(`oracle has no owner/route for ${collectionId}`) + } + return { phase, collectionId, adapterId } + } +} + +const orderArbitrary = fc + .boolean() + .map((forward): [CollectionName, CollectionName] => + forward ? [`alpha`, `beta`] : [`beta`, `alpha`], + ) + +const routingHistoryArbitrary: fc.Arbitrary = fc + .record({ + alphaVersion: fc.integer({ min: 1, max: 50 }), + versionDelta: fc.integer({ min: 1, max: 50 }), + registrationOrder: orderArbitrary, + firstDeliveryOrder: orderArbitrary, + secondDeliveryOrder: orderArbitrary, + }) + .map(({ alphaVersion, versionDelta, ...history }) => ({ + ...history, + alphaVersion, + betaVersion: alphaVersion + versionDelta, + })) + +function mutation( + collectionId: CollectionName, + phase: RoutedApply[`phase`], +): Array { + return [ + { + mutationId: `${phase}:${collectionId}`, + type: `insert`, + key: `${phase}:${collectionId}`, + value: { id: `${phase}:${collectionId}`, collectionId }, + }, + ] +} + +function applyObservation( + phase: RoutedApply[`phase`], + calls: ReadonlyArray, +): Array { + return calls + .filter( + (call): call is AdapterCall & { collectionId: CollectionName } => + call.operation === `apply` && + (call.collectionId === `alpha` || call.collectionId === `beta`), + ) + .map(({ collectionId, adapterId }) => ({ + phase, + collectionId, + adapterId, + })) + .sort((left, right) => left.collectionId.localeCompare(right.collectionId)) +} + +async function requestBothCollections( + requester: BrowserCollectionCoordinator, + phase: RoutedApply[`phase`], + deliveryOrder: [CollectionName, CollectionName], +): Promise { + const pending = ([`alpha`, `beta`] as const).map((collectionId) => + requester.requestApplyLocalMutations( + collectionId, + mutation(collectionId, phase), + ), + ) + for (const collectionId of deliveryOrder) { + expect( + ControlledBroadcastChannel.deliverWhere( + (data) => + payloadType(data) === `rpc:applyLocalMutations:req` && + envelopeCollection(data) === collectionId, + ), + ).toBe(true) + await nextTurn() + } + await pumpNetwork() + const responses = await Promise.all(pending) + expect(responses.every((response) => response.ok)).toBe(true) +} + +describe(`generated collection-route histories`, () => { + it(`calibrates collection-key comparison against one peer-routed apply`, () => { + const expected: Array = [ + { + phase: `initial-owner`, + collectionId: `alpha`, + adapterId: `leader-alpha`, + }, + { + phase: `initial-owner`, + collectionId: `beta`, + adapterId: `leader-beta`, + }, + ] + const misrouted: Array = [ + expected[0]!, + { ...expected[1]!, adapterId: `leader-alpha` }, + ] + + expect(routeViolations(expected, expected)).toEqual([]) + const [violation] = routeViolations(misrouted, expected) + expect(violation).toEqual({ + kind: `misrouted-apply`, + checkpoint: `initial-owner`, + collectionId: `beta`, + expectedAdapterId: `leader-beta`, + actualAdapterIds: [`leader-alpha`], + }) + expect(routeViolationDiscriminant(violation!)).toBe( + `misrouted-apply:initial-owner:beta`, + ) + + const routedCell: CoordinatorRouteObservation = { + phase: `initial-owner`, + operation: `index`, + collectionId: `alpha`, + calls: [ + { + adapterId: `leader-alpha`, + collectionId: `alpha`, + argument: semanticValue({ signature: `alpha-index` }), + }, + ], + result: semanticValue({ status: `fulfilled` }), + } + expect(coordinatorRouteViolations([routedCell], [routedCell])).toEqual([]) + expect( + coordinatorRouteViolations( + [{ ...routedCell, calls: [] }], + [routedCell], + ).map(({ operation, checkpoint, collectionId, field }) => ({ + operation, + checkpoint, + collectionId, + field, + })), + ).toEqual([ + { + operation: `index`, + checkpoint: `initial-owner`, + collectionId: `alpha`, + field: `calls`, + }, + ]) + }) + + it(`routes every adapter-bound RPC by collection before and after leadership transfer`, async () => { + const makePullResult = ( + marker: string, + latestRowVersion: number, + ): PullFixtureResult => ({ + latestRowVersion, + requiresFullReload: false, + changedKeys: [`${marker}:changed`], + deletedKeys: [`${marker}:deleted`], + }) + const leaderPullResults = { + alpha: makePullResult(`leader-alpha`, 101), + beta: makePullResult(`leader-beta`, 202), + } + const followerPullResults = { + alpha: makePullResult(`follower-alpha`, 303), + beta: makePullResult(`follower-beta`, 404), + } + const createPhaseAdapters = ( + tab: TabName, + pullResults: Record, + ): Record => ({ + alpha: createRecordingAdapter({ + id: `${tab}-alpha`, + remoteSubsetOwner: () => Promise.resolve(), + pullResult: pullResults.alpha, + }), + beta: createRecordingAdapter({ + id: `${tab}-beta`, + remoteSubsetOwner: () => Promise.resolve(), + pullResult: pullResults.beta, + }), + }) + const leaderAdapters = createPhaseAdapters(`leader`, leaderPullResults) + const followerAdapters = createPhaseAdapters( + `follower`, + followerPullResults, + ) + const requesterAdapter = createRecordingAdapter({ id: `requester` }) + let leader: BrowserCollectionCoordinator | undefined + let follower: BrowserCollectionCoordinator | undefined + let requester: BrowserCollectionCoordinator | undefined + + await withFailurePreservingCleanup(async () => { + const observed: Array = [] + const expected: Array = [] + const dbName = `adapter-bound-rpc-matrix` + leader = createCoordinator(dbName, leaderAdapters.alpha) + follower = createCoordinator(dbName, followerAdapters.alpha) + for (const collectionId of [`alpha`, `beta`] as const) { + // Register beta last so a global mutable adapter misroutes alpha. + registerCollectionAdapter( + leader, + collectionId, + leaderAdapters[collectionId], + ) + registerCollectionAdapter( + follower, + collectionId, + followerAdapters[collectionId], + ) + leader.subscribe(collectionId, () => {}) + follower.subscribe(collectionId, () => {}) + } + await waitFor( + () => + leader!.isLeader(`alpha`) && + leader!.isLeader(`beta`) && + !follower!.isLeader(`alpha`) && + !follower!.isLeader(`beta`), + `adapter-bound initial leadership`, + ) + + const exercisePhase = async ( + phase: RoutedApply[`phase`], + rpcRequester: BrowserCollectionCoordinator, + ownerAdapters: Record, + pullResults: Record, + ): Promise => { + for (const operation of [`remote-subset`, `index`, `pull`] as const) { + for (const collectionId of [`alpha`, `beta`] as const) { + const before = new Map( + Object.values(ownerAdapters).map((adapter) => [ + adapter, + adapter.calls.length, + ]), + ) + let requestType: string + let argument: SemanticValue + let pending: Promise + let expectedResult: SemanticValue + + if (operation === `remote-subset`) { + const demand: LoadSubsetOptions = { + where: new IR.Func(`eq`, [ + new IR.PropRef([collectionId, `group`]), + new IR.Value(`${phase}:${collectionId}`), + ]), + limit: collectionId === `alpha` ? 1 : 2, + offset: phase === `initial-owner` ? 0 : 1, + } + requestType = `rpc:ensureRemoteSubset:req` + argument = subsetSemantics(demand) + pending = rpcRequester + .requestEnsureRemoteSubset(collectionId, demand) + .then(() => ({ status: `fulfilled` })) + expectedResult = semanticValue({ status: `fulfilled` }) + } else if (operation === `index`) { + const signature = `${phase}:${collectionId}:index` + const spec = { + expressionSql: [`${collectionId}_score`, phase], + } + requestType = `rpc:ensurePersistedIndex:req` + argument = semanticValue({ signature, spec }) + pending = rpcRequester + .requestEnsurePersistedIndex(collectionId, signature, spec) + .then(() => ({ status: `fulfilled` })) + expectedResult = semanticValue({ status: `fulfilled` }) + } else { + const fromRowVersion = + (phase === `initial-owner` ? 0 : 100) + + (collectionId === `alpha` ? 11 : 22) + requestType = `rpc:pullSince:req` + argument = semanticValue({ fromRowVersion }) + pending = rpcRequester.pullSince(collectionId, fromRowVersion) + const pullResult = pullResults[collectionId] + expectedResult = semanticValue({ + ok: true, + latestTerm: 1, + latestSeq: 0, + latestRowVersion: pullResult.latestRowVersion, + requiresFullReload: false, + changedKeys: pullResult.changedKeys, + deletedKeys: pullResult.deletedKeys, + }) + } + + expect( + ControlledBroadcastChannel.deliverWhere( + (data) => + payloadType(data) === requestType && + envelopeCollection(data) === collectionId, + ), + ).toBe(true) + await pumpNetwork() + const result = await pending + const calls = Object.values(ownerAdapters) + .flatMap((adapter) => + adapter.calls + .slice(before.get(adapter) ?? 0) + .filter((call) => call.operation === operation) + .map( + ({ + adapterId, + collectionId: calledId, + argument: callArgument, + }) => ({ + adapterId, + collectionId: calledId, + ...(callArgument === undefined + ? {} + : { argument: callArgument }), + }), + ), + ) + .sort((left, right) => + left.adapterId.localeCompare(right.adapterId), + ) + const resultValue = + operation === `pull` + ? semanticValue( + result && typeof result === `object` + ? Object.fromEntries( + Object.entries(result).filter( + ([key]) => key !== `type` && key !== `rpcId`, + ), + ) + : result, + ) + : semanticValue(result) + observed.push({ + phase, + operation, + collectionId, + calls, + result: resultValue, + }) + expected.push({ + phase, + operation, + collectionId, + calls: [ + { + adapterId: ownerAdapters[collectionId].id, + collectionId, + argument, + }, + ], + result: expectedResult, + }) + } + } + } + + await exercisePhase( + `initial-owner`, + follower, + leaderAdapters, + leaderPullResults, + ) + await disposeCoordinator(leader) + await waitFor( + () => follower!.isLeader(`alpha`) && follower!.isLeader(`beta`), + `adapter-bound follower takeover`, + ) + requester = createCoordinator(dbName, requesterAdapter) + await exercisePhase( + `after-takeover`, + requester, + followerAdapters, + followerPullResults, + ) + + // Each violation retains operation, phase, collection, and whether the + // loss was the exact routed call or the exact response. + expect(coordinatorRouteViolations(observed, expected)).toEqual([]) + }, [ + [ + `adapter-bound initial leader`, + async () => { + await disposeCoordinator(leader) + }, + ], + [ + `adapter-bound takeover leader`, + async () => { + await disposeCoordinator(follower) + }, + ], + [ + `adapter-bound post-takeover requester`, + async () => { + await disposeCoordinator(requester) + }, + ], + ]) + }) + + it(`uses the collection's registered adapter before and after leadership transfer`, async () => { + let originalFailingTrace: RoutingHistory | undefined + let originalViolation: RouteViolation | undefined + let targetDiscriminant: string | undefined + const cleanupDiagnostics: Array< + CleanupDiagnostic & { history: RoutingHistory } + > = [] + const executionDiagnostics: Array< + CleanupDiagnostic & { history: RoutingHistory; checkpoint: string } + > = [] + let propertyFailure: unknown | typeof NO_PRIMARY_FAILURE = + NO_PRIMARY_FAILURE + + try { + await fc.assert( + fc.asyncProperty(routingHistoryArbitrary, async (history) => { + const oracle = new PerCollectionRouteOracle() + const leaderAdapters = { + alpha: createRecordingAdapter({ + id: `leader-alpha-v${history.alphaVersion}-reset`, + schemaVersion: history.alphaVersion, + policy: `sync-present-reset`, + }), + beta: createRecordingAdapter({ + id: `leader-beta-v${history.betaVersion}-error`, + schemaVersion: history.betaVersion, + policy: `sync-absent-error`, + }), + } + const followerAdapters = { + alpha: createRecordingAdapter({ + id: `follower-alpha-v${history.alphaVersion}-reset`, + schemaVersion: history.alphaVersion, + policy: `sync-present-reset`, + }), + beta: createRecordingAdapter({ + id: `follower-beta-v${history.betaVersion}-error`, + schemaVersion: history.betaVersion, + policy: `sync-absent-error`, + }), + } + const dbName = `generated-route-${history.alphaVersion}-${history.betaVersion}` + const initialCollection = history.registrationOrder[0] + let leader: BrowserCollectionCoordinator | undefined + let follower: BrowserCollectionCoordinator | undefined + const observed: Array = [] + const expected: Array = [] + let semanticFailure: unknown | typeof NO_PRIMARY_FAILURE = + NO_PRIMARY_FAILURE + let checkpoint = `coordinator allocation` + + try { + leader = createCoordinator( + dbName, + leaderAdapters[initialCollection], + ) + follower = createCoordinator( + dbName, + followerAdapters[initialCollection], + ) + + for (const collectionId of history.registrationOrder) { + registerCollectionAdapter( + leader, + collectionId, + leaderAdapters[collectionId], + ) + registerCollectionAdapter( + follower, + collectionId, + followerAdapters[collectionId], + ) + oracle.register( + `leader`, + collectionId, + leaderAdapters[collectionId].id, + ) + oracle.register( + `follower`, + collectionId, + followerAdapters[collectionId].id, + ) + } + + checkpoint = `initial leadership` + for (const collectionId of [`alpha`, `beta`] as const) { + leader.subscribe(collectionId, () => {}) + follower.subscribe(collectionId, () => {}) + oracle.setOwner(collectionId, `leader`) + } + await waitFor( + () => + leader!.isLeader(`alpha`) && + leader!.isLeader(`beta`) && + !follower!.isLeader(`alpha`) && + !follower!.isLeader(`beta`), + `generated initial two-collection leadership`, + ) + + checkpoint = `initial routed applies` + await requestBothCollections( + follower, + `initial-owner`, + history.firstDeliveryOrder, + ) + observed.push( + ...applyObservation(`initial-owner`, [ + ...leaderAdapters.alpha.calls, + ...leaderAdapters.beta.calls, + ]), + ) + expected.push( + oracle.expectedApply(`initial-owner`, `alpha`), + oracle.expectedApply(`initial-owner`, `beta`), + ) + + checkpoint = `leadership transfer` + await disposeCoordinator(leader) + for (const collectionId of [`alpha`, `beta`] as const) { + oracle.setOwner(collectionId, `follower`) + } + await waitFor( + () => follower!.isLeader(`alpha`) && follower!.isLeader(`beta`), + `generated follower takeover`, + ) + + checkpoint = `post-takeover routed applies` + // The new leader's direct calls still exercise the same production + // handler and must look up adapters per collection. + for (const collectionId of history.secondDeliveryOrder) { + const response = await follower.requestApplyLocalMutations( + collectionId, + mutation(collectionId, `after-takeover`), + ) + expect(response.ok).toBe(true) + } + observed.push( + ...applyObservation(`after-takeover`, [ + ...followerAdapters.alpha.calls, + ...followerAdapters.beta.calls, + ]), + ) + expected.push( + oracle.expectedApply(`after-takeover`, `alpha`), + oracle.expectedApply(`after-takeover`, `beta`), + ) + + const [violation] = routeViolations(observed, expected) + if (violation) { + const discriminant = routeViolationDiscriminant(violation) + if (targetDiscriminant === undefined) { + targetDiscriminant = discriminant + originalViolation = structuredClone(violation) + originalFailingTrace = structuredClone(history) + } + // A smaller candidate is a valid reduction only when it reaches + // the same checkpoint and misroutes the same collection. + if (discriminant === targetDiscriminant) { + semanticFailure = new Error( + `per-collection route mismatch; discriminant=${discriminant}; originalViolation=${JSON.stringify(originalViolation)}; reducedViolation=${JSON.stringify(violation)}; original=${JSON.stringify(originalFailingTrace)}; reduced=${JSON.stringify(history)}; expected=${JSON.stringify(expected)}; observed=${JSON.stringify(observed)}`, + ) + } + } + } catch (error) { + executionDiagnostics.push({ + resource: `generated route execution`, + error: + error instanceof Error + ? `${error.name}: ${error.message}` + : String(error), + history: structuredClone(history), + checkpoint, + }) + } + + const historyCleanupDiagnostics: Array = [] + await captureCleanup( + historyCleanupDiagnostics, + `leader coordinator`, + async () => { + await disposeCoordinator(leader) + }, + ) + await captureCleanup( + historyCleanupDiagnostics, + `follower coordinator`, + async () => { + await disposeCoordinator(follower) + }, + ) + await captureCleanup( + historyCleanupDiagnostics, + `coordinator locks`, + async () => { + await waitForWithoutDelivery( + () => heldLocks.size === 0 && queuedLocks.size === 0, + `generated history lock cleanup`, + ) + }, + ) + await captureCleanup( + historyCleanupDiagnostics, + `coordinator transport`, + () => { + const snapshot = { + endpoints: ControlledBroadcastChannel.endpoints.size, + delayed: ControlledBroadcastChannel.delayed.length, + } + if (snapshot.endpoints !== 0 || snapshot.delayed !== 0) { + throw new Error(JSON.stringify(snapshot)) + } + }, + ) + cleanupDiagnostics.push( + ...historyCleanupDiagnostics.map((diagnostic) => ({ + ...diagnostic, + history: structuredClone(history), + })), + ) + + if (semanticFailure !== NO_PRIMARY_FAILURE) throw semanticFailure + }), + { + seed: SEED, + numRuns: RUNS, + ...(PATH === undefined ? {} : { path: PATH }), + }, + ) + } catch (error) { + propertyFailure = error + } + + // Cleanup diagnostics are evaluated outside FastCheck, so cleanup cannot + // become the predicate that selects or shrinks a semantic route failure. + expect.soft(cleanupDiagnostics).toEqual([]) + // Setup/reach/fixture failures are also reported outside the property; + // only the locked semantic discriminant is eligible for shrinking. + expect.soft(executionDiagnostics).toEqual([]) + if (propertyFailure !== NO_PRIMARY_FAILURE) throw propertyFailure + }) +}) + +type OwnershipObservation = { + collectionId: string + ownerAtCall?: string +} + +function findUnownedWrites( + observations: ReadonlyArray, + electedOwner: string, +): Array { + return observations.filter( + (observation) => observation.ownerAtCall !== electedOwner, + ) +} + +describe(`sync-ingested write ownership oracle`, () => { + it(`applies a source commit through the elected persistence owner after leadership changes`, async () => { + type Todo = { id: string; title: string } + type SourceParams = Parameters[`sync`]>[0] + + let electedOwner = `tab-a` + let activeWriter: string | undefined + let sourceParams: SourceParams | undefined + let ownerRequests = 0 + const adapter = createRecordingAdapter({ + id: `sync-alpha`, + owner: () => activeWriter, + }) + const coordinator: PersistedCollectionCoordinator = { + getNodeId: () => `tab-a`, + subscribe: () => () => {}, + publish: () => {}, + isLeader: () => electedOwner === `tab-a`, + ensureLeadership: async () => {}, + requestEnsurePersistedIndex: async () => {}, + // Remote subset ownership is unrelated to this write-owner law. + requestEnsureRemoteSubset: async () => {}, + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: () => () => {}, + requestApplyCommittedTx: async (collectionId, tx) => { + ownerRequests++ + const ownerTx = { + ...tx, + term: 1, + seq: ownerRequests, + rowVersion: ownerRequests, + } + activeWriter = electedOwner + try { + await adapter.applyCommittedTx(collectionId, ownerTx) + } finally { + activeWriter = undefined + } + return { + type: `rpc:applyCommittedTx:res`, + rpcId: `owned-${ownerRequests}`, + ok: true, + term: 1, + seq: ownerRequests, + latestRowVersion: ownerRequests, + } + }, + } + let cleanupCollection: (() => Promise) | undefined + + await withFailurePreservingCleanup(async () => { + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-alpha`, + getKey: (todo) => todo.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + cleanupCollection = () => collection.cleanup() + collection.startSyncImmediate() + await waitForWithoutDelivery( + () => sourceParams !== undefined, + `wrapped source controls`, + ) + + // Legal leadership history: the original tab loses ownership before an + // external source callback ingests its next transaction. + electedOwner = `tab-b` + sourceParams!.begin() + sourceParams!.write({ + type: `insert`, + value: { id: `sync-1`, title: `from source` }, + }) + const receipt = sourceParams!.commit() + if (receipt !== true) await receipt + + const applies = adapter.calls + .filter(({ operation }) => operation === `apply`) + .map(({ collectionId, ownerAtCall }) => ({ + collectionId, + ownerAtCall, + })) + + // Positive production reach: the source write became collection-visible + // and reached exactly one persistence apply call. + expect(collection.get(`sync-1`)).toMatchObject({ + id: `sync-1`, + title: `from source`, + }) + expect(applies).toHaveLength(1) + // Ownership checkpoint: a direct-adapter bypass runs outside the elected + // tab-b writer scope. Owner-routed implementations pass this unchanged + // regardless of request count. + expect(findUnownedWrites(applies, electedOwner)).toEqual([]) + }, [ + [ + `sync-ingested collection`, + async () => { + await cleanupCollection?.() + }, + ], + ]) + }) + + it(`calibrates the owner checker against one bypass and one owned write`, () => { + expect( + findUnownedWrites( + [ + { collectionId: `alpha`, ownerAtCall: `tab-b` }, + { collectionId: `beta` }, + ], + `tab-b`, + ), + ).toEqual([{ collectionId: `beta` }]) + }) + + it(`routes a rich source commit through the new elected Browser owner`, async () => { + type Todo = { id: string; title: string } + type SourceParams = Parameters[`sync`]>[0] + type TxAdapter = PersistenceAdapter & { applied: Array } + + const createTxAdapter = (): TxAdapter => { + const applied: Array = [] + return { + applied, + loadSubset: () => Promise.resolve([]), + applyCommittedTx: (_collectionId, tx) => { + applied.push(structuredClone(tx)) + return Promise.resolve() + }, + ensureIndex: () => Promise.resolve(), + getStreamPosition: () => + Promise.resolve({ + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + }), + } + } + + const retiredAdapter = createTxAdapter() + const electedAdapter = createTxAdapter() + const requesterAdapter = createTxAdapter() + const originalElectedApply = + electedAdapter.applyCommittedTx.bind(electedAdapter) + let electedApplyEntered = false + let releaseElectedApply = (): void => {} + const electedApplyGate = new Promise((resolve) => { + releaseElectedApply = resolve + }) + electedAdapter.applyCommittedTx = async (collectionId, tx) => { + electedApplyEntered = true + await electedApplyGate + await originalElectedApply(collectionId, tx) + } + let retired: BrowserCollectionCoordinator | undefined + let elected: BrowserCollectionCoordinator | undefined + let requester: BrowserCollectionCoordinator | undefined + let cleanupCollection: (() => Promise) | undefined + const electedCommitted: Array = [] + const requesterCommitted: Array = [] + + await withFailurePreservingCleanup(async () => { + const dbName = `source-owner-transfer` + retired = createCoordinator(dbName, retiredAdapter) + elected = createCoordinator(dbName, electedAdapter) + requester = createCoordinator(dbName, requesterAdapter) + retired.subscribe(`todos`, () => {}) + elected.subscribe(`todos`, (message) => { + const payload = message.payload as { type?: string } + if (payload.type === `tx:committed`) { + electedCommitted.push(structuredClone(payload)) + } + }) + requester.subscribe(`todos`, (message) => { + const payload = message.payload as { type?: string } + if (payload.type === `tx:committed`) { + requesterCommitted.push(structuredClone(payload)) + } + }) + await waitFor( + () => + retired!.isLeader(`todos`) && + !elected!.isLeader(`todos`) && + !requester!.isLeader(`todos`), + `initial source owner`, + ) + + await disposeCoordinator(retired) + await waitFor( + () => elected!.isLeader(`todos`) && !requester!.isLeader(`todos`), + `transferred source owner`, + ) + + let sourceParams: SourceParams | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `todos`, + getKey: (todo) => todo.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter: requesterAdapter, coordinator: requester }, + }), + ) + cleanupCollection = () => collection.cleanup() + collection.startSyncImmediate() + await waitForWithoutDelivery( + () => sourceParams !== undefined, + `Browser source controls`, + ) + + sourceParams!.begin() + sourceParams!.metadata?.collection.set(`resume`, { offset: 11 }) + sourceParams!.truncate() + sourceParams!.write({ + type: `insert`, + value: { id: `rich`, title: `through Browser owner` }, + metadata: { source: `browser-sync` }, + }) + const receipt = sourceParams!.commit() + let receiptSettled = receipt === true + const receiptPromise = receipt === true ? Promise.resolve() : receipt + void receiptPromise.then( + () => { + receiptSettled = true + }, + () => { + receiptSettled = true + }, + ) + + await pumpNetwork() + await waitForWithoutDelivery( + () => electedApplyEntered, + `elected Browser source apply entry`, + ) + expect({ + receiptSettled, + electedApplies: electedAdapter.applied.length, + requesterInvalidations: requesterCommitted.length, + }).toEqual({ + receiptSettled: false, + electedApplies: 0, + requesterInvalidations: 0, + }) + + releaseElectedApply() + await pumpNetwork() + await receiptPromise + await pumpNetwork() + + expect(collection.get(`rich`)).toMatchObject({ + id: `rich`, + title: `through Browser owner`, + }) + expect(retiredAdapter.applied).toEqual([]) + expect(requesterAdapter.applied).toEqual([]) + expect(electedAdapter.applied).toHaveLength(1) + expect(electedAdapter.applied[0]).toMatchObject({ + truncate: true, + mutations: [ + { + type: `update`, + key: `rich`, + value: { id: `rich`, title: `through Browser owner` }, + }, + ], + rowMetadataMutations: [ + { + type: `set`, + key: `rich`, + value: { source: `browser-sync` }, + }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `resume`, value: { offset: 11 } }, + ], + }) + const committedTxId = electedAdapter.applied[0]!.txId + const expectedCommitted = { + type: `tx:committed`, + term: 1, + seq: 1, + txId: committedTxId, + latestRowVersion: 1, + requiresFullReload: true, + } + expect(electedCommitted).toEqual([expectedCommitted]) + expect(requesterCommitted).toEqual([expectedCommitted]) + }, [ + [ + `elected Browser source apply gate`, + () => { + releaseElectedApply() + }, + ], + [ + `Browser source collection`, + async () => { + await cleanupCollection?.() + }, + ], + [ + `retired Browser source owner`, + async () => { + await disposeCoordinator(retired) + }, + ], + [ + `elected Browser source owner`, + async () => { + await disposeCoordinator(elected) + }, + ], + [ + `Browser source requester`, + async () => { + await disposeCoordinator(requester) + }, + ], + ]) + }) +}) + +describe(`coordinator fixture lifecycle`, () => { + it(`releases delayed transport and leadership state before a fresh lifecycle`, async () => { + let first: BrowserCollectionCoordinator | undefined + let peer: BrowserCollectionCoordinator | undefined + let fresh: BrowserCollectionCoordinator | undefined + + await withFailurePreservingCleanup(async () => { + first = createCoordinator( + `cleanup-first`, + createRecordingAdapter({ id: `first` }), + ) + peer = createCoordinator( + `cleanup-first`, + createRecordingAdapter({ id: `peer` }), + ) + first.subscribe(`alpha`, () => {}) + peer.subscribe(`alpha`, () => {}) + await waitFor( + () => first!.isLeader(`alpha`) && !peer!.isLeader(`alpha`), + `first lifecycle leadership`, + ) + + const retiredBefore = ControlledBroadcastChannel.retiredDeliveries + first.publish(`alpha`, { + v: 1, + dbName: `cleanup-first`, + collectionId: `alpha`, + senderId: first.getNodeId(), + ts: Date.now(), + payload: { type: `lifecycle:pending` }, + }) + expect(ControlledBroadcastChannel.delayed.length).toBeGreaterThan(0) + await disposeCoordinator(first) + await disposeCoordinator(peer) + await waitForWithoutDelivery( + () => heldLocks.size === 0 && queuedLocks.size === 0, + `first lifecycle release`, + ) + expect(ControlledBroadcastChannel.retiredDeliveries).toBeGreaterThan( + retiredBefore, + ) + expect(coordinatorFixtureSnapshot()).toEqual({ + delayed: 0, + endpoints: 0, + heldLocks: 0, + queuedLocks: 0, + }) + + fresh = createCoordinator( + `cleanup-fresh`, + createRecordingAdapter({ id: `fresh` }), + ) + fresh.subscribe(`alpha`, () => {}) + await waitFor( + () => fresh!.isLeader(`alpha`), + `fresh lifecycle leadership`, + ) + expect(fresh.isLeader(`alpha`)).toBe(true) + await disposeCoordinator(fresh) + await waitForWithoutDelivery( + () => heldLocks.size === 0 && queuedLocks.size === 0, + `fresh lifecycle release`, + ) + expect(coordinatorFixtureSnapshot()).toEqual({ + delayed: 0, + endpoints: 0, + heldLocks: 0, + queuedLocks: 0, + }) + }, [ + [ + `first lifecycle coordinator`, + async () => { + await disposeCoordinator(first) + }, + ], + [ + `first lifecycle peer`, + async () => { + await disposeCoordinator(peer) + }, + ], + [ + `fresh lifecycle coordinator`, + async () => { + await disposeCoordinator(fresh) + }, + ], + ]) + }) +}) diff --git a/packages/db-sqlite-persistence-core/README.md b/packages/db-sqlite-persistence-core/README.md index 1ed225957f..87bdc89371 100644 --- a/packages/db-sqlite-persistence-core/README.md +++ b/packages/db-sqlite-persistence-core/README.md @@ -25,17 +25,39 @@ binding. Provide a runtime `SQLiteDriver` implementation from a wrapper package. - `TxCommitted` - `EnsureRemoteSubsetRequest` - `EnsureRemoteSubsetResponse` +- `ReleaseRemoteSubsetRequest` +- `ReleaseRemoteSubsetResponse` +- `RemoteSubsetOwner` +- `DuplicateRemoteSubsetOwnerError` +- `TransportedLoadSubsetOptions` +- `RemoteSubsetWirePrimitive` +- `RemoteSubsetWireTypedArray` +- `RemoteSubsetWireRecord` +- `RemoteSubsetWireValue` +- `RemoteSubsetWireExpression` +- `RemoteSubsetWireCompareOptions` +- `RemoteSubsetWireOrderByClause` +- `RemoteSubsetWireCursor` +- `RemoteSubsetWireValueError` +- `toTransportedLoadSubsetOptions(...)` - `ApplyLocalMutationsRequest` - `ApplyLocalMutationsResponse` +- `ApplyCommittedTxRequest` +- `ApplyCommittedTxResponse` - `PullSinceRequest` - `PullSinceResponse` - `CollectionReset` - `PersistedIndexSpec` +- `PersistedRowMetadataMutation` +- `PersistedCollectionMetadataMutation` +- `ReplayableTxDelta` +- `PersistedScannedRow` +- `PersistedRowScanOptions` - `PersistedTx` -- `PersistenceAdapter` +- `PersistenceAdapter` - `SQLiteDriver` - `PersistedCollectionCoordinator` -- `PersistedCollectionPersistence` +- `PersistedCollectionPersistence` - `PersistedCollectionMode` - `PersistedCollectionLeadershipState` - `PersistedCollectionUtils` @@ -64,12 +86,109 @@ and resolves persistence using: This lets runtime wrappers expose one shared persistence instance per database while still handling per-collection schema versions correctly. +### Coordinator contract + +Every `PersistedCollectionCoordinator` must implement +`requestApplyCommittedTx(collectionId, tx)`. A sync transaction with durable +effects is routed in full to the persistence adapter owned by that collection. +The `PersistedTx` includes any truncate, row mutations, row metadata mutations, +collection metadata mutations, and stream position. Multiprocess coordinators +must preserve the complete transaction when they forward it to the elected +writer. This contract does not decide whether an effect-free source commit +should produce a persistence call. + +This is a required invariant, not a capability to detect at commit time. The +TypeScript interface requires the method, and `persistedCollectionOptions(...)` +validates custom coordinators when it configures a collection. An untyped +integration that omits the method throws +`InvalidPersistedCollectionCoordinatorError` before the sync source starts or +publishes rows. Implementations must not fall back to a row-only route such as +`requestApplyLocalMutations(...)`, because that would discard transaction +semantics. + +After a mutating RPC transport failure, replay is allowed only while the +requester can prove the same non-null leader id and term still own the route. +If either value was unknown for the first attempt, or either value changes, +the request rejects with `IndeterminateCommitError` and requires application +reconciliation. Coordinators never retry an indeterminate mutation against an +unknown or replacement leader. + +`SingleProcessCoordinator` also implements the complete route. It is always +leader and has no cross-process transport, but it applies the transaction to +the resolved adapter registered for the specific collection. The Browser and +Electron coordinators provide the corresponding elected-owner route in their +runtime packages. + +An adapter/storage rejection is surfaced as +`PersistedCollectionDurabilityError`, never as `CONFLICT`. The error preserves +the original local cause and its `code` and `path` fields. A transported +`PERSISTENCE_ERROR` response carries the source code/path that can cross the +coordinator boundary. Source writes remain published before their applied +receipt settles; if persistence then fails, the receipt rejects with the named +error and the collection enters its existing `error` lifecycle state. The +collection does not continue, retry, or fall back to a partial writer. + +### Remote subset wire contract + +`requestEnsureRemoteSubset(collectionId, options)` accepts the full live +`LoadSubsetOptions` input. Before choosing a local owner or remote transport, +every built-in coordinator projects it once to the exported +`TransportedLoadSubsetOptions` contract. A registered remote-subset owner +receives that transported type, never live-only fields such as `signal` or +`subscription`. + +Expression values may contain structured-clone-safe primitives, plain records, +arrays (including sparse arrays), `Date`, zero-position `RegExp`, fixed +`ArrayBuffer`, fixed `DataView`, the views listed by the exported +`RemoteSubsetWireTypedArray` union (`Int8Array`, `Uint8Array`, +`Uint8ClampedArray`, `Int16Array`, `Uint16Array`, `Int32Array`, `Uint32Array`, +`Float32Array`, `Float64Array`, `BigInt64Array`, and `BigUint64Array`), `Map`, +and `Set`. Cycles and aliases are preserved. Other view types; shared, +resizable, growable, or detached buffers; accessors; non-enumerable or symbol +keys; native expandos; custom prototypes; promises; weak collections; +functions; and symbols are outside the contract. A rejected value throws the exported +`RemoteSubsetWireValueError` before owner work or transport, with its exact +path in the request. The wire contract does not provide codecs, coercion, or +lossy normalization for unsupported values. + +Remote subset ownership is lease-based. A coordinator gives each accepted +request object a stable acquisition identity scoped to its collection and +requester. Retrying the same acquisition coalesces with the original; a second +request object with equal values is an independent lease. Releasing a lease +calls the registered owner's required `unloadSubset(...)` method with the exact +transported options object that was loaded. Releasing it again is a no-op. +`EnsureRemoteSubsetRequest.acquisitionId` is the wire identity established by +acquisition; `ReleaseRemoteSubsetRequest.acquisitionId` must carry that same +identity so the elected owner releases the matching lease. + +The owner also provides a required `onError(error)` callback. A load or unload +throw, or a runtime thenable rejection from either operation, is reported +through that callback with the original value before the awaited request +rejects. Retirement and disposal observe and report the same failure while +continuing cleanup of sibling leases; they do not retry or silently recover +the failed lifecycle. + +Only one remote subset owner may be registered per collection in a coordinator. +A second live registration throws `DuplicateRemoteSubsetOwnerError` instead of +replacing the first. When a leader loses ownership or is disposed, it unloads +every transferred lease. A requester that still owns a lease replays that same +acquisition when the next leader is observed. There is no adapter fallback or +direct-writer escape hatch for missing ownership. + +The named-error and exact-path guarantee covers values the boundary can +identify with standard JavaScript reflection. A fully transparent `Proxy` is +outside the public wire type and cannot be portably distinguished from its +target. Untyped callers must not rely on Proxy identity, traps, or a +Proxy-specific diagnostic; no non-standard detection is attempted. + ### SQLite core adapter APIs - `SQLiteCoreAdapterOptions` - `SQLitePullSinceResult` -- `SQLiteCorePersistenceAdapter` -- `createSQLiteCorePersistenceAdapter(...)` +- `SQLiteCorePersistenceAdapter` +- `DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS` +- `DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS` +- `createSQLiteCorePersistenceAdapter(...)` ### Error APIs @@ -78,6 +197,12 @@ while still handling per-collection schema versions correctly. - `InvalidSyncConfigError` - `InvalidPersistedCollectionCoordinatorError` - `InvalidPersistenceAdapterError` +- `PersistedCollectionDurabilityErrorOptions` +- `PersistedCollectionDurabilityError` +- `toPersistedCollectionDurabilityError(...)` +- `IndeterminateCommitRequestType` +- `IndeterminateCommitErrorOptions` +- `IndeterminateCommitError` - `InvalidPersistedStorageKeyError` - `InvalidPersistedStorageKeyEncodingError` - `PersistenceUnavailableError` diff --git a/packages/db-sqlite-persistence-core/src/errors.ts b/packages/db-sqlite-persistence-core/src/errors.ts index f0ffc330eb..632f62a074 100644 --- a/packages/db-sqlite-persistence-core/src/errors.ts +++ b/packages/db-sqlite-persistence-core/src/errors.ts @@ -43,6 +43,107 @@ export class InvalidPersistenceAdapterError extends InvalidPersistedCollectionCo } } +export type PersistedCollectionDurabilityErrorOptions = { + cause?: unknown + code?: unknown + path?: unknown +} + +export class PersistedCollectionDurabilityError extends PersistedCollectionCoreError { + override readonly cause: unknown + readonly code: unknown + readonly path: unknown + + constructor( + message: string, + options: PersistedCollectionDurabilityErrorOptions = {}, + ) { + super(message) + this.name = `PersistedCollectionDurabilityError` + this.cause = options.cause + this.code = options.code + this.path = options.path + } +} + +export function toPersistedCollectionDurabilityError( + collectionId: string, + cause: unknown, +): PersistedCollectionDurabilityError { + if (cause instanceof PersistedCollectionDurabilityError) { + return cause + } + + const details = + typeof cause === `object` && cause !== null + ? (cause as Record) + : undefined + const message = cause instanceof Error ? cause.message : String(cause) + return new PersistedCollectionDurabilityError( + `Failed to durably persist collection "${collectionId}": ${message}`, + { + cause, + code: details?.code, + path: details?.path, + }, + ) +} + +export type IndeterminateCommitRequestType = + | `rpc:applyLocalMutations:req` + | `rpc:applyCommittedTx:req` + +export type IndeterminateCommitErrorOptions = { + collectionId: string + requestType: IndeterminateCommitRequestType + previousLeaderId: string | null + previousTerm: number | null + currentLeaderId: string | null + currentTerm: number | null + cause: unknown +} + +export class IndeterminateCommitError extends PersistedCollectionCoreError { + readonly code: `INDETERMINATE_COMMIT` = `INDETERMINATE_COMMIT` + readonly collectionId: string + readonly requestType: IndeterminateCommitRequestType + readonly previousLeaderId: string | null + readonly previousTerm: number | null + readonly currentLeaderId: string | null + readonly currentTerm: number | null + override readonly cause: unknown + + constructor(options: IndeterminateCommitErrorOptions) { + super( + `Commit outcome is indeterminate for collection "${options.collectionId}": ${options.requestType} crossed leadership from ${formatLeader(options.previousLeaderId, options.previousTerm)} to ${formatLeader(options.currentLeaderId, options.currentTerm)}`, + ) + this.name = `IndeterminateCommitError` + this.collectionId = options.collectionId + this.requestType = options.requestType + this.previousLeaderId = options.previousLeaderId + this.previousTerm = options.previousTerm + this.currentLeaderId = options.currentLeaderId + this.currentTerm = options.currentTerm + this.cause = options.cause + } +} + +function formatLeader(leaderId: string | null, term: number | null): string { + return `${leaderId ?? `unknown leader`} (term ${term ?? `unknown`})` +} + +export class DuplicateRemoteSubsetOwnerError extends PersistedCollectionCoreError { + readonly collectionId: string + + constructor(collectionId: string) { + super( + `A remote subset owner is already registered for collection "${collectionId}"`, + ) + this.name = `DuplicateRemoteSubsetOwnerError` + this.collectionId = collectionId + } +} + export class InvalidPersistedStorageKeyError extends InvalidPersistedCollectionConfigError { constructor(key: string | number) { super( diff --git a/packages/db-sqlite-persistence-core/src/index.ts b/packages/db-sqlite-persistence-core/src/index.ts index 9e2bb9faae..963bd84eef 100644 --- a/packages/db-sqlite-persistence-core/src/index.ts +++ b/packages/db-sqlite-persistence-core/src/index.ts @@ -1,5 +1,6 @@ export * from './persisted' export * from './errors' export * from './sqlite-core-adapter' +export * from './remote-subset-wire' // Re-export for use in non-secure browser contexts (see #1541) export { safeRandomUUID } from '@tanstack/db' diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 0f1e4112ac..f47999dfa9 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -6,13 +6,18 @@ import { withCollectionConfigFactory, } from '@tanstack/db' import { + DuplicateRemoteSubsetOwnerError, InvalidPersistedCollectionConfigError, InvalidPersistedCollectionCoordinatorError, InvalidPersistedStorageKeyEncodingError, InvalidPersistedStorageKeyError, InvalidPersistenceAdapterError, InvalidSyncConfigError, + PersistedCollectionDurabilityError, + toPersistedCollectionDurabilityError, } from './errors' +import { toTransportedLoadSubsetOptions } from './remote-subset-wire' +import type { TransportedLoadSubsetOptions } from './remote-subset-wire' import type { StandardSchemaV1 } from '@standard-schema/spec' import type { ChangeMessageOrDeleteKeyMessage, @@ -38,12 +43,16 @@ export type PersistedMutationEnvelope = type: `insert` key: string | number value: Record + metadata?: unknown + metadataChanged?: boolean } | { mutationId: string type: `update` key: string | number value: Record + metadata?: unknown + metadataChanged?: boolean } | { mutationId: string @@ -96,7 +105,8 @@ export type TxCommitted = { export type EnsureRemoteSubsetRequest = { type: `rpc:ensureRemoteSubset:req` rpcId: string - options: LoadSubsetOptions + acquisitionId: string + options: TransportedLoadSubsetOptions } export type EnsureRemoteSubsetResponse = @@ -104,6 +114,7 @@ export type EnsureRemoteSubsetResponse = type: `rpc:ensureRemoteSubset:res` rpcId: string ok: true + leaderId: string } | { type: `rpc:ensureRemoteSubset:res` @@ -112,6 +123,25 @@ export type EnsureRemoteSubsetResponse = error: string } +export type ReleaseRemoteSubsetRequest = { + type: `rpc:releaseRemoteSubset:req` + rpcId: string + acquisitionId: string +} + +export type ReleaseRemoteSubsetResponse = + | { + type: `rpc:releaseRemoteSubset:res` + rpcId: string + ok: true + } + | { + type: `rpc:releaseRemoteSubset:res` + rpcId: string + ok: false + error: string + } + export type ApplyLocalMutationsRequest = { type: `rpc:applyLocalMutations:req` rpcId: string @@ -136,6 +166,48 @@ export type ApplyLocalMutationsResponse = code: `NOT_LEADER` | `VALIDATION_ERROR` | `CONFLICT` | `TIMEOUT` error: string } + | { + type: `rpc:applyLocalMutations:res` + rpcId: string + ok: false + code: `PERSISTENCE_ERROR` + error: string + sourceCode?: string | number + path?: string | ReadonlyArray + } + +export type ApplyCommittedTxRequest = { + type: `rpc:applyCommittedTx:req` + rpcId: string + envelopeId: string + tx: PersistedTx +} + +export type ApplyCommittedTxResponse = + | { + type: `rpc:applyCommittedTx:res` + rpcId: string + ok: true + term: number + seq: number + latestRowVersion: number + } + | { + type: `rpc:applyCommittedTx:res` + rpcId: string + ok: false + code: `NOT_LEADER` | `CONFLICT` | `TIMEOUT` + error: string + } + | { + type: `rpc:applyCommittedTx:res` + rpcId: string + ok: false + code: `PERSISTENCE_ERROR` + error: string + sourceCode?: string | number + path?: string | ReadonlyArray + } export type PullSinceRequest = { type: `rpc:pullSince:req` @@ -282,6 +354,47 @@ export interface PersistenceAdapter { }> } +export type RemoteSubsetOwner = (( + options: TransportedLoadSubsetOptions, +) => Promise | void) & { + unloadSubset: (options: TransportedLoadSubsetOptions) => void + onError: (error: unknown) => void +} + +function reportRemoteSubsetOwnerError( + owner: RemoteSubsetOwner, + error: unknown, +): void { + try { + owner.onError(error) + } catch { + // Reporting must not replace the original owner failure. + } +} + +async function unloadRemoteSubsetOwner( + owner: RemoteSubsetOwner, + options: TransportedLoadSubsetOptions, +): Promise { + try { + const result = ( + owner.unloadSubset as unknown as ( + options: TransportedLoadSubsetOptions, + ) => unknown + )(options) + await Promise.resolve(result) + } catch (error) { + reportRemoteSubsetOwnerError(owner, error) + throw error + } +} + +type SingleProcessRemoteSubsetAcquisition = { + owner: RemoteSubsetOwner + options: TransportedLoadSubsetOptions + load: Promise +} + export interface SQLiteDriver { exec: (sql: string) => Promise query: ( @@ -306,10 +419,18 @@ export interface PersistedCollectionCoordinator { publish: (collectionId: string, message: ProtocolEnvelope) => void isLeader: (collectionId: string) => boolean ensureLeadership: (collectionId: string) => Promise - requestEnsureRemoteSubset?: ( + requestEnsureRemoteSubset: ( collectionId: string, options: LoadSubsetOptions, ) => Promise + requestReleaseRemoteSubset: ( + collectionId: string, + options: LoadSubsetOptions, + ) => Promise + registerRemoteSubsetOwner: ( + collectionId: string, + owner: RemoteSubsetOwner, + ) => () => void requestEnsurePersistedIndex: ( collectionId: string, signature: string, @@ -319,6 +440,10 @@ export interface PersistedCollectionCoordinator { collectionId: string, mutations: Array, ) => Promise + requestApplyCommittedTx: ( + collectionId: string, + tx: PersistedTx, + ) => Promise pullSince?: ( collectionId: string, fromRowVersion: number, @@ -405,7 +530,11 @@ const REQUIRED_COORDINATOR_METHODS: ReadonlyArray< | `publish` | `isLeader` | `ensureLeadership` + | `requestEnsureRemoteSubset` + | `requestReleaseRemoteSubset` + | `registerRemoteSubsetOwner` | `requestEnsurePersistedIndex` + | `requestApplyCommittedTx` > > = [ `getNodeId`, @@ -413,7 +542,11 @@ const REQUIRED_COORDINATOR_METHODS: ReadonlyArray< `publish`, `isLeader`, `ensureLeadership`, + `requestEnsureRemoteSubset`, + `requestReleaseRemoteSubset`, + `registerRemoteSubsetOwner`, `requestEnsurePersistedIndex`, + `requestApplyCommittedTx`, ] const REQUIRED_ADAPTER_METHODS: ReadonlyArray< @@ -440,6 +573,7 @@ type SyncControlFns = { commit: ((signal?: AbortSignal) => SyncAppliedReceipt) | null truncate: (() => void) | null metadata: SyncMetadataApi | null + markError: ((error: unknown) => void) | null } /** @@ -448,6 +582,12 @@ type SyncControlFns = { */ export class SingleProcessCoordinator implements PersistedCollectionCoordinator { private readonly nodeId: string + private readonly collectionAdapters = new Map() + private readonly remoteSubsetOwners = new Map() + private readonly remoteSubsetAcquisitions = new Map< + string, + Map + >() constructor(nodeId: string = safeRandomUUID()) { this.nodeId = nodeId @@ -469,10 +609,128 @@ export class SingleProcessCoordinator implements PersistedCollectionCoordinator public async ensureLeadership(): Promise {} - public async requestEnsureRemoteSubset(): Promise {} + public async requestEnsureRemoteSubset( + collectionId: string, + options: LoadSubsetOptions, + ): Promise { + const transported = toTransportedLoadSubsetOptions(options) + const owner = this.remoteSubsetOwners.get(collectionId) + if (!owner) { + throw new InvalidPersistedCollectionConfigError( + `SingleProcessCoordinator has no remote subset owner configured for collection "${collectionId}"`, + ) + } + let acquisitions = this.remoteSubsetAcquisitions.get(collectionId) + if (!acquisitions) { + acquisitions = new Map() + this.remoteSubsetAcquisitions.set(collectionId, acquisitions) + } + const existing = acquisitions.get(options) + if (existing) { + await existing.load + return + } + + let resolveLoad!: () => void + let rejectLoad!: (error: unknown) => void + const load = new Promise((resolve, reject) => { + resolveLoad = resolve + rejectLoad = reject + }) + acquisitions.set(options, { owner, options: transported, load }) + try { + const ownerLoad = owner(transported) + void Promise.resolve(ownerLoad).then(resolveLoad, (error) => { + reportRemoteSubsetOwnerError(owner, error) + rejectLoad(error) + }) + } catch (error) { + reportRemoteSubsetOwnerError(owner, error) + rejectLoad(error) + } + await load + } + + public async requestReleaseRemoteSubset( + collectionId: string, + options: LoadSubsetOptions, + ): Promise { + const acquisitions = this.remoteSubsetAcquisitions.get(collectionId) + const acquisition = acquisitions?.get(options) + if (!acquisition) return + acquisitions!.delete(options) + if (acquisitions!.size === 0) { + this.remoteSubsetAcquisitions.delete(collectionId) + } + try { + await acquisition.load + } catch { + // Calling the owner transferred the lease even when its load rejected. + } + await unloadRemoteSubsetOwner(acquisition.owner, acquisition.options) + } + + public registerRemoteSubsetOwner( + collectionId: string, + owner: RemoteSubsetOwner, + ): () => void { + if (this.remoteSubsetOwners.has(collectionId)) { + throw new DuplicateRemoteSubsetOwnerError(collectionId) + } + this.remoteSubsetOwners.set(collectionId, owner) + return () => { + if (this.remoteSubsetOwners.get(collectionId) !== owner) return + const acquisitions = this.remoteSubsetAcquisitions.get(collectionId) + this.remoteSubsetAcquisitions.delete(collectionId) + this.remoteSubsetOwners.delete(collectionId) + for (const acquisition of acquisitions?.values() ?? []) { + void (async () => { + try { + await acquisition.load + } catch { + // Calling the owner transferred the lease even when its load rejected. + } + await unloadRemoteSubsetOwner(owner, acquisition.options) + })().catch(() => undefined) + } + } + } public async requestEnsurePersistedIndex(): Promise {} + public setAdapterForCollection( + collectionId: string, + adapter: PersistenceAdapter, + ): void { + this.collectionAdapters.set(collectionId, adapter) + } + + public async requestApplyCommittedTx( + collectionId: string, + tx: PersistedTx, + ): Promise { + const adapter = this.collectionAdapters.get(collectionId) + if (!adapter) { + throw new InvalidPersistedCollectionConfigError( + `SingleProcessCoordinator has no persistence adapter configured for collection "${collectionId}"`, + ) + } + + try { + await adapter.applyCommittedTx(collectionId, tx) + } catch (error) { + throw toPersistedCollectionDurabilityError(collectionId, error) + } + return { + type: `rpc:applyCommittedTx:res`, + rpcId: safeRandomUUID(), + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + } + } + public pullSince(): Promise { return Promise.resolve({ type: `rpc:pullSince:res`, @@ -509,10 +767,14 @@ function validatePersistenceAdapter(adapter: PersistenceAdapter): void { function resolvePersistence( persistence: PersistedCollectionPersistence, + collectionId: string, ): PersistedResolvedPersistence { validatePersistenceAdapter(persistence.adapter) const coordinator = persistence.coordinator ?? new SingleProcessCoordinator() + if (coordinator instanceof SingleProcessCoordinator) { + coordinator.setAdapterForCollection(collectionId, persistence.adapter) + } validatePersistedCollectionCoordinator(coordinator) return { @@ -524,9 +786,13 @@ function resolvePersistence( function resolvePersistenceForMode( persistence: PersistedCollectionPersistence, mode: PersistedCollectionMode, + collectionId: string, ): PersistedResolvedPersistence { const modeSpecificPersistence = persistence.resolvePersistenceForMode?.(mode) - return resolvePersistence(modeSpecificPersistence ?? persistence) + return resolvePersistence( + modeSpecificPersistence ?? persistence, + collectionId, + ) } function resolvePersistenceForCollection( @@ -540,10 +806,17 @@ function resolvePersistenceForCollection( const collectionSpecificPersistence = persistence.resolvePersistenceForCollection?.(options) if (collectionSpecificPersistence) { - return resolvePersistence(collectionSpecificPersistence) + return resolvePersistence( + collectionSpecificPersistence, + options.collectionId, + ) } - return resolvePersistenceForMode(persistence, options.mode) + return resolvePersistenceForMode( + persistence, + options.mode, + options.collectionId, + ) } function hasOwnSyncKey(options: object): options is { sync: unknown } { @@ -801,6 +1074,7 @@ class PersistedCollectionRuntime< commit: null, truncate: null, metadata: null, + markError: null, } private started = false private startupMetadataPromise: Promise | null = null @@ -809,9 +1083,11 @@ class PersistedCollectionRuntime< private lifecycleGeneration = 0 private internalApplyDepth = 0 private appliedReceiptSequence = 0 + private syncErrorReported = false private readonly pendingAppliedReceipts = new Map>() private hydratingGeneration: number | null = null private coordinatorUnsubscribe: (() => void) | null = null + private remoteSubsetOwnerUnsubscribe: (() => void) | null = null private indexAddedUnsubscribe: (() => void) | null = null private indexRemovedUnsubscribe: (() => void) | null = null private remoteEnsureRetryTimer: ReturnType | null = null @@ -834,6 +1110,7 @@ class PersistedCollectionRuntime< setSyncControls(syncControls: SyncControlFns): void { this.advanceLifecycle() + this.syncErrorReported = false const commit = syncControls.commit this.syncControls = { @@ -844,6 +1121,27 @@ class PersistedCollectionRuntime< } } + reportSyncError(error: unknown): void { + const markError = this.syncControls.markError + if (this.syncErrorReported || !markError) return + + this.syncErrorReported = true + try { + markError(error) + } catch { + // Reporting must not replace the original asynchronous failure. + } + } + + registerRemoteSubsetOwner(owner: RemoteSubsetOwner): void { + this.remoteSubsetOwnerUnsubscribe?.() + this.remoteSubsetOwnerUnsubscribe = + this.persistence.coordinator.registerRemoteSubsetOwner( + this.collectionId, + owner, + ) + } + private trackAppliedReceipt(receipt: SyncAppliedReceipt): SyncAppliedReceipt { const sequence = ++this.appliedReceiptSequence if (receipt === true) { @@ -870,6 +1168,7 @@ class PersistedCollectionRuntime< commit: null, truncate: null, metadata: null, + markError: null, } } @@ -1045,18 +1344,45 @@ class PersistedCollectionRuntime< upstreamLoadSubset?: LoadSubsetFn, ): Promise { const lifecycleGeneration = this.lifecycleGeneration + const routeRemoteDemandThroughCoordinator = + this.mode === `sync-present` && + !(this.persistence.coordinator instanceof SingleProcessCoordinator) this.activeSubsets.set(this.getSubsetKey(options), options) const appliedCursor = this.appliedReceiptSequence await this.applyMutex.run(() => this.hydrateSubsetUnsafe(options, { - requestRemoteEnsure: this.mode === `sync-present`, + requestRemoteEnsure: + this.mode === `sync-present` && !routeRemoteDemandThroughCoordinator, lifecycleGeneration, }), ) if (lifecycleGeneration !== this.lifecycleGeneration) return await this.waitForAppliedReceiptsAfter(appliedCursor) + if (routeRemoteDemandThroughCoordinator) { + try { + await this.persistence.coordinator.requestEnsureRemoteSubset( + this.collectionId, + options, + ) + } catch (error) { + if ( + options.signal?.aborted || + (typeof error === `object` && + error !== null && + `name` in error && + error.name === `AbortError`) + ) { + this.pendingRemoteSubsetEnsures.delete(this.getSubsetKey(options)) + throw error + } + this.queueRemoteSubsetEnsure(options) + throw error + } + return + } + if (upstreamLoadSubset) { try { await upstreamLoadSubset(options) @@ -1085,7 +1411,27 @@ class PersistedCollectionRuntime< ): void { this.activeSubsets.delete(this.getSubsetKey(options)) this.pendingRemoteSubsetEnsures.delete(this.getSubsetKey(options)) - upstreamUnloadSubset?.(options) + if (this.mode === `sync-present`) { + void this.persistence.coordinator + .requestReleaseRemoteSubset(this.collectionId, options) + .catch((error) => { + this.reportSyncError(error) + }) + } + if (upstreamUnloadSubset) { + try { + const result = ( + upstreamUnloadSubset as unknown as ( + options: LoadSubsetOptions, + ) => unknown + )(options) + void Promise.resolve(result).catch((error) => { + this.reportSyncError(error) + }) + } catch (error) { + this.reportSyncError(error) + } + } } async forceReloadSubset(options: LoadSubsetOptions): Promise { @@ -1216,9 +1562,22 @@ class PersistedCollectionRuntime< cleanup(): void { this.advanceLifecycle() + if (this.mode === `sync-present`) { + for (const options of this.activeSubsets.values()) { + void this.persistence.coordinator + .requestReleaseRemoteSubset(this.collectionId, options) + .catch((error) => { + this.reportSyncError(error) + }) + } + } + this.coordinatorUnsubscribe?.() this.coordinatorUnsubscribe = null + this.remoteSubsetOwnerUnsubscribe?.() + this.remoteSubsetOwnerUnsubscribe = null + this.indexAddedUnsubscribe?.() this.indexAddedUnsubscribe = null @@ -1519,27 +1878,39 @@ class PersistedCollectionRuntime< } const tx = this.createPersistedTxFromOperations(transaction, streamPosition) - - await this.persistence.adapter.applyCommittedTx(this.collectionId, tx) - this.publishTxCommittedEvent( - this.createTxCommittedPayload({ - term: tx.term, - seq: tx.seq, - txId: tx.txId, - latestRowVersion: tx.rowVersion, - requiresFullReload: transaction.truncate, - changedRows: transaction.operations - .filter((operation) => operation.type === `update`) - .map((operation) => ({ - key: operation.key, - value: operation.value as Record, - })), - deletedKeys: transaction.operations - .filter((operation) => operation.type === `delete`) - .map((operation) => operation.key), - rowMetadataMutations: tx.rowMetadataMutations, - collectionMetadataMutations: tx.collectionMetadataMutations, - }), + let response: ApplyCommittedTxResponse + try { + response = await this.persistence.coordinator.requestApplyCommittedTx( + this.collectionId, + tx, + ) + } catch (error) { + if (error instanceof PersistedCollectionDurabilityError) { + this.reportSyncError(error) + } + throw error + } + if (!response.ok) { + if (response.code === `PERSISTENCE_ERROR`) { + const error = new PersistedCollectionDurabilityError( + `Failed to durably persist collection "${this.collectionId}": ${response.error}`, + { + cause: response, + code: response.sourceCode ?? response.code, + path: response.path, + }, + ) + this.reportSyncError(error) + throw error + } + throw new Error( + `failed to apply external sync transaction through coordinator: ${response.error}`, + ) + } + this.observeStreamPosition( + response.term, + response.seq, + response.latestRowVersion, ) } @@ -1871,7 +2242,7 @@ class PersistedCollectionRuntime< private queueRemoteSubsetEnsure(options: LoadSubsetOptions): void { if ( this.mode !== `sync-present` || - !this.persistence.coordinator.requestEnsureRemoteSubset || + this.persistence.coordinator instanceof SingleProcessCoordinator || this.activeSubsets.get(this.getSubsetKey(options)) !== options ) { return @@ -1884,7 +2255,7 @@ class PersistedCollectionRuntime< private scheduleRemoteEnsureRetry(): void { if ( this.mode !== `sync-present` || - !this.persistence.coordinator.requestEnsureRemoteSubset + this.persistence.coordinator instanceof SingleProcessCoordinator ) { return } @@ -1905,7 +2276,7 @@ class PersistedCollectionRuntime< private async flushPendingRemoteSubsetEnsures(): Promise { if ( this.mode !== `sync-present` || - !this.persistence.coordinator.requestEnsureRemoteSubset + this.persistence.coordinator instanceof SingleProcessCoordinator ) { return } @@ -2337,6 +2708,7 @@ function createWrappedSyncConfig< commit: params.commit, truncate: params.truncate, metadata: params.metadata ?? null, + markError: params.markError, }) runtime.setCollection( params.collection as Collection, @@ -2640,6 +3012,25 @@ function createWrappedSyncConfig< sourceResult = normalizeSyncFnResult( sourceSyncConfig.sync(wrappedParams), ) + if (sourceResult.loadSubset) { + const loadSubset = async (options: TransportedLoadSubsetOptions) => { + if (startupState.cleanedUp) { + throw new Error(`persisted sync source is no longer active`) + } + await sourceResult.loadSubset?.( + options as unknown as LoadSubsetOptions, + ) + } + runtime.registerRemoteSubsetOwner( + Object.assign(loadSubset, { + unloadSubset: (options: TransportedLoadSubsetOptions) => + sourceResult.unloadSubset?.( + options as unknown as LoadSubsetOptions, + ), + onError: (error: unknown) => runtime.reportSyncError(error), + }), + ) + } return sourceResult })() @@ -2708,6 +3099,7 @@ function createLoopbackSyncConfig< commit: params.commit, truncate: params.truncate, metadata: params.metadata ?? null, + markError: params.markError, }) runtime.setCollection( params.collection as Collection, diff --git a/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts b/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts new file mode 100644 index 0000000000..407a0534c4 --- /dev/null +++ b/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts @@ -0,0 +1,899 @@ +import { IR } from '@tanstack/db' +import type { LoadSubsetOptions } from '@tanstack/db' + +export type RemoteSubsetWirePrimitive = + | undefined + | null + | boolean + | string + | number + | bigint + +export type RemoteSubsetWireTypedArray = + | Int8Array + | Uint8Array + | Uint8ClampedArray + | Int16Array + | Uint16Array + | Int32Array + | Uint32Array + | Float32Array + | Float64Array + | BigInt64Array + | BigUint64Array + +export type RemoteSubsetWireRecord = { + [key: string]: RemoteSubsetWireValue +} + +/** Values whose complete semantics survive a coordinator transport boundary. */ +export type RemoteSubsetWireValue = + | RemoteSubsetWirePrimitive + | Date + | RegExp + | ArrayBuffer + | DataView + | RemoteSubsetWireTypedArray + | Array + | Map + | Set + | RemoteSubsetWireRecord + +export type RemoteSubsetWireExpression = + | { type: `ref`; path: Array } + | { type: `val`; value: RemoteSubsetWireValue } + | { + type: `func` + name: string + args: Array + } + +export type RemoteSubsetWireCompareOptions = { + direction: `asc` | `desc` + nulls: `first` | `last` +} & ( + | { stringSort?: `lexical` } + | { + stringSort?: `locale` + locale?: string + localeOptions?: RemoteSubsetWireRecord + } +) + +export type RemoteSubsetWireOrderByClause = { + expression: RemoteSubsetWireExpression + compareOptions: RemoteSubsetWireCompareOptions +} + +export type RemoteSubsetWireCursor = { + whereFrom: RemoteSubsetWireExpression + whereCurrent: RemoteSubsetWireExpression + lastKey?: string | number +} + +/** The complete data portion transported by a remote-subset request. */ +export type TransportedLoadSubsetOptions = { + where?: RemoteSubsetWireExpression + orderBy?: Array + limit?: number + cursor?: RemoteSubsetWireCursor + offset?: number +} + +export class RemoteSubsetWireValueError extends TypeError { + override readonly name = `RemoteSubsetWireValueError` + + constructor( + readonly path: string, + reason: string, + ) { + super(`Unsupported remote subset wire value at ${path}: ${reason}`) + } +} + +type ProjectionState = { + expressions: WeakMap + expressionArrays: WeakMap> + stringArrays: WeakMap> + orderByArrays: WeakMap> + orderByClauses: WeakMap + compareOptions: WeakMap + cursors: WeakMap + wireValues: WeakMap +} + +type DataProperty = { + present: boolean + value?: unknown +} + +type TypedArrayConstructor = { + readonly prototype: RemoteSubsetWireTypedArray + new ( + buffer: ArrayBuffer, + byteOffset?: number, + length?: number, + ): RemoteSubsetWireTypedArray +} + +const typedArrayConstructors: ReadonlyArray = [ + Int8Array, + Uint8Array, + Uint8ClampedArray, + Int16Array, + Uint16Array, + Int32Array, + Uint32Array, + Float32Array, + Float64Array, + BigInt64Array, + BigUint64Array, +] + +const NativeDate = Date +const identifierPattern = /^[A-Za-z_$][A-Za-z0-9_$]*$/ + +/** + * Projects one live subset request into the exact coordinator wire domain. + * Validation happens before callers choose a local-leader or transported path. + */ +export function toTransportedLoadSubsetOptions( + options: LoadSubsetOptions, +): TransportedLoadSubsetOptions { + assertPlainContainer(options, `options`) + assertAllowedProperties(options, `options`, [ + `where`, + `orderBy`, + `limit`, + `cursor`, + `offset`, + `signal`, + `subscription`, + ]) + + const state: ProjectionState = { + expressions: new WeakMap(), + expressionArrays: new WeakMap(), + stringArrays: new WeakMap(), + orderByArrays: new WeakMap(), + orderByClauses: new WeakMap(), + compareOptions: new WeakMap(), + cursors: new WeakMap(), + wireValues: new WeakMap(), + } + const projected: TransportedLoadSubsetOptions = {} + const where = readDataProperty(options, `where`, `options.where`) + const orderBy = readDataProperty(options, `orderBy`, `options.orderBy`) + const limit = readDataProperty(options, `limit`, `options.limit`) + const cursor = readDataProperty(options, `cursor`, `options.cursor`) + const offset = readDataProperty(options, `offset`, `options.offset`) + + if (where.present && where.value !== undefined) { + projected.where = projectExpression(where.value, `options.where`, state) + } + if (orderBy.present && orderBy.value !== undefined) { + projected.orderBy = projectOrderBy(orderBy.value, `options.orderBy`, state) + } + if (limit.present && limit.value !== undefined) { + projected.limit = projectNumber(limit.value, `options.limit`) + } + if (cursor.present && cursor.value !== undefined) { + projected.cursor = projectCursor(cursor.value, `options.cursor`, state) + } + if (offset.present && offset.value !== undefined) { + projected.offset = projectNumber(offset.value, `options.offset`) + } + + return projected +} + +function projectExpression( + value: unknown, + path: string, + state: ProjectionState, +): RemoteSubsetWireExpression { + const object = requireObject(value, path) + const existing = state.expressions.get(object) + if (existing) return existing + + const type = readDataProperty(object, `type`, `${path}.type`) + if (!type.present || typeof type.value !== `string`) { + throw new RemoteSubsetWireValueError(`${path}.type`, describe(type.value)) + } + + switch (type.value) { + case `ref`: { + assertExpressionPrototype(object, path, IR.PropRef.prototype) + assertAllowedProperties(object, path, [`type`, `path`]) + const projected = { + type: `ref`, + path: [] as Array, + } satisfies RemoteSubsetWireExpression + state.expressions.set(object, projected) + const sourcePath = readRequiredDataProperty( + object, + `path`, + `${path}.path`, + ) + projected.path = projectStringArray(sourcePath, `${path}.path`, state) + return projected + } + case `val`: { + assertExpressionPrototype(object, path, IR.Value.prototype) + assertAllowedProperties(object, path, [`type`, `value`]) + const projected: Extract = { + type: `val`, + value: undefined, + } + state.expressions.set(object, projected) + const sourceValue = readRequiredDataProperty( + object, + `value`, + `${path}.value`, + ) + projected.value = projectWireValue(sourceValue, `${path}.value`, state) + return projected + } + case `func`: { + assertExpressionPrototype(object, path, IR.Func.prototype) + assertAllowedProperties(object, path, [`type`, `name`, `args`]) + const projected = { + type: `func`, + name: ``, + args: [] as Array, + } satisfies RemoteSubsetWireExpression + state.expressions.set(object, projected) + const name = readRequiredDataProperty(object, `name`, `${path}.name`) + if (typeof name !== `string`) { + throw new RemoteSubsetWireValueError(`${path}.name`, describe(name)) + } + projected.name = name + const args = readRequiredDataProperty(object, `args`, `${path}.args`) + projected.args = projectExpressionArray(args, `${path}.args`, state) + return projected + } + default: + throw new RemoteSubsetWireValueError(`${path}.type`, String(type.value)) + } +} + +function projectExpressionArray( + value: unknown, + path: string, + state: ProjectionState, +): Array { + const array = requireArray(value, path) + const existing = state.expressionArrays.get(array) + if (existing) return existing + assertArrayShape(array, path) + const projected = new Array(array.length) + state.expressionArrays.set(array, projected) + for (let index = 0; index < array.length; index++) { + if (!(index in array)) { + throw new RemoteSubsetWireValueError( + `${path}[${index}]`, + `missing expression`, + ) + } + projected[index] = projectExpression( + array[index], + `${path}[${index}]`, + state, + ) + } + return projected +} + +function projectStringArray( + value: unknown, + path: string, + state: ProjectionState, +): Array { + const array = requireArray(value, path) + const existing = state.stringArrays.get(array) + if (existing) return existing + assertArrayShape(array, path) + const projected = new Array(array.length) + state.stringArrays.set(array, projected) + for (let index = 0; index < array.length; index++) { + const entry = array[index] + if (!(index in array) || typeof entry !== `string`) { + throw new RemoteSubsetWireValueError(`${path}[${index}]`, describe(entry)) + } + projected[index] = entry + } + return projected +} + +function projectOrderBy( + value: unknown, + path: string, + state: ProjectionState, +): Array { + const array = requireArray(value, path) + const existing = state.orderByArrays.get(array) + if (existing) return existing + assertArrayShape(array, path) + const projected = new Array(array.length) + state.orderByArrays.set(array, projected) + + for (let index = 0; index < array.length; index++) { + const clausePath = `${path}[${index}]` + const clause = requirePlainRecord(array[index], clausePath) + const existingClause = state.orderByClauses.get(clause) + if (existingClause) { + projected[index] = existingClause + continue + } + assertAllowedProperties(clause, clausePath, [ + `expression`, + `compareOptions`, + ]) + const projectedClause = { + expression: undefined as unknown as RemoteSubsetWireExpression, + compareOptions: undefined as unknown as RemoteSubsetWireCompareOptions, + } + projected[index] = projectedClause + state.orderByClauses.set(clause, projectedClause) + projectedClause.expression = projectExpression( + readRequiredDataProperty( + clause, + `expression`, + `${clausePath}.expression`, + ), + `${clausePath}.expression`, + state, + ) + projectedClause.compareOptions = projectCompareOptions( + readRequiredDataProperty( + clause, + `compareOptions`, + `${clausePath}.compareOptions`, + ), + `${clausePath}.compareOptions`, + state, + ) + } + return projected +} + +function projectCompareOptions( + value: unknown, + path: string, + state: ProjectionState, +): RemoteSubsetWireCompareOptions { + const source = requirePlainRecord(value, path) + const existing = state.compareOptions.get(source) + if (existing) return existing + assertAllowedProperties(source, path, [ + `direction`, + `nulls`, + `stringSort`, + `locale`, + `localeOptions`, + ]) + const projected: { + direction?: `asc` | `desc` + nulls?: `first` | `last` + stringSort?: `lexical` | `locale` + locale?: string + localeOptions?: RemoteSubsetWireRecord + } = {} + + const direction = readRequiredDataProperty( + source, + `direction`, + `${path}.direction`, + ) + if (direction !== `asc` && direction !== `desc`) { + throw new RemoteSubsetWireValueError( + `${path}.direction`, + describe(direction), + ) + } + projected.direction = direction + + const nulls = readRequiredDataProperty(source, `nulls`, `${path}.nulls`) + if (nulls !== `first` && nulls !== `last`) { + throw new RemoteSubsetWireValueError(`${path}.nulls`, describe(nulls)) + } + projected.nulls = nulls + + const stringSort = readDataProperty( + source, + `stringSort`, + `${path}.stringSort`, + ) + if (stringSort.present && stringSort.value !== undefined) { + if (stringSort.value !== `lexical` && stringSort.value !== `locale`) { + throw new RemoteSubsetWireValueError( + `${path}.stringSort`, + describe(stringSort.value), + ) + } + projected.stringSort = stringSort.value + } + const locale = readDataProperty(source, `locale`, `${path}.locale`) + if (locale.present && locale.value !== undefined) { + if (typeof locale.value !== `string`) { + throw new RemoteSubsetWireValueError( + `${path}.locale`, + describe(locale.value), + ) + } + projected.locale = locale.value + } + const localeOptions = readDataProperty( + source, + `localeOptions`, + `${path}.localeOptions`, + ) + if (localeOptions.present && localeOptions.value !== undefined) { + projected.localeOptions = projectWireRecord( + localeOptions.value, + `${path}.localeOptions`, + state, + ) + } + const result = projected as RemoteSubsetWireCompareOptions + state.compareOptions.set(source, result) + return result +} + +function projectCursor( + value: unknown, + path: string, + state: ProjectionState, +): RemoteSubsetWireCursor { + const source = requirePlainRecord(value, path) + const existing = state.cursors.get(source) + if (existing) return existing + assertAllowedProperties(source, path, [ + `whereFrom`, + `whereCurrent`, + `lastKey`, + ]) + const projected: RemoteSubsetWireCursor = { + whereFrom: undefined as unknown as RemoteSubsetWireExpression, + whereCurrent: undefined as unknown as RemoteSubsetWireExpression, + } + state.cursors.set(source, projected) + projected.whereFrom = projectExpression( + readRequiredDataProperty(source, `whereFrom`, `${path}.whereFrom`), + `${path}.whereFrom`, + state, + ) + projected.whereCurrent = projectExpression( + readRequiredDataProperty(source, `whereCurrent`, `${path}.whereCurrent`), + `${path}.whereCurrent`, + state, + ) + const lastKey = readDataProperty(source, `lastKey`, `${path}.lastKey`) + if (lastKey.present && lastKey.value !== undefined) { + if ( + typeof lastKey.value !== `string` && + typeof lastKey.value !== `number` + ) { + throw new RemoteSubsetWireValueError( + `${path}.lastKey`, + describe(lastKey.value), + ) + } + projected.lastKey = lastKey.value + } + return projected +} + +function projectWireValue( + value: unknown, + path: string, + state: ProjectionState, +): RemoteSubsetWireValue { + if ( + value === undefined || + value === null || + typeof value === `boolean` || + typeof value === `string` || + typeof value === `number` || + typeof value === `bigint` + ) { + return value + } + if (typeof value !== `object`) { + throw new RemoteSubsetWireValueError(path, describe(value)) + } + + const existing = state.wireValues.get(value) + if (existing) return existing + + if ( + typeof SharedArrayBuffer !== `undefined` && + value instanceof SharedArrayBuffer + ) { + throw new RemoteSubsetWireValueError(path, `SharedArrayBuffer`) + } + if (value instanceof NativeDate) { + assertExactPrototype(value, NativeDate.prototype, path) + assertNoOwnProperties(value, path) + const projected = new NativeDate(value.getTime()) + state.wireValues.set(value, projected) + return projected + } + if (value instanceof RegExp) { + assertExactPrototype(value, RegExp.prototype, path) + assertRegExpShape(value, path) + const projected = new RegExp(value.source, value.flags) + state.wireValues.set(value, projected) + return projected + } + if (value instanceof ArrayBuffer) { + assertExactPrototype(value, ArrayBuffer.prototype, path) + assertNoOwnProperties(value, path) + assertFixedArrayBuffer(value, path) + let projected: ArrayBuffer + try { + projected = value.slice(0) + } catch { + throw new RemoteSubsetWireValueError(path, `detached ArrayBuffer`) + } + state.wireValues.set(value, projected) + return projected + } + if (value instanceof DataView) { + assertExactPrototype(value, DataView.prototype, path) + assertNoOwnProperties(value, path) + let byteOffset: number + let byteLength: number + let buffer: ArrayBufferLike + try { + byteOffset = value.byteOffset + byteLength = value.byteLength + buffer = value.buffer + } catch { + throw new RemoteSubsetWireValueError(path, `detached DataView`) + } + const projectedBuffer = projectWireValue(buffer, path, state) + if (!(projectedBuffer instanceof ArrayBuffer)) { + throw new RemoteSubsetWireValueError(path, `shared DataView buffer`) + } + const projected = new DataView(projectedBuffer, byteOffset, byteLength) + state.wireValues.set(value, projected) + return projected + } + + const typedArrayConstructor = getTypedArrayConstructor(value) + if (typedArrayConstructor) { + const typedArray = value as RemoteSubsetWireTypedArray + assertTypedArrayShape(typedArray, path) + let byteOffset: number + let length: number + let buffer: ArrayBufferLike + try { + byteOffset = typedArray.byteOffset + length = typedArray.length + buffer = typedArray.buffer + } catch { + throw new RemoteSubsetWireValueError(path, `detached typed array`) + } + if (!(buffer instanceof ArrayBuffer)) { + throw new RemoteSubsetWireValueError(path, `shared typed-array buffer`) + } + const projectedBuffer = projectWireValue(buffer, path, state) + if (!(projectedBuffer instanceof ArrayBuffer)) { + throw new RemoteSubsetWireValueError(path, `shared typed-array buffer`) + } + let projected: RemoteSubsetWireTypedArray + try { + projected = new typedArrayConstructor(projectedBuffer, byteOffset, length) + } catch { + throw new RemoteSubsetWireValueError(path, `detached typed array`) + } + state.wireValues.set(value, projected) + return projected + } + if (ArrayBuffer.isView(value)) { + throw new RemoteSubsetWireValueError(path, describe(value)) + } + if (Array.isArray(value)) return projectWireArray(value, path, state) + if (value instanceof Map) return projectWireMap(value, path, state) + if (value instanceof Set) return projectWireSet(value, path, state) + return projectWireRecord(value, path, state) +} + +function projectWireArray( + value: Array, + path: string, + state: ProjectionState, +): Array { + assertArrayShape(value, path) + const projected = new Array(value.length) + state.wireValues.set(value, projected) + for (let index = 0; index < value.length; index++) { + if (index in value) { + projected[index] = projectWireValue( + value[index], + `${path}[${index}]`, + state, + ) + } + } + return projected +} + +function projectWireMap( + value: Map, + path: string, + state: ProjectionState, +): Map { + assertExactPrototype(value, Map.prototype, path) + assertNoOwnProperties(value, path) + const projected = new Map() + state.wireValues.set(value, projected) + let index = 0 + for (const [key, entry] of value) { + projected.set( + projectWireValue(key, `${path}.entries[${index}].key`, state), + projectWireValue(entry, `${path}.entries[${index}].value`, state), + ) + index++ + } + return projected +} + +function projectWireSet( + value: Set, + path: string, + state: ProjectionState, +): Set { + assertExactPrototype(value, Set.prototype, path) + assertNoOwnProperties(value, path) + const projected = new Set() + state.wireValues.set(value, projected) + let index = 0 + for (const entry of value) { + projected.add(projectWireValue(entry, `${path}.values[${index}]`, state)) + index++ + } + return projected +} + +function projectWireRecord( + value: unknown, + path: string, + state: ProjectionState, +): RemoteSubsetWireRecord { + const source = requirePlainRecord(value, path) + const existing = state.wireValues.get(source) + if (existing) return existing as RemoteSubsetWireRecord + const projected: RemoteSubsetWireRecord = {} + state.wireValues.set(source, projected) + + for (const key of ownKeys(source, path)) { + const childPath = propertyPath(path, key) + if (typeof key === `symbol`) { + throw new RemoteSubsetWireValueError(childPath, `symbol property key`) + } + const descriptor = Object.getOwnPropertyDescriptor(source, key) + if (!descriptor || !descriptor.enumerable || !(`value` in descriptor)) { + throw new RemoteSubsetWireValueError(childPath, `non-data property`) + } + Object.defineProperty(projected, key, { + value: projectWireValue(descriptor.value, childPath, state), + enumerable: true, + writable: true, + configurable: true, + }) + } + return projected +} + +function projectNumber(value: unknown, path: string): number { + if (typeof value !== `number`) { + throw new RemoteSubsetWireValueError(path, describe(value)) + } + return value +} + +function readRequiredDataProperty( + value: object, + key: string, + path: string, +): unknown { + const property = readDataProperty(value, key, path) + if (!property.present) { + throw new RemoteSubsetWireValueError(path, `missing property`) + } + return property.value +} + +function readDataProperty( + value: object, + key: string, + path: string, +): DataProperty { + let descriptor: PropertyDescriptor | undefined + try { + descriptor = Object.getOwnPropertyDescriptor(value, key) + } catch { + throw new RemoteSubsetWireValueError(path, `unreadable property`) + } + if (!descriptor) return { present: false } + if (!descriptor.enumerable || !(`value` in descriptor)) { + throw new RemoteSubsetWireValueError(path, `non-data property`) + } + return { present: true, value: descriptor.value } +} + +function assertAllowedProperties( + value: object, + path: string, + allowed: ReadonlyArray, +): void { + const allowedSet = new Set(allowed) + for (const key of ownKeys(value, path)) { + if (typeof key === `symbol` || !allowedSet.has(key)) { + throw new RemoteSubsetWireValueError(propertyPath(path, key), `property`) + } + readDataProperty(value, key, propertyPath(path, key)) + } +} + +function assertPlainContainer(value: object, path: string): void { + assertExactPrototype(value, Object.prototype, path) +} + +function assertExpressionPrototype( + value: object, + path: string, + expressionPrototype: object, +): void { + let prototype: object | null + try { + prototype = Object.getPrototypeOf(value) + } catch { + throw new RemoteSubsetWireValueError(path, `unreadable prototype`) + } + if (prototype !== Object.prototype && prototype !== expressionPrototype) { + throw new RemoteSubsetWireValueError(path, describe(value)) + } +} + +function assertExactPrototype( + value: object, + expected: object, + path: string, +): void { + let prototype: object | null + try { + prototype = Object.getPrototypeOf(value) + } catch { + throw new RemoteSubsetWireValueError(path, `unreadable prototype`) + } + if (prototype !== expected) { + throw new RemoteSubsetWireValueError(path, describe(value)) + } +} + +function assertNoOwnProperties(value: object, path: string): void { + const keys = ownKeys(value, path) + if (keys.length > 0) { + throw new RemoteSubsetWireValueError( + propertyPath(path, keys[0]!), + `property`, + ) + } +} + +function assertFixedArrayBuffer(value: ArrayBuffer, path: string): void { + if ((value as ArrayBuffer & { readonly resizable?: boolean }).resizable) { + throw new RemoteSubsetWireValueError(path, `resizable ArrayBuffer`) + } +} + +function assertRegExpShape(value: RegExp, path: string): void { + const keys = ownKeys(value, path) + for (const key of keys) { + if (key !== `lastIndex`) { + throw new RemoteSubsetWireValueError(propertyPath(path, key), `property`) + } + } + const descriptor = Object.getOwnPropertyDescriptor(value, `lastIndex`) + if (!descriptor || !(`value` in descriptor) || descriptor.value !== 0) { + throw new RemoteSubsetWireValueError( + `${path}.lastIndex`, + `RegExp lastIndex must be 0`, + ) + } +} + +function assertTypedArrayShape( + value: RemoteSubsetWireTypedArray, + path: string, +): void { + const keys = ownKeys(value, path) + for (const key of keys) { + if (typeof key !== `string` || !isCanonicalArrayIndex(key, value.length)) { + throw new RemoteSubsetWireValueError(propertyPath(path, key), `property`) + } + } +} + +function assertArrayShape(value: Array, path: string): void { + assertExactPrototype(value, Array.prototype, path) + const keys = ownKeys(value, path) + for (const key of keys) { + if ( + key !== `length` && + (typeof key !== `string` || !isCanonicalArrayIndex(key, value.length)) + ) { + throw new RemoteSubsetWireValueError(propertyPath(path, key), `property`) + } + if (key !== `length`) { + readDataProperty(value, key, propertyPath(path, key)) + } + } +} + +function isCanonicalArrayIndex(key: string, length: number): boolean { + if (!/^(0|[1-9][0-9]*)$/.test(key)) return false + const index = Number(key) + return Number.isSafeInteger(index) && index >= 0 && index < length +} + +function getTypedArrayConstructor( + value: object, +): TypedArrayConstructor | undefined { + let prototype: object | null + try { + prototype = Object.getPrototypeOf(value) + } catch { + return undefined + } + return typedArrayConstructors.find( + (constructor) => prototype === constructor.prototype, + ) +} + +function requireArray(value: unknown, path: string): Array { + if (!Array.isArray(value)) { + throw new RemoteSubsetWireValueError(path, describe(value)) + } + return value +} + +function requireObject(value: unknown, path: string): object { + if (value === null || typeof value !== `object`) { + throw new RemoteSubsetWireValueError(path, describe(value)) + } + return value +} + +function requirePlainRecord( + value: unknown, + path: string, +): Record { + const object = requireObject(value, path) + assertExactPrototype(object, Object.prototype, path) + return object as Record +} + +function ownKeys(value: object, path: string): Array { + try { + return Reflect.ownKeys(value) + } catch { + throw new RemoteSubsetWireValueError(path, `unreadable properties`) + } +} + +function propertyPath(path: string, key: string | symbol): string { + if (typeof key === `symbol`) return `${path}[${String(key)}]` + return identifierPattern.test(key) + ? `${path}.${key}` + : `${path}[${JSON.stringify(key)}]` +} + +function describe(value: unknown): string { + if (value === null) return `null` + if (typeof value !== `object`) return typeof value + return `object` +} diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index 69f29fc604..5cd5305911 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -1884,10 +1884,15 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { } } - private async ensureCollectionReadyInternal( - collectionId: string, - ): Promise { - const existingRows = await this.driver.query<{ + private async loadCollectionRegistration(collectionId: string): Promise< + | { + table_name: string + tombstone_table_name: string + schema_version: number + } + | undefined + > { + const rows = await this.driver.query<{ table_name: string tombstone_table_name: string schema_version: number @@ -1899,25 +1904,17 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { [collectionId], ) - let tableName: string - let tombstoneTableName: string + return rows[0] + } - if (existingRows.length > 0) { - tableName = existingRows[0]!.table_name - tombstoneTableName = existingRows[0]!.tombstone_table_name + private async ensureCollectionReadyInternal( + collectionId: string, + ): Promise { + let registration = await this.loadCollectionRegistration(collectionId) - if (existingRows[0]!.schema_version !== this.schemaVersion) { - await this.handleSchemaMismatch( - collectionId, - existingRows[0]!.schema_version, - this.schemaVersion, - tableName, - tombstoneTableName, - ) - } - } else { - tableName = createPersistedTableName(collectionId, `c`) - tombstoneTableName = createPersistedTableName(collectionId, `t`) + if (!registration) { + const tableName = createPersistedTableName(collectionId, `c`) + const tombstoneTableName = createPersistedTableName(collectionId, `t`) await this.driver.run( `INSERT INTO collection_registry ( collection_id, @@ -1926,9 +1923,29 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { schema_version, updated_at ) - VALUES (?, ?, ?, ?, CAST(strftime('%s', 'now') AS INTEGER))`, + VALUES (?, ?, ?, ?, CAST(strftime('%s', 'now') AS INTEGER)) + ON CONFLICT DO NOTHING`, [collectionId, tableName, tombstoneTableName, this.schemaVersion], ) + + registration = await this.loadCollectionRegistration(collectionId) + if (!registration) { + throw new InvalidPersistedCollectionConfigError( + `Unable to register persistence tables for collection "${collectionId}"`, + ) + } + } + + const tableName = registration.table_name + const tombstoneTableName = registration.tombstone_table_name + if (registration.schema_version !== this.schemaVersion) { + await this.handleSchemaMismatch( + collectionId, + registration.schema_version, + this.schemaVersion, + tableName, + tombstoneTableName, + ) } const collectionTableSql = quoteIdentifier(tableName) diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts b/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts index b07f9f787d..1305fd9274 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts @@ -1,7 +1,22 @@ import { describe, expectTypeOf, it } from 'vitest' import { createCollection } from '@tanstack/db' -import { persistedCollectionOptions } from '../src' -import type { PersistedCollectionUtils, PersistenceAdapter } from '../src' +import { + IndeterminateCommitError, + PersistedCollectionDurabilityError, + persistedCollectionOptions, +} from '../src' +import type { + ApplyCommittedTxResponse, + ApplyLocalMutationsResponse, + EnsureRemoteSubsetRequest, + EnsureRemoteSubsetResponse, + IndeterminateCommitRequestType, + PersistedCollectionCoordinator, + PersistedCollectionUtils, + PersistenceAdapter, + RemoteSubsetWireValue, + TransportedLoadSubsetOptions, +} from '../src' import type { SyncConfig, UtilsRecord } from '@tanstack/db' type Todo = { @@ -24,6 +39,212 @@ const adapter: PersistenceAdapter = { } describe(`persisted collection types`, () => { + it(`exports the indeterminate commit reconciliation contract`, () => { + const requestType: IndeterminateCommitRequestType = `rpc:applyLocalMutations:req` + const error = new IndeterminateCommitError({ + collectionId: `todos`, + requestType, + previousLeaderId: `leader-a`, + previousTerm: 1, + currentLeaderId: null, + currentTerm: null, + cause: new Error(`channel closed`), + }) + + expectTypeOf(error.code).toEqualTypeOf<`INDETERMINATE_COMMIT`>() + expectTypeOf( + error.requestType, + ).toEqualTypeOf() + expectTypeOf(error.previousLeaderId).toEqualTypeOf() + expectTypeOf(error.previousTerm).toEqualTypeOf() + expectTypeOf(error.currentLeaderId).toEqualTypeOf() + expectTypeOf(error.currentTerm).toEqualTypeOf() + expectTypeOf(error.cause).toEqualTypeOf() + }) + + it(`exports the durability error and persistence response details`, () => { + const cause = Object.assign(new Error(`disk failed`), { + code: `SQLITE_FULL`, + path: `todos.sqlite`, + }) + const error = new PersistedCollectionDurabilityError(`durability failed`, { + cause, + code: cause.code, + path: cause.path, + }) + const response: Extract< + ApplyCommittedTxResponse, + { ok: false; code: `PERSISTENCE_ERROR` } + > = { + type: `rpc:applyCommittedTx:res`, + rpcId: `durability`, + ok: false, + code: `PERSISTENCE_ERROR`, + error: `disk failed`, + sourceCode: `SQLITE_FULL`, + path: [`database`, `todos.sqlite`], + } + const localMutationResponse: Extract< + ApplyLocalMutationsResponse, + { ok: false; code: `PERSISTENCE_ERROR` } + > = { + type: `rpc:applyLocalMutations:res`, + rpcId: `local-durability`, + ok: false, + code: `PERSISTENCE_ERROR`, + error: `disk failed`, + sourceCode: `SQLITE_FULL`, + path: [`database`, `todos.sqlite`], + } + + expectTypeOf(error.cause).toEqualTypeOf() + expectTypeOf(error.code).toEqualTypeOf() + expectTypeOf(error.path).toEqualTypeOf() + expectTypeOf(response.sourceCode).toEqualTypeOf< + string | number | undefined + >() + expectTypeOf(response.path).toEqualTypeOf< + string | ReadonlyArray | undefined + >() + expectTypeOf(localMutationResponse.sourceCode).toEqualTypeOf< + string | number | undefined + >() + expectTypeOf(localMutationResponse.path).toEqualTypeOf< + string | ReadonlyArray | undefined + >() + }) + + it(`exports the exact remote-subset transport domain`, () => { + type WireRecord = { [key: string]: RemoteSubsetWireValue } + const cyclic: WireRecord = {} + cyclic.self = cyclic + const supported: TransportedLoadSubsetOptions = { + where: { + type: `func`, + name: `eq`, + args: [ + { type: `ref`, path: [`todos`, `payload`] }, + { + type: `val`, + value: { + missing: undefined, + precise: 9_007_199_254_740_993n, + date: new Date(Number.NaN), + regexp: /wire/giu, + buffer: new ArrayBuffer(4), + view: new DataView(new ArrayBuffer(4)), + typed: new Uint8Array([1, 2]), + map: new Map([ + [{ key: `object` }, cyclic], + ]), + set: new Set([cyclic]), + }, + }, + ], + }, + orderBy: [ + { + expression: { type: `ref`, path: [`todos`, `title`] }, + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `en`, + localeOptions: { sensitivity: `base` }, + }, + }, + ], + cursor: { + whereFrom: { type: `val`, value: `from` }, + whereCurrent: { type: `val`, value: `current` }, + lastKey: `key`, + }, + limit: 10, + offset: 2, + } + const request: EnsureRemoteSubsetRequest = { + type: `rpc:ensureRemoteSubset:req`, + rpcId: `wire`, + acquisitionId: `wire-acquisition`, + options: supported, + } + + // @ts-expect-error every transported acquisition has a stable identity + const requestWithoutAcquisition: EnsureRemoteSubsetRequest = { + type: `rpc:ensureRemoteSubset:req`, + rpcId: `wire-without-acquisition`, + options: supported, + } + + expectTypeOf(request.options).toEqualTypeOf() + expectTypeOf(request.acquisitionId).toEqualTypeOf() + expectTypeOf< + Extract + >().toHaveProperty(`leaderId`) + expectTypeOf( + requestWithoutAcquisition, + ).toEqualTypeOf() + type RegisteredOwner = Parameters< + NonNullable + >[1] + expectTypeOf() + .parameter(0) + .toEqualTypeOf() + + // @ts-expect-error functions are not remote-subset wire values + const functionValue: RemoteSubsetWireValue = () => {} + // @ts-expect-error symbols are not remote-subset wire values + const symbolValue: RemoteSubsetWireValue = Symbol(`unsupported`) + // @ts-expect-error promises are not remote-subset wire values + const promiseValue: RemoteSubsetWireValue = Promise.resolve() + // @ts-expect-error weak collections are not remote-subset wire values + const weakValue: RemoteSubsetWireValue = new WeakMap() + // @ts-expect-error custom prototypes are not remote-subset wire values + const customValue: RemoteSubsetWireValue = new (class Custom { + value = `custom` + })() + // @ts-expect-error shared buffers are not remote-subset wire values + const sharedValue: RemoteSubsetWireValue = new SharedArrayBuffer(4) + const signalOptions: TransportedLoadSubsetOptions = { + // @ts-expect-error live signals never cross the coordinator wire + signal: new AbortController().signal, + } + const subscriptionOptions: TransportedLoadSubsetOptions = { + // @ts-expect-error live subscriptions never cross the coordinator wire + subscription: {}, + } + + expectTypeOf(functionValue).toEqualTypeOf() + expectTypeOf(symbolValue).toEqualTypeOf() + expectTypeOf(promiseValue).toEqualTypeOf() + expectTypeOf(weakValue).toEqualTypeOf() + expectTypeOf(customValue).toEqualTypeOf() + expectTypeOf(sharedValue).toEqualTypeOf() + expectTypeOf(signalOptions).toEqualTypeOf() + expectTypeOf( + subscriptionOptions, + ).toEqualTypeOf() + }) + + it(`requires complete committed-transaction routing from coordinators`, () => { + // @ts-expect-error requestApplyCommittedTx is required for every coordinator + const incompleteCoordinator: PersistedCollectionCoordinator = { + getNodeId: () => `incomplete`, + subscribe: () => () => {}, + publish: () => {}, + isLeader: () => true, + ensureLeadership: () => Promise.resolve(), + requestEnsureRemoteSubset: () => Promise.resolve(), + requestReleaseRemoteSubset: () => Promise.resolve(), + registerRemoteSubsetOwner: () => () => {}, + requestEnsurePersistedIndex: () => Promise.resolve(), + } + + expectTypeOf( + incompleteCoordinator, + ).toMatchTypeOf() + }) + it(`adds persisted utils in sync-absent mode`, () => { const options = persistedCollectionOptions< Todo, diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 0895e52375..0862daa2ef 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -8,6 +8,7 @@ import { createTransaction, } from '@tanstack/db' import { + IndeterminateCommitError, InvalidPersistedCollectionCoordinatorError, InvalidPersistedStorageKeyEncodingError, InvalidPersistedStorageKeyError, @@ -25,9 +26,10 @@ import type { PersistenceAdapter, ProtocolEnvelope, PullSinceResponse, + RemoteSubsetOwner, TxCommitted, } from '../src' -import type { LoadSubsetOptions, SyncConfig } from '@tanstack/db' +import type { Collection, LoadSubsetOptions, SyncConfig } from '@tanstack/db' type Todo = { id: string @@ -218,6 +220,18 @@ function createCoordinatorHarness(): CoordinatorHarness { ensureLeadership: async () => {}, requestEnsurePersistedIndex: async () => {}, requestEnsureRemoteSubset: async () => {}, + // Remote subset ownership is outside this protocol-delivery harness. + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: () => () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), pullSince: () => { harness.pullSinceCalls++ return Promise.resolve(pullSinceResponse) @@ -259,6 +273,34 @@ async function flushAsyncWork(delayMs: number = 0): Promise { } describe(`persistedCollectionOptions`, () => { + it(`preserves exact reconciliation context for an indeterminate commit`, () => { + const cause = new Error(`response channel closed`) + const error = new IndeterminateCommitError({ + collectionId: `todos`, + requestType: `rpc:applyCommittedTx:req`, + previousLeaderId: `leader-a`, + previousTerm: 4, + currentLeaderId: `leader-b`, + currentTerm: 5, + cause, + }) + + expect(error).toMatchObject({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyCommittedTx:req`, + previousLeaderId: `leader-a`, + previousTerm: 4, + currentLeaderId: `leader-b`, + currentTerm: 5, + cause, + }) + expect(error.message).toContain( + `rpc:applyCommittedTx:req crossed leadership from leader-a (term 4) to leader-b (term 5)`, + ) + }) + it(`provides a sync-absent loopback configuration with persisted utils`, async () => { const adapter = createRecordingAdapter() const collection = createCollection( @@ -701,6 +743,326 @@ describe(`persistedCollectionOptions`, () => { ) }) + it(`coalesces duplicate single-process subset acknowledgements until owner work finishes`, async () => { + const coordinator = new SingleProcessCoordinator(`single-coalesced`) + let releaseLoad = (): void => {} + const loadGate = new Promise((resolve) => { + releaseLoad = resolve + }) + const owner = Object.assign( + vi.fn(() => loadGate), + { unloadSubset: vi.fn(), onError: vi.fn() }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { limit: 1 } + + try { + let firstSettled = false + let duplicateSettled = false + const first = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then(() => { + firstSettled = true + }) + await Promise.resolve() + const duplicate = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then(() => { + duplicateSettled = true + }) + await Promise.resolve() + + expect({ + ownerCalls: owner.mock.calls.length, + firstSettled, + duplicateSettled, + }).toEqual({ + ownerCalls: 1, + firstSettled: false, + duplicateSettled: false, + }) + + releaseLoad() + await Promise.all([first, duplicate]) + } finally { + releaseLoad() + unregisterOwner() + } + }) + + it(`coalesces same-stack single-process subset reentry until owner work finishes`, async () => { + const coordinator = new SingleProcessCoordinator(`single-reentrant`) + let releaseLoad = (): void => {} + const loadGate = new Promise((resolve) => { + releaseLoad = resolve + }) + const options: LoadSubsetOptions = { limit: 1 } + let duplicate: Promise | undefined + let didReenter = false + const owner = Object.assign( + vi.fn(() => { + if (!didReenter) { + didReenter = true + duplicate = coordinator.requestEnsureRemoteSubset(`todos`, options) + } + return loadGate + }), + { unloadSubset: vi.fn(), onError: vi.fn() }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + + try { + let firstSettled = false + let duplicateSettled = false + const first = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then(() => { + firstSettled = true + }) + await Promise.resolve() + const duplicateResult = duplicate!.then(() => { + duplicateSettled = true + }) + await Promise.resolve() + + expect({ + ownerCalls: owner.mock.calls.length, + firstSettled, + duplicateSettled, + }).toEqual({ + ownerCalls: 1, + firstSettled: false, + duplicateSettled: false, + }) + + releaseLoad() + await Promise.all([first, duplicateResult]) + } finally { + releaseLoad() + unregisterOwner() + } + }) + + it(`waits for a pending single-process subset load before unloading it`, async () => { + const coordinator = new SingleProcessCoordinator(`single-release-order`) + let releaseLoad = (): void => {} + const loadGate = new Promise((resolve) => { + releaseLoad = resolve + }) + const events: Array = [] + const owner = Object.assign( + vi.fn(async () => { + events.push(`load-start`) + await loadGate + events.push(`load-finish`) + }), + { + unloadSubset: vi.fn(() => { + events.push(`unload`) + }), + onError: vi.fn(), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { limit: 1 } + + try { + const load = coordinator.requestEnsureRemoteSubset(`todos`, options) + await Promise.resolve() + let releaseSettled = false + const release = coordinator + .requestReleaseRemoteSubset(`todos`, options) + .then(() => { + releaseSettled = true + }) + await Promise.resolve() + + const beforeLoadRelease = { events: [...events], releaseSettled } + expect(beforeLoadRelease).toEqual({ + events: [`load-start`], + releaseSettled: false, + }) + + releaseLoad() + await Promise.all([load, release]) + expect(events).toEqual([`load-start`, `load-finish`, `unload`]) + } finally { + releaseLoad() + unregisterOwner() + } + }) + + it(`rejects unsupported single-process subset values before owner work`, async () => { + const coordinator = new SingleProcessCoordinator(`single-wire-domain`) + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + + try { + await expect( + coordinator.requestEnsureRemoteSubset(`todos`, { + where: new IR.Func(`in`, [ + new IR.PropRef([`todos`, `status`]), + new IR.Value([`kept`, () => {}]), + ]), + }), + ).rejects.toMatchObject({ + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1].value[1]`, + }) + expect(owner).not.toHaveBeenCalled() + } finally { + unregisterOwner() + } + }) + + it(`rejects a sparse function argument at its exact expression path`, async () => { + const coordinator = new SingleProcessCoordinator(`single-wire-hole`) + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const args = [new IR.PropRef([`todos`, `status`])] + args.length = 2 + + try { + await expect( + coordinator.requestEnsureRemoteSubset(`todos`, { + where: new IR.Func(`eq`, args), + }), + ).rejects.toMatchObject({ + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1]`, + }) + expect(owner).not.toHaveBeenCalled() + } finally { + unregisterOwner() + } + }) + + it(`reports and rethrows a single-process owner unload rejection`, async () => { + const coordinator = new SingleProcessCoordinator(`single-unload-error`) + const unloadError = new Error(`single-process owner unload failed`) + const ownerErrors: Array = [] + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(() => + Promise.reject(unloadError), + ) as unknown as RemoteSubsetOwner[`unloadSubset`], + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { limit: 1 } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, options) + await expect( + coordinator.requestReleaseRemoteSubset(`todos`, options), + ).rejects.toBe(unloadError) + expect(ownerErrors).toEqual([unloadError]) + } finally { + unregisterOwner() + } + }) + + it(`reports one fail-stop lifecycle error when a single-process async unload rejects`, async () => { + const unloadError = new Error(`remote owner unload failed`) + let unloadThenCalls = 0 + const rejectingThenable = { + then: ( + _resolve: (value?: void) => void, + reject: (error: unknown) => void, + ) => { + unloadThenCalls++ + queueMicrotask(() => reject(unloadError)) + }, + } + const sourceUnloadMock = vi.fn((options: LoadSubsetOptions) => + options.limit === 1 ? rejectingThenable : undefined, + ) + const sourceUnload = sourceUnloadMock as unknown as ( + options: LoadSubsetOptions, + ) => void + const collection = createCollection( + persistedCollectionOptions({ + id: `owner-unload-error`, + getKey: (todo) => todo.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => true, + unloadSubset: sourceUnload, + } + }, + }, + persistence: { + adapter: createRecordingAdapter(), + }, + }), + ) + const failing: LoadSubsetOptions = { limit: 1 } + const sibling: LoadSubsetOptions = { limit: 2 } + const markError = vi.spyOn(collection._lifecycle, `markError`) + const unhandled: Array = [] + const captureUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, captureUnhandled) + + try { + collection.startSyncImmediate() + await collection._sync.loadSubset(failing) + await collection._sync.loadSubset(sibling) + collection._sync.unloadSubset(failing) + collection._sync.unloadSubset(sibling) + await flushAsyncWork() + await flushAsyncWork() + + expect({ + status: collection.status, + error: collection._lifecycle.getSyncError(), + unloadThenCalls, + unloadOptions: sourceUnloadMock.mock.calls.map(([options]) => options), + lifecycleErrors: markError.mock.calls.map(([error]) => error), + unhandled, + }).toEqual({ + status: `error`, + error: unloadError, + unloadThenCalls: 1, + unloadOptions: [failing, sibling], + lifecycleErrors: [unloadError], + unhandled: [], + }) + + await flushAsyncWork() + expect(sourceUnloadMock).toHaveBeenCalledTimes(2) + expect(markError).toHaveBeenCalledTimes(1) + } finally { + process.off(`unhandledRejection`, captureUnhandled) + await collection.cleanup() + } + }) + it(`resolves persistence per collection and forwards schemaVersion`, () => { const baseAdapter = createNoopAdapter() const syncAdapter = createNoopAdapter() @@ -778,6 +1140,114 @@ describe(`persistedCollectionOptions`, () => { ).toThrow(InvalidPersistedCollectionCoordinatorError) }) + it(`rejects an incomplete coordinator before starting a rich sync source`, async () => { + type SourceParams = Parameters[`sync`]>[0] + + const adapter = createRecordingAdapter() + const applyCommittedTx = adapter.applyCommittedTx + let adapterApplies = 0 + adapter.applyCommittedTx = (...args) => { + adapterApplies++ + return applyCommittedTx(...args) + } + let sourceCalls = 0 + let legacyCalls = 0 + let sourceParams: SourceParams | undefined + let collectionConstructed = false + let visible = false + let configurationError: { name: string; message: string } | null = null + let collection: Collection | undefined + + const incompleteCoordinator = { + getNodeId: () => `incomplete-rich-sync`, + subscribe: () => () => {}, + publish: () => {}, + isLeader: () => false, + ensureLeadership: () => Promise.resolve(), + requestEnsurePersistedIndex: () => Promise.resolve(), + // Remote subset routing is complete so this fixture isolates the + // required committed-transaction method named by the assertion. + requestEnsureRemoteSubset: () => Promise.resolve(), + requestReleaseRemoteSubset: () => Promise.resolve(), + registerRemoteSubsetOwner: () => () => {}, + requestApplyLocalMutations: () => { + legacyCalls++ + return Promise.resolve({ + type: `rpc:applyLocalMutations:res` as const, + rpcId: `legacy`, + ok: true as const, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: [], + }) + }, + } as unknown as PersistedCollectionCoordinator + + try { + try { + const options = persistedCollectionOptions({ + id: `incomplete-rich-sync`, + getKey: (todo) => todo.id, + sync: { + sync: (params) => { + sourceCalls++ + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter, coordinator: incompleteCoordinator }, + }) + collection = createCollection(options) + collectionConstructed = true + } catch (error) { + const resolvedError = + error instanceof Error ? error : new Error(String(error)) + configurationError = { + name: resolvedError.name, + message: resolvedError.message, + } + } + + if (collection) { + collection.startSyncImmediate() + await vi.waitFor(() => expect(sourceParams).toBeDefined()) + sourceParams!.begin() + sourceParams!.metadata?.collection.set(`resume`, { offset: 7 }) + sourceParams!.truncate() + sourceParams!.write({ + type: `insert`, + value: { id: `rich`, title: `Visible before rejection` }, + metadata: { source: `remote` }, + }) + await Promise.resolve(sourceParams!.commit()).catch(() => undefined) + visible = collection.has(`rich`) + } + } finally { + await collection?.cleanup() + } + + expect({ + configurationError, + sourceCalls, + collectionConstructed, + visible, + adapterApplies, + legacyCalls, + }).toEqual({ + configurationError: { + name: `InvalidPersistedCollectionCoordinatorError`, + message: + 'Invalid persisted collection coordinator: missing required "requestApplyCommittedTx" method', + }, + sourceCalls: 0, + collectionConstructed: false, + visible: false, + adapterApplies: 0, + legacyCalls: 0, + }) + }) + it(`preserves valid sync config in sync-present mode`, async () => { const adapter = createRecordingAdapter() const sync: SyncConfig = { @@ -943,10 +1413,17 @@ describe(`persistedCollectionOptions`, () => { } }) - it(`rejects a wrapped sync receipt when persistence fails`, async () => { + it(`publishes before persistence settles, then fails with the named durability error`, async () => { const adapter = createRecordingAdapter() - const persistenceError = new Error(`persistence failed`) - adapter.applyCommittedTx = () => Promise.reject(persistenceError) + const persistenceError = Object.assign(new Error(`persistence failed`), { + code: `SQLITE_IOERR_WRITE`, + path: `todos.sqlite-wal`, + }) + let rejectPersistence: ((error: unknown) => void) | undefined + adapter.applyCommittedTx = () => + new Promise((_resolve, reject) => { + rejectPersistence = reject + }) let remoteBegin: (() => void) | undefined let remoteWrite: | ((message: { type: `insert`; value: Todo }) => void) @@ -978,15 +1455,209 @@ describe(`persistedCollectionOptions`, () => { type: `insert`, value: { id: `failed`, title: `Not durable` }, }) + const receipt = Promise.resolve(remoteCommit?.()) + let settled = false + void receipt.then( + () => { + settled = true + }, + () => { + settled = true + }, + ) - await expect(Promise.resolve(remoteCommit?.())).rejects.toBe( - persistenceError, + expect(stripVirtualProps(collection.get(`failed`))).toEqual({ + id: `failed`, + title: `Not durable`, + }) + expect(settled).toBe(false) + await vi.waitFor(() => expect(rejectPersistence).toBeTypeOf(`function`)) + + rejectPersistence!(persistenceError) + const rejection = await receipt.then( + () => undefined, + (error: unknown) => error, + ) + + expect(rejection).not.toBe(persistenceError) + expect(rejection).toMatchObject({ + name: `PersistedCollectionDurabilityError`, + code: `SQLITE_IOERR_WRITE`, + path: `todos.sqlite-wal`, + cause: persistenceError, + }) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(rejection) + } finally { + rejectPersistence?.(persistenceError) + await collection.cleanup() + } + }) + + it(`preserves a coordinator persistence classification without converting it to conflict`, async () => { + const coordinator = createCoordinatorHarness() + const persistenceResponse = { + type: `rpc:applyCommittedTx:res` as const, + rpcId: `persistence-response`, + ok: false as const, + code: `PERSISTENCE_ERROR` as const, + error: `disk write failed`, + sourceCode: `SQLITE_IOERR_FSYNC`, + path: [`database`, `wal`] as const, + } + coordinator.requestApplyCommittedTx = () => + Promise.resolve(persistenceResponse) + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `coordinator-persistence-error`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter: createRecordingAdapter(), coordinator }, + }), + ) + + try { + await collection.stateWhenReady() + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `remote-failure`, title: `Published once` }, + }) + const rejection = await Promise.resolve(remoteCommit?.()).then( + () => undefined, + (error: unknown) => error, ) + + expect(rejection).toMatchObject({ + name: `PersistedCollectionDurabilityError`, + code: `SQLITE_IOERR_FSYNC`, + path: [`database`, `wal`], + cause: persistenceResponse, + }) + expect(String(rejection)).not.toContain(`CONFLICT`) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(rejection) } finally { await collection.cleanup() } }) + it(`replays a hydration-buffered write before its named durability rejection`, async () => { + const adapter = createRecordingAdapter() + let releaseHydration: (() => void) | undefined + adapter.loadSubset = () => + new Promise>((resolve) => { + releaseHydration = () => resolve([]) + }) + const persistenceError = Object.assign( + new Error(`buffered persistence failed`), + { + code: `SQLITE_FULL`, + path: `todos.sqlite`, + }, + ) + let rejectPersistence: ((error: unknown) => void) | undefined + adapter.applyCommittedTx = () => + new Promise((_resolve, reject) => { + rejectPersistence = reject + }) + + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + const collection = createCollection( + persistedCollectionOptions({ + id: `hydration-buffered-persistence-error`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + + try { + void collection.stateWhenReady().catch(() => undefined) + await vi.waitFor(() => { + expect(releaseHydration).toBeTypeOf(`function`) + expect(remoteCommit).toBeTypeOf(`function`) + }) + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `buffered`, title: `Published before failure` }, + }) + const receipt = Promise.resolve(remoteCommit?.()) + let settled = false + void receipt.then( + () => { + settled = true + }, + () => { + settled = true + }, + ) + + expect(collection.get(`buffered`)).toBeUndefined() + expect(settled).toBe(false) + releaseHydration!() + await vi.waitFor(() => expect(rejectPersistence).toBeTypeOf(`function`)) + + expect(stripVirtualProps(collection.get(`buffered`))).toEqual({ + id: `buffered`, + title: `Published before failure`, + }) + expect(settled).toBe(false) + + rejectPersistence!(persistenceError) + const rejection = await receipt.then( + () => undefined, + (error: unknown) => error, + ) + + expect(rejection).not.toBe(persistenceError) + expect(rejection).toMatchObject({ + name: `PersistedCollectionDurabilityError`, + code: `SQLITE_FULL`, + path: `todos.sqlite`, + cause: persistenceError, + }) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(rejection) + } finally { + releaseHydration?.() + rejectPersistence?.(persistenceError) + await collection.cleanup() + warning.mockRestore() + } + }) + it(`preserves row metadata set before a metadata-less insert in the same sync transaction`, async () => { const adapter = createRecordingAdapter() const ownership = { queryCollection: { owners: [`gc:q1`] } } @@ -2181,6 +2852,107 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`hydrates a multiprocess follower locally while only the elected owner acquires remote demand`, async () => { + const adapter = createRecordingAdapter([ + { id: `persisted`, title: `Persisted follower row` }, + ]) + let releaseRemoteLoad = (): void => {} + const remoteLoadGate = new Promise((resolve) => { + releaseRemoteLoad = resolve + }) + let registeredOwner: RemoteSubsetOwner | undefined + const ensureRemote = vi.fn(() => remoteLoadGate) + const coordinator: PersistedCollectionCoordinator = { + getNodeId: () => `follower-node`, + subscribe: () => () => {}, + publish: () => {}, + isLeader: () => false, + ensureLeadership: async () => {}, + requestEnsurePersistedIndex: async () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), + requestEnsureRemoteSubset: ensureRemote, + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: (_collectionId, owner) => { + registeredOwner = owner + return () => { + if (registeredOwner === owner) registeredOwner = undefined + } + }, + } + const followerUpstreamLoad = vi.fn(() => Promise.resolve()) + const collection = createCollection( + persistedCollectionOptions({ + id: `multiprocess-follower-subset`, + getKey: (todo) => todo.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: followerUpstreamLoad, + unloadSubset: vi.fn(), + } + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const options: LoadSubsetOptions = { limit: 1 } + + try { + collection.startSyncImmediate() + let settled = false + const load = Promise.resolve(collection._sync.loadSubset(options)).then( + () => { + settled = true + }, + ) + + await vi.waitFor(() => { + expect(adapter.loadSubsetCalls.length).toBeGreaterThan(0) + expect(ensureRemote).toHaveBeenCalledWith( + `multiprocess-follower-subset`, + options, + ) + }) + const persistedRow = collection.get(`persisted`) + expect({ + persistedRow: persistedRow + ? { id: persistedRow.id, title: persistedRow.title } + : undefined, + followerUpstreamCalls: followerUpstreamLoad.mock.calls.length, + registeredOwner: registeredOwner !== undefined, + settled, + }).toEqual({ + persistedRow: { id: `persisted`, title: `Persisted follower row` }, + followerUpstreamCalls: 0, + registeredOwner: true, + settled: false, + }) + + releaseRemoteLoad() + await load + expect({ + routedEnsures: ensureRemote.mock.calls.length, + followerUpstreamCalls: followerUpstreamLoad.mock.calls.length, + }).toEqual({ + routedEnsures: 1, + followerUpstreamCalls: 0, + }) + } finally { + releaseRemoteLoad() + await collection.cleanup() + } + }) + it.each([`abort`, `release`, `offline`] as const)( `handles remote ensure after %s without resurrecting cancelled demand`, async (action) => { @@ -2190,7 +2962,10 @@ describe(`persistedCollectionOptions`, () => { name: action === `abort` ? `AbortError` : `Error`, }) const ensure = vi.fn(async () => { - throw new Error(`offline`) + throw failure + }) + const followerUpstreamLoad = vi.fn(async () => { + throw new Error(`follower-local source must not own remote demand`) }) const coordinator: PersistedCollectionCoordinator = { getNodeId: () => `cancel-ensure`, @@ -2199,7 +2974,19 @@ describe(`persistedCollectionOptions`, () => { isLeader: () => true, ensureLeadership: async () => {}, requestEnsurePersistedIndex: async () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), requestEnsureRemoteSubset: ensure, + // This fixture isolates ensure retry/cancellation, not ownership. + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: () => () => {}, } const collection = createCollection( persistedCollectionOptions({ @@ -2210,9 +2997,7 @@ describe(`persistedCollectionOptions`, () => { sync: ({ markReady }) => { markReady() return { - loadSubset: async () => { - throw failure - }, + loadSubset: followerUpstreamLoad, } }, }, @@ -2231,11 +3016,11 @@ describe(`persistedCollectionOptions`, () => { if (action === `release`) collection._sync.unloadSubset(options) const callsBeforeRetry = ensure.mock.calls.length await vi.advanceTimersByTimeAsync(200) + expect(result).toBe(failure) + expect(followerUpstreamLoad).not.toHaveBeenCalled() if (action === `offline`) { - expect(result).toBe(failure) expect(ensure.mock.calls.length).toBeGreaterThan(callsBeforeRetry) } else { - if (action === `abort`) expect(result).toBe(failure) expect(ensure).toHaveBeenCalledTimes(callsBeforeRetry) } } finally { @@ -2249,6 +3034,7 @@ describe(`persistedCollectionOptions`, () => { it(`retries queued remote subset ensure after transient failures`, async () => { const adapter = createRecordingAdapter() let ensureCalls = 0 + const offline = new Error(`offline`) const coordinator: PersistedCollectionCoordinator = { getNodeId: () => `retry-node`, @@ -2257,12 +3043,24 @@ describe(`persistedCollectionOptions`, () => { isLeader: () => true, ensureLeadership: async () => {}, requestEnsurePersistedIndex: async () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), requestEnsureRemoteSubset: async () => { ensureCalls++ if (ensureCalls === 1) { - throw new Error(`offline`) + throw offline } }, + // This fixture isolates ensure retry behavior, not ownership. + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: () => () => {}, } const collection = createCollection( @@ -2286,9 +3084,15 @@ describe(`persistedCollectionOptions`, () => { collection.startSyncImmediate() await flushAsyncWork() - await (collection as any)._sync.loadSubset({ limit: 1 }) + const firstResult = await Promise.resolve( + (collection as any)._sync.loadSubset({ limit: 1 }), + ).then( + () => `ready`, + (error: unknown) => error, + ) await flushAsyncWork(120) + expect(firstResult).toBe(offline) expect(ensureCalls).toBeGreaterThanOrEqual(2) }) @@ -2301,6 +3105,19 @@ describe(`persistedCollectionOptions`, () => { isLeader: () => false, ensureLeadership: async () => {}, requestEnsurePersistedIndex: async () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), + // This fixture isolates follower transaction acknowledgements. + requestEnsureRemoteSubset: async () => {}, + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: () => () => {}, requestApplyLocalMutations: async () => ({ type: `rpc:applyLocalMutations:res`, rpcId: `ack-1`, diff --git a/packages/db/skills/db-core/persistence/SKILL.md b/packages/db/skills/db-core/persistence/SKILL.md index 8a5f657af0..66330ffcc5 100644 --- a/packages/db/skills/db-core/persistence/SKILL.md +++ b/packages/db/skills/db-core/persistence/SKILL.md @@ -110,11 +110,24 @@ This works with any adapter: `electricCollectionOptions`, `queryCollectionOption Coordinators handle leader election and cross-instance communication so only one tab/process owns the database writer. -| Platform | Coordinator | Mechanism | -| ------------------------------------- | ------------------------------- | ---------------------------- | -| Browser | `BrowserCollectionCoordinator` | BroadcastChannel + Web Locks | -| Electron | `ElectronCollectionCoordinator` | BroadcastChannel + Web Locks | -| Single-process (RN, Expo, Node, etc.) | `SingleProcessCoordinator` | No-op (always leader) | +| Platform | Coordinator | Mechanism | +| ------------------------------------- | ------------------------------- | ---------------------------------------------------- | +| Browser | `BrowserCollectionCoordinator` | BroadcastChannel + Web Locks | +| Electron | `ElectronCollectionCoordinator` | BroadcastChannel + Web Locks | +| Single-process (RN, Expo, Node, etc.) | `SingleProcessCoordinator` | Always leader; direct per-collection adapter routing | + +Every coordinator must implement +`requestApplyCommittedTx(collectionId, tx)`. The method routes a complete +committed transaction to the collection's supported writer, including +truncate, rows, row metadata, collection metadata, and stream position. This +is a required contract, not an optional capability. Do not feature-detect it, +fall back to row-only mutation routing, or bypass the coordinator after source +publication. Untyped custom coordinators that omit the method fail during +collection configuration. + +`SingleProcessCoordinator` has no election or cross-process communication, but +it is not a no-op persistence owner. It registers the resolved adapter for each +collection and applies complete committed transactions through that adapter. Browser persistence uses single-process semantics by default. That is correct when the app runs in one tab at a time or each tab has its own database. Pass a @@ -163,7 +176,9 @@ const persistence = createElectronSQLitePersistence({ Electron persistence calls cross the renderer/main boundary through IPC. The `ElectronCollectionCoordinator` separately coordinates renderer instances with -`BroadcastChannel` and Web Locks. +`BroadcastChannel` and Web Locks. It elects an owner per collection and routes +the complete committed transaction through that collection's resolved renderer +adapter to the main-process persistence owner. ## Schema Versioning diff --git a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts index 00b3fc09b4..eed134f1f0 100644 --- a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts +++ b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts @@ -171,6 +171,7 @@ function fixture( } return { collection, + adapter, rows, metadata, commits, @@ -239,6 +240,7 @@ describe(`persisted Electric recovery laws`, () => { function externalPublisher() { let receive: ((message: ProtocolEnvelope) => void) | undefined + let ownerAdapter: PersistenceAdapter | undefined let id = `` let term = 100 const coordinator: PersistedCollectionCoordinator = { @@ -255,9 +257,29 @@ describe(`persisted Electric recovery laws`, () => { ensureLeadership: () => Promise.resolve(), requestEnsurePersistedIndex: () => Promise.resolve(), requestEnsureRemoteSubset: () => Promise.resolve(), + // This fixture owns external publication/recovery, not subset leases. + requestReleaseRemoteSubset: () => Promise.resolve(), + registerRemoteSubsetOwner: () => () => {}, + requestApplyCommittedTx: async (collectionId, tx) => { + if (!ownerAdapter) { + throw new Error(`external publisher has no persistence owner`) + } + await ownerAdapter.applyCommittedTx(collectionId, tx) + return { + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + } + }, } return { coordinator, + bindAdapter: (adapter: PersistenceAdapter) => { + ownerAdapter = adapter + }, publish: ( row: Item, deleted: boolean, @@ -300,6 +322,7 @@ describe(`persisted Electric recovery laws`, () => { async ({ syncMode, fullReload }) => { const peer = externalPublisher() const f = fixture(syncMode, peer.coordinator) + peer.bindAdapter(f.adapter) try { f.start() await vi.waitFor(() => expect(subscribers).toHaveLength(1)) @@ -369,6 +392,7 @@ describe(`persisted Electric recovery laws`, () => { subscribers.length = 0 const peer = externalPublisher() const f = fixture(`on-demand`, peer.coordinator) + peer.bindAdapter(f.adapter) const expected = new Map([[oldRow.id, structuredClone(oldRow)]]) const expectedRows = () => structuredClone([...expected.values()].sort((a, b) => a.id - b.id)) diff --git a/packages/electron-db-sqlite-persistence/README.md b/packages/electron-db-sqlite-persistence/README.md index 8f288dab3b..c53dd7493e 100644 --- a/packages/electron-db-sqlite-persistence/README.md +++ b/packages/electron-db-sqlite-persistence/README.md @@ -6,6 +6,7 @@ Thin Electron bridge for TanStack DB SQLite persistence. - `exposeElectronSQLitePersistence(...)` (main process) - `createElectronSQLitePersistence(...)` (renderer process) +- `ElectronCollectionCoordinator` - `persistedCollectionOptions(...)` (re-exported from core) Use `@tanstack/electron-db-sqlite-persistence/main` and @@ -64,8 +65,72 @@ export const todosCollection = createCollection( ) ``` +## Multi-renderer ownership + +Pass an `ElectronCollectionCoordinator` when multiple renderers share the same +database. Coordinators using the same `dbName` elect a writer independently for +each collection. + +```ts +import { + ElectronCollectionCoordinator, + createElectronSQLitePersistence, +} from '@tanstack/electron-db-sqlite-persistence' + +const coordinator = new ElectronCollectionCoordinator({ + dbName: `tanstack-db`, +}) + +const persistence = createElectronSQLitePersistence({ + ipcRenderer, + coordinator, +}) + +// On teardown: +// coordinator.dispose() +``` + +The persisted sync wrapper routes every source transaction with durable effects +through the coordinator's required +`requestApplyCommittedTx(collectionId, tx)` method. The elected renderer applies +the complete `PersistedTx` through its resolved per-collection renderer adapter, +which forwards it over IPC to the main-process persistence owner. Resolution +remains specific to the collection id, mode, and `schemaVersion`. + +Truncation, row changes, row metadata, collection metadata, and stream position +stay in the same transaction. Electron does not feature-detect a partial route +or fall back to row-only mutation RPC. A custom coordinator that omits +`requestApplyCommittedTx` is rejected while the collection is configured, +before its sync source can publish rows. + +If a mutating RPC loses its response, Electron coordination replays it only +while the requester still knows the same non-null leader id and term. An +unknown initial route or any leader/term change rejects with +`IndeterminateCommitError`; the application must reconcile the outcome. The +coordinator does not retry that mutation against an unknown or replacement +leader. + +Remote-subset requests use the same crash-only boundary. The coordinator +projects a live `LoadSubsetOptions` input to the exported +`TransportedLoadSubsetOptions` type before either local completion or renderer +transport. Live `signal` and `subscription` fields do not cross the boundary. +Unsupported nested values throw `RemoteSubsetWireValueError` with the exact +value path before publication; Electron does not add codecs, coercion, or a +fallback representation. + +Electron uses the same explicit lease lifecycle as Browser coordination. The +same request object reuses one acquisition identity, distinct equal objects +remain independent, and release unloads the exact transported options once. +Leadership loss unloads the retiring renderer's leases; still-live requester +leases replay against the next leader. A second owner registration for one +collection throws `DuplicateRemoteSubsetOwnerError`, and there is no adapter +fallback for a missing owner. + ## Notes - The renderer API mirrors other runtimes: one shared `create...Persistence`. - Collection mode (`sync-present` vs `sync-absent`) and `schemaVersion` are resolved per collection and forwarded across IPC automatically. +- Without an `ElectronCollectionCoordinator`, single-renderer mode uses + `SingleProcessCoordinator`: it is always leader, but still routes complete + transactions to the collection's resolved IPC adapter. diff --git a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts index a4c6bb7fe8..1c3ea73f0a 100644 --- a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts +++ b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts @@ -1,12 +1,31 @@ -import { safeRandomUUID } from '@tanstack/db-sqlite-persistence-core' +import { + DuplicateRemoteSubsetOwnerError, + IndeterminateCommitError, + PersistedCollectionDurabilityError, + safeRandomUUID, + toPersistedCollectionDurabilityError, + toTransportedLoadSubsetOptions, +} from '@tanstack/db-sqlite-persistence-core' import type { + ApplyCommittedTxRequest, + ApplyCommittedTxResponse, ApplyLocalMutationsResponse, + EnsureRemoteSubsetRequest, + EnsureRemoteSubsetResponse, + IndeterminateCommitRequestType, PersistedCollectionCoordinator, PersistedIndexSpec, PersistedMutationEnvelope, + PersistedRowMetadataMutation, + PersistedTx, PersistenceAdapter, ProtocolEnvelope, PullSinceResponse, + ReleaseRemoteSubsetRequest, + ReleaseRemoteSubsetResponse, + RemoteSubsetOwner, + TransportedLoadSubsetOptions, + TxCommitted, } from '@tanstack/db-sqlite-persistence-core' import type { LoadSubsetOptions } from '@tanstack/db' @@ -26,11 +45,8 @@ const WRITER_LOCK_MAX_RETRIES = 20 // --------------------------------------------------------------------------- type RPCRequest = - | { - type: `rpc:ensureRemoteSubset:req` - rpcId: string - options: LoadSubsetOptions - } + | EnsureRemoteSubsetRequest + | ReleaseRemoteSubsetRequest | { type: `rpc:ensurePersistedIndex:req` rpcId: string @@ -43,6 +59,7 @@ type RPCRequest = envelopeId: string mutations: Array } + | ApplyCommittedTxRequest | { type: `rpc:pullSince:req` rpcId: string @@ -50,12 +67,8 @@ type RPCRequest = } type RPCResponse = - | { - type: `rpc:ensureRemoteSubset:res` - rpcId: string - ok: boolean - error?: string - } + | EnsureRemoteSubsetResponse + | ReleaseRemoteSubsetResponse | { type: `rpc:ensurePersistedIndex:res` rpcId: string @@ -63,6 +76,7 @@ type RPCResponse = error?: string } | ApplyLocalMutationsResponse + | ApplyCommittedTxResponse | PullSinceResponse type PendingRPC = { @@ -73,6 +87,7 @@ type PendingRPC = { type CollectionState = { isLeader: boolean + leaderId: string | null lockAbortController: AbortController | null heartbeatTimer: ReturnType | null latestTerm: number @@ -81,6 +96,47 @@ type CollectionState = { subscribers: Set<(message: ProtocolEnvelope) => void> } +type ActiveRemoteSubsetAcquisition = { + collectionId: string + requesterId: string + acquisitionId: string + owner: RemoteSubsetOwner + options: TransportedLoadSubsetOptions + load: Promise + transferred: boolean + released: boolean + terminalRelease: boolean + release: Promise | null +} + +type AwaitingRemoteSubsetOwnerAcquisition = { + collectionId: string + requesterId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + released: true + awaitingOwner: true +} + +type RemoteSubsetAcquisition = + | ActiveRemoteSubsetAcquisition + | AwaitingRemoteSubsetOwnerAcquisition + | { + collectionId: string + requesterId: string + acquisitionId: string + released: true + } + +type OutboundRemoteSubsetAcquisition = { + collectionId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + acquiredLeaderId: string | null + inFlight: Promise | null + forceReplay: boolean +} + // Adapter with pullSince support type AdapterWithPullSince = PersistenceAdapter & { pullSince?: ( @@ -121,11 +177,40 @@ export type ElectronCollectionCoordinatorOptions = { export class ElectronCollectionCoordinator implements PersistedCollectionCoordinator { private readonly nodeId = safeRandomUUID() private readonly dbName: string - private adapter: AdapterWithPullSince | null + private defaultAdapter: AdapterWithPullSince | null + private readonly collectionAdapters = new Map() + private readonly remoteSubsetOwners = new Map() + private readonly remoteSubsetIds = new Map< + string, + WeakMap + >() + private readonly outboundRemoteSubsetAcquisitions = new Map< + string, + OutboundRemoteSubsetAcquisition + >() + private readonly inboundRemoteSubsetAcquisitions = new Map< + string, + RemoteSubsetAcquisition + >() private readonly channel: BroadcastChannel private readonly collections = new Map() private readonly pendingRPCs = new Map() - private readonly appliedEnvelopeIds = new Map() + private readonly appliedEnvelopeIds = new Map< + string, + { appliedAt: number; response: ApplyLocalMutationsResponse } + >() + private readonly inFlightLocalMutationEnvelopes = new Map< + string, + Promise + >() + private readonly appliedCommittedTxEnvelopes = new Map< + string, + { appliedAt: number; response: ApplyCommittedTxResponse } + >() + private readonly inFlightCommittedTxEnvelopes = new Map< + string, + Promise + >() private disposed = false /** Method indirection to prevent TypeScript from narrowing `disposed` across awaits */ @@ -133,18 +218,20 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin return this.disposed } - private requireAdapter(): AdapterWithPullSince { - if (!this.adapter) { + private requireAdapter(collectionId: string): AdapterWithPullSince { + const adapter = + this.collectionAdapters.get(collectionId) ?? this.defaultAdapter + if (!adapter) { throw new Error( - `ElectronCollectionCoordinator: adapter not set. Call setAdapter() before using leader-side operations.`, + `ElectronCollectionCoordinator: adapter not set for collection "${collectionId}". Call setAdapterForCollection() before using leader-side operations.`, ) } - return this.adapter + return adapter } constructor(options: ElectronCollectionCoordinatorOptions) { this.dbName = options.dbName - this.adapter = options.adapter ?? null + this.defaultAdapter = options.adapter ?? null this.channel = new BroadcastChannel(`tsdb:coord:${this.dbName}`) this.channel.onmessage = (event: MessageEvent) => { this.onChannelMessage(event.data) @@ -157,7 +244,36 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin * adapter into the coordinator. */ setAdapter(adapter: AdapterWithPullSince): void { - this.adapter = adapter + this.defaultAdapter = adapter + } + + setAdapterForCollection( + collectionId: string, + adapter: AdapterWithPullSince, + ): void { + this.collectionAdapters.set(collectionId, adapter) + } + + registerRemoteSubsetOwner( + collectionId: string, + owner: RemoteSubsetOwner, + ): () => void { + if (this.remoteSubsetOwners.has(collectionId)) { + throw new DuplicateRemoteSubsetOwnerError(collectionId) + } + this.remoteSubsetOwners.set(collectionId, owner) + for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + if (acquisition.collectionId !== collectionId) continue + acquisition.acquiredLeaderId = null + acquisition.forceReplay = true + } + void this.replayRemoteSubsetAcquisitions(collectionId) + this.rebindRemoteInboundSubsetAcquisitions(collectionId, owner) + return () => { + if (this.remoteSubsetOwners.get(collectionId) !== owner) return + this.remoteSubsetOwners.delete(collectionId) + this.releaseInboundRemoteSubsetAcquisitions(collectionId, owner) + } } // ----------------------------------------------------------------------- @@ -197,23 +313,138 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin collectionId: string, options: LoadSubsetOptions, ): Promise { - if (this.isLeader(collectionId)) return + const transportedOptions = toTransportedLoadSubsetOptions(options) + let collectionIds = this.remoteSubsetIds.get(collectionId) + if (!collectionIds) { + collectionIds = new WeakMap() + this.remoteSubsetIds.set(collectionId, collectionIds) + } + let acquisitionId = collectionIds.get(options) + let acquisition = acquisitionId + ? this.outboundRemoteSubsetAcquisitions.get( + remoteSubsetAcquisitionKey(collectionId, acquisitionId), + ) + : undefined + if (!acquisition) { + acquisitionId = safeRandomUUID() + collectionIds.set(options, acquisitionId) + acquisition = { + collectionId, + acquisitionId, + options: transportedOptions, + acquiredLeaderId: null, + inFlight: null, + forceReplay: false, + } + this.outboundRemoteSubsetAcquisitions.set( + remoteSubsetAcquisitionKey(collectionId, acquisitionId), + acquisition, + ) + } - const response = await this.sendRPC<{ - type: `rpc:ensureRemoteSubset:res` - rpcId: string - ok: boolean - error?: string - }>(collectionId, { - type: `rpc:ensureRemoteSubset:req`, + await this.acquireRemoteSubset(acquisition) + } + + async requestReleaseRemoteSubset( + collectionId: string, + options: LoadSubsetOptions, + ): Promise { + const collectionIds = this.remoteSubsetIds.get(collectionId) + const acquisitionId = collectionIds?.get(options) + if (!acquisitionId) return + const key = remoteSubsetAcquisitionKey(collectionId, acquisitionId) + if (!this.outboundRemoteSubsetAcquisitions.delete(key)) return + collectionIds!.delete(options) + + const request: Extract< + RPCRequest, + { type: `rpc:releaseRemoteSubset:req` } + > = { + type: `rpc:releaseRemoteSubset:req`, rpcId: safeRandomUUID(), - options, - }) + acquisitionId, + } + const response = this.isLeader(collectionId) + ? await this.handleReleaseRemoteSubset(collectionId, request, this.nodeId) + : await this.sendRPC(collectionId, request) if (!response.ok) { - throw new Error( - `ensureRemoteSubset failed: ${response.error ?? `unknown error`}`, + throw new Error(`releaseRemoteSubset failed: ${response.error}`) + } + } + + private async acquireRemoteSubset( + acquisition: OutboundRemoteSubsetAcquisition, + ): Promise { + if (acquisition.inFlight) return acquisition.inFlight + + const routedToLocalOwner = this.isLeader(acquisition.collectionId) + let resolveWork!: () => void + let rejectWork!: (error: unknown) => void + const work = new Promise((resolve, reject) => { + resolveWork = resolve + rejectWork = reject + }) + acquisition.inFlight = work + const run = async (): Promise => { + const request: Extract< + RPCRequest, + { type: `rpc:ensureRemoteSubset:req` } + > = { + type: `rpc:ensureRemoteSubset:req`, + rpcId: safeRandomUUID(), + acquisitionId: acquisition.acquisitionId, + options: acquisition.options, + } + const response = routedToLocalOwner + ? await this.handleEnsureRemoteSubset( + acquisition.collectionId, + request, + this.nodeId, + ) + : await this.sendRPC( + acquisition.collectionId, + request, + ) + + if (!response.ok) { + throw new Error(`ensureRemoteSubset failed: ${response.error}`) + } + acquisition.acquiredLeaderId = response.leaderId + } + void run().then(resolveWork, rejectWork) + let acquired = false + try { + await work + acquired = true + } catch (error) { + if (!routedToLocalOwner) { + const owner = this.remoteSubsetOwners.get(acquisition.collectionId) + if (owner) reportRemoteSubsetOwnerError(owner, error) + } + throw error + } finally { + if (acquisition.inFlight === work) acquisition.inFlight = null + const current = this.collections.get(acquisition.collectionId) + const currentLeaderId = current?.isLeader + ? this.nodeId + : (current?.leaderId ?? null) + const key = remoteSubsetAcquisitionKey( + acquisition.collectionId, + acquisition.acquisitionId, ) + if ( + acquired && + this.outboundRemoteSubsetAcquisitions.get(key) === acquisition && + (acquisition.forceReplay || + (currentLeaderId !== null && + acquisition.acquiredLeaderId !== currentLeaderId)) + ) { + acquisition.forceReplay = false + void this.acquireRemoteSubset(acquisition).catch(() => { + // Failure is already reported; only new demand or ownership change retries. + }) + } } } @@ -223,7 +454,11 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin spec: PersistedIndexSpec, ): Promise { if (this.isLeader(collectionId)) { - await this.requireAdapter().ensureIndex(collectionId, signature, spec) + await this.requireAdapter(collectionId).ensureIndex( + collectionId, + signature, + spec, + ) return } @@ -267,6 +502,23 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin }) } + async requestApplyCommittedTx( + collectionId: string, + tx: PersistedTx, + ): Promise { + const request: ApplyCommittedTxRequest = { + type: `rpc:applyCommittedTx:req`, + rpcId: safeRandomUUID(), + envelopeId: safeRandomUUID(), + tx, + } + if (this.isLeader(collectionId)) { + return this.handleApplyCommittedTx(collectionId, request) + } + + return this.sendRPC(collectionId, request) + } + async pullSince( collectionId: string, fromRowVersion: number, @@ -293,6 +545,12 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin dispose(): void { this.disposed = true + for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + this.postRemoteSubsetRelease(acquisition) + } + this.outboundRemoteSubsetAcquisitions.clear() + this.remoteSubsetIds.clear() + for (const [collectionId, state] of this.collections) { this.releaseLeadership(collectionId, state) } @@ -305,6 +563,16 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin this.channel.close() this.collections.clear() + this.collectionAdapters.clear() + for (const collectionId of this.remoteSubsetOwners.keys()) { + this.releaseInboundRemoteSubsetAcquisitions(collectionId) + } + this.remoteSubsetOwners.clear() + this.inboundRemoteSubsetAcquisitions.clear() + this.appliedEnvelopeIds.clear() + this.inFlightLocalMutationEnvelopes.clear() + this.appliedCommittedTxEnvelopes.clear() + this.inFlightCommittedTxEnvelopes.clear() } // ----------------------------------------------------------------------- @@ -316,6 +584,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin if (!state) { state = { isLeader: false, + leaderId: null, lockAbortController: null, heartbeatTimer: null, latestTerm: 0, @@ -348,7 +617,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin try { // Restore stream position from DB before claiming leadership - const adapter = this.requireAdapter() + const adapter = this.requireAdapter(collectionId) if (adapter.getStreamPosition) { const pos = await adapter.getStreamPosition(collectionId) state.latestTerm = pos.latestTerm @@ -358,8 +627,10 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin state.latestTerm++ state.isLeader = true + state.leaderId = this.nodeId this.emitHeartbeat(collectionId, state) + void this.replayRemoteSubsetAcquisitions(collectionId) state.heartbeatTimer = setInterval(() => { this.emitHeartbeat(collectionId, state) }, HEARTBEAT_INTERVAL_MS) @@ -377,7 +648,9 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin abortController.signal.addEventListener(`abort`, onAbort) }) } finally { + this.releaseInboundRemoteSubsetAcquisitions(collectionId) state.isLeader = false + state.leaderId = null if (state.heartbeatTimer) { clearInterval(state.heartbeatTimer) state.heartbeatTimer = null @@ -399,9 +672,10 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin } private releaseLeadership( - _collectionId: string, + collectionId: string, state: CollectionState, ): void { + this.releaseInboundRemoteSubsetAcquisitions(collectionId) if (state.lockAbortController) { state.lockAbortController.abort() state.lockAbortController = null @@ -411,6 +685,68 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin state.heartbeatTimer = null } state.isLeader = false + state.leaderId = null + } + + private postRemoteSubsetRelease( + acquisition: OutboundRemoteSubsetAcquisition, + ): void { + const request: Extract< + RPCRequest, + { type: `rpc:releaseRemoteSubset:req` } + > = { + type: `rpc:releaseRemoteSubset:req`, + rpcId: safeRandomUUID(), + acquisitionId: acquisition.acquisitionId, + } + if (this.isLeader(acquisition.collectionId)) { + void this.handleReleaseRemoteSubset( + acquisition.collectionId, + request, + this.nodeId, + ).catch(() => { + // The owner already received the exact unload failure through onError. + }) + return + } + this.channel.postMessage({ + v: 1, + dbName: this.dbName, + collectionId: acquisition.collectionId, + senderId: this.nodeId, + ts: Date.now(), + payload: request, + } satisfies ProtocolEnvelope) + } + + private async replayRemoteSubsetAcquisitions( + collectionId: string, + ): Promise { + if (this.isDisposed()) return + const state = this.collections.get(collectionId) + const leaderId = state?.isLeader ? this.nodeId : state?.leaderId + if (!leaderId) return + + const replays: Array> = [] + for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + if ( + acquisition.collectionId !== collectionId || + (!acquisition.forceReplay && acquisition.acquiredLeaderId === leaderId) + ) { + continue + } + if (acquisition.inFlight) { + acquisition.forceReplay = true + continue + } + acquisition.forceReplay = false + replays.push( + this.acquireRemoteSubset(acquisition).catch(() => { + // Failure is already reported; only new demand or ownership change retries. + }), + ) + } + await Promise.all(replays) } private emitHeartbeat(collectionId: string, state: CollectionState): void { @@ -448,6 +784,35 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin const type = (payload as Record).type as string | undefined + if (type === `leader:heartbeat`) { + const heartbeat = payload as { + leaderId?: unknown + term?: unknown + latestSeq?: unknown + latestRowVersion?: unknown + } + if ( + typeof heartbeat.leaderId === `string` && + typeof heartbeat.term === `number` && + typeof heartbeat.latestSeq === `number` && + typeof heartbeat.latestRowVersion === `number` + ) { + const state = this.ensureCollectionState(envelope.collectionId) + if (heartbeat.term < state.latestTerm) return + const changedLeader = state.leaderId !== heartbeat.leaderId + state.leaderId = heartbeat.leaderId + state.latestTerm = Math.max(state.latestTerm, heartbeat.term) + state.latestSeq = Math.max(state.latestSeq, heartbeat.latestSeq) + state.latestRowVersion = Math.max( + state.latestRowVersion, + heartbeat.latestRowVersion, + ) + if (changedLeader) { + void this.replayRemoteSubsetAcquisitions(envelope.collectionId) + } + } + } + // Handle RPC responses (for pending outbound RPCs) if (type && type.endsWith(`:res`)) { const rpcId = (payload as { rpcId?: string }).rpcId @@ -464,7 +829,11 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin if (type && type.endsWith(`:req`)) { const collectionId = envelope.collectionId if (this.isLeader(collectionId)) { - void this.handleRPCRequest(collectionId, payload as RPCRequest) + void this.handleRPCRequest( + collectionId, + payload as RPCRequest, + envelope.senderId, + ) } return } @@ -487,15 +856,53 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin request: RPCRequest, ): Promise { let lastError: Error | undefined + let firstTransportCause: unknown + const mutationRequestType = isMutatingRPCRequest(request) + ? request.type + : undefined + const mutationRoute = mutationRequestType + ? this.captureMutationRoute(collectionId) + : undefined for (let attempt = 0; attempt <= RPC_RETRY_ATTEMPTS; attempt++) { if (attempt > 0) { await sleep(RPC_RETRY_DELAY_MS * attempt) } + if ( + mutationRoute && + mutationRequestType && + firstTransportCause !== undefined + ) { + this.assertMutationRouteUnchanged( + collectionId, + mutationRequestType, + mutationRoute, + firstTransportCause, + ) + } + + if (this.isLeader(collectionId)) { + return (await this.dispatchRPCRequest( + collectionId, + request, + this.nodeId, + )) as T + } + try { return await this.sendRPCOnce(collectionId, request) } catch (error) { + if (this.isDisposed()) throw error + firstTransportCause ??= error + if (mutationRoute && mutationRequestType) { + this.assertMutationRouteUnchanged( + collectionId, + mutationRequestType, + mutationRoute, + firstTransportCause, + ) + } lastError = error instanceof Error ? error : new Error(String(error)) } } @@ -503,6 +910,43 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin throw lastError ?? new Error(`RPC failed after retries`) } + private captureMutationRoute(collectionId: string): { + leaderId: string | null + term: number | null + } { + const state = this.collections.get(collectionId) + return { + leaderId: state?.isLeader ? this.nodeId : (state?.leaderId ?? null), + term: state?.latestTerm ?? null, + } + } + + private assertMutationRouteUnchanged( + collectionId: string, + requestType: IndeterminateCommitRequestType, + previous: { leaderId: string | null; term: number | null }, + cause: unknown, + ): void { + const current = this.captureMutationRoute(collectionId) + if ( + previous.leaderId !== null && + previous.term !== null && + current.leaderId === previous.leaderId && + current.term === previous.term + ) { + return + } + throw new IndeterminateCommitError({ + collectionId, + requestType, + previousLeaderId: previous.leaderId, + previousTerm: previous.term, + currentLeaderId: current.leaderId, + currentTerm: current.term, + cause, + }) + } + private sendRPCOnce( collectionId: string, request: RPCRequest, @@ -542,38 +986,22 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin private async handleRPCRequest( collectionId: string, request: RPCRequest, + requesterId: string, ): Promise { let response: RPCResponse try { - switch (request.type) { - case `rpc:ensureRemoteSubset:req`: - response = await this.handleEnsureRemoteSubset(collectionId, request) - break - case `rpc:ensurePersistedIndex:req`: - response = await this.handleEnsurePersistedIndex( - collectionId, - request, - ) - break - case `rpc:applyLocalMutations:req`: - response = await this.handleApplyLocalMutations(collectionId, request) - break - case `rpc:pullSince:req`: - response = await this.handlePullSince(collectionId, request) - break - default: - return - } + response = await this.dispatchRPCRequest( + collectionId, + request, + requesterId, + ) } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error) - response = { - type: request.type.replace(`:req`, `:res`) as RPCResponse[`type`], - rpcId: request.rpcId, - ok: false, - error: errorMessage, - } as RPCResponse + response = createRPCErrorResponse(request, error) + } + + if (this.isDisposed()) { + return } const envelope: ProtocolEnvelope = { @@ -587,16 +1015,357 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin this.channel.postMessage(envelope) } - private handleEnsureRemoteSubset( - _collectionId: string, - request: { type: `rpc:ensureRemoteSubset:req`; rpcId: string }, - ): RPCResponse { - // Leader doesn't need to do anything special — the remote subset - // is ensured by the leader's own sync connection + private dispatchRPCRequest( + collectionId: string, + request: RPCRequest, + requesterId: string, + ): Promise { + switch (request.type) { + case `rpc:ensureRemoteSubset:req`: + return this.handleEnsureRemoteSubset(collectionId, request, requesterId) + case `rpc:releaseRemoteSubset:req`: + return this.handleReleaseRemoteSubset( + collectionId, + request, + requesterId, + ) + case `rpc:ensurePersistedIndex:req`: + return this.handleEnsurePersistedIndex(collectionId, request) + case `rpc:applyLocalMutations:req`: + return this.handleApplyLocalMutations(collectionId, request) + case `rpc:applyCommittedTx:req`: + return this.handleApplyCommittedTx(collectionId, request) + case `rpc:pullSince:req`: + return this.handlePullSince(collectionId, request) + } + } + + private async handleEnsureRemoteSubset( + collectionId: string, + request: Extract, + requesterId: string, + ): Promise { + const key = inboundRemoteSubsetAcquisitionKey( + collectionId, + requesterId, + request.acquisitionId, + ) + const existing = this.inboundRemoteSubsetAcquisitions.get(key) + const awaitingOwner = + existing && `awaitingOwner` in existing ? existing : undefined + if (existing) { + if (`owner` in existing) { + if (!existing.released) { + await existing.load + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + await existing.release + if (existing.terminalRelease) { + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + if (this.inboundRemoteSubsetAcquisitions.get(key) === existing) { + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } else if (!(`awaitingOwner` in existing)) { + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + } + + const owner = this.remoteSubsetOwners.get(collectionId) + if (!owner) { + throw new Error( + `ElectronCollectionCoordinator: no remote subset owner registered for collection "${collectionId}"`, + ) + } + + const acquisition: ActiveRemoteSubsetAcquisition = { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + owner, + options: awaitingOwner?.options ?? request.options, + load: Promise.resolve(), + transferred: false, + released: false, + terminalRelease: false, + release: null, + } + this.inboundRemoteSubsetAcquisitions.set(key, acquisition) + let resolveLoad!: () => void + let rejectLoad!: (error: unknown) => void + acquisition.load = new Promise((resolve, reject) => { + resolveLoad = resolve + rejectLoad = reject + }) + try { + const load = owner(acquisition.options) + acquisition.transferred = true + void Promise.resolve(load).then(resolveLoad, rejectLoad) + } catch (error) { + rejectLoad(error) + } + try { + await acquisition.load + } catch (error) { + if ( + !acquisition.transferred && + this.inboundRemoteSubsetAcquisitions.get(key) === acquisition + ) { + if (awaitingOwner) { + this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + } else { + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } + reportRemoteSubsetOwnerError(owner, error) + throw error + } return { type: `rpc:ensureRemoteSubset:res`, rpcId: request.rpcId, ok: true, + leaderId: this.nodeId, + } + } + + private async handleReleaseRemoteSubset( + collectionId: string, + request: Extract, + requesterId: string, + ): Promise { + const key = inboundRemoteSubsetAcquisitionKey( + collectionId, + requesterId, + request.acquisitionId, + ) + const acquisition = this.inboundRemoteSubsetAcquisitions.get(key) + if (!acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } else if (`owner` in acquisition) { + acquisition.terminalRelease = true + await this.releaseRemoteSubsetAcquisition(acquisition) + if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } + } else if (`awaitingOwner` in acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } + return { + type: `rpc:releaseRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + } + } + + private releaseRemoteSubsetAcquisition( + acquisition: ActiveRemoteSubsetAcquisition, + ): Promise { + if (acquisition.release) return acquisition.release + acquisition.released = true + acquisition.release = (async () => { + try { + await acquisition.load + } catch { + // A returned promise transfers the lease even when initial loading fails. + } + try { + if (acquisition.transferred) { + await unloadRemoteSubsetOwner(acquisition.owner, acquisition.options) + } + } finally { + if (!acquisition.terminalRelease) { + const key = inboundRemoteSubsetAcquisitionKey( + acquisition.collectionId, + acquisition.requesterId, + acquisition.acquisitionId, + ) + if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId: acquisition.collectionId, + requesterId: acquisition.requesterId, + acquisitionId: acquisition.acquisitionId, + options: acquisition.options, + released: true, + awaitingOwner: true, + }) + } + } + } + })() + return acquisition.release + } + + private releaseInboundRemoteSubsetAcquisitions( + collectionId: string, + owner?: RemoteSubsetOwner, + ): void { + for (const acquisition of this.inboundRemoteSubsetAcquisitions.values()) { + if ( + !(`owner` in acquisition) || + acquisition.collectionId !== collectionId || + (owner && acquisition.owner !== owner) + ) { + continue + } + void this.releaseRemoteSubsetAcquisition(acquisition).catch( + () => undefined, + ) + } + } + + private rebindRemoteInboundSubsetAcquisitions( + collectionId: string, + owner: RemoteSubsetOwner, + ): void { + for (const acquisition of this.inboundRemoteSubsetAcquisitions.values()) { + if (`awaitingOwner` in acquisition) { + if ( + acquisition.collectionId === collectionId && + acquisition.requesterId !== this.nodeId + ) { + void this.bindAwaitingRemoteSubsetAcquisition( + acquisition, + owner, + ).catch(() => { + // The owner receives the exact load failure through onError. + }) + } + continue + } + if ( + !(`owner` in acquisition) || + acquisition.collectionId !== collectionId || + acquisition.requesterId === this.nodeId || + !acquisition.released || + acquisition.terminalRelease + ) { + continue + } + + void this.rebindRemoteInboundSubsetAcquisition(acquisition, owner).catch( + () => undefined, + ) + } + } + + private async rebindRemoteInboundSubsetAcquisition( + previous: ActiveRemoteSubsetAcquisition, + owner: RemoteSubsetOwner, + ): Promise { + await previous.release + if ( + previous.terminalRelease || + this.remoteSubsetOwners.get(previous.collectionId) !== owner + ) { + return + } + + const key = inboundRemoteSubsetAcquisitionKey( + previous.collectionId, + previous.requesterId, + previous.acquisitionId, + ) + const current = this.inboundRemoteSubsetAcquisitions.get(key) + if (current && current !== previous) { + if (`awaitingOwner` in current) { + await this.bindAwaitingRemoteSubsetAcquisition(current, owner) + } + return + } + + const awaitingOwner: AwaitingRemoteSubsetOwnerAcquisition = { + collectionId: previous.collectionId, + requesterId: previous.requesterId, + acquisitionId: previous.acquisitionId, + options: previous.options, + released: true, + awaitingOwner: true, + } + this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + await this.bindAwaitingRemoteSubsetAcquisition(awaitingOwner, owner) + } + + private async bindAwaitingRemoteSubsetAcquisition( + awaitingOwner: AwaitingRemoteSubsetOwnerAcquisition, + owner: RemoteSubsetOwner, + ): Promise { + if (this.remoteSubsetOwners.get(awaitingOwner.collectionId) !== owner) { + return + } + const key = inboundRemoteSubsetAcquisitionKey( + awaitingOwner.collectionId, + awaitingOwner.requesterId, + awaitingOwner.acquisitionId, + ) + if (this.inboundRemoteSubsetAcquisitions.get(key) !== awaitingOwner) return + + const acquisition: ActiveRemoteSubsetAcquisition = { + collectionId: awaitingOwner.collectionId, + requesterId: awaitingOwner.requesterId, + acquisitionId: awaitingOwner.acquisitionId, + owner, + options: awaitingOwner.options, + load: Promise.resolve(), + transferred: false, + released: false, + terminalRelease: false, + release: null, + } + this.inboundRemoteSubsetAcquisitions.set(key, acquisition) + let resolveLoad!: () => void + let rejectLoad!: (error: unknown) => void + acquisition.load = new Promise((resolve, reject) => { + resolveLoad = resolve + rejectLoad = reject + }) + try { + const load = owner(acquisition.options) + acquisition.transferred = true + void Promise.resolve(load).then(resolveLoad, rejectLoad) + } catch (error) { + rejectLoad(error) + } + try { + await acquisition.load + } catch (error) { + if ( + !acquisition.transferred && + this.inboundRemoteSubsetAcquisitions.get(key) === acquisition + ) { + this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + } + reportRemoteSubsetOwnerError(owner, error) + throw error } } @@ -610,7 +1379,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin }, ): Promise { await this.withWriterLock(() => - this.requireAdapter().ensureIndex( + this.requireAdapter(collectionId).ensureIndex( collectionId, request.signature, request.spec, @@ -632,17 +1401,39 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin mutations: Array }, ): Promise { - // Dedupe by envelopeId - if (this.appliedEnvelopeIds.has(request.envelopeId)) { - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: false, - code: `CONFLICT`, - error: `envelope ${request.envelopeId} already applied`, + const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) + const appliedEnvelope = this.appliedEnvelopeIds.get(envelopeKey) + if (appliedEnvelope) { + return { ...appliedEnvelope.response, rpcId: request.rpcId } + } + + const inFlightEnvelope = + this.inFlightLocalMutationEnvelopes.get(envelopeKey) + if (inFlightEnvelope) { + const response = await inFlightEnvelope + return { ...response, rpcId: request.rpcId } + } + + const response = this.applyLocalMutationsOnce(collectionId, request) + this.inFlightLocalMutationEnvelopes.set(envelopeKey, response) + try { + return await response + } finally { + if (this.inFlightLocalMutationEnvelopes.get(envelopeKey) === response) { + this.inFlightLocalMutationEnvelopes.delete(envelopeKey) } } + } + private async applyLocalMutationsOnce( + collectionId: string, + request: { + type: `rpc:applyLocalMutations:req` + rpcId: string + envelopeId: string + mutations: Array + }, + ): Promise { const state = this.collections.get(collectionId) if (!state || !state.isLeader) { return { @@ -663,6 +1454,21 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin const rowVersion = state.latestRowVersion // Build and apply the persisted transaction + const rowMetadataMutations: Array = [] + for (const mutation of request.mutations) { + if (!(`metadataChanged` in mutation) || !mutation.metadataChanged) { + continue + } + rowMetadataMutations.push( + mutation.metadata === undefined + ? { type: `delete`, key: mutation.key } + : { + type: `set`, + key: mutation.key, + value: mutation.metadata, + }, + ) + } const tx = { txId: safeRandomUUID(), term, @@ -672,15 +1478,38 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin type: m.type, key: m.key, value: m.value, + ...(`metadataChanged` in m + ? { metadata: m.metadata, metadataChanged: m.metadataChanged } + : {}), })), + rowMetadataMutations, } - await this.withWriterLock(() => - this.requireAdapter().applyCommittedTx(collectionId, tx), - ) + try { + await this.withWriterLock(() => + this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), + ) + } catch (error) { + throw toPersistedCollectionDurabilityError(collectionId, error) + } - // Track envelope for dedup - this.appliedEnvelopeIds.set(request.envelopeId, Date.now()) + const response: ApplyLocalMutationsResponse = { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: true, + term, + seq, + latestRowVersion: rowVersion, + acceptedMutationIds: request.mutations.map((m) => m.mutationId), + } + if (this.isDisposed()) { + return response + } + + this.appliedEnvelopeIds.set( + appliedEnvelopeKey(collectionId, request.envelopeId), + { appliedAt: Date.now(), response }, + ) this.pruneAppliedEnvelopeIds() // Broadcast tx:committed to all tabs @@ -706,6 +1535,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin requiresFullReload: false, changedRows, deletedKeys, + rowMetadataMutations, }, } this.channel.postMessage(txCommitted) @@ -715,15 +1545,123 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin subscriber(txCommitted) } - return { - type: `rpc:applyLocalMutations:res`, + return response + } + + private async handleApplyCommittedTx( + collectionId: string, + request: ApplyCommittedTxRequest, + ): Promise { + const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) + const appliedEnvelope = this.appliedCommittedTxEnvelopes.get(envelopeKey) + if (appliedEnvelope) { + return { ...appliedEnvelope.response, rpcId: request.rpcId } + } + + const inFlightEnvelope = this.inFlightCommittedTxEnvelopes.get(envelopeKey) + if (inFlightEnvelope) { + const response = await inFlightEnvelope + return { ...response, rpcId: request.rpcId } + } + + const response = this.applyCommittedTxOnce(collectionId, request) + this.inFlightCommittedTxEnvelopes.set(envelopeKey, response) + try { + return await response + } finally { + if (this.inFlightCommittedTxEnvelopes.get(envelopeKey) === response) { + this.inFlightCommittedTxEnvelopes.delete(envelopeKey) + } + } + } + + private async applyCommittedTxOnce( + collectionId: string, + request: ApplyCommittedTxRequest, + ): Promise { + const state = this.collections.get(collectionId) + if (!state || !state.isLeader) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `NOT_LEADER`, + error: `not the leader for ${collectionId}`, + } + } + + state.latestSeq++ + state.latestRowVersion++ + const tx: PersistedTx = { + ...request.tx, + term: state.latestTerm, + seq: state.latestSeq, + rowVersion: state.latestRowVersion, + } + + try { + await this.withWriterLock(() => + this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), + ) + } catch (error) { + throw toPersistedCollectionDurabilityError(collectionId, error) + } + + const response: ApplyCommittedTxResponse = { + type: `rpc:applyCommittedTx:res`, rpcId: request.rpcId, ok: true, - term, - seq, - latestRowVersion: rowVersion, - acceptedMutationIds: request.mutations.map((m) => m.mutationId), + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, } + if (this.isDisposed()) { + return response + } + this.appliedCommittedTxEnvelopes.set( + appliedEnvelopeKey(collectionId, request.envelopeId), + { appliedAt: Date.now(), response }, + ) + this.pruneAppliedEnvelopeIds() + + const committedBase = { + type: `tx:committed` as const, + term: tx.term, + seq: tx.seq, + txId: tx.txId, + latestRowVersion: tx.rowVersion, + } + const committedPayload: TxCommitted = tx.truncate + ? { ...committedBase, requiresFullReload: true } + : { + ...committedBase, + requiresFullReload: false, + changedRows: tx.mutations + .filter((mutation) => mutation.type !== `delete`) + .map((mutation) => ({ + key: mutation.key, + value: mutation.value, + })), + deletedKeys: tx.mutations + .filter((mutation) => mutation.type === `delete`) + .map((mutation) => mutation.key), + rowMetadataMutations: tx.rowMetadataMutations, + collectionMetadataMutations: tx.collectionMetadataMutations, + } + const committed: ProtocolEnvelope = { + v: 1, + dbName: this.dbName, + collectionId, + senderId: this.nodeId, + ts: Date.now(), + payload: committedPayload, + } + this.channel.postMessage(committed) + for (const subscriber of state.subscribers) { + subscriber(committed) + } + + return response } private async handlePullSince( @@ -736,7 +1674,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin ): Promise { const state = this.collections.get(collectionId) - const adapter = this.requireAdapter() + const adapter = this.requireAdapter(collectionId) if (!adapter.pullSince) { return { type: `rpc:pullSince:res`, @@ -784,9 +1722,16 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin const lockName = `tsdb:writer:${this.dbName}` for (let attempt = 0; attempt <= WRITER_LOCK_MAX_RETRIES; attempt++) { + const callbackState = { entered: false } try { - return await navigator.locks.request(lockName, async () => fn()) + return await navigator.locks.request(lockName, async () => { + callbackState.entered = true + return fn() + }) } catch (error) { + if (callbackState.entered) { + throw error + } if (error instanceof DOMException && error.name === `AbortError`) { throw error } @@ -811,11 +1756,16 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin private pruneAppliedEnvelopeIds(): void { // Keep envelopes for 60 seconds for dedup const cutoff = Date.now() - 60_000 - for (const [id, ts] of this.appliedEnvelopeIds) { - if (ts < cutoff) { + for (const [id, envelope] of this.appliedEnvelopeIds) { + if (envelope.appliedAt < cutoff) { this.appliedEnvelopeIds.delete(id) } } + for (const [key, envelope] of this.appliedCommittedTxEnvelopes) { + if (envelope.appliedAt < cutoff) { + this.appliedCommittedTxEnvelopes.delete(key) + } + } } } @@ -838,3 +1788,157 @@ function isProtocolEnvelope(data: unknown): data is ProtocolEnvelope { function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)) } + +async function unloadRemoteSubsetOwner( + owner: RemoteSubsetOwner, + options: TransportedLoadSubsetOptions, +): Promise { + try { + const result = ( + owner.unloadSubset as unknown as ( + options: TransportedLoadSubsetOptions, + ) => unknown + )(options) + await Promise.resolve(result) + } catch (error) { + reportRemoteSubsetOwnerError(owner, error) + throw error + } +} + +function reportRemoteSubsetOwnerError( + owner: RemoteSubsetOwner, + error: unknown, +): void { + try { + owner.onError(error) + } catch { + // Reporting must not replace the original owner failure. + } +} + +function appliedEnvelopeKey(collectionId: string, envelopeId: string): string { + return JSON.stringify([collectionId, envelopeId]) +} + +function remoteSubsetAcquisitionKey( + collectionId: string, + acquisitionId: string, +): string { + return JSON.stringify([collectionId, acquisitionId]) +} + +function inboundRemoteSubsetAcquisitionKey( + collectionId: string, + requesterId: string, + acquisitionId: string, +): string { + return JSON.stringify([collectionId, requesterId, acquisitionId]) +} + +function createRPCErrorResponse( + request: RPCRequest, + cause: unknown, +): RPCResponse { + const error = cause instanceof Error ? cause.message : String(cause) + switch (request.type) { + case `rpc:ensureRemoteSubset:req`: + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: false, + error, + } + case `rpc:releaseRemoteSubset:req`: + return { + type: `rpc:releaseRemoteSubset:res`, + rpcId: request.rpcId, + ok: false, + error, + } + case `rpc:ensurePersistedIndex:req`: + return { + type: `rpc:ensurePersistedIndex:res`, + rpcId: request.rpcId, + ok: false, + error, + } + case `rpc:applyLocalMutations:req`: + if (cause instanceof PersistedCollectionDurabilityError) { + return { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: false, + code: `PERSISTENCE_ERROR`, + error, + ...toSafeDurabilityDetails(cause), + } + } + return { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error, + } + case `rpc:applyCommittedTx:req`: + if (cause instanceof PersistedCollectionDurabilityError) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `PERSISTENCE_ERROR`, + error, + ...toSafeDurabilityDetails(cause), + } + } + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error, + } + case `rpc:pullSince:req`: + return { + type: `rpc:pullSince:res`, + rpcId: request.rpcId, + ok: false, + error, + } + } +} + +function isMutatingRPCRequest(request: RPCRequest): request is Extract< + RPCRequest, + { + type: IndeterminateCommitRequestType + } +> { + return ( + request.type === `rpc:applyLocalMutations:req` || + request.type === `rpc:applyCommittedTx:req` + ) +} + +function toSafeDurabilityDetails(error: PersistedCollectionDurabilityError): { + sourceCode?: string | number + path?: string | ReadonlyArray +} { + const sourceCode = + typeof error.code === `string` || typeof error.code === `number` + ? error.code + : undefined + const path = + typeof error.path === `string` || + (Array.isArray(error.path) && + error.path.every( + (part) => typeof part === `string` || typeof part === `number`, + )) + ? (error.path as string | ReadonlyArray) + : undefined + return { + ...(sourceCode === undefined ? {} : { sourceCode }), + ...(path === undefined ? {} : { path }), + } +} diff --git a/packages/electron-db-sqlite-persistence/src/index.ts b/packages/electron-db-sqlite-persistence/src/index.ts index 65c5717261..ab41419fed 100644 --- a/packages/electron-db-sqlite-persistence/src/index.ts +++ b/packages/electron-db-sqlite-persistence/src/index.ts @@ -10,8 +10,19 @@ export type { } from './renderer' export { ElectronCollectionCoordinator } from './electron-coordinator' export type { ElectronCollectionCoordinatorOptions } from './electron-coordinator' -export { persistedCollectionOptions } from '@tanstack/db-sqlite-persistence-core' +export { + DuplicateRemoteSubsetOwnerError, + IndeterminateCommitError, + PersistedCollectionDurabilityError, + RemoteSubsetWireValueError, + persistedCollectionOptions, +} from '@tanstack/db-sqlite-persistence-core' export type { PersistedCollectionCoordinator, PersistedCollectionPersistence, + IndeterminateCommitRequestType, + RemoteSubsetOwner, + RemoteSubsetWireExpression, + RemoteSubsetWireValue, + TransportedLoadSubsetOptions, } from '@tanstack/db-sqlite-persistence-core' diff --git a/packages/electron-db-sqlite-persistence/src/renderer.ts b/packages/electron-db-sqlite-persistence/src/renderer.ts index 2ac7203000..50e459cc95 100644 --- a/packages/electron-db-sqlite-persistence/src/renderer.ts +++ b/packages/electron-db-sqlite-persistence/src/renderer.ts @@ -360,22 +360,24 @@ export function createElectronSQLitePersistence( }) adapterCache.set(cacheKey, adapter) - // Wire the adapter into the coordinator so it can handle - // leader-side RPCs (applyCommittedTx, pullSince, getStreamPosition, etc.) - if (coordinator instanceof ElectronCollectionCoordinator) { - coordinator.setAdapter(adapter) - } - return adapter } const createCollectionPersistence = ( mode: PersistedCollectionMode, schemaVersion: number | undefined, - ): PersistedCollectionPersistence => ({ - adapter: getAdapterForCollection(mode, schemaVersion), - coordinator, - }) + collectionId?: string, + ): PersistedCollectionPersistence => { + const adapter = getAdapterForCollection(mode, schemaVersion) + if (coordinator instanceof ElectronCollectionCoordinator) { + if (collectionId === undefined) { + coordinator.setAdapter(adapter) + } else { + coordinator.setAdapterForCollection(collectionId, adapter) + } + } + return { adapter, coordinator } + } const defaultPersistence = createCollectionPersistence( `sync-absent`, @@ -384,8 +386,8 @@ export function createElectronSQLitePersistence( return { ...defaultPersistence, - resolvePersistenceForCollection: ({ mode, schemaVersion }) => - createCollectionPersistence(mode, schemaVersion), + resolvePersistenceForCollection: ({ collectionId, mode, schemaVersion }) => + createCollectionPersistence(mode, schemaVersion, collectionId), // Backward compatible fallback for older callers. resolvePersistenceForMode: (mode) => createCollectionPersistence(mode, undefined), diff --git a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts index d1c330fb6f..c2913a0833 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts @@ -1,11 +1,16 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterEach, describe, expect, it } from 'vitest' -import { InvalidPersistedCollectionConfigError } from '@tanstack/db-sqlite-persistence-core' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + InvalidPersistedCollectionConfigError, + persistedCollectionOptions, +} from '@tanstack/db-sqlite-persistence-core' import { createNodeSQLitePersistence } from '@tanstack/node-db-sqlite-persistence' +import { IR, createCollection } from '../../db/src' import { BetterSqlite3SQLiteDriver } from '../../node-db-sqlite-persistence/src/node-driver' import { + ElectronCollectionCoordinator, createElectronSQLitePersistence, exposeElectronSQLitePersistence, } from '../src' @@ -17,7 +22,18 @@ import { createElectronRuntimeBridgeInvoke, isElectronFullE2EEnabled, } from './e2e/electron-process-client' -import type { PersistedCollectionPersistence } from '@tanstack/db-sqlite-persistence-core' +import type { + ApplyCommittedTxRequest, + ApplyLocalMutationsRequest, + IndeterminateCommitError, + PersistedCollectionDurabilityError, + PersistedCollectionPersistence, + PersistedTx, + PersistenceAdapter, + RemoteSubsetOwner, + TransportedLoadSubsetOptions, +} from '@tanstack/db-sqlite-persistence-core' +import type { LoadSubsetOptions, Subscription, SyncConfig } from '../../db/src' import type { ElectronPersistenceInvoke, ElectronPersistenceRequestEnvelope, @@ -29,6 +45,15 @@ type InvokeHarness = { close: () => void } +type CoordinatorInvokeHarness = { + coordinator: ElectronCollectionCoordinator + persistence: PersistedCollectionPersistence + appliedTransactions: Array + committedPayloads: Array> + start: () => void + close: () => void +} + type ElectronMainPersistence = PersistedCollectionPersistence const electronRuntimeBridgeTimeoutMs = isElectronFullE2EEnabled() @@ -151,6 +176,17 @@ function createInvokeHarness( const activeCleanupFns: Array<() => void> = [] +function registerCleanup(cleanupFn: () => void): () => void { + let cleanupPending = true + const cleanupOnce = () => { + if (!cleanupPending) return + cleanupPending = false + cleanupFn() + } + activeCleanupFns.push(cleanupOnce) + return cleanupOnce +} + afterEach(() => { while (activeCleanupFns.length > 0) { const cleanupFn = activeCleanupFns.pop() @@ -167,6 +203,117 @@ function createTempDbPath(): string { return dbPath } +function installImmediatelyGrantedWebLocks(): () => void { + const originalNavigator = Object.getOwnPropertyDescriptor( + globalThis, + `navigator`, + ) + const navigatorValue = globalThis.navigator + + Object.defineProperty(globalThis, `navigator`, { + value: { + ...navigatorValue, + locks: { + request: async ( + name: string, + optionsOrCallback: + | { signal?: AbortSignal } + | ((lock: { name: string }) => Promise), + maybeCallback?: (lock: { name: string }) => Promise, + ): Promise => { + const callback = + typeof optionsOrCallback === `function` + ? optionsOrCallback + : maybeCallback! + return callback({ name }) + }, + }, + }, + writable: true, + configurable: true, + }) + + return () => { + if (originalNavigator) { + Object.defineProperty(globalThis, `navigator`, originalNavigator) + } else { + Reflect.deleteProperty(globalThis, `navigator`) + } + } +} + +async function waitForLeadership( + coordinator: ElectronCollectionCoordinator, + collectionId: string, +): Promise { + const deadline = Date.now() + electronRuntimeBridgeTimeoutMs + while (Date.now() < deadline) { + if (coordinator.isLeader(collectionId)) return + await new Promise((resolve) => setTimeout(resolve, 10)) + } + throw new Error(`Electron coordinator did not acquire leadership`) +} + +function electronSubsetWithNestedValue(value: unknown): LoadSubsetOptions { + return { + where: new IR.Func(`in`, [ + new IR.PropRef([`todos`, `status`]), + new IR.Value([`kept`, value]), + ]), + } +} + +function createElectronCoordinatorTestAdapter(): PersistenceAdapter { + return { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: () => Promise.resolve(), + ensureIndex: () => Promise.resolve(), + } +} + +function createCoordinatorInvokeHarness( + dbPath: string, + dbName: string, +): CoordinatorInvokeHarness { + const invokeHarness = createInvokeHarness(dbPath, `todos`) + const coordinator = new ElectronCollectionCoordinator({ dbName }) + const appliedTransactions: Array = [] + const committedPayloads: Array> = [] + const persistence = createElectronSQLitePersistence({ + coordinator, + invoke: async (channel, request) => { + if (request.method === `applyCommittedTx`) { + appliedTransactions.push(structuredClone(request.payload.tx)) + } + return invokeHarness.invoke(channel, request) + }, + timeoutMs: electronRuntimeBridgeTimeoutMs, + }) + const close = registerCleanup(() => { + try { + coordinator.dispose() + } finally { + invokeHarness.close() + } + }) + + return { + coordinator, + persistence, + appliedTransactions, + committedPayloads, + start: () => { + coordinator.subscribe(`todos`, (message) => { + const payload = message.payload as Record + if (payload.type === `tx:committed`) { + committedPayloads.push(payload) + } + }) + }, + close, + } +} + describe(`electron sqlite persistence bridge`, () => { it(`round-trips reads and writes through main process`, async () => { const dbPath = createTempDbPath() @@ -283,6 +430,2841 @@ describe(`electron sqlite persistence bridge`, () => { expect(rows[0]?.value.title).toBe(`Survives restart`) }) + it(`preserves coordinator row metadata updates across restarts`, async () => { + const dbPath = createTempDbPath() + registerCleanup(installImmediatelyGrantedWebLocks()) + const metadataSetHarness = createCoordinatorInvokeHarness( + dbPath, + `electron-metadata-set`, + ) + + await metadataSetHarness.persistence.adapter.applyCommittedTx(`todos`, { + txId: `seed-row`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `todo-1`, + value: { id: `todo-1`, title: `Before` }, + }, + ], + }) + metadataSetHarness.appliedTransactions.length = 0 + metadataSetHarness.start() + await waitForLeadership(metadataSetHarness.coordinator, `todos`) + + const setResponse = + await metadataSetHarness.coordinator.requestApplyLocalMutations(`todos`, [ + { + mutationId: `metadata-set`, + type: `update`, + key: `todo-1`, + value: { id: `todo-1`, title: `After set` }, + metadataChanged: true, + metadata: { source: `local`, revision: 2 }, + }, + ]) + const setScannedRows = + await metadataSetHarness.persistence.adapter.scanRows?.(`todos`) + const setAppliedTransaction = metadataSetHarness.appliedTransactions[0] + const setCommittedPayload = metadataSetHarness.committedPayloads[0] + + expect({ + response: + setResponse.ok === true + ? { + ok: setResponse.ok, + acceptedMutationIds: setResponse.acceptedMutationIds, + } + : setResponse, + appliedTransactionCount: metadataSetHarness.appliedTransactions.length, + appliedMutation: setAppliedTransaction?.mutations[0], + appliedRowMetadataMutations: + setAppliedTransaction?.rowMetadataMutations ?? null, + committedPayloadCount: metadataSetHarness.committedPayloads.length, + committedType: setCommittedPayload?.type, + committedRowMetadataMutations: + setCommittedPayload?.rowMetadataMutations ?? null, + committedChangedRows: setCommittedPayload?.changedRows, + committedDeletedKeys: setCommittedPayload?.deletedKeys, + scannedRows: setScannedRows, + }).toEqual({ + response: { + ok: true, + acceptedMutationIds: [`metadata-set`], + }, + appliedTransactionCount: 1, + appliedMutation: { + type: `update`, + key: `todo-1`, + value: { id: `todo-1`, title: `After set` }, + metadataChanged: true, + metadata: { source: `local`, revision: 2 }, + }, + appliedRowMetadataMutations: [ + { + type: `set`, + key: `todo-1`, + value: { source: `local`, revision: 2 }, + }, + ], + committedPayloadCount: 1, + committedType: `tx:committed`, + committedRowMetadataMutations: [ + { + type: `set`, + key: `todo-1`, + value: { source: `local`, revision: 2 }, + }, + ], + committedChangedRows: [ + { + key: `todo-1`, + value: { id: `todo-1`, title: `After set` }, + }, + ], + committedDeletedKeys: [], + scannedRows: [ + { + key: `todo-1`, + value: { id: `todo-1`, title: `After set` }, + metadata: { source: `local`, revision: 2 }, + }, + ], + }) + + metadataSetHarness.close() + + const metadataDeleteHarness = createCoordinatorInvokeHarness( + dbPath, + `electron-metadata-delete`, + ) + expect( + await metadataDeleteHarness.persistence.adapter.scanRows?.(`todos`), + ).toEqual([ + { + key: `todo-1`, + value: { id: `todo-1`, title: `After set` }, + metadata: { source: `local`, revision: 2 }, + }, + ]) + + metadataDeleteHarness.start() + await waitForLeadership(metadataDeleteHarness.coordinator, `todos`) + const deleteResponse = + await metadataDeleteHarness.coordinator.requestApplyLocalMutations( + `todos`, + [ + { + mutationId: `metadata-delete`, + type: `update`, + key: `todo-1`, + value: { id: `todo-1`, title: `After delete` }, + metadataChanged: true, + }, + ], + ) + const deleteScannedRows = + await metadataDeleteHarness.persistence.adapter.scanRows?.(`todos`) + const deleteAppliedTransaction = + metadataDeleteHarness.appliedTransactions[0] + const deleteCommittedPayload = metadataDeleteHarness.committedPayloads[0] + + expect({ + response: + deleteResponse.ok === true + ? { + ok: deleteResponse.ok, + acceptedMutationIds: deleteResponse.acceptedMutationIds, + } + : deleteResponse, + appliedTransactionCount: metadataDeleteHarness.appliedTransactions.length, + appliedMutation: deleteAppliedTransaction?.mutations[0], + appliedRowMetadataMutations: + deleteAppliedTransaction?.rowMetadataMutations ?? null, + committedPayloadCount: metadataDeleteHarness.committedPayloads.length, + committedType: deleteCommittedPayload?.type, + committedRowMetadataMutations: + deleteCommittedPayload?.rowMetadataMutations ?? null, + committedChangedRows: deleteCommittedPayload?.changedRows, + committedDeletedKeys: deleteCommittedPayload?.deletedKeys, + scannedRows: deleteScannedRows, + }).toEqual({ + response: { + ok: true, + acceptedMutationIds: [`metadata-delete`], + }, + appliedTransactionCount: 1, + appliedMutation: { + type: `update`, + key: `todo-1`, + value: { id: `todo-1`, title: `After delete` }, + metadataChanged: true, + metadata: undefined, + }, + appliedRowMetadataMutations: [{ type: `delete`, key: `todo-1` }], + committedPayloadCount: 1, + committedType: `tx:committed`, + committedRowMetadataMutations: [{ type: `delete`, key: `todo-1` }], + committedChangedRows: [ + { + key: `todo-1`, + value: { id: `todo-1`, title: `After delete` }, + }, + ], + committedDeletedKeys: [], + scannedRows: [ + { + key: `todo-1`, + value: { id: `todo-1`, title: `After delete` }, + metadata: undefined, + }, + ], + }) + + metadataDeleteHarness.close() + + const reopenedInvokeHarness = createInvokeHarness(dbPath, `todos`) + const closeReopenedHarness = registerCleanup(reopenedInvokeHarness.close) + const reopenedPersistence = createElectronSQLitePersistence({ + invoke: reopenedInvokeHarness.invoke, + timeoutMs: electronRuntimeBridgeTimeoutMs, + }) + expect(await reopenedPersistence.adapter.scanRows?.(`todos`)).toEqual([ + { + key: `todo-1`, + value: { id: `todo-1`, title: `After delete` }, + metadata: undefined, + }, + ]) + closeReopenedHarness() + }) + + it(`routes a rich source transaction through the Electron owner and durable reopen`, async () => { + type Todo = { id: string; title: string } + type SourceParams = Parameters[`sync`]>[0] + + const dbPath = createTempDbPath() + registerCleanup(installImmediatelyGrantedWebLocks()) + const ownerHarness = createCoordinatorInvokeHarness( + dbPath, + `electron-rich-source`, + ) + + await ownerHarness.persistence.adapter.applyCommittedTx(`todos`, { + txId: `stale-seed`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `stale`, + value: { id: `stale`, title: `Must be truncated` }, + }, + ], + }) + ownerHarness.appliedTransactions.length = 0 + ownerHarness.start() + await waitForLeadership(ownerHarness.coordinator, `todos`) + + let sourceParams: SourceParams | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `todos`, + getKey: (todo) => todo.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: ownerHarness.persistence, + }), + ) + + try { + await collection.stateWhenReady() + sourceParams!.begin() + sourceParams!.metadata?.collection.set(`resume`, { offset: 9 }) + sourceParams!.truncate() + sourceParams!.write({ + type: `insert`, + value: { id: `fresh`, title: `Durable through owner` }, + metadata: { source: `electron-sync` }, + }) + const receipt = sourceParams!.commit() + if (receipt !== true) await receipt + + expect(collection.has(`stale`)).toBe(false) + expect(collection.get(`fresh`)).toMatchObject({ + id: `fresh`, + title: `Durable through owner`, + }) + expect(ownerHarness.appliedTransactions).toHaveLength(1) + expect(ownerHarness.appliedTransactions[0]).toMatchObject({ + truncate: true, + mutations: [ + { + type: `update`, + key: `fresh`, + value: { id: `fresh`, title: `Durable through owner` }, + }, + ], + rowMetadataMutations: [ + { + type: `set`, + key: `fresh`, + value: { source: `electron-sync` }, + }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `resume`, value: { offset: 9 } }, + ], + }) + expect(ownerHarness.committedPayloads).toHaveLength(1) + expect(ownerHarness.committedPayloads[0]).toMatchObject({ + type: `tx:committed`, + requiresFullReload: true, + }) + } finally { + await collection.cleanup() + } + + ownerHarness.close() + + const reopenedInvokeHarness = createInvokeHarness(dbPath, `todos`) + const closeReopenedHarness = registerCleanup(reopenedInvokeHarness.close) + const reopenedPersistence = createElectronSQLitePersistence({ + invoke: reopenedInvokeHarness.invoke, + timeoutMs: electronRuntimeBridgeTimeoutMs, + }) + expect(await reopenedPersistence.adapter.scanRows?.(`todos`)).toEqual([ + { + key: `fresh`, + value: { id: `fresh`, title: `Durable through owner` }, + metadata: { source: `electron-sync` }, + }, + ]) + expect( + await reopenedPersistence.adapter.loadCollectionMetadata?.(`todos`), + ).toEqual([{ key: `resume`, value: { offset: 9 } }]) + closeReopenedHarness() + }) + + it( + `does not retry a committed transaction after the writer callback starts`, + { timeout: 10_000 }, + async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + const persistenceError = new Error(`irreversible apply failed`) + let applyCalls = 0 + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-writer-callback-failure`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: () => { + applyCalls++ + return Promise.reject(persistenceError) + }, + ensureIndex: () => Promise.resolve(), + }, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.subscribe(`todos`, () => {}) + await waitForLeadership(coordinator, `todos`) + + await expect( + coordinator.requestApplyCommittedTx(`todos`, { + txId: `irreversible-effect`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }), + ).rejects.toMatchObject({ + name: `PersistedCollectionDurabilityError`, + cause: persistenceError, + } satisfies Partial) + expect(applyCalls).toBe(1) + }, + ) + + it(`fails indeterminate instead of retrying a committed mutation across leaders`, async () => { + const bEffects: Array = [] + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-requester-takeover`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: (_collectionId, tx) => { + bEffects.push(tx) + return Promise.resolve() + }, + ensureIndex: () => Promise.resolve(), + }, + }) + registerCleanup(() => coordinator.dispose()) + const internals = coordinator as unknown as { + nodeId: string + onChannelMessage: (message: unknown) => void + acquireLeadership: () => Promise + } + internals.acquireLeadership = async () => {} + const heartbeat = (leaderId: string, term: number) => ({ + v: 1, + dbName: `electron-requester-takeover`, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + internals.onChannelMessage(heartbeat(`electron-leader-a`, 4)) + const transportError = new Error(`leader A response was lost`) + const aEffects: Array = [] + let aPublications = 0 + let bPublications = 0 + let transportCalls = 0 + const failedTransport = vi.fn( + (_collectionId: string, request: ApplyCommittedTxRequest) => { + transportCalls++ + if (transportCalls === 1) { + aEffects.push(structuredClone(request.tx)) + aPublications++ + internals.onChannelMessage(heartbeat(`electron-leader-b`, 5)) + return Promise.reject(transportError) + } + bEffects.push(structuredClone(request.tx)) + bPublications++ + return Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: true, + term: 5, + seq: 1, + latestRowVersion: 1, + }) + }, + ) + Object.defineProperty(coordinator, `sendRPCOnce`, { + value: failedTransport, + configurable: true, + }) + + vi.useFakeTimers() + try { + const outcomePromise = coordinator + .requestApplyCommittedTx(`todos`, { + txId: `requester-takeover`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `takeover`, + value: { id: `takeover` }, + }, + ], + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + await vi.advanceTimersByTimeAsync(1_000) + const outcome = await outcomePromise + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyCommittedTx:req`, + previousLeaderId: `electron-leader-a`, + previousTerm: 4, + currentLeaderId: `electron-leader-b`, + currentTerm: 5, + cause: transportError, + } satisfies Partial), + }) + expect(failedTransport).toHaveBeenCalledTimes(1) + expect(aEffects).toHaveLength(1) + expect(aPublications).toBe(1) + expect(aEffects[0]?.txId).toBe(`requester-takeover`) + expect(bEffects).toEqual([]) + expect(bPublications).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it(`fails indeterminate instead of retrying a committed transaction without an initial leader route`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-unknown-leader-committed`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + const transportError = new Error(`unknown leader response was lost`) + const attemptedTransactions: Array = [] + const failedTransport = vi.fn( + (_collectionId: string, request: ApplyCommittedTxRequest) => { + attemptedTransactions.push(structuredClone(request.tx)) + if (attemptedTransactions.length === 1) { + return Promise.reject(transportError) + } + return Promise.resolve({ + type: `rpc:applyCommittedTx:res` as const, + rpcId: request.rpcId, + ok: true as const, + term: 1, + seq: 1, + latestRowVersion: 1, + }) + }, + ) + Object.defineProperty(coordinator, `sendRPCOnce`, { + value: failedTransport, + configurable: true, + }) + + const outcome = await coordinator + .requestApplyCommittedTx(`todos`, { + txId: `electron-unknown-leader-committed`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `electron-unknown-leader-committed`, + value: { id: `electron-unknown-leader-committed` }, + }, + ], + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyCommittedTx:req`, + previousLeaderId: null, + previousTerm: null, + currentLeaderId: null, + currentTerm: null, + cause: transportError, + } satisfies Partial), + }) + expect(failedTransport).toHaveBeenCalledTimes(1) + expect(attemptedTransactions).toEqual([ + { + txId: `electron-unknown-leader-committed`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `electron-unknown-leader-committed`, + value: { id: `electron-unknown-leader-committed` }, + }, + ], + }, + ]) + }) + + it(`fails indeterminate instead of retrying local mutations across leaders`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-local-requester-takeover`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + const internals = coordinator as unknown as { + onChannelMessage: (message: unknown) => void + acquireLeadership: () => Promise + } + internals.acquireLeadership = async () => {} + const heartbeat = (leaderId: string, term: number) => ({ + v: 1, + dbName: `electron-local-requester-takeover`, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + internals.onChannelMessage(heartbeat(`electron-local-leader-a`, 7)) + const transportError = new Error(`leader A local response was lost`) + const aEffects: Array = [] + const bEffects: Array = [] + let aPublications = 0 + let bPublications = 0 + let transportCalls = 0 + const failedTransport = vi.fn( + (_collectionId: string, request: ApplyLocalMutationsRequest) => { + transportCalls++ + if (transportCalls === 1) { + aEffects.push(structuredClone(request.mutations)) + aPublications++ + internals.onChannelMessage(heartbeat(`electron-local-leader-b`, 8)) + return Promise.reject(transportError) + } + bEffects.push(structuredClone(request.mutations)) + bPublications++ + return Promise.resolve({ + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: true, + term: 8, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: request.mutations.map( + (mutation) => mutation.mutationId, + ), + }) + }, + ) + Object.defineProperty(coordinator, `sendRPCOnce`, { + value: failedTransport, + configurable: true, + }) + + const outcome = await coordinator + .requestApplyLocalMutations(`todos`, [ + { + mutationId: `electron-local-takeover`, + type: `insert`, + key: `electron-local-takeover`, + value: { id: `electron-local-takeover` }, + }, + ]) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyLocalMutations:req`, + previousLeaderId: `electron-local-leader-a`, + previousTerm: 7, + currentLeaderId: `electron-local-leader-b`, + currentTerm: 8, + cause: transportError, + } satisfies Partial), + }) + expect(failedTransport).toHaveBeenCalledTimes(1) + expect(aEffects).toHaveLength(1) + expect(aPublications).toBe(1) + expect(bEffects).toEqual([]) + expect(bPublications).toBe(0) + }) + + it(`fails indeterminate instead of retrying local mutations without an initial leader route`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-unknown-leader-local`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + const transportError = new Error(`unknown leader response was lost`) + const attemptedMutations: Array = + [] + const failedTransport = vi.fn( + (_collectionId: string, request: ApplyLocalMutationsRequest) => { + attemptedMutations.push(structuredClone(request.mutations)) + if (attemptedMutations.length === 1) { + return Promise.reject(transportError) + } + return Promise.resolve({ + type: `rpc:applyLocalMutations:res` as const, + rpcId: request.rpcId, + ok: true as const, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: request.mutations.map( + (mutation) => mutation.mutationId, + ), + }) + }, + ) + Object.defineProperty(coordinator, `sendRPCOnce`, { + value: failedTransport, + configurable: true, + }) + + const outcome = await coordinator + .requestApplyLocalMutations(`todos`, [ + { + mutationId: `electron-unknown-leader-local`, + type: `insert`, + key: `electron-unknown-leader-local`, + value: { id: `electron-unknown-leader-local` }, + }, + ]) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyLocalMutations:req`, + previousLeaderId: null, + previousTerm: null, + currentLeaderId: null, + currentTerm: null, + cause: transportError, + } satisfies Partial), + }) + expect(failedTransport).toHaveBeenCalledTimes(1) + expect(attemptedMutations).toEqual([ + [ + { + mutationId: `electron-unknown-leader-local`, + type: `insert`, + key: `electron-unknown-leader-local`, + value: { id: `electron-unknown-leader-local` }, + }, + ], + ]) + }) + + it(`binds committed transaction owners to each resolved collection adapter`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + const requests: Array = [] + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-per-collection-adapters`, + }) + registerCleanup(() => coordinator.dispose()) + const persistence = createElectronSQLitePersistence({ + coordinator, + invoke: (_channel, request) => { + requests.push(structuredClone(request)) + if (request.method === `getStreamPosition`) { + return Promise.resolve({ + v: ELECTRON_PERSISTENCE_PROTOCOL_VERSION, + requestId: request.requestId, + method: request.method, + ok: true, + result: { + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + } + if (request.method === `applyCommittedTx`) { + return Promise.resolve({ + v: ELECTRON_PERSISTENCE_PROTOCOL_VERSION, + requestId: request.requestId, + method: request.method, + ok: true, + result: null, + }) + } + throw new Error(`unexpected method ${request.method}`) + }, + }) + + persistence.resolvePersistenceForCollection?.({ + collectionId: `alpha`, + mode: `sync-present`, + schemaVersion: 1, + }) + persistence.resolvePersistenceForCollection?.({ + collectionId: `beta`, + mode: `sync-present`, + schemaVersion: 2, + }) + coordinator.subscribe(`alpha`, () => {}) + coordinator.subscribe(`beta`, () => {}) + await waitForLeadership(coordinator, `alpha`) + await waitForLeadership(coordinator, `beta`) + + await coordinator.requestApplyCommittedTx(`alpha`, { + txId: `alpha-tx`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }) + await coordinator.requestApplyCommittedTx(`beta`, { + txId: `beta-tx`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }) + + expect( + requests + .filter((request) => request.method === `applyCommittedTx`) + .map((request) => ({ + collectionId: request.collectionId, + resolution: request.resolution, + txId: request.payload.tx.txId, + })), + ).toEqual([ + { + collectionId: `alpha`, + resolution: { mode: `sync-present`, schemaVersion: 1 }, + txId: `alpha-tx`, + }, + { + collectionId: `beta`, + resolution: { mode: `sync-present`, schemaVersion: 2 }, + txId: `beta-tx`, + }, + ]) + }) + + it(`coalesces and replays the exact committed success on the same leader`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + let releaseApply = (): void => {} + const applyGate = new Promise((resolve) => { + releaseApply = resolve + }) + let applyCalls = 0 + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-committed-envelope-replay`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: async () => { + applyCalls++ + await applyGate + }, + ensureIndex: () => Promise.resolve(), + }, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.subscribe(`todos`, () => {}) + await waitForLeadership(coordinator, `todos`) + + const handleApplyCommittedTx = ( + coordinator as unknown as { + handleApplyCommittedTx: ( + collectionId: string, + request: ApplyCommittedTxRequest, + ) => Promise + } + ).handleApplyCommittedTx.bind(coordinator) + const request: ApplyCommittedTxRequest = { + type: `rpc:applyCommittedTx:req`, + rpcId: `first-rpc`, + envelopeId: `stable-envelope`, + tx: { + txId: `stable-tx`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }, + } + + const first = handleApplyCommittedTx(`todos`, request) + const duplicate = handleApplyCommittedTx(`todos`, { + ...request, + rpcId: `duplicate-rpc`, + }) + await Promise.resolve() + expect(applyCalls).toBe(1) + releaseApply() + expect(await Promise.all([first, duplicate])).toMatchObject([ + { ok: true, rpcId: `first-rpc` }, + { ok: true, rpcId: `duplicate-rpc` }, + ]) + + await expect( + handleApplyCommittedTx(`todos`, { + ...request, + rpcId: `replay-rpc`, + }), + ).resolves.toMatchObject({ ok: true, rpcId: `replay-rpc` }) + expect(applyCalls).toBe(1) + }) + + it(`replays the exact local-mutation success on the same leader`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + let applyCalls = 0 + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-local-envelope-replay`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: () => { + applyCalls++ + return Promise.resolve() + }, + ensureIndex: () => Promise.resolve(), + }, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.subscribe(`todos`, () => {}) + await waitForLeadership(coordinator, `todos`) + + const handleApplyLocalMutations = ( + coordinator as unknown as { + handleApplyLocalMutations: ( + collectionId: string, + request: ApplyLocalMutationsRequest, + ) => Promise + } + ).handleApplyLocalMutations.bind(coordinator) + const request: ApplyLocalMutationsRequest = { + type: `rpc:applyLocalMutations:req`, + rpcId: `first-local-rpc`, + envelopeId: `stable-local-envelope`, + mutations: [ + { + mutationId: `stable-local-mutation`, + type: `insert`, + key: `stable-local-row`, + value: { id: `stable-local-row` }, + }, + ], + } + + const first = await handleApplyLocalMutations(`todos`, request) + const replay = await handleApplyLocalMutations(`todos`, { + ...request, + rpcId: `replayed-local-rpc`, + }) + + expect([first, replay]).toEqual([ + { + type: `rpc:applyLocalMutations:res`, + rpcId: `first-local-rpc`, + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: [`stable-local-mutation`], + }, + { + type: `rpc:applyLocalMutations:res`, + rpcId: `replayed-local-rpc`, + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: [`stable-local-mutation`], + }, + ]) + expect(applyCalls).toBe(1) + }) + + it(`classifies Electron durability failures on local and follower routes`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + const persistenceError = Object.assign(new Error(`electron disk failed`), { + code: `SQLITE_IOERR`, + path: [`electron`, `todos`], + }) + const adapter = createElectronCoordinatorTestAdapter() + adapter.applyCommittedTx = vi.fn().mockRejectedValue(persistenceError) + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-durability-classification`, + adapter, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.subscribe(`todos`, () => {}) + await waitForLeadership(coordinator, `todos`) + + const responses: Array = [] + const internals = coordinator as unknown as { + channel: { postMessage: (message: unknown) => void } + handleRPCRequest: ( + collectionId: string, + request: ApplyLocalMutationsRequest | ApplyCommittedTxRequest, + requesterId: string, + ) => Promise + } + internals.channel.postMessage = (message) => { + responses.push((message as { payload: unknown }).payload) + } + + await internals.handleRPCRequest( + `todos`, + { + type: `rpc:applyLocalMutations:req`, + rpcId: `remote-local-failure-rpc`, + envelopeId: `remote-local-failure-envelope`, + mutations: [], + }, + `remote-electron-follower`, + ) + await internals.handleRPCRequest( + `todos`, + { + type: `rpc:applyCommittedTx:req`, + rpcId: `remote-committed-failure-rpc`, + envelopeId: `remote-committed-failure-envelope`, + tx: { + txId: `remote-committed-failure`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }, + }, + `remote-electron-follower`, + ) + + expect(responses).toEqual([ + { + type: `rpc:applyLocalMutations:res`, + rpcId: `remote-local-failure-rpc`, + ok: false, + code: `PERSISTENCE_ERROR`, + error: expect.stringContaining(persistenceError.message), + sourceCode: `SQLITE_IOERR`, + path: [`electron`, `todos`], + }, + { + type: `rpc:applyCommittedTx:res`, + rpcId: `remote-committed-failure-rpc`, + ok: false, + code: `PERSISTENCE_ERROR`, + error: expect.stringContaining(persistenceError.message), + sourceCode: `SQLITE_IOERR`, + path: [`electron`, `todos`], + }, + ]) + + await expect( + coordinator.requestApplyLocalMutations(`todos`, []), + ).rejects.toMatchObject({ + name: `PersistedCollectionDurabilityError`, + code: `SQLITE_IOERR`, + path: [`electron`, `todos`], + cause: persistenceError, + } satisfies Partial) + await expect( + coordinator.requestApplyCommittedTx(`todos`, { + txId: `local-committed-failure`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }), + ).rejects.toMatchObject({ + name: `PersistedCollectionDurabilityError`, + code: `SQLITE_IOERR`, + path: [`electron`, `todos`], + cause: persistenceError, + } satisfies Partial) + expect(adapter.applyCommittedTx).toHaveBeenCalledTimes(4) + }) + + it(`preserves a held committed apply without retaining or publishing after disposal`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + let applyEntered = false + let releaseApply = (): void => {} + const applyGate = new Promise((resolve) => { + releaseApply = resolve + }) + const durableTransactions: Array = [] + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-dispose-held-committed`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: async (_collectionId, tx) => { + applyEntered = true + await applyGate + durableTransactions.push(structuredClone(tx)) + }, + ensureIndex: () => Promise.resolve(), + }, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.subscribe(`todos`, () => {}) + await waitForLeadership(coordinator, `todos`) + + const internals = coordinator as unknown as { + handleApplyCommittedTx: ( + collectionId: string, + request: ApplyCommittedTxRequest, + ) => Promise + appliedCommittedTxEnvelopes: Map + inFlightCommittedTxEnvelopes: Map + } + const outcomePromise = internals + .handleApplyCommittedTx(`todos`, { + type: `rpc:applyCommittedTx:req`, + rpcId: `held-committed-rpc`, + envelopeId: `held-committed-envelope`, + tx: { + txId: `held-committed-tx`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }, + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + try { + await vi.waitFor(() => expect(applyEntered).toBe(true)) + coordinator.dispose() + releaseApply() + const outcome = await outcomePromise + + expect(outcome).toEqual({ + response: { + type: `rpc:applyCommittedTx:res`, + rpcId: `held-committed-rpc`, + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + }, + }) + expect(durableTransactions).toHaveLength(1) + expect(durableTransactions[0]?.txId).toBe(`held-committed-tx`) + expect({ + completed: internals.appliedCommittedTxEnvelopes.size, + inFlight: internals.inFlightCommittedTxEnvelopes.size, + }).toEqual({ completed: 0, inFlight: 0 }) + } finally { + releaseApply() + coordinator.dispose() + } + }) + + it(`preserves a held local apply without retaining or publishing after disposal`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + let applyEntered = false + let releaseApply = (): void => {} + const applyGate = new Promise((resolve) => { + releaseApply = resolve + }) + const durableTransactions: Array = [] + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-dispose-held-local`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: async (_collectionId, tx) => { + applyEntered = true + await applyGate + durableTransactions.push(structuredClone(tx)) + }, + ensureIndex: () => Promise.resolve(), + }, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.subscribe(`todos`, () => {}) + await waitForLeadership(coordinator, `todos`) + + const internals = coordinator as unknown as { + handleApplyLocalMutations: ( + collectionId: string, + request: ApplyLocalMutationsRequest, + ) => Promise + appliedEnvelopeIds: Map + } + const outcomePromise = internals + .handleApplyLocalMutations(`todos`, { + type: `rpc:applyLocalMutations:req`, + rpcId: `held-local-rpc`, + envelopeId: `held-local-envelope`, + mutations: [ + { + mutationId: `held-local-mutation`, + type: `insert`, + key: `held-local-row`, + value: { id: `held-local-row` }, + }, + ], + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + try { + await vi.waitFor(() => expect(applyEntered).toBe(true)) + coordinator.dispose() + releaseApply() + const outcome = await outcomePromise + + expect(outcome).toEqual({ + response: { + type: `rpc:applyLocalMutations:res`, + rpcId: `held-local-rpc`, + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: [`held-local-mutation`], + }, + }) + expect(durableTransactions).toHaveLength(1) + expect(durableTransactions[0]?.mutations).toEqual([ + { + type: `insert`, + key: `held-local-row`, + value: { id: `held-local-row` }, + }, + ]) + expect(internals.appliedEnvelopeIds.size).toBe(0) + } finally { + releaseApply() + coordinator.dispose() + } + }) + + it(`validates remote-subset values before Electron leader completion`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + const leader = new ElectronCollectionCoordinator({ + dbName: `electron-subset-wire-leader`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: () => Promise.resolve(), + ensureIndex: () => Promise.resolve(), + }, + }) + registerCleanup(() => leader.dispose()) + leader.subscribe(`todos`, () => {}) + await waitForLeadership(leader, `todos`) + + const invalid = electronSubsetWithNestedValue(() => {}) + await expect( + leader.requestEnsureRemoteSubset(`todos`, invalid), + ).rejects.toMatchObject({ + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1].value[1]`, + }) + }) + + it(`validates remote-subset values before Electron follower transport`, async () => { + const follower = new ElectronCollectionCoordinator({ + dbName: `electron-subset-wire-follower`, + }) + registerCleanup(() => follower.dispose()) + follower.isLeader = () => false + const followerInternals = follower as unknown as { + channel: { postMessage: (message: unknown) => void } + } + let subsetPosts = 0 + followerInternals.channel.postMessage = (message) => { + const payload = (message as { payload?: { type?: string } }).payload + if (payload?.type === `rpc:ensureRemoteSubset:req`) subsetPosts++ + structuredClone(message) + } + const outcome = await follower + .requestEnsureRemoteSubset( + `todos`, + electronSubsetWithNestedValue(() => {}), + ) + .then( + () => ({ ok: true as const }), + (error: unknown) => ({ ok: false as const, error }), + ) + expect({ outcome, subsetPosts }).toMatchObject({ + outcome: { + ok: false, + error: { + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1].value[1]`, + }, + }, + subsetPosts: 0, + }) + }) + + it(`rejects a sparse function argument at its exact Electron wire path`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-wire-hole`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const args = [new IR.PropRef([`todos`, `status`])] + args.length = 2 + + try { + await expect( + coordinator.requestEnsureRemoteSubset(`todos`, { + where: new IR.Func(`eq`, args), + }), + ).rejects.toMatchObject({ + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1]`, + }) + expect(owner).not.toHaveBeenCalled() + } finally { + unregisterOwner() + } + }) + + it(`holds same-stack Electron subset reentry behind the original owner load`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-same-stack`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const options: LoadSubsetOptions = { limit: 1 } + let releaseLoad = (): void => {} + const loadGate = new Promise((resolve) => { + releaseLoad = resolve + }) + let duplicate: Promise | undefined + let didReenter = false + const owner = Object.assign( + vi.fn(() => { + if (!didReenter) { + didReenter = true + duplicate = coordinator.requestEnsureRemoteSubset(`todos`, options) + } + return loadGate + }), + { unloadSubset: vi.fn(), onError: vi.fn() }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + + try { + let firstSettled = false + let duplicateSettled = false + const first = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then(() => { + firstSettled = true + }) + await vi.waitFor(() => expect(duplicate).toBeDefined()) + const duplicateResult = duplicate!.then(() => { + duplicateSettled = true + }) + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect({ + ownerCalls: owner.mock.calls.length, + firstSettled, + duplicateSettled, + }).toEqual({ + ownerCalls: 1, + firstSettled: false, + duplicateSettled: false, + }) + + releaseLoad() + await Promise.all([first, duplicateResult]) + } finally { + releaseLoad() + unregisterOwner() + } + }) + + it(`compacts a terminal same-stack Electron release after the real owner load finishes`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-terminal-release`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const options: LoadSubsetOptions = { limit: 1 } + let releaseLoad = (): void => {} + const loadGate = new Promise((resolve) => { + releaseLoad = resolve + }) + let release: Promise | undefined + const events: Array = [] + const owner = Object.assign( + vi.fn(() => { + events.push(`load`) + release = coordinator.requestReleaseRemoteSubset(`todos`, options) + return loadGate + }), + { + unloadSubset: vi.fn(() => { + events.push(`unload`) + }), + onError: vi.fn(), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const internals = coordinator as unknown as { + outboundRemoteSubsetAcquisitions: Map + inboundRemoteSubsetAcquisitions: Map> + } + + try { + let releaseSettled = false + const load = coordinator.requestEnsureRemoteSubset(`todos`, options) + await vi.waitFor(() => expect(release).toBeDefined()) + const terminalRelease = release!.then(() => { + releaseSettled = true + }) + await Promise.resolve() + expect({ events: [...events], releaseSettled }).toEqual({ + events: [`load`], + releaseSettled: false, + }) + + releaseLoad() + await Promise.all([load, terminalRelease]) + const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() + expect({ + events, + outbound: internals.outboundRemoteSubsetAcquisitions.size, + inbound: internals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + events: [`load`, `unload`], + outbound: 0, + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + releaseLoad() + unregisterOwner() + coordinator.dispose() + } + }) + + it(`releases a transferred Electron lease whose initial load rejected`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-rejected-transfer`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const loadError = new Error(`electron transferred load failed`) + const ownerErrors: Array = [] + const owner = Object.assign( + vi.fn((_options: TransportedLoadSubsetOptions) => + Promise.reject(loadError), + ), + { + unloadSubset: vi.fn( + (_options: TransportedLoadSubsetOptions) => undefined, + ), + onError: (error: unknown) => ownerErrors.push(error), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { offset: 20 } + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + inboundRemoteSubsetAcquisitions: Map> + } + + try { + const ensureError = await coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then( + () => undefined, + (error: unknown) => error, + ) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + await new Promise((resolve) => setTimeout(resolve, 0)) + const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() + + expect(ensureError).toBe(loadError) + expect(owner).toHaveBeenCalledTimes(1) + expect(owner.unloadSubset).toHaveBeenCalledTimes(1) + expect(owner.unloadSubset.mock.calls[0]?.[0]).toBe( + owner.mock.calls[0]?.[0], + ) + expect(ownerErrors).toEqual([loadError]) + expect(unhandled).toEqual([]) + expect({ + inbound: internals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`keeps an Electron release tombstone when a transferred load rejects concurrently`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-concurrent-rejected-transfer`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const loadError = new Error(`electron concurrent transferred load failed`) + let rejectLoad = (_error: unknown): void => {} + const loadGate = new Promise((_resolve, reject) => { + rejectLoad = reject + }) + const ownerErrors: Array = [] + const owner = Object.assign( + vi.fn((_options: TransportedLoadSubsetOptions) => loadGate), + { + unloadSubset: vi.fn( + (_options: TransportedLoadSubsetOptions) => undefined, + ), + onError: (error: unknown) => ownerErrors.push(error), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { offset: 21 } + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + nodeId: string + outboundRemoteSubsetAcquisitions: Map + inboundRemoteSubsetAcquisitions: Map> + handleEnsureRemoteSubset: ( + collectionId: string, + request: { + type: `rpc:ensureRemoteSubset:req` + rpcId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + }, + requesterId: string, + ) => Promise<{ ok: boolean }> + } + + try { + const ensure = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then( + () => ({ status: `fulfilled` as const }), + (error: unknown) => ({ status: `rejected` as const, error }), + ) + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(1)) + const [outbound] = internals.outboundRemoteSubsetAcquisitions.values() + const release = coordinator + .requestReleaseRemoteSubset(`todos`, options) + .then( + () => ({ status: `fulfilled` as const }), + (error: unknown) => ({ status: `rejected` as const, error }), + ) + rejectLoad(loadError) + const outcomes = await Promise.all([ensure, release]) + const duplicate = await internals + .handleEnsureRemoteSubset( + `todos`, + { + type: `rpc:ensureRemoteSubset:req`, + rpcId: `delayed-electron-duplicate`, + acquisitionId: outbound!.acquisitionId, + options: owner.mock.calls[0]![0], + }, + internals.nodeId, + ) + .then( + (response) => ({ status: `fulfilled` as const, response }), + (error: unknown) => ({ status: `rejected` as const, error }), + ) + await new Promise((resolve) => setTimeout(resolve, 0)) + const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() + + expect(outcomes).toEqual([ + { status: `rejected`, error: loadError }, + { status: `fulfilled` }, + ]) + expect(duplicate).toEqual({ + status: `fulfilled`, + response: expect.objectContaining({ ok: true }), + }) + expect(owner).toHaveBeenCalledTimes(1) + expect(owner.unloadSubset).toHaveBeenCalledTimes(1) + expect(ownerErrors).toEqual([loadError]) + expect(unhandled).toEqual([]) + expect({ + inbound: internals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + rejectLoad(loadError) + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`posts only the validated remote-subset wire domain from Electron`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-wire-projection`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => false + + const shared = { label: `shared` } + const cycle: Record = { label: `cycle` } + cycle.self = cycle + const options: LoadSubsetOptions = { + where: new IR.Func(`eq`, [ + new IR.PropRef([`todos`, `payload`]), + new IR.Value({ + date: new Date(`2026-09-16T12:34:56.000Z`), + regexp: /wire/giu, + bytes: new Uint8Array([0, 127, 255]), + map: new Map([[shared, cycle]]), + set: new Set([shared, cycle]), + first: shared, + second: shared, + cycle, + }), + ]), + signal: new AbortController().signal, + subscription: { + on: () => () => {}, + } as unknown as Subscription, + } + const internals = coordinator as unknown as { + channel: { postMessage: (message: unknown) => void } + } + let posted: unknown + internals.channel.postMessage = (message) => { + posted = structuredClone(message) + } + + const pending = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .catch((error: unknown) => error) + await Promise.resolve() + + const wireOptions = ( + posted as + | { + payload?: { + type?: string + options?: LoadSubsetOptions + } + } + | undefined + )?.payload?.options + expect( + (posted as { payload?: { type?: string } } | undefined)?.payload?.type, + ).toBe(`rpc:ensureRemoteSubset:req`) + expect(wireOptions).not.toHaveProperty(`signal`) + expect(wireOptions).not.toHaveProperty(`subscription`) + const wireValue = ( + wireOptions?.where as unknown as + | { + args: Array<{ + value?: { + date: Date + regexp: RegExp + bytes: Uint8Array + map: Map + set: Set + first: unknown + second: unknown + cycle: Record + } + }> + } + | undefined + )?.args[1]?.value + expect(wireValue?.date.getTime()).toBe( + new Date(`2026-09-16T12:34:56.000Z`).getTime(), + ) + expect(wireValue?.regexp).toEqual(/wire/giu) + expect(Array.from(wireValue?.bytes ?? [])).toEqual([0, 127, 255]) + expect(wireValue?.first).toBe(wireValue?.second) + expect([...wireValue!.map.keys()]).toEqual([wireValue?.first]) + expect(wireValue?.map.get(wireValue.first)).toBe(wireValue?.cycle) + expect(wireValue?.set.has(wireValue.first)).toBe(true) + expect(wireValue?.cycle.self).toBe(wireValue?.cycle) + + coordinator.dispose() + await pending + }) + + it(`loads and releases exact Electron remote-subset acquisitions`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-owner-routing`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + + const loads: Array = [] + const unloads: Array = [] + const owner = Object.assign( + vi.fn((options: TransportedLoadSubsetOptions) => { + loads.push(options) + }), + { + unloadSubset: vi.fn((options: TransportedLoadSubsetOptions) => { + unloads.push(options) + }), + onError: vi.fn(), + }, + ) satisfies RemoteSubsetOwner + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const first = { limit: 2, offset: 1 } + const sibling = { limit: 2, offset: 1 } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, first) + await coordinator.requestEnsureRemoteSubset(`todos`, first) + await coordinator.requestEnsureRemoteSubset(`todos`, sibling) + + expect(loads).toEqual([ + { limit: 2, offset: 1 }, + { limit: 2, offset: 1 }, + ]) + + await coordinator.requestReleaseRemoteSubset(`todos`, first) + await coordinator.requestReleaseRemoteSubset(`todos`, first) + expect(unloads).toEqual([{ limit: 2, offset: 1 }]) + + await coordinator.requestReleaseRemoteSubset(`todos`, sibling) + expect(unloads).toEqual([ + { limit: 2, offset: 1 }, + { limit: 2, offset: 1 }, + ]) + } finally { + unregisterOwner() + } + }) + + it(`scopes a reused subset options object to each Electron collection`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-owner-collection-identity`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const alpha = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const beta = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterAlpha = coordinator.registerRemoteSubsetOwner( + `alpha`, + alpha, + ) + const unregisterBeta = coordinator.registerRemoteSubsetOwner(`beta`, beta) + const shared: LoadSubsetOptions = { limit: 2, offset: 1 } + + try { + await coordinator.requestEnsureRemoteSubset(`alpha`, shared) + await coordinator.requestEnsureRemoteSubset(`beta`, shared) + await coordinator.requestEnsureRemoteSubset(`alpha`, shared) + + const afterAcquire = { + alphaLoads: alpha.mock.calls.length, + betaLoads: beta.mock.calls.length, + } + await coordinator.requestReleaseRemoteSubset(`alpha`, shared) + await coordinator.requestReleaseRemoteSubset(`beta`, shared) + + expect({ + afterAcquire, + alphaUnloads: alpha.unloadSubset.mock.calls.length, + betaUnloads: beta.unloadSubset.mock.calls.length, + }).toEqual({ + afterAcquire: { alphaLoads: 1, betaLoads: 1 }, + alphaUnloads: 1, + betaUnloads: 1, + }) + } finally { + unregisterAlpha() + unregisterBeta() + } + }) + + it(`replays after an earlier Electron leader responds behind a new heartbeat`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + const dbName = `electron-subset-stale-response-${process.pid}` + const leader = new ElectronCollectionCoordinator({ + dbName, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: () => Promise.resolve(), + ensureIndex: () => Promise.resolve(), + }, + }) + const follower = new ElectronCollectionCoordinator({ + dbName, + }) + ;( + follower as unknown as { acquireLeadership: () => Promise } + ).acquireLeadership = async () => {} + registerCleanup(() => leader.dispose()) + registerCleanup(() => follower.dispose()) + let releaseOwner = (): void => {} + const owner = Object.assign( + vi.fn( + () => + new Promise((resolve) => { + releaseOwner = resolve + }), + ), + { unloadSubset: vi.fn(), onError: vi.fn() }, + ) satisfies RemoteSubsetOwner + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + leader.subscribe(`todos`, () => {}) + await waitForLeadership(leader, `todos`) + + const leaderInternals = leader as unknown as { + nodeId: string + channel: BroadcastChannel + } + const followerInternals = follower as unknown as { + onChannelMessage: (message: unknown) => void + channel: BroadcastChannel + outboundRemoteSubsetAcquisitions: Map< + string, + { acquiredLeaderId: string | null } + > + } + const originalLeaderPost = leaderInternals.channel.postMessage.bind( + leaderInternals.channel, + ) + let heldResponse: unknown + leaderInternals.channel.postMessage = (message: unknown) => { + const type = (message as { payload?: { type?: string } }).payload?.type + if (type === `rpc:ensureRemoteSubset:res`) { + heldResponse = structuredClone(message) + return + } + originalLeaderPost(message) + } + const originalFollowerPost = followerInternals.channel.postMessage.bind( + followerInternals.channel, + ) + let requestPosts = 0 + followerInternals.channel.postMessage = (message: unknown) => { + if ( + (message as { payload?: { type?: string } }).payload?.type === + `rpc:ensureRemoteSubset:req` + ) { + requestPosts++ + } + originalFollowerPost(message) + } + + try { + const request = follower.requestEnsureRemoteSubset(`todos`, { limit: 1 }) + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(1)) + followerInternals.onChannelMessage({ + v: 1, + dbName, + collectionId: `todos`, + senderId: `replacement-electron-leader`, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term: 2, + leaderId: `replacement-electron-leader`, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + releaseOwner() + await vi.waitFor(() => expect(heldResponse).toBeDefined()) + leader.isLeader = () => false + followerInternals.onChannelMessage(heldResponse) + await request + await new Promise((resolve) => setTimeout(resolve, 0)) + + const [acquisition] = + followerInternals.outboundRemoteSubsetAcquisitions.values() + expect({ + requestPosts, + acquiredLeaderId: acquisition?.acquiredLeaderId, + }).toEqual({ + requestPosts: 2, + acquiredLeaderId: leaderInternals.nodeId, + }) + } finally { + releaseOwner() + unregisterOwner() + leader.dispose() + follower.dispose() + } + }) + + it(`replays a held Electron lease after an A to B to A leader cycle`, async () => { + const dbName = `electron-subset-a-b-a` + const leader = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + const follower = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => leader.dispose()) + registerCleanup(() => follower.dispose()) + leader.isLeader = () => true + follower.isLeader = () => false + let releaseFirstLoad = (): void => {} + const firstLoadGate = new Promise((resolve) => { + releaseFirstLoad = resolve + }) + const owner = Object.assign( + vi.fn().mockImplementationOnce(() => firstLoadGate), + { unloadSubset: vi.fn(), onError: vi.fn() }, + ) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + const leaderInternals = leader as unknown as { + nodeId: string + channel: BroadcastChannel + inboundRemoteSubsetAcquisitions: Map + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + } + const followerInternals = follower as unknown as { + onChannelMessage: (message: unknown) => void + channel: BroadcastChannel + acquireLeadership: () => Promise + outboundRemoteSubsetAcquisitions: Map< + string, + { acquiredLeaderId: string | null } + > + } + followerInternals.acquireLeadership = async () => {} + const originalLeaderPost = leaderInternals.channel.postMessage.bind( + leaderInternals.channel, + ) + let heldResponse: unknown + leaderInternals.channel.postMessage = (message) => { + const type = (message as { payload?: { type?: string } }).payload?.type + if (type === `rpc:ensureRemoteSubset:res` && heldResponse === undefined) { + heldResponse = structuredClone(message) + return + } + originalLeaderPost(message) + } + const originalFollowerPost = followerInternals.channel.postMessage.bind( + followerInternals.channel, + ) + let requestPosts = 0 + const wireRequests: Array<{ + acquisitionId: string + rpcId: string + }> = [] + followerInternals.channel.postMessage = (message) => { + const payload = ( + message as { + payload?: { + type?: string + acquisitionId?: string + rpcId?: string + } + } + ).payload + if (payload?.type === `rpc:ensureRemoteSubset:req`) { + requestPosts++ + wireRequests.push({ + acquisitionId: payload.acquisitionId!, + rpcId: payload.rpcId!, + }) + } + originalFollowerPost(message) + } + const heartbeat = (leaderId: string, term: number) => ({ + v: 1, + dbName, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + const options: LoadSubsetOptions = { limit: 1 } + + try { + const request = follower.requestEnsureRemoteSubset(`todos`, options) + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(1)) + followerInternals.onChannelMessage( + heartbeat(`replacement-electron-leader`, 2), + ) + leaderInternals.releaseInboundRemoteSubsetAcquisitions(`todos`) + followerInternals.onChannelMessage(heartbeat(leaderInternals.nodeId, 3)) + releaseFirstLoad() + await vi.waitFor(() => expect(heldResponse).toBeDefined()) + followerInternals.onChannelMessage(heldResponse) + await request + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(2)) + + const [acquisition] = + followerInternals.outboundRemoteSubsetAcquisitions.values() + expect({ + requestPosts, + loads: owner.mock.calls.length, + unloads: owner.unloadSubset.mock.calls.length, + inbound: leaderInternals.inboundRemoteSubsetAcquisitions.size, + acquiredLeaderId: acquisition?.acquiredLeaderId, + }).toEqual({ + requestPosts: 2, + loads: 2, + unloads: 1, + inbound: 1, + acquiredLeaderId: leaderInternals.nodeId, + }) + expect(wireRequests).toHaveLength(2) + expect(wireRequests[0]!.acquisitionId).not.toBe(``) + expect(wireRequests[1]!.acquisitionId).toBe( + wireRequests[0]!.acquisitionId, + ) + expect(wireRequests[1]!.rpcId).not.toBe(wireRequests[0]!.rpcId) + + await follower.requestReleaseRemoteSubset(`todos`, options) + expect(owner.unloadSubset).toHaveBeenCalledTimes(2) + } finally { + releaseFirstLoad() + unregisterOwner() + leader.dispose() + follower.dispose() + } + }) + + it(`rebinds a live Electron acquisition when its owner is replaced`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-owner-replacement`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + let releaseFirstLoad = (): void => {} + const firstLoadGate = new Promise((resolve) => { + releaseFirstLoad = resolve + }) + const firstOwner = Object.assign( + vi.fn(() => firstLoadGate), + { + unloadSubset: vi.fn(), + onError: vi.fn(), + }, + ) + const secondOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn( + (_options: TransportedLoadSubsetOptions) => undefined, + ), + onError: vi.fn(), + }) + const unregisterFirst = coordinator.registerRemoteSubsetOwner( + `todos`, + firstOwner, + ) + let unregisterSecond: (() => void) | undefined + const options: LoadSubsetOptions = { limit: 1 } + + try { + const initialAcquire = coordinator.requestEnsureRemoteSubset( + `todos`, + options, + ) + await vi.waitFor(() => expect(firstOwner).toHaveBeenCalledTimes(1)) + unregisterFirst() + unregisterSecond = coordinator.registerRemoteSubsetOwner( + `todos`, + secondOwner, + ) + releaseFirstLoad() + await initialAcquire + await vi.waitFor(() => expect(secondOwner).toHaveBeenCalledTimes(1)) + + expect(firstOwner.unloadSubset).toHaveBeenCalledTimes(1) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + expect(secondOwner.unloadSubset).toHaveBeenCalledTimes(1) + } finally { + releaseFirstLoad() + unregisterFirst() + unregisterSecond?.() + coordinator.dispose() + } + }) + + it(`rebinds a live remote Electron follower lease when the same leader replaces its owner`, async () => { + const dbName = `electron-remote-owner-replacement` + const leader = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + const follower = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => leader.dispose()) + registerCleanup(() => follower.dispose()) + leader.isLeader = () => true + follower.isLeader = () => false + let releaseFirstLoad = (): void => {} + const firstLoadGate = new Promise((resolve) => { + releaseFirstLoad = resolve + }) + const firstOwner = Object.assign( + vi.fn(() => firstLoadGate), + { + unloadSubset: vi.fn(), + onError: vi.fn(), + }, + ) + const secondOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterFirst = leader.registerRemoteSubsetOwner( + `todos`, + firstOwner, + ) + let unregisterSecond: (() => void) | undefined + const options: LoadSubsetOptions = { limit: 1 } + const followerInternals = follower as unknown as { + channel: BroadcastChannel + } + const originalFollowerPost = followerInternals.channel.postMessage.bind( + followerInternals.channel, + ) + let requestPosts = 0 + followerInternals.channel.postMessage = (message) => { + if ( + (message as { payload?: { type?: string } }).payload?.type === + `rpc:ensureRemoteSubset:req` + ) { + requestPosts++ + } + originalFollowerPost(message) + } + + try { + const initialAcquire = follower.requestEnsureRemoteSubset( + `todos`, + options, + ) + await vi.waitFor(() => expect(firstOwner).toHaveBeenCalledTimes(1)) + unregisterFirst() + unregisterSecond = leader.registerRemoteSubsetOwner(`todos`, secondOwner) + releaseFirstLoad() + await initialAcquire + await vi.waitFor(() => expect(secondOwner).toHaveBeenCalledTimes(1)) + + expect({ + requestPosts, + firstUnloads: firstOwner.unloadSubset.mock.calls.length, + }).toEqual({ requestPosts: 1, firstUnloads: 1 }) + await follower.requestReleaseRemoteSubset(`todos`, options) + expect(secondOwner.unloadSubset).toHaveBeenCalledTimes(1) + } finally { + releaseFirstLoad() + unregisterFirst() + unregisterSecond?.() + leader.dispose() + follower.dispose() + } + }) + + it(`rebinds remote Electron demand when its replacement owner registers after release settlement`, async () => { + const dbName = `electron-remote-owner-settled-replacement` + const leader = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + const follower = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => leader.dispose()) + registerCleanup(() => follower.dispose()) + leader.isLeader = () => true + follower.isLeader = () => false + const firstOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const secondOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterFirst = leader.registerRemoteSubsetOwner( + `todos`, + firstOwner, + ) + let unregisterSecond: (() => void) | undefined + const options: LoadSubsetOptions = { offset: 22 } + const leaderInternals = leader as unknown as { + inboundRemoteSubsetAcquisitions: Map> + } + + try { + await follower.requestEnsureRemoteSubset(`todos`, options) + unregisterFirst() + await vi.waitFor(() => + expect(firstOwner.unloadSubset).toHaveBeenCalledTimes(1), + ) + await new Promise((resolve) => setTimeout(resolve, 0)) + + unregisterSecond = leader.registerRemoteSubsetOwner(`todos`, secondOwner) + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect({ + firstLoads: firstOwner.mock.calls.length, + firstUnloads: firstOwner.unloadSubset.mock.calls.length, + secondLoads: secondOwner.mock.calls.length, + inbound: leaderInternals.inboundRemoteSubsetAcquisitions.size, + }).toEqual({ + firstLoads: 1, + firstUnloads: 1, + secondLoads: 1, + inbound: 1, + }) + + await follower.requestReleaseRemoteSubset(`todos`, options) + const [terminal] = + leaderInternals.inboundRemoteSubsetAcquisitions.values() + expect(secondOwner.unloadSubset).toHaveBeenCalledTimes(1) + expect(secondOwner.unloadSubset.mock.calls[0]?.[0]).toBe( + secondOwner.mock.calls[0]?.[0], + ) + expect({ + inbound: leaderInternals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + unregisterFirst() + unregisterSecond?.() + leader.dispose() + follower.dispose() + } + }) + + it(`reacquires the same Electron lease after leadership retirement`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-leadership-return`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const internals = coordinator as unknown as { + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + } + const options: LoadSubsetOptions = { limit: 1 } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, options) + internals.releaseInboundRemoteSubsetAcquisitions(`todos`) + await vi.waitFor(() => + expect(owner.unloadSubset).toHaveBeenCalledTimes(1), + ) + + await coordinator.requestEnsureRemoteSubset(`todos`, options) + expect(owner).toHaveBeenCalledTimes(2) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + expect(owner.unloadSubset).toHaveBeenCalledTimes(2) + } finally { + unregisterOwner() + coordinator.dispose() + } + }) + + it(`starts independent Electron lease replays without sibling head-of-line blocking`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-parallel-replay`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 1 }) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 2 }) + + type Acquisition = { + collectionId: string + options: TransportedLoadSubsetOptions + acquiredLeaderId: string | null + } + const internals = coordinator as unknown as { + nodeId: string + collections: Map + outboundRemoteSubsetAcquisitions: Map + acquireRemoteSubset: (acquisition: Acquisition) => Promise + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + internals.collections.set(`todos`, { + isLeader: true, + leaderId: internals.nodeId, + }) + const acquisitions = [ + ...internals.outboundRemoteSubsetAcquisitions.values(), + ] + for (const acquisition of acquisitions) { + acquisition.acquiredLeaderId = `retired-electron-leader` + } + let releaseFirstReplay = (): void => {} + const firstReplayGate = new Promise((resolve) => { + releaseFirstReplay = resolve + }) + const starts: Array = [] + internals.acquireRemoteSubset = async (acquisition) => { + starts.push(acquisition.options.offset) + if (acquisition.options.offset === 1) await firstReplayGate + } + + try { + const replay = internals.replayRemoteSubsetAcquisitions(`todos`) + await Promise.resolve() + const startsBeforeFirstFinished = [...starts] + expect(startsBeforeFirstFinished).toEqual([1, 2]) + releaseFirstReplay() + await replay + } finally { + releaseFirstReplay() + unregisterOwner() + coordinator.dispose() + } + }) + + it(`reports a failed Electron replay once without self-retrying`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-replay-error`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const replayError = new Error(`replacement Electron owner is not ready`) + const ownerErrors: Array = [] + const owner = Object.assign( + vi + .fn() + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(replayError), + { + unloadSubset: vi.fn(), + onError: (error: unknown) => ownerErrors.push(error), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { limit: 1 } + await coordinator.requestEnsureRemoteSubset(`todos`, options) + type Acquisition = { + acquiredLeaderId: string | null + inFlight: Promise | null + } + const internals = coordinator as unknown as { + nodeId: string + collections: Map + outboundRemoteSubsetAcquisitions: Map + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + internals.collections.set(`todos`, { + isLeader: true, + leaderId: internals.nodeId, + }) + const [acquisition] = internals.outboundRemoteSubsetAcquisitions.values() + internals.releaseInboundRemoteSubsetAcquisitions(`todos`) + await vi.waitFor(() => expect(owner.unloadSubset).toHaveBeenCalledTimes(1)) + acquisition!.acquiredLeaderId = `retired-electron-leader` + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + + try { + await internals.replayRemoteSubsetAcquisitions(`todos`) + await new Promise((resolve) => setTimeout(resolve, 0)) + expect({ + attempts: owner.mock.calls.length - 1, + ownerErrors, + acquiredLeaderId: acquisition!.acquiredLeaderId, + inFlight: acquisition!.inFlight, + unhandled, + }).toEqual({ + attempts: 1, + ownerErrors: [replayError], + acquiredLeaderId: `retired-electron-leader`, + inFlight: null, + unhandled: [], + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`reports a failed remote Electron follower replay to its local owner once`, async () => { + const dbName = `electron-remote-replay-error` + const coordinator = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => false + const internals = coordinator as unknown as { + onChannelMessage: (message: unknown) => void + acquireLeadership: () => Promise + sendRPC: (collectionId: string, request: unknown) => Promise + outboundRemoteSubsetAcquisitions: Map< + string, + { acquiredLeaderId: string | null; inFlight: Promise | null } + > + } + internals.acquireLeadership = async () => {} + const heartbeat = (leaderId: string, term: number) => + internals.onChannelMessage({ + v: 1, + dbName, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + heartbeat(`remote-electron-a`, 1) + const ownerErrors: Array = [] + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + internals.sendRPC = vi.fn().mockResolvedValue({ + type: `rpc:ensureRemoteSubset:res`, + rpcId: `initial-electron`, + ok: true, + leaderId: `remote-electron-a`, + }) + const options: LoadSubsetOptions = { limit: 1 } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, options) + const replayError = new Error(`remote Electron replay transport failed`) + const replayRPC = vi.fn().mockRejectedValue(replayError) + internals.sendRPC = replayRPC + heartbeat(`remote-electron-b`, 2) + await vi.waitFor(() => expect(replayRPC).toHaveBeenCalledTimes(1)) + await vi.waitFor(() => { + const [acquisition] = + internals.outboundRemoteSubsetAcquisitions.values() + expect(acquisition?.inFlight).toBeNull() + }) + + const [acquisition] = internals.outboundRemoteSubsetAcquisitions.values() + expect({ + ownerErrors, + attempts: replayRPC.mock.calls.length, + acquiredLeaderId: acquisition?.acquiredLeaderId, + retained: internals.outboundRemoteSubsetAcquisitions.size, + }).toEqual({ + ownerErrors: [replayError], + attempts: 1, + acquiredLeaderId: `remote-electron-a`, + retained: 1, + }) + } finally { + unregisterOwner() + coordinator.dispose() + } + }) + + it(`ignores a lower-term Electron heartbeat without replaying to its stale leader`, () => { + const dbName = `electron-stale-heartbeat` + const coordinator = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => false + const internals = coordinator as unknown as { + onChannelMessage: (message: unknown) => void + acquireLeadership: () => Promise + collections: Map + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + internals.acquireLeadership = async () => {} + const replay = vi.fn(() => Promise.resolve()) + internals.replayRemoteSubsetAcquisitions = replay + const heartbeat = (leaderId: string, term: number) => + internals.onChannelMessage({ + v: 1, + dbName, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + + heartbeat(`remote-electron-b`, 2) + replay.mockClear() + heartbeat(`retired-electron-a`, 1) + const state = internals.collections.get(`todos`) + + expect({ + leaderId: state?.leaderId, + latestTerm: state?.latestTerm, + replayCalls: replay.mock.calls.length, + }).toEqual({ + leaderId: `remote-electron-b`, + latestTerm: 2, + replayCalls: 0, + }) + }) + + it(`reports owner unload rejection while completing sibling Electron cleanup`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-owner-unload-error`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const unloadError = new Error(`electron owner unload failed`) + const ownerErrors: Array = [] + const unloadSubset = vi.fn((options: TransportedLoadSubsetOptions) => + options.offset === 1 ? Promise.reject(unloadError) : undefined, + ) + const owner = Object.assign(vi.fn(), { + unloadSubset, + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 1 }) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 2 }) + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + inboundRemoteSubsetAcquisitions: Map< + string, + { release: Promise | null } + > + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + } + + try { + internals.releaseInboundRemoteSubsetAcquisitions(`todos`) + const releases = [ + ...internals.inboundRemoteSubsetAcquisitions.values(), + ].map((acquisition) => acquisition.release!) + const outcomes = await Promise.allSettled(releases) + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect({ + outcomes: outcomes.map((outcome) => + outcome.status === `rejected` ? outcome.reason : outcome.status, + ), + unloadOffsets: unloadSubset.mock.calls.map( + ([options]) => options.offset, + ), + ownerErrors, + unhandled, + }).toEqual({ + outcomes: [unloadError, `fulfilled`], + unloadOffsets: [1, 2], + ownerErrors: [unloadError], + unhandled: [], + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`reports local Electron disposal unload rejection once and completes cleanup`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-disposal-unload-error`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const unloadError = new Error(`electron disposal unload failed`) + const ownerErrors: Array = [] + const unloadSubset = vi.fn((options: TransportedLoadSubsetOptions) => + options.offset === 23 ? Promise.reject(unloadError) : undefined, + ) + const owner = Object.assign(vi.fn(), { + unloadSubset, + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 23 }) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 24 }) + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + collections: Map + pendingRPCs: Map + remoteSubsetOwners: Map + outboundRemoteSubsetAcquisitions: Map + inboundRemoteSubsetAcquisitions: Map + } + + try { + coordinator.dispose() + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect({ + unloadOffsets: unloadSubset.mock.calls.map( + ([options]) => options.offset, + ), + ownerErrors, + unhandled, + collections: internals.collections.size, + pendingRPCs: internals.pendingRPCs.size, + owners: internals.remoteSubsetOwners.size, + outbound: internals.outboundRemoteSubsetAcquisitions.size, + inbound: internals.inboundRemoteSubsetAcquisitions.size, + }).toEqual({ + unloadOffsets: [23, 24], + ownerErrors: [unloadError], + unhandled: [], + collections: 0, + pendingRPCs: 0, + owners: 0, + outbound: 0, + inbound: 0, + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`rejects duplicate Electron remote-subset owners`, () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-owner-duplicate`, + }) + registerCleanup(() => coordinator.dispose()) + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) satisfies RemoteSubsetOwner + const duplicate = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) satisfies RemoteSubsetOwner + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + + try { + expect(() => + coordinator.registerRemoteSubsetOwner(`todos`, duplicate), + ).toThrowError( + expect.objectContaining({ + name: `DuplicateRemoteSubsetOwnerError`, + collectionId: `todos`, + }), + ) + } finally { + unregisterOwner() + } + }) + + it(`does not answer a held inbound request after its Electron leader is disposed`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + let applyEntered = false + let releaseApply = (): void => {} + const applyGate = new Promise((resolve) => { + releaseApply = resolve + }) + const durableTransactions: Array = [] + const leader = new ElectronCollectionCoordinator({ + dbName: `electron-dispose-held-inbound`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: async (_collectionId, tx) => { + applyEntered = true + await applyGate + durableTransactions.push(structuredClone(tx)) + }, + ensureIndex: () => Promise.resolve(), + }, + }) + const follower = new ElectronCollectionCoordinator({ + dbName: `electron-dispose-held-inbound`, + }) + // This fixture isolates the leader's post-disposal response boundary. Its + // immediate-grant lock shim cannot model a blocked follower election. + ;( + follower as unknown as { + acquireLeadership: () => Promise + } + ).acquireLeadership = async () => {} + registerCleanup(() => leader.dispose()) + registerCleanup(() => follower.dispose()) + leader.subscribe(`todos`, () => {}) + await waitForLeadership(leader, `todos`) + + const leaderInternals = leader as unknown as { + channel: BroadcastChannel + appliedCommittedTxEnvelopes: Map + inFlightCommittedTxEnvelopes: Map + } + const followerInternals = follower as unknown as { + sendRPCOnce: ( + collectionId: string, + request: ApplyCommittedTxRequest, + ) => Promise + } + const originalPostMessage = leaderInternals.channel.postMessage.bind( + leaderInternals.channel, + ) + let leaderDisposed = false + let postAfterDispose = 0 + leaderInternals.channel.postMessage = (message: unknown) => { + if (leaderDisposed) postAfterDispose++ + originalPostMessage(message) + } + + const outcomePromise = followerInternals + .sendRPCOnce(`todos`, { + type: `rpc:applyCommittedTx:req`, + rpcId: `held-inbound-rpc`, + envelopeId: `held-inbound-envelope`, + tx: { + txId: `held-inbound-tx`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }, + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + try { + await vi.waitFor(() => expect(applyEntered).toBe(true)) + leaderDisposed = true + leader.dispose() + releaseApply() + await vi.waitFor(() => expect(durableTransactions).toHaveLength(1)) + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect({ + postAfterDispose, + completed: leaderInternals.appliedCommittedTxEnvelopes.size, + inFlight: leaderInternals.inFlightCommittedTxEnvelopes.size, + }).toEqual({ postAfterDispose: 0, completed: 0, inFlight: 0 }) + + follower.dispose() + expect(await outcomePromise).toEqual({ + error: new Error(`coordinator disposed`), + }) + } finally { + releaseApply() + leader.dispose() + follower.dispose() + } + }) + it(`returns deterministic timeout errors`, async () => { const neverInvoke: ElectronPersistenceInvoke = async () => await new Promise(() => {}) From 56ac0f46a6703fe58ffa474d1efe286187207f68 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 17 Sep 2026 06:42:20 -0600 Subject: [PATCH 02/18] chore: add persistence coordination changeset --- .changeset/enforce-crash-only-persistence-coordination.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changeset/enforce-crash-only-persistence-coordination.md diff --git a/.changeset/enforce-crash-only-persistence-coordination.md b/.changeset/enforce-crash-only-persistence-coordination.md new file mode 100644 index 0000000000..bf368d4375 --- /dev/null +++ b/.changeset/enforce-crash-only-persistence-coordination.md @@ -0,0 +1,8 @@ +--- +'@tanstack/browser-db-sqlite-persistence': patch +'@tanstack/db-sqlite-persistence-core': patch +'@tanstack/electron-db-sqlite-persistence': patch +'@tanstack/db': patch +--- + +Require coordinators to route complete committed transactions through the per-collection persistence owner, with named fail-stop errors for indeterminate commits and durability failures. Add clone-safe remote-subset leases with exact release, recursive wire validation, and matching Browser and Electron coordination. From 5ff207b5aaf3f96e9751c7a4219fbdc2a739dc50 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 17 Sep 2026 11:39:14 -0600 Subject: [PATCH 03/18] fix(sqlite): address coordinator review findings --- .../browser-db-sqlite-persistence/README.md | 7 +- .../src/browser-coordinator.ts | 45 +++-- .../tests/browser-coordinator.test.ts | 42 ++++- packages/db-sqlite-persistence-core/README.md | 29 ++-- .../src/persisted.ts | 38 +++-- .../tests/persisted.test.ts | 157 ++++++++++++++++++ .../electron-db-sqlite-persistence/README.md | 9 +- .../src/electron-coordinator.ts | 45 +++-- .../tests/electron-ipc.test.ts | 44 ++++- 9 files changed, 358 insertions(+), 58 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/README.md b/packages/browser-db-sqlite-persistence/README.md index 08a5222935..e6fcf38ec8 100644 --- a/packages/browser-db-sqlite-persistence/README.md +++ b/packages/browser-db-sqlite-persistence/README.md @@ -112,7 +112,8 @@ The route preserves truncation, row changes, row metadata, collection metadata, and stream position as one `PersistedTx`. It does not feature-detect a partial route or fall back to row-only mutation RPC. A custom coordinator that omits `requestApplyCommittedTx` is rejected while the collection is configured, -before its sync source can publish rows. +before its sync source can publish rows. An effect-free source commit neither +publishes nor consumes a coordinator sequence. Single-tab mode uses the same complete transaction contract. Its `SingleProcessCoordinator` skips election and channel traffic but still routes @@ -144,6 +145,10 @@ exact acquired options once. Leadership loss unloads the retiring owner's live leases, and requesters replay still-live acquisitions against the next leader. Registering a second owner for one collection throws `DuplicateRemoteSubsetOwnerError`; no adapter fallback replaces the owner. +Remote follower transport/admission failures reject and retain demand for its +normal retry without entering the follower's local owner lifecycle. Terminal +release identities expire after the existing RPC dedupe horizon while delayed +duplicates inside that horizon remain idempotent. ## Notes diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index e5c8c419cb..3d3072666f 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -36,6 +36,7 @@ const HEARTBEAT_INTERVAL_MS = 3_000 const RPC_TIMEOUT_MS = 10_000 const RPC_RETRY_ATTEMPTS = 2 const RPC_RETRY_DELAY_MS = 200 +const RPC_DEDUPE_RETENTION_MS = 60_000 const WRITER_LOCK_BUSY_RETRY_MS = 50 const WRITER_LOCK_MAX_RETRIES = 20 @@ -218,6 +219,10 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina string, RemoteSubsetAcquisition >() + private readonly releasedRemoteSubsetAcquisitionTimes = new Map< + string, + number + >() private readonly channel: BroadcastChannel private readonly collections = new Map() private readonly pendingRPCs = new Map() @@ -431,12 +436,6 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina try { await work acquired = true - } catch (error) { - if (!route.localOwner) { - const owner = this.remoteSubsetOwners.get(acquisition.collectionId) - if (owner) reportRemoteSubsetOwnerError(owner, error) - } - throw error } finally { if (acquisition.inFlight === work) acquisition.inFlight = null const current = this.collections.get(acquisition.collectionId) @@ -456,7 +455,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina ) { acquisition.forceReplay = false void this.acquireRemoteSubset(acquisition).catch(() => { - // Failure is already reported; only new demand or ownership change retries. + // Demand stays retained; only new demand or ownership change retries. }) } } @@ -583,6 +582,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina } this.remoteSubsetOwners.clear() this.inboundRemoteSubsetAcquisitions.clear() + this.releasedRemoteSubsetAcquisitionTimes.clear() this.appliedEnvelopes.clear() this.inFlightEnvelopes.clear() } @@ -754,7 +754,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina acquisition.forceReplay = false replays.push( this.acquireRemoteSubset(acquisition).catch(() => { - // Failure is already reported; only new demand or ownership change retries. + // Demand stays retained; only new demand or ownership change retries. }), ) } @@ -1057,6 +1057,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina request: Extract, requesterId: string, ): Promise { + this.pruneReleasedRemoteSubsetAcquisitions() const key = inboundRemoteSubsetAcquisitionKey( collectionId, requesterId, @@ -1117,6 +1118,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina terminalRelease: false, release: null, } + this.releasedRemoteSubsetAcquisitionTimes.delete(key) this.inboundRemoteSubsetAcquisitions.set(key, acquisition) let resolveLoad!: () => void let rejectLoad!: (error: unknown) => void @@ -1160,6 +1162,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina request: Extract, requesterId: string, ): Promise { + this.pruneReleasedRemoteSubsetAcquisitions() const key = inboundRemoteSubsetAcquisitionKey( collectionId, requesterId, @@ -1167,7 +1170,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina ) const acquisition = this.inboundRemoteSubsetAcquisitions.get(key) if (!acquisition) { - this.inboundRemoteSubsetAcquisitions.set(key, { + this.setReleasedRemoteSubsetAcquisition(key, { collectionId, requesterId, acquisitionId: request.acquisitionId, @@ -1177,7 +1180,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina acquisition.terminalRelease = true await this.releaseRemoteSubsetAcquisition(acquisition) if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { - this.inboundRemoteSubsetAcquisitions.set(key, { + this.setReleasedRemoteSubsetAcquisition(key, { collectionId, requesterId, acquisitionId: request.acquisitionId, @@ -1185,7 +1188,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina }) } } else if (`awaitingOwner` in acquisition) { - this.inboundRemoteSubsetAcquisitions.set(key, { + this.setReleasedRemoteSubsetAcquisition(key, { collectionId, requesterId, acquisitionId: request.acquisitionId, @@ -1814,13 +1817,31 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina private pruneAppliedEnvelopes(): void { // Keep envelopes for 60 seconds for dedup - const cutoff = Date.now() - 60_000 + const cutoff = Date.now() - RPC_DEDUPE_RETENTION_MS for (const [key, envelope] of this.appliedEnvelopes) { if (envelope.appliedAt < cutoff) { this.appliedEnvelopes.delete(key) } } } + + private setReleasedRemoteSubsetAcquisition( + key: string, + acquisition: RemoteSubsetAcquisition, + ): void { + this.inboundRemoteSubsetAcquisitions.set(key, acquisition) + this.releasedRemoteSubsetAcquisitionTimes.set(key, Date.now()) + } + + private pruneReleasedRemoteSubsetAcquisitions(): void { + const cutoff = Date.now() - RPC_DEDUPE_RETENTION_MS + for (const [key, releasedAt] of this.releasedRemoteSubsetAcquisitionTimes) { + if (releasedAt < cutoff) { + this.releasedRemoteSubsetAcquisitionTimes.delete(key) + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } + } } // --------------------------------------------------------------------------- diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 98fbcd751b..5860c5ee23 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -3313,7 +3313,7 @@ describe(`BrowserCollectionCoordinator`, () => { } }) - it(`reports a failed remote Browser follower replay to its local owner once`, async () => { + it(`keeps a failed remote Browser follower replay out of its local owner lifecycle`, async () => { const coordinator = createCoordinator() coordinator.subscribe(`todos`, () => {}) const internals = coordinator as unknown as { @@ -3379,7 +3379,7 @@ describe(`BrowserCollectionCoordinator`, () => { acquiredLeaderId: acquisition?.acquiredLeaderId, retained: internals.outboundRemoteSubsetAcquisitions.size, }).toEqual({ - ownerErrors: [replayError], + ownerErrors: [], attempts: 1, acquiredLeaderId: `remote-browser-a`, retained: 1, @@ -3390,6 +3390,44 @@ describe(`BrowserCollectionCoordinator`, () => { } }) + it(`expires Browser release tombstones after the existing RPC dedupe horizon`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const now = vi.spyOn(Date, `now`) + const internals = coordinator as unknown as { + inboundRemoteSubsetAcquisitions: Map + } + + try { + for (let index = 0; index < 8; index++) { + now.mockReturnValue(index * 60_001) + const options = { offset: index } + await coordinator.requestEnsureRemoteSubset(`todos`, options) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + } + + expect({ + loads: owner.mock.calls.length, + unloads: owner.unloadSubset.mock.calls.length, + retained: internals.inboundRemoteSubsetAcquisitions.size, + }).toEqual({ loads: 8, unloads: 8, retained: 1 }) + } finally { + now.mockRestore() + unregisterOwner() + coordinator.dispose() + } + }) + it(`ignores a lower-term Browser heartbeat without replaying to its stale leader`, async () => { const coordinator = createCoordinator() coordinator.subscribe(`todos`, () => {}) diff --git a/packages/db-sqlite-persistence-core/README.md b/packages/db-sqlite-persistence-core/README.md index 87bdc89371..a1da823079 100644 --- a/packages/db-sqlite-persistence-core/README.md +++ b/packages/db-sqlite-persistence-core/README.md @@ -94,8 +94,9 @@ effects is routed in full to the persistence adapter owned by that collection. The `PersistedTx` includes any truncate, row mutations, row metadata mutations, collection metadata mutations, and stream position. Multiprocess coordinators must preserve the complete transaction when they forward it to the elected -writer. This contract does not decide whether an effect-free source commit -should produce a persistence call. +writer. An effect-free source commit does not allocate a stream position, +publish an invalidation, or call persistence; the next durable transaction +retains the next coordinator-owned sequence. This is a required invariant, not a capability to detect at commit time. The TypeScript interface requires the method, and `persistedCollectionOptions(...)` @@ -162,18 +163,24 @@ acquisition; `ReleaseRemoteSubsetRequest.acquisitionId` must carry that same identity so the elected owner releases the matching lease. The owner also provides a required `onError(error)` callback. A load or unload -throw, or a runtime thenable rejection from either operation, is reported -through that callback with the original value before the awaited request -rejects. Retirement and disposal observe and report the same failure while -continuing cleanup of sibling leases; they do not retry or silently recover -the failed lifecycle. +throw by the actual elected owner, or a runtime thenable rejection from either +operation, is reported through that callback with the original value before +the awaited request rejects. A follower transport or remote-owner admission +failure rejects the acquisition and remains eligible for the existing demand +retry; it is not a failure of that follower's local owner lifecycle. Retirement +and disposal observe and report actual owner failures while continuing cleanup +of sibling leases; they do not retry or silently recover the failed lifecycle. Only one remote subset owner may be registered per collection in a coordinator. A second live registration throws `DuplicateRemoteSubsetOwnerError` instead of -replacing the first. When a leader loses ownership or is disposed, it unloads -every transferred lease. A requester that still owns a lease replays that same -acquisition when the next leader is observed. There is no adapter fallback or -direct-writer escape hatch for missing ownership. +replacing the first, and startup reports the named failure before readiness +rather than leaving a rejected registration promise unobserved. Released +acquisition identities remain as dedupe tombstones for the coordinator's +existing RPC dedupe horizon, then expire during later subset traffic. When a +leader loses ownership or is disposed, it unloads every transferred lease. A +requester that still owns a lease replays that same acquisition when the next +leader is observed. There is no adapter fallback or direct-writer escape hatch +for missing ownership. The named-error and exact-path guarantee covers values the boundary can identify with standard JavaScript reflection. A fully transparent `Proxy` is diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index f47999dfa9..c8e3d9c35f 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -1855,28 +1855,16 @@ class PersistedCollectionRuntime< return } - const streamPosition = this.nextLocalStreamPosition() - if ( !transaction.truncate && transaction.operations.length === 0 && transaction.rowMetadataWrites.size === 0 && transaction.collectionMetadataWrites.size === 0 ) { - this.publishTxCommittedEvent( - this.createTxCommittedPayload({ - term: streamPosition.term, - seq: streamPosition.seq, - txId: safeRandomUUID(), - latestRowVersion: streamPosition.rowVersion, - changedRows: [], - deletedKeys: [], - requiresFullReload: true, - }), - ) return } + const streamPosition = this.nextLocalStreamPosition() const tx = this.createPersistedTxFromOperations(transaction, streamPosition) let response: ApplyCommittedTxResponse try { @@ -2700,7 +2688,13 @@ function createWrappedSyncConfig< const getOpenTransaction = () => transactionStack[transactionStack.length - 1] let fullStartPromise: Promise | null = null + let sourceResultPromise: Promise | null = null + let resolveSourceResultAssigned!: () => void + const sourceResultAssigned = new Promise((resolve) => { + resolveSourceResultAssigned = resolve + }) const startupState = { cleanedUp: false } + const isCleanedUp = () => startupState.cleanedUp const acquisitions = new Map() runtime.setSyncControls({ begin: params.begin, @@ -2719,8 +2713,16 @@ function createWrappedSyncConfig< markReady: () => { if (startupState.cleanedUp) return void (fullStartPromise ?? runtime.ensureStarted()) - .then(() => { - if (startupState.cleanedUp) return + .then(async () => { + if (isCleanedUp()) return + await sourceResultAssigned + try { + await sourceResultPromise + } catch (error) { + runtime.reportSyncError(error) + return + } + if (isCleanedUp()) return params.markReady() }) .catch((error) => { @@ -3002,7 +3004,7 @@ function createWrappedSyncConfig< let sourceResult: SyncConfigRes = {} fullStartPromise = runtime.ensureStarted() - const sourceResultPromise = (async () => { + sourceResultPromise = (async () => { await runtime.ensureStartupMetadataLoaded() if (startupState.cleanedUp) { @@ -3033,6 +3035,10 @@ function createWrappedSyncConfig< } return sourceResult })() + resolveSourceResultAssigned() + void sourceResultPromise.catch((error) => { + runtime.reportSyncError(error) + }) return { cleanup: () => { diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 0862daa2ef..c893975092 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -1063,6 +1063,69 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`surfaces duplicate remote-subset owner registration through the collection lifecycle`, async () => { + const coordinator = new SingleProcessCoordinator() + const unhandled: Array = [] + const sourceLoads: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const create = (label: string) => + createCollection( + persistedCollectionOptions({ + id: `duplicate-owner-lifecycle`, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => { + sourceLoads.push(label) + return true + }, + unloadSubset: () => {}, + } + }, + }, + persistence: { adapter: createRecordingAdapter(), coordinator }, + }), + ) + const first = create(`first`) + let duplicate: ReturnType | undefined + + try { + first.startSyncImmediate() + await first.stateWhenReady() + await vi.waitFor(() => + expect( + ( + coordinator as unknown as { + remoteSubsetOwners: Map + } + ).remoteSubsetOwners.size, + ).toBe(1), + ) + duplicate = create(`duplicate`) + duplicate.startSyncImmediate() + const readinessError = await duplicate.stateWhenReady().then( + () => undefined, + (error: unknown) => error, + ) + await flushAsyncWork() + + expect(readinessError).toMatchObject({ + name: `DuplicateRemoteSubsetOwnerError`, + collectionId: `duplicate-owner-lifecycle`, + }) + expect(duplicate.status).toBe(`error`) + expect(sourceLoads).toEqual([]) + expect(unhandled).toEqual([]) + } finally { + process.off(`unhandledRejection`, onUnhandled) + await duplicate?.cleanup() + await first.cleanup() + } + }) + it(`resolves persistence per collection and forwards schemaVersion`, () => { const baseAdapter = createNoopAdapter() const syncAdapter = createNoopAdapter() @@ -1337,6 +1400,100 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`does not publish or allocate a sequence for an empty external source transaction`, async () => { + const adapter = Object.assign(createRecordingAdapter(), { + getStreamPosition: () => + Promise.resolve({ + latestTerm: 1, + latestSeq: 10, + latestRowVersion: 10, + }), + }) + let subscriber: ((message: ProtocolEnvelope) => void) | undefined + const published: Array> = [] + const coordinator: PersistedCollectionCoordinator = { + getNodeId: () => `empty-source-follower`, + subscribe: (_collectionId, onMessage) => { + subscriber = onMessage + return () => { + subscriber = undefined + } + }, + publish: (_collectionId, message) => published.push(message), + isLeader: () => false, + ensureLeadership: async () => {}, + requestEnsurePersistedIndex: async () => {}, + requestEnsureRemoteSubset: async () => {}, + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: () => () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), + } + let sourceBegin: (() => void) | undefined + let sourceCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `empty-source-sequence`, + getKey: (row) => row.id, + sync: { + sync: ({ begin, commit, markReady }) => { + sourceBegin = begin + sourceCommit = commit + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + try { + await collection.stateWhenReady() + sourceBegin?.() + await sourceCommit?.() + await flushAsyncWork() + + expect(published).toEqual([]) + + subscriber?.({ + v: 1, + dbName: `empty-source-sequence`, + collectionId: `empty-source-sequence`, + senderId: `elected-owner`, + ts: Date.now(), + payload: { + type: `tx:committed`, + term: 1, + seq: 11, + txId: `first-real-coordinator-sequence`, + latestRowVersion: 11, + requiresFullReload: false, + changedRows: [ + { + key: `kept`, + value: { id: `kept`, title: `First real write` }, + }, + ], + deletedKeys: [], + }, + }) + await flushAsyncWork() + + expect(stripVirtualProps(collection.get(`kept`))).toEqual({ + id: `kept`, + title: `First real write`, + }) + } finally { + await collection.cleanup() + } + }) + it(`persists a wrapped sync transaction when abort follows application`, async () => { const adapter = createRecordingAdapter() let remoteBegin: (() => void) | undefined diff --git a/packages/electron-db-sqlite-persistence/README.md b/packages/electron-db-sqlite-persistence/README.md index c53dd7493e..cb3e6c56a8 100644 --- a/packages/electron-db-sqlite-persistence/README.md +++ b/packages/electron-db-sqlite-persistence/README.md @@ -101,7 +101,8 @@ Truncation, row changes, row metadata, collection metadata, and stream position stay in the same transaction. Electron does not feature-detect a partial route or fall back to row-only mutation RPC. A custom coordinator that omits `requestApplyCommittedTx` is rejected while the collection is configured, -before its sync source can publish rows. +before its sync source can publish rows. An effect-free source commit neither +publishes nor consumes a coordinator sequence. If a mutating RPC loses its response, Electron coordination replays it only while the requester still knows the same non-null leader id and term. An @@ -124,7 +125,11 @@ remain independent, and release unloads the exact transported options once. Leadership loss unloads the retiring renderer's leases; still-live requester leases replay against the next leader. A second owner registration for one collection throws `DuplicateRemoteSubsetOwnerError`, and there is no adapter -fallback for a missing owner. +fallback for a missing owner. Remote follower transport/admission failures +reject and retain demand for its normal retry without entering the follower's +local owner lifecycle. Terminal release identities expire after the existing +RPC dedupe horizon while delayed duplicates inside that horizon remain +idempotent. ## Notes diff --git a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts index 1c3ea73f0a..5614bc410e 100644 --- a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts +++ b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts @@ -37,6 +37,7 @@ const HEARTBEAT_INTERVAL_MS = 3_000 const RPC_TIMEOUT_MS = 10_000 const RPC_RETRY_ATTEMPTS = 2 const RPC_RETRY_DELAY_MS = 200 +const RPC_DEDUPE_RETENTION_MS = 60_000 const WRITER_LOCK_BUSY_RETRY_MS = 50 const WRITER_LOCK_MAX_RETRIES = 20 @@ -192,6 +193,10 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin string, RemoteSubsetAcquisition >() + private readonly releasedRemoteSubsetAcquisitionTimes = new Map< + string, + number + >() private readonly channel: BroadcastChannel private readonly collections = new Map() private readonly pendingRPCs = new Map() @@ -417,12 +422,6 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin try { await work acquired = true - } catch (error) { - if (!routedToLocalOwner) { - const owner = this.remoteSubsetOwners.get(acquisition.collectionId) - if (owner) reportRemoteSubsetOwnerError(owner, error) - } - throw error } finally { if (acquisition.inFlight === work) acquisition.inFlight = null const current = this.collections.get(acquisition.collectionId) @@ -442,7 +441,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin ) { acquisition.forceReplay = false void this.acquireRemoteSubset(acquisition).catch(() => { - // Failure is already reported; only new demand or ownership change retries. + // Demand stays retained; only new demand or ownership change retries. }) } } @@ -569,6 +568,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin } this.remoteSubsetOwners.clear() this.inboundRemoteSubsetAcquisitions.clear() + this.releasedRemoteSubsetAcquisitionTimes.clear() this.appliedEnvelopeIds.clear() this.inFlightLocalMutationEnvelopes.clear() this.appliedCommittedTxEnvelopes.clear() @@ -742,7 +742,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin acquisition.forceReplay = false replays.push( this.acquireRemoteSubset(acquisition).catch(() => { - // Failure is already reported; only new demand or ownership change retries. + // Demand stays retained; only new demand or ownership change retries. }), ) } @@ -1045,6 +1045,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin request: Extract, requesterId: string, ): Promise { + this.pruneReleasedRemoteSubsetAcquisitions() const key = inboundRemoteSubsetAcquisitionKey( collectionId, requesterId, @@ -1105,6 +1106,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin terminalRelease: false, release: null, } + this.releasedRemoteSubsetAcquisitionTimes.delete(key) this.inboundRemoteSubsetAcquisitions.set(key, acquisition) let resolveLoad!: () => void let rejectLoad!: (error: unknown) => void @@ -1148,6 +1150,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin request: Extract, requesterId: string, ): Promise { + this.pruneReleasedRemoteSubsetAcquisitions() const key = inboundRemoteSubsetAcquisitionKey( collectionId, requesterId, @@ -1155,7 +1158,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin ) const acquisition = this.inboundRemoteSubsetAcquisitions.get(key) if (!acquisition) { - this.inboundRemoteSubsetAcquisitions.set(key, { + this.setReleasedRemoteSubsetAcquisition(key, { collectionId, requesterId, acquisitionId: request.acquisitionId, @@ -1165,7 +1168,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin acquisition.terminalRelease = true await this.releaseRemoteSubsetAcquisition(acquisition) if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { - this.inboundRemoteSubsetAcquisitions.set(key, { + this.setReleasedRemoteSubsetAcquisition(key, { collectionId, requesterId, acquisitionId: request.acquisitionId, @@ -1173,7 +1176,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin }) } } else if (`awaitingOwner` in acquisition) { - this.inboundRemoteSubsetAcquisitions.set(key, { + this.setReleasedRemoteSubsetAcquisition(key, { collectionId, requesterId, acquisitionId: request.acquisitionId, @@ -1755,7 +1758,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin private pruneAppliedEnvelopeIds(): void { // Keep envelopes for 60 seconds for dedup - const cutoff = Date.now() - 60_000 + const cutoff = Date.now() - RPC_DEDUPE_RETENTION_MS for (const [id, envelope] of this.appliedEnvelopeIds) { if (envelope.appliedAt < cutoff) { this.appliedEnvelopeIds.delete(id) @@ -1767,6 +1770,24 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin } } } + + private setReleasedRemoteSubsetAcquisition( + key: string, + acquisition: RemoteSubsetAcquisition, + ): void { + this.inboundRemoteSubsetAcquisitions.set(key, acquisition) + this.releasedRemoteSubsetAcquisitionTimes.set(key, Date.now()) + } + + private pruneReleasedRemoteSubsetAcquisitions(): void { + const cutoff = Date.now() - RPC_DEDUPE_RETENTION_MS + for (const [key, releasedAt] of this.releasedRemoteSubsetAcquisitionTimes) { + if (releasedAt < cutoff) { + this.releasedRemoteSubsetAcquisitionTimes.delete(key) + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } + } } // --------------------------------------------------------------------------- diff --git a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts index c2913a0833..25c5b3e854 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts @@ -2876,7 +2876,7 @@ describe(`electron sqlite persistence bridge`, () => { } }) - it(`reports a failed remote Electron follower replay to its local owner once`, async () => { + it(`keeps a failed remote Electron follower replay out of its local owner lifecycle`, async () => { const dbName = `electron-remote-replay-error` const coordinator = new ElectronCollectionCoordinator({ dbName, @@ -2947,7 +2947,7 @@ describe(`electron sqlite persistence bridge`, () => { acquiredLeaderId: acquisition?.acquiredLeaderId, retained: internals.outboundRemoteSubsetAcquisitions.size, }).toEqual({ - ownerErrors: [replayError], + ownerErrors: [], attempts: 1, acquiredLeaderId: `remote-electron-a`, retained: 1, @@ -2958,6 +2958,46 @@ describe(`electron sqlite persistence bridge`, () => { } }) + it(`expires Electron release tombstones after the existing RPC dedupe horizon`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-release-tombstone-expiry`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const now = vi.spyOn(Date, `now`) + const internals = coordinator as unknown as { + inboundRemoteSubsetAcquisitions: Map + } + + try { + for (let index = 0; index < 8; index++) { + now.mockReturnValue(index * 60_001) + const options = { offset: index } + await coordinator.requestEnsureRemoteSubset(`todos`, options) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + } + + expect({ + loads: owner.mock.calls.length, + unloads: owner.unloadSubset.mock.calls.length, + retained: internals.inboundRemoteSubsetAcquisitions.size, + }).toEqual({ loads: 8, unloads: 8, retained: 1 }) + } finally { + now.mockRestore() + unregisterOwner() + coordinator.dispose() + } + }) + it(`ignores a lower-term Electron heartbeat without replaying to its stale leader`, () => { const dbName = `electron-stale-heartbeat` const coordinator = new ElectronCollectionCoordinator({ From 2934f15e6ee7929316426f449cf146abec1efb7f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 18 Sep 2026 17:10:53 -0600 Subject: [PATCH 04/18] ci: trigger PR checks From ca446bc40bc196f98299e0d9feeb67526f5c1429 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 10:08:43 +0100 Subject: [PATCH 05/18] docs: make persistence oracles literate --- docs/contributing/oracle-coverage.md | 2 +- .../tests/browser-coordinator.test.ts | 29 ++++++++ .../per-collection-coordinator-oracle.test.ts | 66 +++++++++++++------ .../tests/persisted.test-d.ts | 15 +++++ .../tests/persisted.test.ts | 28 ++++++++ .../tests/electric-recovery-oracle.test.ts | 17 +++++ .../tests/electron-ipc.test.ts | 28 ++++++++ 7 files changed, 164 insertions(+), 21 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index d95cf11712..f11185e405 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -38,7 +38,7 @@ comment and the current API/architecture contract before extending its model. | Opaque backend pagination | [window oracle](../../packages/query-db-collection/tests/cursor-pagination.oracle.test.ts), [cache histories](../../packages/query-db-collection/tests/cursor-pagination.cache-oracle.test.ts), [cache publication](../../packages/query-db-collection/tests/cursor-pagination.publication-oracle.test.ts), [browser acquisition boundaries](../../packages/query-db-collection/tests/cursor-pagination.boundary-oracle.test.ts), [QueryCollection integration](../../packages/query-db-collection/tests/cursor-pagination.integration.test.ts) | Full filter/sort/slice reference, opaque token transport, actual Query cache expiry/invalidation/GC, forced refresh during growth, protocol failure publication/recovery, bounded slice work, nested cancellation/replacement, reader abort, browser retry defaults, manual-write cache isolation, and production window publications. Stable backend sequences; not snapshot guarantees for changing endpoints. Peek-ahead remains enabled. | | Electric and TrailBase | [Electric histories](../../packages/electric-db-collection/tests/electric-oracle.property.test.ts), [PostgreSQL semantics](../../packages/electric-db-collection/e2e/sql-predicate-semantics.e2e.test.ts), [TrailBase contract](../../packages/trailbase-db-collection/tests/ORACLE.md) | Installed SDK delivery/framing, independent predicates, exact subscription arguments and late errors. SDK fixtures and a real service test earn different credit. | | PowerSync | [tests](../../packages/powersync-db-collection/tests) | Applied receipt positions crossed with held peers, native SQLite/SDK and cleanup evidence. A timeout mutant proves a progress failure, not every value assertion. | -| SQLite persistence and native hosts | [persisted histories](../../packages/db-sqlite-persistence-core/tests/persisted.test.ts), [Browser composed-owner histories](../../packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts), [Browser coordinator RPC](../../packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts), [Electron IPC and composed owner](../../packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts), [driver contracts](../../packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [113-law manifest](../../packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Core cache/remote rejection/peer/reopen histories and exact driver results. Browser composes public source commits with per-collection elected-owner routing and covers the complete committed-transaction wire partition through deterministic Node transport seams. Electron composes source commits with a per-collection renderer owner, IPC adapter, real SQLite, and reopen checks. The Browser seams are not real multi-context/OPFS-worker execution; the Electron harness is not an actual Electron process unless its explicit runtime-bridge mode runs. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | +| SQLite persistence and native hosts | [persisted histories](../../packages/db-sqlite-persistence-core/tests/persisted.test.ts), [Browser composed-owner histories](../../packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts), [Browser coordinator RPC](../../packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts), [Electron IPC and composed owner](../../packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts), [driver contracts](../../packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [113-law manifest](../../packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Core cache/remote rejection/peer/reopen histories and exact driver results. Browser composes public source commits with per-collection elected-owner routing and covers the complete committed-transaction wire partition through deterministic Node transport seams. Remote-subset histories distinguish logical demand, physical acquisitions, exact acquisition leases, and released replay tombstones. Electron composes source commits with a per-collection renderer owner, IPC adapter, real SQLite, and reopen checks. The Browser seams are not real multi-context/OPFS-worker execution; the Electron harness is not an actual Electron process unless its explicit runtime-bridge mode runs. Registration-gated core remote demand and bounded host retry after transport/admission failure remain open review gaps. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | | Offline execution | [scheduler](../../packages/offline-transactions/tests/KeyScheduler.property.test.ts), [leadership](../../packages/offline-transactions/tests/leadership-replay.property.test.ts), [settlement](../../packages/offline-transactions/tests/transaction-settlement.property.test.ts), [serialization](../../packages/offline-transactions/tests/transaction-serializer.property.test.ts) | Declarative FIFO eligibility, per-transaction outcomes, durable state and typed wire trees. Issued work may finish after ownership loss, but new work must not start. Exactly-once network execution is not promised. | | Frameworks | [React conformance](../../packages/react-db/tests/conformance.test.tsx), [React pagination](../../packages/react-db/tests/infinite-query-conformance.test.tsx), [shared suites](../../packages/db-collection-e2e/src/suites) | Exact exposed rows/pages and each framework's own lifecycle cuts. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. | | Small structures and test mechanics | [SortedMap](../../packages/db/tests/SortedMap.test.ts), [cleanup queue](../../packages/db/tests/cleanup-queue.property.test.ts), [guarded replay](../../packages/db/tests/oracle-replay.test.ts) | Map/full-sort and appointment-list models; executed target/seed/path checks. Callback-reentrant scheduling is outside the initial cleanup-queue domain. | diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 5860c5ee23..f8390f9238 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -14,6 +14,35 @@ import type { } from '@tanstack/db-sqlite-persistence-core' import type { BrowserCollectionCoordinatorOptions } from '../src/browser-coordinator' +/** + * # Which Browser coordinator outcomes are safe to acknowledge? + * + * RFC #1659 requires every adapter operation to use the elected owner for its + * collection. Complete committed transactions must cross that boundary + * without losing metadata. A mutating RPC may replay only through the same + * known leader and term. Remote subset request data must be clone-safe, and + * each accepted physical acquisition creates one exact acquisition lease. + * + * The adapter call logs, transport controls, owner callbacks, and internal-map + * snapshots are focused reference ledgers. Histories vary local and follower + * routes, response loss, leadership change, owner replacement, duplicate + * delivery, release, failure, and disposal. The production driver is the real + * `BrowserCollectionCoordinator`; only BroadcastChannel and Web Locks are + * replaced with deterministic seams. + * + * Checkpoints sit at adapter entry, RPC response delivery, acquisition + * acceptance, acquisition release, lifecycle failure, and disposal. Fault + * controls drop or duplicate messages, reject owner work, change leaders, and + * advance the released-tombstone clock. Exact call counts and error identities + * prevent a final-state-only false green. + * + * The composed public-Collection and generated route models live in + * `per-collection-coordinator-oracle.test.ts`. These seams do not prove real + * browser scheduling, Web Locks, BroadcastChannel, OPFS ownership, or worker + * behavior. They also do not yet prove bounded retry after follower transport + * or remote-owner admission failure; that review finding remains open. + */ + // --------------------------------------------------------------------------- // BroadcastChannel mock // --------------------------------------------------------------------------- diff --git a/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts index f2ff086f13..1fcc42d252 100644 --- a/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts @@ -57,9 +57,9 @@ Routing law and source: collectionId-bearing PersistedCollectionCoordinator API; issues #1589 and #1753. Configuration and adapter effects for collection A cannot be handled by collection B's adapter, before or after leadership transfer. -- Domain: two distinct collections, distinct schema versions/policies, two - tabs, reordered request delivery, local writes, and leader -> follower - takeover. The reference is a Map keyed by (tab, collection), not the +- History grammar: two distinct collections, distinct schema versions/policies, + two tabs, reordered request delivery, local writes, and leadership transfer. + The reference is a Map keyed by (tab, collection), not the coordinator's mutable adapter slot. - Production/checkpoint: real coordinator RPC plus Web Locks seam; compare the adapter identity that applies each collection's transaction after both the @@ -74,16 +74,30 @@ Wire/ack law and source: wire; live signal/subscription ownership stays local. A successful follower response is permitted only after the leader's exact upstream subset load has completed. -- Domain: a callback-bearing Subscription witness plus clone-safe expression, - limit, and offset requests. Requests and responses may be held separately. +- History grammar: a callback-bearing Subscription witness plus clone-safe + expression, limit, and offset request data. Requests and responses may be + held separately. - Production/checkpoint: BrowserCollectionCoordinator.sendRPC and the public persisted on-demand leader collection. Record postMessage clone failures, upstream load entry/completion, and follower settlement at response delivery. +Acquisition law and source: +- RFC invariant 4 and the remote-subset owner contract. Demand can survive + leadership transfer, but every accepted physical acquisition creates an + acquisition lease with one exact release obligation. +- History grammar: sibling logical owners, reused and distinct request-data + objects, prefix and duplicate release, owner replacement, and leadership + transfer. `RemoteLeaseOwnerLedger` is a model-only call ledger whose active + entries represent physical acquisitions and their acquisition leases. +- Production/checkpoint: real coordinator ensure/release calls. Compare owner + load/unload counts, exact request-data identity, active acquisitions, and + cleanup after each release and takeover cut. + Write-ownership law and source: - RFC invariants 3 and 9 and issue #1753. A sync-ingested persistent write has the same supported-owner requirement as a local write. -- Domain: public source begin/write/commit after a modeled leadership change. +- History grammar: public source begin/write/commit after a modeled leadership + change. - Production/checkpoint: persistedCollectionOptions' wrapped source commit; record collection-visible rows, adapter calls, and the owner active at the exact applyCommittedTx call. @@ -91,18 +105,19 @@ Write-ownership law and source: Reach, challenge, replay, cleanup, and limits: - Green calibration tests prove structuredClone rejects a function payload, the leader upstream fixture can really enter/complete, and the ownership - checker rejects an unowned apply. Every RED test records a reached production - checkpoint before comparing the independent ledger. -- FastCheck prints seed/path. Replay with TANSTACK_DB_COORDINATOR_ORACLE_{SEED, - PATH,RUNS}; the thrown report retains the first failing (original) and final - shrunk candidate traces. -- Coordinators/collections/databases are disposed by failure-preserving cleanup. A final - lifecycle test proves no channel, held lock, queued lock, or delayed delivery - remains and that a fresh db name elects normally. + checker rejects an unowned apply. Every production-path test records a + reached checkpoint before comparing the independent ledger. +- The default is one fixed fast-check campaign. Replay with + TANSTACK_DB_COORDINATOR_ORACLE_{SEED,PATH,RUNS}; the thrown report retains the + first failing trace and final shrunk candidate. +- Coordinators, Collections, and databases use failure-preserving cleanup. A + final lifecycle test proves no channel, held lock, queued lock, or delayed + delivery remains and that a fresh database name elects normally. - The BroadcastChannel and Web Locks seams below perform real structuredClone and real coordinator code but are deterministic Node controls. They do not earn real-browser, multi-context, OPFS exclusive-handle, worker, Electric, - PowerSync, or service credit. Those cells remain required for GREEN closeout. + PowerSync, or service credit. Bounded retry after follower transport or + remote-owner admission failure also remains an open review finding. */ type MessageHandler = (event: { data: unknown }) => void @@ -626,7 +641,7 @@ function registerCollectionAdapter( (options: TransportedLoadSubsetOptions) => adapter.ensureRemoteSubset(collectionId, options), { - // These route/wire laws do not exercise lease release semantics; the + // These route/wire laws do not exercise acquisition-release semantics; the // dedicated remote-ownership oracle below owns exact unload behavior. unloadSubset: () => {}, onError: () => {}, @@ -716,6 +731,8 @@ type SubsetWireTrace = { expected: SemanticValue } +// Clone-safe projection may remove live ownership callbacks. It must preserve +// every supported request-data value that determines subset semantics. function subsetWireViolations(trace: SubsetWireTrace): Array { const violations: Array = [] if (trace.outcome?.ok !== true) violations.push(`subset-transport-failed`) @@ -728,6 +745,8 @@ function subsetWireViolations(trace: SubsetWireTrace): Array { return violations } +// A follower acknowledgement is valid only after the registered leader owner +// has accepted and completed the exact remote subset acquisition attempt. function subsetAckViolations(trace: SubsetAckTrace): Array { const violations: Array = [] if (trace.ownerInvocation !== `coordinator`) { @@ -829,7 +848,7 @@ describe(`per-collection coordinator wire and acknowledgement oracle`, () => { if (outcome?.ok === false) { expect(outcome.errorName).toBe(observedCloneFault?.errorName) } - // RED D3: after clone-safe projection, the registered leader owner must + // D3 refinement: after clone-safe projection, the registered leader owner must // receive the exact supported semantics and the follower call succeeds. expect( subsetWireViolations({ @@ -1297,7 +1316,7 @@ describe(`per-collection coordinator wire and acknowledgement oracle`, () => { ), acknowledgedAfterResponseDelivery: acknowledged, } - // RED D13: one comparator owns both the production verdict and hostile + // D13 refinement: one comparator owns both the production verdict and hostile // entry-only/false-success calibrations above. expect(subsetAckViolations(trace)).toEqual([]) }, [ @@ -1509,6 +1528,9 @@ const remoteLeaseHistoryArbitrary = fc.record({ reverseFinalRelease: fc.boolean(), }) +// Every accepted physical acquisition remains active until its exact +// acquisition lease is released. Leadership transfer releases retired-owner +// acquisitions and establishes replacements only for surviving demand. function remoteLeaseHistoryViolations( history: RemoteLeaseHistory, actual: RemoteLeaseHistoryTrace, @@ -2293,7 +2315,7 @@ describe(`per-collection adapter and SQLite-state oracle`, () => { ) const after = await readRawCollectionSnapshot(createdDatabase, `alpha`) - // RED D2 fixed cross-collection contamination witness. The shared + // D2 fixed cross-collection contamination witness. The shared // comparator reports schema, reset, and row losses independently. expect(rawSnapshotViolations(after, expected)).toEqual([]) }, [ @@ -2564,6 +2586,8 @@ function routeViolations( return violations } +// Collection identity and elected ownership jointly select the adapter. The +// model deliberately uses Maps instead of the coordinator's routing storage. class PerCollectionRouteOracle { private readonly routes = new Map() private readonly owners = new Map() @@ -3215,7 +3239,7 @@ describe(`generated collection-route histories`, () => { propertyFailure = error } - // Cleanup diagnostics are evaluated outside FastCheck, so cleanup cannot + // Cleanup diagnostics are evaluated outside fast-check, so cleanup cannot // become the predicate that selects or shrinks a semantic route failure. expect.soft(cleanupDiagnostics).toEqual([]) // Setup/reach/fixture failures are also reported outside the property; @@ -3230,6 +3254,8 @@ type OwnershipObservation = { ownerAtCall?: string } +// A source commit may apply only while the elected persistence owner holds the +// writer boundary. Collection visibility alone does not establish ownership. function findUnownedWrites( observations: ReadonlyArray, electedOwner: string, diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts b/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts index 1305fd9274..6219c8a0b8 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts @@ -19,6 +19,21 @@ import type { } from '../src' import type { SyncConfig, UtilsRecord } from '@tanstack/db' +/** + * # Which persistence contracts are visible to TypeScript callers? + * + * The exported types require complete committed-transaction routing, preserve + * named reconciliation and durability errors, and admit exactly the declared + * remote-subset wire domain. Positive assignments are the reference type + * shapes. `@ts-expect-error` cases are the fault controls for omitted + * acquisition identity, live signals/subscriptions, unsupported values, and + * incomplete coordinators. + * + * The TypeScript compile is the production checkpoint. These checks establish + * assignability and inference only. Runtime validation, structured cloning, + * routing, settlement, and publication belong to the executable owners. + */ + type Todo = { id: string title: string diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index c893975092..37c3308d45 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -31,6 +31,34 @@ import type { } from '../src' import type { Collection, LoadSubsetOptions, SyncConfig } from '@tanstack/db' +/** + * # Does persisted wrapping preserve one Collection history? + * + * Persistence adds a durable replica beneath an optional upstream sync source. + * Startup hydrates rows and metadata, buffers concurrent remote work, then + * publishes one coherent public snapshot. Complete committed transactions + * route through the configured collection owner. Publication may precede + * durability settlement, but a later failure must stay observable. + * + * The recording adapter is a plain durable-state model: Maps for rows and + * metadata plus ordered transaction, index, load, and reload calls. Histories + * vary hydration, source commits, applied receipts, remote subset demand, + * acquisition release, retry, failure, cleanup, and restart. The driver uses + * the real persisted wrapper, Collection, coordinator, transactions, and + * indexes. + * + * Refinement checkpoints compare public rows, durable state, metadata, request + * data, sequence evidence, exact errors, and late-work fencing. Fixed hostile + * values challenge wire admission. Controlled failures challenge publication + * and durability classification. Focused Browser and Electron suites own the + * multiprocess transport and host-specific replay partitions. + * + * Driver SQL behavior, native host ownership, and the shared conformance + * portfolio have separate owners. This file does not yet prove that + * non-single-process remote demand is gated by successful remote-subset owner + * registration; that completed-review finding remains open. + */ + type Todo = { id: string title: string diff --git a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts index eed134f1f0..dc62979dfd 100644 --- a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts +++ b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts @@ -14,6 +14,23 @@ import type { } from '../../db-sqlite-persistence-core/src' import type { ElectricCollectionUtils, ElectricSyncMode } from '../src/electric' +/** + * # What remains visible while a persisted Electric replica repairs itself? + * + * Hydrated rows and resume metadata provide the last complete public snapshot. + * A must-refetch starts a private replacement. Until that replacement is fully + * applied, readers may see an earlier permitted snapshot but never a torn mix. + * Failure keeps the old public rows and records repair debt; later success may + * replace them atomically. + * + * A plain persisted row Map and metadata Map form the reference snapshots. The + * driver controls hydration, SDK callbacks, applied receipts, cleanup, restart, + * and eager or progressive mode through the real persistence coordinator and + * Electric adapter. It records every exposed observation cut, not only final + * rows. The external-publisher fixture routes complete committed transactions + * through its bound adapter; that wiring does not add native-host evidence. + */ + type Item = Row & { id: number; name: string; stable: string } type Subscriber = (messages: Array>) => void type Exposure = { cut: string; rows: Array } diff --git a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts index 25c5b3e854..9ce912d865 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts @@ -40,6 +40,34 @@ import type { ElectronPersistenceResponseEnvelope, } from '../src/protocol' +/** + * # Does Electron preserve one collection's transaction and acquisition laws? + * + * RFC #1659 requires renderer work to reach the exact collection adapter in + * the elected main-process owner. Complete committed transactions retain row + * and collection metadata. Mutating RPC replay is limited to the same known + * leader and term. Remote subset request data stays inside the clone-safe wire + * domain, and each accepted physical acquisition has one acquisition lease. + * + * Expected transactions, adapter call logs, SQLite rows, metadata, owner + * callbacks, and coordinator snapshots form the reference observations. + * Histories vary response loss, leadership change, owner replacement, + * duplicate delivery, acquisition release, durability failure, cleanup, and + * reopen. The driver crosses the real Electron coordinator and IPC persistence + * adapter; the durable witness reopens a real SQLite database. + * + * Checkpoints sit at adapter entry, RPC settlement, acquisition acceptance and + * release, lifecycle error, disposal, and durable reopen. Hostile wire values, + * held work, changed leaders, rejected owner operations, and released-tombstone + * clock advances challenge the assertions. + * + * The default invoke and Web Locks seams are deterministic process-local + * controls. They prove an actual Electron process only when explicit runtime- + * bridge mode runs. This suite also does not yet prove bounded retry after + * follower transport or remote-owner admission failure; that review finding + * remains open. + */ + type InvokeHarness = { invoke: ElectronPersistenceInvoke close: () => void From dd228052c577fd92c9a7b21caedf3dedef6160d9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 19:47:46 +0100 Subject: [PATCH 06/18] fix(sqlite): recover retained remote subset demand --- .../src/browser-coordinator.ts | 106 +++++++++++--- .../tests/browser-coordinator.test.ts | 125 +++++++++++++++- .../db-sqlite-persistence-core/src/errors.ts | 10 ++ .../src/persisted.ts | 3 + .../tests/persisted.test.ts | 59 +++++++- .../src/electron-coordinator.ts | 106 +++++++++++--- .../tests/electron-ipc.test.ts | 135 +++++++++++++++++- 7 files changed, 506 insertions(+), 38 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index 3d3072666f..2b8348de17 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -2,6 +2,7 @@ import { DuplicateRemoteSubsetOwnerError, IndeterminateCommitError, PersistedCollectionDurabilityError, + RetryableRemoteSubsetAcquisitionError, safeRandomUUID, toPersistedCollectionDurabilityError, toTransportedLoadSubsetOptions, @@ -36,6 +37,7 @@ const HEARTBEAT_INTERVAL_MS = 3_000 const RPC_TIMEOUT_MS = 10_000 const RPC_RETRY_ATTEMPTS = 2 const RPC_RETRY_DELAY_MS = 200 +const REMOTE_SUBSET_REPLAY_RETRY_ATTEMPTS = 2 const RPC_DEDUPE_RETENTION_MS = 60_000 const WRITER_LOCK_BUSY_RETRY_MS = 50 const WRITER_LOCK_MAX_RETRIES = 20 @@ -164,6 +166,8 @@ type OutboundRemoteSubsetAcquisition = { acquiredLeaderId: string | null inFlight: Promise | null forceReplay: boolean + retryTimer: ReturnType | null + retryAttempts: number } type AppliedEnvelope = @@ -353,6 +357,8 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina acquiredLeaderId: null, inFlight: null, forceReplay: false, + retryTimer: null, + retryAttempts: 0, } this.outboundRemoteSubsetAcquisitions.set( remoteSubsetAcquisitionKey(collectionId, acquisitionId), @@ -371,7 +377,10 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina const acquisitionId = collectionIds?.get(options) if (!acquisitionId) return const key = remoteSubsetAcquisitionKey(collectionId, acquisitionId) - if (!this.outboundRemoteSubsetAcquisitions.delete(key)) return + const acquisition = this.outboundRemoteSubsetAcquisitions.get(key) + if (!acquisition) return + this.outboundRemoteSubsetAcquisitions.delete(key) + this.cancelRemoteSubsetReplayRetry(acquisition) collectionIds!.delete(options) const request: Extract< @@ -415,18 +424,35 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina options: acquisition.options, } route.localOwner = this.isLeader(acquisition.collectionId) - const response = route.localOwner - ? await this.handleEnsureRemoteSubset( - acquisition.collectionId, - request, - this.nodeId, - ) - : await this.sendRPC( - acquisition.collectionId, - request, - ) + let response: EnsureRemoteSubsetResponse + try { + response = route.localOwner + ? await this.handleEnsureRemoteSubset( + acquisition.collectionId, + request, + this.nodeId, + ) + : await this.sendRPC( + acquisition.collectionId, + request, + ) + } catch (error) { + if ( + route.localOwner || + error instanceof RetryableRemoteSubsetAcquisitionError + ) { + throw error + } + throw new RetryableRemoteSubsetAcquisitionError( + `Remote subset transport failed`, + error, + ) + } if (!response.ok) { + if (response.retryable) { + throw new RetryableRemoteSubsetAcquisitionError(response.error) + } throw new Error(`ensureRemoteSubset failed: ${response.error}`) } acquisition.acquiredLeaderId = response.leaderId @@ -436,6 +462,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina try { await work acquired = true + this.cancelRemoteSubsetReplayRetry(acquisition) } finally { if (acquisition.inFlight === work) acquisition.inFlight = null const current = this.collections.get(acquisition.collectionId) @@ -454,8 +481,8 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina acquisition.acquiredLeaderId !== currentLeaderId)) ) { acquisition.forceReplay = false - void this.acquireRemoteSubset(acquisition).catch(() => { - // Demand stays retained; only new demand or ownership change retries. + void this.acquireRemoteSubset(acquisition).catch((error) => { + this.scheduleRemoteSubsetReplayRetry(acquisition, error) }) } } @@ -559,6 +586,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina this.disposed = true for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + this.cancelRemoteSubsetReplayRetry(acquisition) this.postRemoteSubsetRelease(acquisition) } this.outboundRemoteSubsetAcquisitions.clear() @@ -731,6 +759,49 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina } satisfies ProtocolEnvelope) } + private cancelRemoteSubsetReplayRetry( + acquisition: OutboundRemoteSubsetAcquisition, + ): void { + if (acquisition.retryTimer !== null) { + clearTimeout(acquisition.retryTimer) + acquisition.retryTimer = null + } + acquisition.retryAttempts = 0 + } + + private scheduleRemoteSubsetReplayRetry( + acquisition: OutboundRemoteSubsetAcquisition, + error: unknown, + ): void { + const key = remoteSubsetAcquisitionKey( + acquisition.collectionId, + acquisition.acquisitionId, + ) + if ( + !(error instanceof RetryableRemoteSubsetAcquisitionError) || + this.isDisposed() || + this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition || + acquisition.retryTimer !== null || + acquisition.retryAttempts >= REMOTE_SUBSET_REPLAY_RETRY_ATTEMPTS + ) { + return + } + + acquisition.retryAttempts++ + acquisition.retryTimer = setTimeout(() => { + acquisition.retryTimer = null + if ( + this.isDisposed() || + this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition + ) { + return + } + void this.acquireRemoteSubset(acquisition).catch((retryError) => { + this.scheduleRemoteSubsetReplayRetry(acquisition, retryError) + }) + }, RPC_RETRY_DELAY_MS) + } + private async replayRemoteSubsetAcquisitions( collectionId: string, ): Promise { @@ -753,8 +824,8 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina } acquisition.forceReplay = false replays.push( - this.acquireRemoteSubset(acquisition).catch(() => { - // Demand stays retained; only new demand or ownership change retries. + this.acquireRemoteSubset(acquisition).catch((error) => { + this.scheduleRemoteSubsetReplayRetry(acquisition, error) }), ) } @@ -1101,7 +1172,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina const owner = this.remoteSubsetOwners.get(collectionId) if (!owner) { - throw new Error( + throw new RetryableRemoteSubsetAcquisitionError( `BrowserCollectionCoordinator: no remote subset owner registered for collection "${collectionId}"`, ) } @@ -1923,6 +1994,9 @@ function createRPCErrorResponse( rpcId: request.rpcId, ok: false, error, + ...(cause instanceof RetryableRemoteSubsetAcquisitionError + ? { retryable: true as const } + : {}), } case `rpc:releaseRemoteSubset:req`: return { diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index f8390f9238..d2b0b3d378 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { IR } from '@tanstack/db' +import { RetryableRemoteSubsetAcquisitionError } from '@tanstack/db-sqlite-persistence-core' import { BrowserCollectionCoordinator } from '../src/browser-coordinator' import type { LoadSubsetOptions, Subscription } from '@tanstack/db' import type { @@ -39,8 +40,8 @@ import type { BrowserCollectionCoordinatorOptions } from '../src/browser-coordin * The composed public-Collection and generated route models live in * `per-collection-coordinator-oracle.test.ts`. These seams do not prove real * browser scheduling, Web Locks, BroadcastChannel, OPFS ownership, or worker - * behavior. They also do not yet prove bounded retry after follower transport - * or remote-owner admission failure; that review finding remains open. + * behavior. Bounded replay retry is covered for retryable transport and + * admission failures, including cancellation on release and disposal. */ // --------------------------------------------------------------------------- @@ -3750,6 +3751,126 @@ describe(`BrowserCollectionCoordinator`, () => { } }) + it(`retries retained Browser demand after a retryable replay failure`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + + type Acquisition = { + collectionId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + acquiredLeaderId: string | null + inFlight: Promise | null + forceReplay: boolean + retryTimer: ReturnType | null + retryAttempts: number + } + const acquisition: Acquisition = { + collectionId: `todos`, + acquisitionId: `retryable-browser-replay`, + options: { limit: 1 }, + acquiredLeaderId: `retired-browser-leader`, + inFlight: null, + forceReplay: false, + retryTimer: null, + retryAttempts: 0, + } + const internals = coordinator as unknown as { + nodeId: string + outboundRemoteSubsetAcquisitions: Map + acquireRemoteSubset: (acquisition: Acquisition) => Promise + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + internals.outboundRemoteSubsetAcquisitions.set( + JSON.stringify([acquisition.collectionId, acquisition.acquisitionId]), + acquisition, + ) + let attempts = 0 + internals.acquireRemoteSubset = vi.fn((current) => { + attempts++ + if (attempts === 1) { + return Promise.reject( + new RetryableRemoteSubsetAcquisitionError(`offline`), + ) + } + current.acquiredLeaderId = internals.nodeId + return Promise.resolve() + }) + + await internals.replayRemoteSubsetAcquisitions(`todos`) + expect(attempts).toBe(1) + await vi.waitFor(() => expect(attempts).toBe(2)) + }) + + it.each([`release`, `dispose`] as const)( + `cancels a scheduled Browser replay retry on %s`, + async (stop) => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + + type Acquisition = { + collectionId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + acquiredLeaderId: string | null + inFlight: Promise | null + forceReplay: boolean + retryTimer: ReturnType | null + retryAttempts: number + } + const requestedOptions: LoadSubsetOptions = { limit: 1 } + const acquisition: Acquisition = { + collectionId: `todos`, + acquisitionId: `cancelled-browser-replay`, + options: { limit: 1 }, + acquiredLeaderId: `retired-browser-leader`, + inFlight: null, + forceReplay: false, + retryTimer: null, + retryAttempts: 0, + } + const internals = coordinator as unknown as { + remoteSubsetIds: Map> + outboundRemoteSubsetAcquisitions: Map + acquireRemoteSubset: (acquisition: Acquisition) => Promise + replayRemoteSubsetAcquisitions: ( + collectionId: string, + ) => Promise + } + internals.remoteSubsetIds.set( + `todos`, + new WeakMap([[requestedOptions, acquisition.acquisitionId]]), + ) + internals.outboundRemoteSubsetAcquisitions.set( + JSON.stringify([acquisition.collectionId, acquisition.acquisitionId]), + acquisition, + ) + let attempts = 0 + internals.acquireRemoteSubset = vi.fn(() => { + attempts++ + return Promise.reject( + new RetryableRemoteSubsetAcquisitionError(`offline`), + ) + }) + + await internals.replayRemoteSubsetAcquisitions(`todos`) + expect(acquisition.retryTimer).not.toBeNull() + if (stop === `release`) { + await coordinator.requestReleaseRemoteSubset( + `todos`, + requestedOptions, + ) + } else { + coordinator.dispose() + } + expect(acquisition.retryTimer).toBeNull() + await new Promise((resolve) => setTimeout(resolve, 250)) + expect(attempts).toBe(1) + }, + ) + it(`starts independent Browser lease replays without sibling head-of-line blocking`, async () => { const coordinator = createCoordinator() coordinator.subscribe(`todos`, () => {}) diff --git a/packages/db-sqlite-persistence-core/src/errors.ts b/packages/db-sqlite-persistence-core/src/errors.ts index 632f62a074..e69b1e5b7c 100644 --- a/packages/db-sqlite-persistence-core/src/errors.ts +++ b/packages/db-sqlite-persistence-core/src/errors.ts @@ -144,6 +144,16 @@ export class DuplicateRemoteSubsetOwnerError extends PersistedCollectionCoreErro } } +export class RetryableRemoteSubsetAcquisitionError extends PersistedCollectionCoreError { + override readonly cause: unknown + + constructor(message: string, cause?: unknown) { + super(message) + this.name = `RetryableRemoteSubsetAcquisitionError` + this.cause = cause + } +} + export class InvalidPersistedStorageKeyError extends InvalidPersistedCollectionConfigError { constructor(key: string | number) { super( diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index c8e3d9c35f..29bb25caca 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -121,6 +121,7 @@ export type EnsureRemoteSubsetResponse = rpcId: string ok: false error: string + retryable?: true } export type ReleaseRemoteSubsetRequest = { @@ -1346,6 +1347,7 @@ class PersistedCollectionRuntime< const lifecycleGeneration = this.lifecycleGeneration const routeRemoteDemandThroughCoordinator = this.mode === `sync-present` && + this.remoteSubsetOwnerUnsubscribe !== null && !(this.persistence.coordinator instanceof SingleProcessCoordinator) this.activeSubsets.set(this.getSubsetKey(options), options) @@ -2230,6 +2232,7 @@ class PersistedCollectionRuntime< private queueRemoteSubsetEnsure(options: LoadSubsetOptions): void { if ( this.mode !== `sync-present` || + this.remoteSubsetOwnerUnsubscribe === null || this.persistence.coordinator instanceof SingleProcessCoordinator || this.activeSubsets.get(this.getSubsetKey(options)) !== options ) { diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 37c3308d45..f6a6b4b5f8 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -3256,7 +3256,7 @@ describe(`persistedCollectionOptions`, () => { sync: { sync: ({ markReady }) => { markReady() - return {} + return { loadSubset: vi.fn() } }, }, persistence: { @@ -3281,6 +3281,63 @@ describe(`persistedCollectionOptions`, () => { expect(ensureCalls).toBeGreaterThanOrEqual(2) }) + it(`does not route remote demand without a subset owner`, async () => { + const ensure = vi.fn(async () => { + throw new Error(`no remote subset owner registered`) + }) + const coordinator: PersistedCollectionCoordinator = { + getNodeId: () => `ownerless-node`, + subscribe: () => () => {}, + publish: () => {}, + isLeader: () => true, + ensureLeadership: async () => {}, + requestEnsurePersistedIndex: async () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), + requestEnsureRemoteSubset: ensure, + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: () => () => {}, + } + const collection = createCollection( + persistedCollectionOptions({ + id: `ownerless-on-demand-source`, + syncMode: `on-demand`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + return {} + }, + }, + persistence: { + adapter: createRecordingAdapter(), + coordinator, + }, + }), + ) + + try { + collection.startSyncImmediate() + await flushAsyncWork() + + await expect( + Promise.resolve(collection._sync.loadSubset({ limit: 1 })), + ).resolves.toBeUndefined() + await flushAsyncWork(120) + + expect(ensure).not.toHaveBeenCalled() + } finally { + await collection.cleanup() + } + }) + it(`fails sync-absent persistence when follower ack omits mutation ids`, async () => { const adapter = createRecordingAdapter() const coordinator: PersistedCollectionCoordinator = { diff --git a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts index 5614bc410e..551306a86f 100644 --- a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts +++ b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts @@ -2,6 +2,7 @@ import { DuplicateRemoteSubsetOwnerError, IndeterminateCommitError, PersistedCollectionDurabilityError, + RetryableRemoteSubsetAcquisitionError, safeRandomUUID, toPersistedCollectionDurabilityError, toTransportedLoadSubsetOptions, @@ -37,6 +38,7 @@ const HEARTBEAT_INTERVAL_MS = 3_000 const RPC_TIMEOUT_MS = 10_000 const RPC_RETRY_ATTEMPTS = 2 const RPC_RETRY_DELAY_MS = 200 +const REMOTE_SUBSET_REPLAY_RETRY_ATTEMPTS = 2 const RPC_DEDUPE_RETENTION_MS = 60_000 const WRITER_LOCK_BUSY_RETRY_MS = 50 const WRITER_LOCK_MAX_RETRIES = 20 @@ -136,6 +138,8 @@ type OutboundRemoteSubsetAcquisition = { acquiredLeaderId: string | null inFlight: Promise | null forceReplay: boolean + retryTimer: ReturnType | null + retryAttempts: number } // Adapter with pullSince support @@ -340,6 +344,8 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin acquiredLeaderId: null, inFlight: null, forceReplay: false, + retryTimer: null, + retryAttempts: 0, } this.outboundRemoteSubsetAcquisitions.set( remoteSubsetAcquisitionKey(collectionId, acquisitionId), @@ -358,7 +364,10 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin const acquisitionId = collectionIds?.get(options) if (!acquisitionId) return const key = remoteSubsetAcquisitionKey(collectionId, acquisitionId) - if (!this.outboundRemoteSubsetAcquisitions.delete(key)) return + const acquisition = this.outboundRemoteSubsetAcquisitions.get(key) + if (!acquisition) return + this.outboundRemoteSubsetAcquisitions.delete(key) + this.cancelRemoteSubsetReplayRetry(acquisition) collectionIds!.delete(options) const request: Extract< @@ -401,18 +410,35 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin acquisitionId: acquisition.acquisitionId, options: acquisition.options, } - const response = routedToLocalOwner - ? await this.handleEnsureRemoteSubset( - acquisition.collectionId, - request, - this.nodeId, - ) - : await this.sendRPC( - acquisition.collectionId, - request, - ) + let response: EnsureRemoteSubsetResponse + try { + response = routedToLocalOwner + ? await this.handleEnsureRemoteSubset( + acquisition.collectionId, + request, + this.nodeId, + ) + : await this.sendRPC( + acquisition.collectionId, + request, + ) + } catch (error) { + if ( + routedToLocalOwner || + error instanceof RetryableRemoteSubsetAcquisitionError + ) { + throw error + } + throw new RetryableRemoteSubsetAcquisitionError( + `Remote subset transport failed`, + error, + ) + } if (!response.ok) { + if (response.retryable) { + throw new RetryableRemoteSubsetAcquisitionError(response.error) + } throw new Error(`ensureRemoteSubset failed: ${response.error}`) } acquisition.acquiredLeaderId = response.leaderId @@ -422,6 +448,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin try { await work acquired = true + this.cancelRemoteSubsetReplayRetry(acquisition) } finally { if (acquisition.inFlight === work) acquisition.inFlight = null const current = this.collections.get(acquisition.collectionId) @@ -440,8 +467,8 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin acquisition.acquiredLeaderId !== currentLeaderId)) ) { acquisition.forceReplay = false - void this.acquireRemoteSubset(acquisition).catch(() => { - // Demand stays retained; only new demand or ownership change retries. + void this.acquireRemoteSubset(acquisition).catch((error) => { + this.scheduleRemoteSubsetReplayRetry(acquisition, error) }) } } @@ -545,6 +572,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin this.disposed = true for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + this.cancelRemoteSubsetReplayRetry(acquisition) this.postRemoteSubsetRelease(acquisition) } this.outboundRemoteSubsetAcquisitions.clear() @@ -719,6 +747,49 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin } satisfies ProtocolEnvelope) } + private cancelRemoteSubsetReplayRetry( + acquisition: OutboundRemoteSubsetAcquisition, + ): void { + if (acquisition.retryTimer !== null) { + clearTimeout(acquisition.retryTimer) + acquisition.retryTimer = null + } + acquisition.retryAttempts = 0 + } + + private scheduleRemoteSubsetReplayRetry( + acquisition: OutboundRemoteSubsetAcquisition, + error: unknown, + ): void { + const key = remoteSubsetAcquisitionKey( + acquisition.collectionId, + acquisition.acquisitionId, + ) + if ( + !(error instanceof RetryableRemoteSubsetAcquisitionError) || + this.isDisposed() || + this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition || + acquisition.retryTimer !== null || + acquisition.retryAttempts >= REMOTE_SUBSET_REPLAY_RETRY_ATTEMPTS + ) { + return + } + + acquisition.retryAttempts++ + acquisition.retryTimer = setTimeout(() => { + acquisition.retryTimer = null + if ( + this.isDisposed() || + this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition + ) { + return + } + void this.acquireRemoteSubset(acquisition).catch((retryError) => { + this.scheduleRemoteSubsetReplayRetry(acquisition, retryError) + }) + }, RPC_RETRY_DELAY_MS) + } + private async replayRemoteSubsetAcquisitions( collectionId: string, ): Promise { @@ -741,8 +812,8 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin } acquisition.forceReplay = false replays.push( - this.acquireRemoteSubset(acquisition).catch(() => { - // Demand stays retained; only new demand or ownership change retries. + this.acquireRemoteSubset(acquisition).catch((error) => { + this.scheduleRemoteSubsetReplayRetry(acquisition, error) }), ) } @@ -1089,7 +1160,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin const owner = this.remoteSubsetOwners.get(collectionId) if (!owner) { - throw new Error( + throw new RetryableRemoteSubsetAcquisitionError( `ElectronCollectionCoordinator: no remote subset owner registered for collection "${collectionId}"`, ) } @@ -1869,6 +1940,9 @@ function createRPCErrorResponse( rpcId: request.rpcId, ok: false, error, + ...(cause instanceof RetryableRemoteSubsetAcquisitionError + ? { retryable: true as const } + : {}), } case `rpc:releaseRemoteSubset:req`: return { diff --git a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts index 9ce912d865..690d65dfdd 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts @@ -4,6 +4,7 @@ import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' import { InvalidPersistedCollectionConfigError, + RetryableRemoteSubsetAcquisitionError, persistedCollectionOptions, } from '@tanstack/db-sqlite-persistence-core' import { createNodeSQLitePersistence } from '@tanstack/node-db-sqlite-persistence' @@ -63,9 +64,8 @@ import type { * * The default invoke and Web Locks seams are deterministic process-local * controls. They prove an actual Electron process only when explicit runtime- - * bridge mode runs. This suite also does not yet prove bounded retry after - * follower transport or remote-owner admission failure; that review finding - * remains open. + * bridge mode runs. Bounded replay retry is covered for retryable transport + * and admission failures, including cancellation on release and disposal. */ type InvokeHarness = { @@ -2771,6 +2771,135 @@ describe(`electron sqlite persistence bridge`, () => { } }) + it(`retries retained Electron demand after a retryable replay failure`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-retryable-replay`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + + type Acquisition = { + collectionId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + acquiredLeaderId: string | null + inFlight: Promise | null + forceReplay: boolean + retryTimer: ReturnType | null + retryAttempts: number + } + const acquisition: Acquisition = { + collectionId: `todos`, + acquisitionId: `retryable-electron-replay`, + options: { limit: 1 }, + acquiredLeaderId: `retired-electron-leader`, + inFlight: null, + forceReplay: false, + retryTimer: null, + retryAttempts: 0, + } + const internals = coordinator as unknown as { + nodeId: string + collections: Map + outboundRemoteSubsetAcquisitions: Map + acquireRemoteSubset: (acquisition: Acquisition) => Promise + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + internals.collections.set(`todos`, { + isLeader: true, + leaderId: internals.nodeId, + }) + internals.outboundRemoteSubsetAcquisitions.set( + JSON.stringify([acquisition.collectionId, acquisition.acquisitionId]), + acquisition, + ) + let attempts = 0 + internals.acquireRemoteSubset = vi.fn((current) => { + attempts++ + if (attempts === 1) { + return Promise.reject( + new RetryableRemoteSubsetAcquisitionError(`offline`), + ) + } + current.acquiredLeaderId = internals.nodeId + return Promise.resolve() + }) + + await internals.replayRemoteSubsetAcquisitions(`todos`) + expect(attempts).toBe(1) + await vi.waitFor(() => expect(attempts).toBe(2)) + }) + + it.each([`release`, `dispose`] as const)( + `cancels a scheduled Electron replay retry on %s`, + async (stop) => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-cancel-retry-${stop}`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + + type Acquisition = { + collectionId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + acquiredLeaderId: string | null + inFlight: Promise | null + forceReplay: boolean + retryTimer: ReturnType | null + retryAttempts: number + } + const requestedOptions: LoadSubsetOptions = { limit: 1 } + const acquisition: Acquisition = { + collectionId: `todos`, + acquisitionId: `cancelled-electron-replay`, + options: { limit: 1 }, + acquiredLeaderId: `retired-electron-leader`, + inFlight: null, + forceReplay: false, + retryTimer: null, + retryAttempts: 0, + } + const internals = coordinator as unknown as { + remoteSubsetIds: Map> + collections: Map + outboundRemoteSubsetAcquisitions: Map + acquireRemoteSubset: (acquisition: Acquisition) => Promise + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + internals.collections.set(`todos`, { + isLeader: true, + leaderId: null, + }) + internals.remoteSubsetIds.set( + `todos`, + new WeakMap([[requestedOptions, acquisition.acquisitionId]]), + ) + internals.outboundRemoteSubsetAcquisitions.set( + JSON.stringify([acquisition.collectionId, acquisition.acquisitionId]), + acquisition, + ) + let attempts = 0 + internals.acquireRemoteSubset = vi.fn(() => { + attempts++ + return Promise.reject( + new RetryableRemoteSubsetAcquisitionError(`offline`), + ) + }) + + await internals.replayRemoteSubsetAcquisitions(`todos`) + expect(acquisition.retryTimer).not.toBeNull() + if (stop === `release`) { + await coordinator.requestReleaseRemoteSubset(`todos`, requestedOptions) + } else { + coordinator.dispose() + } + expect(acquisition.retryTimer).toBeNull() + await new Promise((resolve) => setTimeout(resolve, 250)) + expect(attempts).toBe(1) + }, + ) + it(`starts independent Electron lease replays without sibling head-of-line blocking`, async () => { const coordinator = new ElectronCollectionCoordinator({ dbName: `electron-subset-parallel-replay`, From 805da90113ba419eab674fd6f4c1e7413134f5fb Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 20:18:42 +0100 Subject: [PATCH 07/18] test(electric): bind external recovery coordinator --- .../tests/electric-recovery-oracle.test.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts index 2b41742fec..ca4686b9fd 100644 --- a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts +++ b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts @@ -144,6 +144,19 @@ function fixture( }, ensureIndex: () => Promise.resolve(), } + if (coordinator) { + coordinator.requestApplyCommittedTx = async (collectionId, tx) => { + await adapter.applyCommittedTx(collectionId, tx) + return { + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + } + } + } const collection = createCollection( persistedCollectionOptions< Item, @@ -272,6 +285,17 @@ describe(`persisted Electric recovery laws`, () => { ensureLeadership: () => Promise.resolve(), requestEnsurePersistedIndex: () => Promise.resolve(), requestEnsureRemoteSubset: () => Promise.resolve(), + requestReleaseRemoteSubset: () => Promise.resolve(), + registerRemoteSubsetOwner: () => () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), } return { coordinator, From de2da98913a34fb74fcbc0f22d0efc8367663ddb Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 20:23:35 +0100 Subject: [PATCH 08/18] fix(sqlite): preserve follower remote subset routing --- .../src/persisted.ts | 24 ++++++-- .../tests/persisted.test.ts | 59 ++++++++++++++++++- 2 files changed, 76 insertions(+), 7 deletions(-) diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 29bb25caca..7aecc2c61d 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -1346,9 +1346,7 @@ class PersistedCollectionRuntime< ): Promise { const lifecycleGeneration = this.lifecycleGeneration const routeRemoteDemandThroughCoordinator = - this.mode === `sync-present` && - this.remoteSubsetOwnerUnsubscribe !== null && - !(this.persistence.coordinator instanceof SingleProcessCoordinator) + this.canRouteRemoteDemandThroughCoordinator() this.activeSubsets.set(this.getSubsetKey(options), options) const appliedCursor = this.appliedReceiptSequence @@ -2229,11 +2227,25 @@ class PersistedCollectionRuntime< return id } - private queueRemoteSubsetEnsure(options: LoadSubsetOptions): void { + private canRouteRemoteDemandThroughCoordinator(): boolean { if ( this.mode !== `sync-present` || - this.remoteSubsetOwnerUnsubscribe === null || - this.persistence.coordinator instanceof SingleProcessCoordinator || + this.persistence.coordinator instanceof SingleProcessCoordinator + ) { + return false + } + + // A follower routes demand to the elected owner even when its own source + // cannot own acquisitions. Only an elected node needs a local owner. + return ( + !this.persistence.coordinator.isLeader(this.collectionId) || + this.remoteSubsetOwnerUnsubscribe !== null + ) + } + + private queueRemoteSubsetEnsure(options: LoadSubsetOptions): void { + if ( + !this.canRouteRemoteDemandThroughCoordinator() || this.activeSubsets.get(this.getSubsetKey(options)) !== options ) { return diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 7d9ec07c8f..b496ea733b 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -3302,7 +3302,7 @@ describe(`persistedCollectionOptions`, () => { expect(ensureCalls).toBeGreaterThanOrEqual(2) }) - it(`does not route remote demand without a subset owner`, async () => { + it(`does not route remote demand from an ownerless elected leader`, async () => { const ensure = vi.fn(async () => { throw new Error(`no remote subset owner registered`) }) @@ -3359,6 +3359,63 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`routes follower demand without a local subset owner`, async () => { + const ensure = vi.fn(async () => {}) + const coordinator: PersistedCollectionCoordinator = { + getNodeId: () => `ownerless-follower`, + subscribe: () => () => {}, + publish: () => {}, + isLeader: () => false, + ensureLeadership: async () => {}, + requestEnsurePersistedIndex: async () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), + requestEnsureRemoteSubset: ensure, + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: () => () => {}, + } + const collectionId = `ownerless-on-demand-follower` + const collection = createCollection( + persistedCollectionOptions({ + id: collectionId, + syncMode: `on-demand`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + return {} + }, + }, + persistence: { + adapter: createRecordingAdapter(), + coordinator, + }, + }), + ) + const options = { limit: 1 } + + try { + collection.startSyncImmediate() + await flushAsyncWork() + + await expect( + Promise.resolve(collection._sync.loadSubset(options)), + ).resolves.toBeUndefined() + + expect(ensure).toHaveBeenCalledTimes(1) + expect(ensure).toHaveBeenCalledWith(collectionId, options) + } finally { + await collection.cleanup() + } + }) + it(`fails sync-absent persistence when follower ack omits mutation ids`, async () => { const adapter = createRecordingAdapter() const coordinator: PersistedCollectionCoordinator = { From 10320f259c96738f5e88239cbf70ce9cff5b1a69 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 20:54:39 +0100 Subject: [PATCH 09/18] docs: close resolved persistence review gaps --- .../tests/per-collection-coordinator-oracle.test.ts | 5 +++-- packages/db-sqlite-persistence-core/tests/persisted.test.ts | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts index 1fcc42d252..bb8c3af720 100644 --- a/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts @@ -116,8 +116,9 @@ Reach, challenge, replay, cleanup, and limits: - The BroadcastChannel and Web Locks seams below perform real structuredClone and real coordinator code but are deterministic Node controls. They do not earn real-browser, multi-context, OPFS exclusive-handle, worker, Electric, - PowerSync, or service credit. Bounded retry after follower transport or - remote-owner admission failure also remains an open review finding. + PowerSync, or service credit. The focused Browser coordinator owner proves + bounded replay after retryable follower transport or remote-owner admission + failure, plus cancellation on release and disposal. */ type MessageHandler = (event: { data: unknown }) => void diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index b496ea733b..b55a7e1f39 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -54,9 +54,9 @@ import type { Collection, LoadSubsetOptions, SyncConfig } from '@tanstack/db' * multiprocess transport and host-specific replay partitions. * * Driver SQL behavior, native host ownership, and the shared conformance - * portfolio have separate owners. This file does not yet prove that - * non-single-process remote demand is gated by successful remote-subset owner - * registration; that completed-review finding remains open. + * portfolio have separate owners. This file proves the role partition for + * non-single-process remote demand: an ownerless elected node does not route, + * while a follower may route to the elected owner's registered source. */ /** From 3665806c2771cd0c292c088507dfa785d0ab395e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 13:12:44 +0100 Subject: [PATCH 10/18] fix(sqlite): harden coordinator protocol boundaries --- .../src/browser-coordinator.ts | 22 +++++++-- .../tests/browser-coordinator.test.ts | 38 ++++++++++++++- .../src/remote-subset-wire.ts | 38 ++++++++------- .../tests/persisted.test.ts | 47 +++++++++++++++++++ .../src/electron-coordinator.ts | 22 +++++++-- .../tests/electron-ipc.test.ts | 39 +++++++++++++++ 6 files changed, 177 insertions(+), 29 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index 2b8348de17..109676ef0b 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -910,13 +910,10 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina // Handle RPC requests (leader only) if (type && type.endsWith(`:req`)) { + if (!isRPCRequest(payload)) return const collectionId = envelope.collectionId if (this.isLeader(collectionId)) { - void this.handleRPCRequest( - collectionId, - payload as RPCRequest, - envelope.senderId, - ) + void this.handleRPCRequest(collectionId, payload, envelope.senderId) } return } @@ -1931,6 +1928,21 @@ function isProtocolEnvelope(data: unknown): data is ProtocolEnvelope { ) } +function isRPCRequest(payload: unknown): payload is RPCRequest { + if (!payload || typeof payload !== `object`) return false + switch ((payload as { type?: unknown }).type) { + case `rpc:ensureRemoteSubset:req`: + case `rpc:releaseRemoteSubset:req`: + case `rpc:ensurePersistedIndex:req`: + case `rpc:applyLocalMutations:req`: + case `rpc:applyCommittedTx:req`: + case `rpc:pullSince:req`: + return true + default: + return false + } +} + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)) } diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index d2b0b3d378..eeb833414d 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -695,6 +695,33 @@ describe(`BrowserCollectionCoordinator`, () => { expect(received.length).toBe(0) coord.dispose() }) + + it(`ignores unsupported Browser RPC request types without answering`, async () => { + const coordinator = createCoordinator() + coordinator.isLeader = () => true + const postMessage = vi.fn() + const internals = coordinator as unknown as { + onChannelMessage: (message: unknown) => void + channel: { postMessage: (message: unknown) => void } + } + internals.channel.postMessage = postMessage + + internals.onChannelMessage({ + v: 1, + dbName: `test-db`, + collectionId: `todos`, + senderId: `future-browser-peer`, + ts: Date.now(), + payload: { + type: `rpc:futureProtocol:req`, + rpcId: `future-browser-rpc`, + }, + }) + await flush(0) + + expect(postMessage).not.toHaveBeenCalled() + coordinator.dispose() + }) }) describe(`RPC - applyLocalMutations`, () => { @@ -3460,7 +3487,14 @@ describe(`BrowserCollectionCoordinator`, () => { it(`ignores a lower-term Browser heartbeat without replaying to its stale leader`, async () => { const coordinator = createCoordinator() - coordinator.subscribe(`todos`, () => {}) + let forwardedHeartbeats = 0 + coordinator.subscribe(`todos`, (message) => { + if ( + (message.payload as { type?: unknown }).type === `leader:heartbeat` + ) { + forwardedHeartbeats++ + } + }) coordinator.isLeader = () => false const internals = coordinator as unknown as { onChannelMessage: (message: unknown) => void @@ -3498,10 +3532,12 @@ describe(`BrowserCollectionCoordinator`, () => { leaderId: state?.leaderId, latestTerm: state?.latestTerm, replayCalls: replay.mock.calls.length, + forwardedHeartbeats, }).toEqual({ leaderId: `remote-browser-b`, latestTerm: 2, replayCalls: 0, + forwardedHeartbeats: 1, }) } finally { coordinator.dispose() diff --git a/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts b/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts index 407a0534c4..7227d4c4ea 100644 --- a/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts +++ b/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts @@ -413,27 +413,29 @@ function projectCompareOptions( } projected.stringSort = stringSort.value } - const locale = readDataProperty(source, `locale`, `${path}.locale`) - if (locale.present && locale.value !== undefined) { - if (typeof locale.value !== `string`) { - throw new RemoteSubsetWireValueError( - `${path}.locale`, - describe(locale.value), - ) + if (projected.stringSort !== `lexical`) { + const locale = readDataProperty(source, `locale`, `${path}.locale`) + if (locale.present && locale.value !== undefined) { + if (typeof locale.value !== `string`) { + throw new RemoteSubsetWireValueError( + `${path}.locale`, + describe(locale.value), + ) + } + projected.locale = locale.value } - projected.locale = locale.value - } - const localeOptions = readDataProperty( - source, - `localeOptions`, - `${path}.localeOptions`, - ) - if (localeOptions.present && localeOptions.value !== undefined) { - projected.localeOptions = projectWireRecord( - localeOptions.value, + const localeOptions = readDataProperty( + source, + `localeOptions`, `${path}.localeOptions`, - state, ) + if (localeOptions.present && localeOptions.value !== undefined) { + projected.localeOptions = projectWireRecord( + localeOptions.value, + `${path}.localeOptions`, + state, + ) + } } const result = projected as RemoteSubsetWireCompareOptions state.compareOptions.set(source, result) diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index b55a7e1f39..e7d97249a0 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -18,6 +18,7 @@ import { decodePersistedStorageKey, encodePersistedStorageKey, persistedCollectionOptions, + toTransportedLoadSubsetOptions, } from '../src' import type { PersistedCollectionCoordinator, @@ -1007,6 +1008,52 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`projects lexical comparison options without locale-only wire fields`, () => { + const projected = toTransportedLoadSubsetOptions({ + orderBy: [ + { + expression: new IR.PropRef([`todos`, `title`]), + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `lexical`, + locale: `en`, + localeOptions: { sensitivity: `base` }, + }, + }, + ], + } as unknown as LoadSubsetOptions) + + expect(projected.orderBy?.[0]?.compareOptions).toEqual({ + direction: `asc`, + nulls: `last`, + stringSort: `lexical`, + }) + }) + + it(`preserves locale comparison fields when the optional sort mode is omitted`, () => { + const projected = toTransportedLoadSubsetOptions({ + orderBy: [ + { + expression: new IR.PropRef([`todos`, `title`]), + compareOptions: { + direction: `asc`, + nulls: `last`, + locale: `en`, + localeOptions: { sensitivity: `base` }, + }, + }, + ], + } as LoadSubsetOptions) + + expect(projected.orderBy?.[0]?.compareOptions).toEqual({ + direction: `asc`, + nulls: `last`, + locale: `en`, + localeOptions: { sensitivity: `base` }, + }) + }) + it(`reports and rethrows a single-process owner unload rejection`, async () => { const coordinator = new SingleProcessCoordinator(`single-unload-error`) const unloadError = new Error(`single-process owner unload failed`) diff --git a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts index 551306a86f..765f171c8a 100644 --- a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts +++ b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts @@ -898,13 +898,10 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin // Handle RPC requests (leader only) if (type && type.endsWith(`:req`)) { + if (!isRPCRequest(payload)) return const collectionId = envelope.collectionId if (this.isLeader(collectionId)) { - void this.handleRPCRequest( - collectionId, - payload as RPCRequest, - envelope.senderId, - ) + void this.handleRPCRequest(collectionId, payload, envelope.senderId) } return } @@ -1877,6 +1874,21 @@ function isProtocolEnvelope(data: unknown): data is ProtocolEnvelope { ) } +function isRPCRequest(payload: unknown): payload is RPCRequest { + if (!payload || typeof payload !== `object`) return false + switch ((payload as { type?: unknown }).type) { + case `rpc:ensureRemoteSubset:req`: + case `rpc:releaseRemoteSubset:req`: + case `rpc:ensurePersistedIndex:req`: + case `rpc:applyLocalMutations:req`: + case `rpc:applyCommittedTx:req`: + case `rpc:pullSince:req`: + return true + default: + return false + } +} + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)) } diff --git a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts index 690d65dfdd..366d4f9703 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts @@ -820,6 +820,37 @@ describe(`electron sqlite persistence bridge`, () => { }, ) + it(`ignores unsupported Electron RPC request types without answering`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-future-rpc`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const postMessage = vi.fn() + const internals = coordinator as unknown as { + onChannelMessage: (message: unknown) => void + channel: { postMessage: (message: unknown) => void } + } + internals.channel.postMessage = postMessage + + internals.onChannelMessage({ + v: 1, + dbName: `electron-future-rpc`, + collectionId: `todos`, + senderId: `future-electron-peer`, + ts: Date.now(), + payload: { + type: `rpc:futureProtocol:req`, + rpcId: `future-electron-rpc`, + }, + }) + await Promise.resolve() + await Promise.resolve() + + expect(postMessage).not.toHaveBeenCalled() + }) + it(`fails indeterminate instead of retrying a committed mutation across leaders`, async () => { const bEffects: Array = [] const coordinator = new ElectronCollectionCoordinator({ @@ -3170,6 +3201,12 @@ describe(`electron sqlite persistence bridge`, () => { replayRemoteSubsetAcquisitions: (collectionId: string) => Promise } internals.acquireLeadership = async () => {} + let forwardedHeartbeats = 0 + coordinator.subscribe(`todos`, (message) => { + if ((message.payload as { type?: unknown }).type === `leader:heartbeat`) { + forwardedHeartbeats++ + } + }) const replay = vi.fn(() => Promise.resolve()) internals.replayRemoteSubsetAcquisitions = replay const heartbeat = (leaderId: string, term: number) => @@ -3197,10 +3234,12 @@ describe(`electron sqlite persistence bridge`, () => { leaderId: state?.leaderId, latestTerm: state?.latestTerm, replayCalls: replay.mock.calls.length, + forwardedHeartbeats, }).toEqual({ leaderId: `remote-electron-b`, latestTerm: 2, replayCalls: 0, + forwardedHeartbeats: 1, }) }) From e064fc5b38b994bee9565f997c75b16baea0ba6d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 15:58:46 +0100 Subject: [PATCH 11/18] fix(sqlite): preserve local subset lifecycle options --- .../src/browser-coordinator.ts | 11 +- .../tests/browser-coordinator.test.ts | 31 +++- .../src/persisted.ts | 16 +- .../src/remote-subset-wire.ts | 32 +++- .../tests/persisted.test.ts | 146 +++++++++++++++++- .../src/electron-coordinator.ts | 11 +- .../tests/electron-ipc.test.ts | 43 ++++++ 7 files changed, 271 insertions(+), 19 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index 109676ef0b..cb9f66d590 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -5,6 +5,7 @@ import { RetryableRemoteSubsetAcquisitionError, safeRandomUUID, toPersistedCollectionDurabilityError, + toProcessLocalLoadSubsetOptions, toTransportedLoadSubsetOptions, } from '@tanstack/db-sqlite-persistence-core' import type { @@ -163,6 +164,7 @@ type OutboundRemoteSubsetAcquisition = { collectionId: string acquisitionId: string options: TransportedLoadSubsetOptions + localOptions: TransportedLoadSubsetOptions acquiredLeaderId: string | null inFlight: Promise | null forceReplay: boolean @@ -336,6 +338,10 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina options: LoadSubsetOptions, ): Promise { const transportedOptions = toTransportedLoadSubsetOptions(options) + const localOptions = toProcessLocalLoadSubsetOptions( + options, + transportedOptions, + ) let collectionIds = this.remoteSubsetIds.get(collectionId) if (!collectionIds) { collectionIds = new WeakMap() @@ -354,6 +360,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina collectionId, acquisitionId, options: transportedOptions, + localOptions, acquiredLeaderId: null, inFlight: null, forceReplay: false, @@ -431,6 +438,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina acquisition.collectionId, request, this.nodeId, + acquisition.localOptions, ) : await this.sendRPC( acquisition.collectionId, @@ -1124,6 +1132,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina collectionId: string, request: Extract, requesterId: string, + localOptions?: TransportedLoadSubsetOptions, ): Promise { this.pruneReleasedRemoteSubsetAcquisitions() const key = inboundRemoteSubsetAcquisitionKey( @@ -1179,7 +1188,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina requesterId, acquisitionId: request.acquisitionId, owner, - options: awaitingOwner?.options ?? request.options, + options: awaitingOwner?.options ?? localOptions ?? request.options, load: Promise.resolve(), transferred: false, released: false, diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index eeb833414d..61b8c2b6a6 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -2758,16 +2758,18 @@ describe(`BrowserCollectionCoordinator`, () => { }, ) - it(`preserves supported native values, aliases, and cycles for both routes`, async () => { + it(`preserves local lifecycle fields while transporting only wire values`, async () => { const leader = createCoordinator() const follower = createCoordinator() const received: Array = [] - const owner = withUnusedUnloadSubset( + const unloadSubset = vi.fn((_options: TransportedLoadSubsetOptions) => {}) + const owner = Object.assign( vi.fn((options: TransportedLoadSubsetOptions) => { received.push(options) return Promise.resolve() }), - ) + { unloadSubset, onError: vi.fn() }, + ) satisfies RemoteSubsetOwner const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) leader.subscribe(`todos`, () => {}) follower.subscribe(`todos`, () => {}) @@ -2843,7 +2845,7 @@ describe(`BrowserCollectionCoordinator`, () => { await follower.requestEnsureRemoteSubset(`todos`, options) expect(received).toHaveLength(2) - for (const decoded of received) { + for (const [routeIndex, decoded] of received.entries()) { const value = ( decoded.where as unknown as { args: Array<{ value?: typeof richValue }> @@ -2930,12 +2932,25 @@ describe(`BrowserCollectionCoordinator`, () => { configurable: true, }) expect(value.reservedKeys.constructor).toBe(`own constructor value`) - expect(decoded).not.toHaveProperty(`signal`) - expect(decoded).not.toHaveProperty(`subscription`) + if (routeIndex === 0) { + const local = decoded as TransportedLoadSubsetOptions & + Pick + expect(local.signal).toBe(options.signal) + expect(local.subscription).toBe(options.subscription) + } else { + expect(decoded).not.toHaveProperty(`signal`) + expect(decoded).not.toHaveProperty(`subscription`) + } } - // Local delivery need not clone identity, but it must receive the same - // validated, live-field-free wire domain as follower delivery. + // Both routes validate immutable request data. Only process-local + // delivery retains live lifecycle references. expect(received[1]).not.toBe(options) + + await leader.requestReleaseRemoteSubset(`todos`, options) + await follower.requestReleaseRemoteSubset(`todos`, options) + expect(unloadSubset.mock.calls.map(([value]) => value)).toEqual( + received, + ) } finally { unregisterOwner() leader.dispose() diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 7aecc2c61d..934fcc7469 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -16,7 +16,10 @@ import { PersistedCollectionDurabilityError, toPersistedCollectionDurabilityError, } from './errors' -import { toTransportedLoadSubsetOptions } from './remote-subset-wire' +import { + toProcessLocalLoadSubsetOptions, + toTransportedLoadSubsetOptions, +} from './remote-subset-wire' import type { TransportedLoadSubsetOptions } from './remote-subset-wire' import type { StandardSchemaV1 } from '@standard-schema/spec' import type { @@ -615,6 +618,7 @@ export class SingleProcessCoordinator implements PersistedCollectionCoordinator options: LoadSubsetOptions, ): Promise { const transported = toTransportedLoadSubsetOptions(options) + const localOptions = toProcessLocalLoadSubsetOptions(options, transported) const owner = this.remoteSubsetOwners.get(collectionId) if (!owner) { throw new InvalidPersistedCollectionConfigError( @@ -638,9 +642,9 @@ export class SingleProcessCoordinator implements PersistedCollectionCoordinator resolveLoad = resolve rejectLoad = reject }) - acquisitions.set(options, { owner, options: transported, load }) + acquisitions.set(options, { owner, options: localOptions, load }) try { - const ownerLoad = owner(transported) + const ownerLoad = owner(localOptions) void Promise.resolve(ownerLoad).then(resolveLoad, (error) => { reportRemoteSubsetOwnerError(owner, error) rejectLoad(error) @@ -1345,7 +1349,7 @@ class PersistedCollectionRuntime< upstreamLoadSubset?: LoadSubsetFn, ): Promise { const lifecycleGeneration = this.lifecycleGeneration - const routeRemoteDemandThroughCoordinator = + const routeRemoteDemandDuringHydration = this.canRouteRemoteDemandThroughCoordinator() this.activeSubsets.set(this.getSubsetKey(options), options) @@ -1353,14 +1357,14 @@ class PersistedCollectionRuntime< await this.applyMutex.run(() => this.hydrateSubsetUnsafe(options, { requestRemoteEnsure: - this.mode === `sync-present` && !routeRemoteDemandThroughCoordinator, + this.mode === `sync-present` && !routeRemoteDemandDuringHydration, lifecycleGeneration, }), ) if (lifecycleGeneration !== this.lifecycleGeneration) return await this.waitForAppliedReceiptsAfter(appliedCursor) - if (routeRemoteDemandThroughCoordinator) { + if (this.canRouteRemoteDemandThroughCoordinator()) { try { await this.persistence.coordinator.requestEnsureRemoteSubset( this.collectionId, diff --git a/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts b/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts index 7227d4c4ea..6fd85e76e6 100644 --- a/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts +++ b/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts @@ -187,6 +187,34 @@ export function toTransportedLoadSubsetOptions( return projected } +/** Adds live lifecycle references only to an already validated local request. */ +export function toProcessLocalLoadSubsetOptions( + options: LoadSubsetOptions, + transported = toTransportedLoadSubsetOptions(options), +): TransportedLoadSubsetOptions & + Pick { + const projected = { + ...transported, + } as TransportedLoadSubsetOptions & + Pick + const signal = readDataProperty(options, `signal`, `options.signal`) + const subscription = readDataProperty( + options, + `subscription`, + `options.subscription`, + ) + + if (signal.present) { + projected.signal = signal.value as LoadSubsetOptions[`signal`] + } + if (subscription.present) { + projected.subscription = + subscription.value as LoadSubsetOptions[`subscription`] + } + + return projected +} + function projectExpression( value: unknown, path: string, @@ -690,8 +718,8 @@ function projectWireRecord( } function projectNumber(value: unknown, path: string): number { - if (typeof value !== `number`) { - throw new RemoteSubsetWireValueError(path, describe(value)) + if (typeof value !== `number` || !Number.isSafeInteger(value) || value < 0) { + throw new RemoteSubsetWireValueError(path, `non-negative safe integer`) } return value } diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index e7d97249a0..a63bfdf24a 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -30,7 +30,12 @@ import type { RemoteSubsetOwner, TxCommitted, } from '../src' -import type { Collection, LoadSubsetOptions, SyncConfig } from '@tanstack/db' +import type { + Collection, + LoadSubsetOptions, + Subscription, + SyncConfig, +} from '@tanstack/db' /** * # Does persisted wrapping preserve one Collection history? @@ -843,6 +848,38 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`preserves process-local subset lifecycle fields for owner load and unload`, async () => { + const coordinator = new SingleProcessCoordinator(`single-live-fields`) + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const signal = new AbortController().signal + const subscription = { + on: () => () => {}, + } as unknown as Subscription + const options: LoadSubsetOptions = { limit: 1, signal, subscription } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, options) + expect(owner).toHaveBeenCalledTimes(1) + const delivered = owner.mock.calls[0]![0] as LoadSubsetOptions + expect(delivered).toMatchObject({ limit: 1 }) + expect(delivered.signal).toBe(signal) + expect(delivered.subscription).toBe(subscription) + + await coordinator.requestReleaseRemoteSubset(`todos`, options) + expect(owner.unloadSubset).toHaveBeenCalledTimes(1) + expect(owner.unloadSubset).toHaveBeenCalledWith(delivered) + } finally { + unregisterOwner() + } + }) + it(`coalesces same-stack single-process subset reentry until owner work finishes`, async () => { const coordinator = new SingleProcessCoordinator(`single-reentrant`) let releaseLoad = (): void => {} @@ -1054,6 +1091,42 @@ describe(`persistedCollectionOptions`, () => { }) }) + it.each([ + [`limit`, { limit: Number.NaN }, `options.limit`], + [`limit`, { limit: Number.POSITIVE_INFINITY }, `options.limit`], + [`limit`, { limit: -1 }, `options.limit`], + [`limit`, { limit: 0.5 }, `options.limit`], + [`limit`, { limit: Number.MAX_SAFE_INTEGER + 1 }, `options.limit`], + [`offset`, { offset: Number.NaN }, `options.offset`], + [`offset`, { offset: Number.NEGATIVE_INFINITY }, `options.offset`], + [`offset`, { offset: -1 }, `options.offset`], + [`offset`, { offset: 0.5 }, `options.offset`], + [`offset`, { offset: Number.MAX_SAFE_INTEGER + 1 }, `options.offset`], + ] as const)( + `rejects an invalid transported subset %s before owner work`, + (_field, options, path) => { + let error: unknown + try { + toTransportedLoadSubsetOptions(options) + } catch (caught) { + error = caught + } + expect(error).toMatchObject({ + name: `RemoteSubsetWireValueError`, + path, + }) + }, + ) + + it(`preserves valid transported subset window boundaries`, () => { + expect( + toTransportedLoadSubsetOptions({ + limit: 0, + offset: Number.MAX_SAFE_INTEGER, + }), + ).toEqual({ limit: 0, offset: Number.MAX_SAFE_INTEGER }) + }) + it(`reports and rethrows a single-process owner unload rejection`, async () => { const coordinator = new SingleProcessCoordinator(`single-unload-error`) const unloadError = new Error(`single-process owner unload failed`) @@ -3406,6 +3479,77 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`uses dispatch-time ownership after follower hydration becomes ownerless leader`, async () => { + const adapter = createRecordingAdapter() + const loadSubset = adapter.loadSubset + let markHydrationStarted = (): void => {} + const hydrationStarted = new Promise((resolve) => { + markHydrationStarted = resolve + }) + let releaseHydration = (): void => {} + const hydrationGate = new Promise((resolve) => { + releaseHydration = resolve + }) + adapter.loadSubset = async (...args) => { + markHydrationStarted() + await hydrationGate + return loadSubset(...args) + } + + let isLeader = false + const staleEnsure = new Error(`ownerless leader must not route demand`) + const ensure = vi.fn(() => Promise.reject(staleEnsure)) + const coordinator: PersistedCollectionCoordinator = { + getNodeId: () => `transitioning-node`, + subscribe: () => () => {}, + publish: () => {}, + isLeader: () => isLeader, + ensureLeadership: async () => {}, + requestEnsurePersistedIndex: async () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), + requestEnsureRemoteSubset: ensure, + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: () => () => {}, + } + const collection = createCollection( + persistedCollectionOptions({ + id: `dispatch-time-ownerless-leader`, + syncMode: `on-demand`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + return {} + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + try { + collection.startSyncImmediate() + await flushAsyncWork() + const load = Promise.resolve(collection._sync.loadSubset({ limit: 1 })) + await hydrationStarted + isLeader = true + releaseHydration() + + await expect(load).resolves.toBeUndefined() + expect(ensure).not.toHaveBeenCalled() + } finally { + releaseHydration() + await collection.cleanup() + } + }) + it(`routes follower demand without a local subset owner`, async () => { const ensure = vi.fn(async () => {}) const coordinator: PersistedCollectionCoordinator = { diff --git a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts index 765f171c8a..fd5f3b4769 100644 --- a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts +++ b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts @@ -5,6 +5,7 @@ import { RetryableRemoteSubsetAcquisitionError, safeRandomUUID, toPersistedCollectionDurabilityError, + toProcessLocalLoadSubsetOptions, toTransportedLoadSubsetOptions, } from '@tanstack/db-sqlite-persistence-core' import type { @@ -135,6 +136,7 @@ type OutboundRemoteSubsetAcquisition = { collectionId: string acquisitionId: string options: TransportedLoadSubsetOptions + localOptions: TransportedLoadSubsetOptions acquiredLeaderId: string | null inFlight: Promise | null forceReplay: boolean @@ -323,6 +325,10 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin options: LoadSubsetOptions, ): Promise { const transportedOptions = toTransportedLoadSubsetOptions(options) + const localOptions = toProcessLocalLoadSubsetOptions( + options, + transportedOptions, + ) let collectionIds = this.remoteSubsetIds.get(collectionId) if (!collectionIds) { collectionIds = new WeakMap() @@ -341,6 +347,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin collectionId, acquisitionId, options: transportedOptions, + localOptions, acquiredLeaderId: null, inFlight: null, forceReplay: false, @@ -417,6 +424,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin acquisition.collectionId, request, this.nodeId, + acquisition.localOptions, ) : await this.sendRPC( acquisition.collectionId, @@ -1112,6 +1120,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin collectionId: string, request: Extract, requesterId: string, + localOptions?: TransportedLoadSubsetOptions, ): Promise { this.pruneReleasedRemoteSubsetAcquisitions() const key = inboundRemoteSubsetAcquisitionKey( @@ -1167,7 +1176,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin requesterId, acquisitionId: request.acquisitionId, owner, - options: awaitingOwner?.options ?? request.options, + options: awaitingOwner?.options ?? localOptions ?? request.options, load: Promise.resolve(), transferred: false, released: false, diff --git a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts index 366d4f9703..b2f106bbea 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts @@ -2198,6 +2198,49 @@ describe(`electron sqlite persistence bridge`, () => { await pending }) + it(`preserves process-local Electron subset lifecycle fields`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-local-lifecycle-fields`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + + const received: Array = [] + const owner = Object.assign( + vi.fn((options: TransportedLoadSubsetOptions) => { + received.push(options) + }), + { + unloadSubset: vi.fn(), + onError: vi.fn(), + }, + ) satisfies RemoteSubsetOwner + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const signal = new AbortController().signal + const subscription = { + on: () => () => {}, + } as unknown as Subscription + const options: LoadSubsetOptions = { limit: 1, signal, subscription } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, options) + expect(received).toHaveLength(1) + expect(received[0]).toMatchObject({ limit: 1 }) + const local = received[0] as TransportedLoadSubsetOptions & + Pick + expect(local.signal).toBe(signal) + expect(local.subscription).toBe(subscription) + + await coordinator.requestReleaseRemoteSubset(`todos`, options) + expect(owner.unloadSubset).toHaveBeenCalledWith(received[0]) + } finally { + unregisterOwner() + } + }) + it(`loads and releases exact Electron remote-subset acquisitions`, async () => { const coordinator = new ElectronCollectionCoordinator({ dbName: `electron-subset-owner-routing`, From a5b707777391369196b1016f5c0a14ea944e9066 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 17:42:43 +0100 Subject: [PATCH 12/18] docs(oracles): label coordinator coverage limits --- .../tests/browser-coordinator.test.ts | 8 ++++---- .../per-collection-coordinator-oracle.test.ts | 2 +- .../tests/persisted.test.ts | 15 ++++++++------- .../tests/electron-ipc.test.ts | 9 +++++---- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 61b8c2b6a6..7b3b78580c 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -37,10 +37,10 @@ import type { BrowserCollectionCoordinatorOptions } from '../src/browser-coordin * advance the released-tombstone clock. Exact call counts and error identities * prevent a final-state-only false green. * - * The composed public-Collection and generated route models live in - * `per-collection-coordinator-oracle.test.ts`. These seams do not prove real - * browser scheduling, Web Locks, BroadcastChannel, OPFS ownership, or worker - * behavior. Bounded replay retry is covered for retryable transport and + * Known omissions: the composed public-Collection and generated route models + * live in `per-collection-coordinator-oracle.test.ts`. These seams do not prove + * real browser scheduling, Web Locks, BroadcastChannel, OPFS ownership, or + * worker behavior. Bounded replay retry is covered for retryable transport and * admission failures, including cancellation on release and disposal. */ diff --git a/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts index bb8c3af720..5e6bd28a6a 100644 --- a/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts @@ -113,7 +113,7 @@ Reach, challenge, replay, cleanup, and limits: - Coordinators, Collections, and databases use failure-preserving cleanup. A final lifecycle test proves no channel, held lock, queued lock, or delayed delivery remains and that a fresh database name elects normally. -- The BroadcastChannel and Web Locks seams below perform real structuredClone +- Known omissions: the BroadcastChannel and Web Locks seams below perform real structuredClone and real coordinator code but are deterministic Node controls. They do not earn real-browser, multi-context, OPFS exclusive-handle, worker, Electric, PowerSync, or service credit. The focused Browser coordinator owner proves diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index a63bfdf24a..40223ca225 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -59,10 +59,11 @@ import type { * and durability classification. Focused Browser and Electron suites own the * multiprocess transport and host-specific replay partitions. * - * Driver SQL behavior, native host ownership, and the shared conformance - * portfolio have separate owners. This file proves the role partition for - * non-single-process remote demand: an ownerless elected node does not route, - * while a follower may route to the elected owner's registered source. + * Known omissions: driver SQL behavior, native host ownership, and the shared + * conformance portfolio have separate owners. This file proves the role + * partition for non-single-process remote demand: an ownerless elected node + * does not route, while a follower may route to the elected owner's registered + * source. */ /** @@ -81,9 +82,9 @@ import type { * cleanup, and restart. They compare durable state, public rows, metadata, * request options, sequence evidence, errors, and late-work fencing. * - * Driver SQL behavior, browser page ownership, native runtimes, and the shared - * conformance portfolio have separate owners. This file models persistence - * protocol state, not a particular SQLite engine. + * Known omissions: driver SQL behavior, browser page ownership, native + * runtimes, and the shared conformance portfolio have separate owners. This + * file models persistence protocol state, not a particular SQLite engine. */ type Todo = { diff --git a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts index b2f106bbea..1276608eb9 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts @@ -62,10 +62,11 @@ import type { * held work, changed leaders, rejected owner operations, and released-tombstone * clock advances challenge the assertions. * - * The default invoke and Web Locks seams are deterministic process-local - * controls. They prove an actual Electron process only when explicit runtime- - * bridge mode runs. Bounded replay retry is covered for retryable transport - * and admission failures, including cancellation on release and disposal. + * Known omissions: the default invoke and Web Locks seams are deterministic + * process-local controls. They prove an actual Electron process only when + * explicit runtime-bridge mode runs. Bounded replay retry is covered for + * retryable transport and admission failures, including cancellation on + * release and disposal. */ type InvokeHarness = { From 60d4c38558dc54f8e4eb51104ecc77de4fb6b57a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 18:12:58 +0100 Subject: [PATCH 13/18] refactor(sqlite): share host coordinator engine --- .../src/browser-coordinator.ts | 2118 +---------------- .../tests/browser-coordinator.test.ts | 8 +- .../tsconfig.json | 3 + .../db-sqlite-persistence-core/package.json | 10 + .../src/broadcast-coordinator.ts | 2096 ++++++++++++++++ .../src/persisted.ts | 40 +- .../src/remote-subset-owner.ts | 36 + .../db-sqlite-persistence-core/vite.config.ts | 2 +- .../src/electron-coordinator.ts | 2064 +--------------- .../tests/electron-ipc.test.ts | 92 +- .../tsconfig.json | 3 + 11 files changed, 2258 insertions(+), 4214 deletions(-) create mode 100644 packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts create mode 100644 packages/db-sqlite-persistence-core/src/remote-subset-owner.ts diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index cb9f66d590..f20d8f96fc 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -1,2114 +1,16 @@ -import { - DuplicateRemoteSubsetOwnerError, - IndeterminateCommitError, - PersistedCollectionDurabilityError, - RetryableRemoteSubsetAcquisitionError, - safeRandomUUID, - toPersistedCollectionDurabilityError, - toProcessLocalLoadSubsetOptions, - toTransportedLoadSubsetOptions, -} from '@tanstack/db-sqlite-persistence-core' -import type { - ApplyCommittedTxResponse, - ApplyLocalMutationsResponse, - EnsureRemoteSubsetRequest, - EnsureRemoteSubsetResponse, - IndeterminateCommitRequestType, - PersistedCollectionCoordinator, - PersistedIndexSpec, - PersistedMutationEnvelope, - PersistedRowMetadataMutation, - PersistedTx, - PersistenceAdapter, - ProtocolEnvelope, - PullSinceResponse, - ReleaseRemoteSubsetRequest, - ReleaseRemoteSubsetResponse, - RemoteSubsetOwner, - TransportedLoadSubsetOptions, - TxCommitted, -} from '@tanstack/db-sqlite-persistence-core' -import type { LoadSubsetOptions } from '@tanstack/db' +import { BroadcastCollectionCoordinator } from '@tanstack/db-sqlite-persistence-core/broadcast-coordinator' +import type { BroadcastCollectionCoordinatorOptions } from '@tanstack/db-sqlite-persistence-core/broadcast-coordinator' -// --------------------------------------------------------------------------- -// Constants -// --------------------------------------------------------------------------- - -const HEARTBEAT_INTERVAL_MS = 3_000 -const RPC_TIMEOUT_MS = 10_000 -const RPC_RETRY_ATTEMPTS = 2 -const RPC_RETRY_DELAY_MS = 200 -const REMOTE_SUBSET_REPLAY_RETRY_ATTEMPTS = 2 -const RPC_DEDUPE_RETENTION_MS = 60_000 -const WRITER_LOCK_BUSY_RETRY_MS = 50 -const WRITER_LOCK_MAX_RETRIES = 20 - -// --------------------------------------------------------------------------- -// Internal types -// --------------------------------------------------------------------------- - -type RPCRequest = - | EnsureRemoteSubsetRequest - | ReleaseRemoteSubsetRequest - | { - type: `rpc:ensurePersistedIndex:req` - rpcId: string - signature: string - spec: PersistedIndexSpec - } - | { - type: `rpc:applyLocalMutations:req` - rpcId: string - envelopeId: string - mutations: Array - } - | { - type: `rpc:applyCommittedTx:req` - rpcId: string - envelopeId: string - tx: PersistedTx - } - | { - type: `rpc:pullSince:req` - rpcId: string - fromRowVersion: number - } - -type RPCResponse = - | EnsureRemoteSubsetResponse - | ReleaseRemoteSubsetResponse - | { - type: `rpc:ensurePersistedIndex:res` - rpcId: string - ok: boolean - error?: string - } - | ApplyLocalMutationsResponse - | ApplyCommittedTxResponse - | PullSinceResponse - -type PendingRPC = { - resolve: (response: RPCResponse) => void - reject: (error: Error) => void - timer: ReturnType -} - -type CollectionState = { - isLeader: boolean - leaderId: string | null - lockAbortController: AbortController | null - heartbeatTimer: ReturnType | null - latestTerm: number - latestSeq: number - latestRowVersion: number - subscribers: Set<(message: ProtocolEnvelope) => void> -} - -// Adapter capabilities used by coordinator-side operations -type CoordinatorAdapter = PersistenceAdapter & { - pullSince?: ( - collectionId: string, - fromRowVersion: number, - ) => Promise< - | { - latestRowVersion: number - requiresFullReload: true - } - | { - latestRowVersion: number - requiresFullReload: false - changedKeys: Array - deletedKeys: Array - } - > - getStreamPosition?: (collectionId: string) => Promise<{ - latestTerm: number - latestSeq: number - latestRowVersion: number - }> -} - -type ActiveRemoteSubsetAcquisition = { - collectionId: string - requesterId: string - acquisitionId: string - owner: RemoteSubsetOwner - options: TransportedLoadSubsetOptions - load: Promise - transferred: boolean - released: boolean - terminalRelease: boolean - release: Promise | null -} - -type AwaitingRemoteSubsetOwnerAcquisition = { - collectionId: string - requesterId: string - acquisitionId: string - options: TransportedLoadSubsetOptions - released: true - awaitingOwner: true -} - -type RemoteSubsetAcquisition = - | ActiveRemoteSubsetAcquisition - | AwaitingRemoteSubsetOwnerAcquisition - | { - collectionId: string - requesterId: string - acquisitionId: string - released: true - } - -type OutboundRemoteSubsetAcquisition = { - collectionId: string - acquisitionId: string - options: TransportedLoadSubsetOptions - localOptions: TransportedLoadSubsetOptions - acquiredLeaderId: string | null - inFlight: Promise | null - forceReplay: boolean - retryTimer: ReturnType | null - retryAttempts: number -} - -type AppliedEnvelope = - | { - appliedAt: number - requestType: `rpc:applyLocalMutations:req` - response: ApplyLocalMutationsResponse - } - | { - appliedAt: number - requestType: `rpc:applyCommittedTx:req` - response: ApplyCommittedTxResponse - } - -type InFlightEnvelope = - | { - requestType: `rpc:applyLocalMutations:req` - response: Promise - } - | { - requestType: `rpc:applyCommittedTx:req` - response: Promise - } - -// --------------------------------------------------------------------------- -// Options -// --------------------------------------------------------------------------- - -export type BrowserCollectionCoordinatorOptions = { - dbName: string - adapter?: CoordinatorAdapter -} - -// --------------------------------------------------------------------------- -// BrowserCollectionCoordinator -// --------------------------------------------------------------------------- - -export class BrowserCollectionCoordinator implements PersistedCollectionCoordinator { - private readonly nodeId = safeRandomUUID() - private readonly dbName: string - private defaultAdapter: CoordinatorAdapter | null - private readonly collectionAdapters = new Map() - private readonly remoteSubsetOwners = new Map() - private readonly remoteSubsetIds = new Map< - string, - WeakMap - >() - private readonly outboundRemoteSubsetAcquisitions = new Map< - string, - OutboundRemoteSubsetAcquisition - >() - private readonly inboundRemoteSubsetAcquisitions = new Map< - string, - RemoteSubsetAcquisition - >() - private readonly releasedRemoteSubsetAcquisitionTimes = new Map< - string, - number - >() - private readonly channel: BroadcastChannel - private readonly collections = new Map() - private readonly pendingRPCs = new Map() - private readonly appliedEnvelopes = new Map() - private readonly inFlightEnvelopes = new Map() - private disposed = false - - /** Method indirection to prevent TypeScript from narrowing `disposed` across awaits */ - private isDisposed(): boolean { - return this.disposed - } - - private requireAdapter(collectionId: string): CoordinatorAdapter { - const adapter = - this.collectionAdapters.get(collectionId) ?? this.defaultAdapter - if (!adapter) { - throw new Error( - `BrowserCollectionCoordinator: adapter not set for collection "${collectionId}". Call setAdapterForCollection() before using leader-side operations.`, - ) - } - return adapter - } +export type BrowserCollectionCoordinatorOptions = Omit< + BroadcastCollectionCoordinatorOptions, + `coordinatorName` +> +export class BrowserCollectionCoordinator extends BroadcastCollectionCoordinator { constructor(options: BrowserCollectionCoordinatorOptions) { - this.dbName = options.dbName - this.defaultAdapter = options.adapter ?? null - this.channel = new BroadcastChannel(`tsdb:coord:${this.dbName}`) - this.channel.onmessage = (event: MessageEvent) => { - this.onChannelMessage(event.data) - } - } - - /** - * Set or replace the persistence adapter used for leader-side RPC handling. - * Called by `createBrowserWASQLitePersistence` to wire the internally-created - * adapter into the coordinator. - */ - setAdapter(adapter: CoordinatorAdapter): void { - this.defaultAdapter = adapter - } - - /** Register the persistence adapter that owns one collection. */ - setAdapterForCollection( - collectionId: string, - adapter: CoordinatorAdapter, - ): void { - this.collectionAdapters.set(collectionId, adapter) - } - - registerRemoteSubsetOwner( - collectionId: string, - owner: RemoteSubsetOwner, - ): () => void { - if (this.remoteSubsetOwners.has(collectionId)) { - throw new DuplicateRemoteSubsetOwnerError(collectionId) - } - this.remoteSubsetOwners.set(collectionId, owner) - for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { - if (acquisition.collectionId !== collectionId) continue - acquisition.acquiredLeaderId = null - acquisition.forceReplay = true - } - void this.replayRemoteSubsetAcquisitions(collectionId) - this.rebindRemoteInboundSubsetAcquisitions(collectionId, owner) - return () => { - if (this.remoteSubsetOwners.get(collectionId) !== owner) return - this.remoteSubsetOwners.delete(collectionId) - this.releaseInboundRemoteSubsetAcquisitions(collectionId, owner) - } - } - - // ----------------------------------------------------------------------- - // PersistedCollectionCoordinator interface - // ----------------------------------------------------------------------- - - getNodeId(): string { - return this.nodeId - } - - subscribe( - collectionId: string, - onMessage: (message: ProtocolEnvelope) => void, - ): () => void { - const state = this.ensureCollectionState(collectionId) - state.subscribers.add(onMessage) - return () => { - state.subscribers.delete(onMessage) - } - } - - publish(_collectionId: string, message: ProtocolEnvelope): void { - this.channel.postMessage(message) - } - - isLeader(collectionId: string): boolean { - return this.collections.get(collectionId)?.isLeader ?? false - } - - async ensureLeadership(collectionId: string): Promise { - const state = this.ensureCollectionState(collectionId) - if (state.isLeader) return - await this.acquireLeadership(collectionId, state) - } - - async requestEnsureRemoteSubset( - collectionId: string, - options: LoadSubsetOptions, - ): Promise { - const transportedOptions = toTransportedLoadSubsetOptions(options) - const localOptions = toProcessLocalLoadSubsetOptions( - options, - transportedOptions, - ) - let collectionIds = this.remoteSubsetIds.get(collectionId) - if (!collectionIds) { - collectionIds = new WeakMap() - this.remoteSubsetIds.set(collectionId, collectionIds) - } - let acquisitionId = collectionIds.get(options) - let acquisition = acquisitionId - ? this.outboundRemoteSubsetAcquisitions.get( - remoteSubsetAcquisitionKey(collectionId, acquisitionId), - ) - : undefined - if (!acquisition) { - acquisitionId = safeRandomUUID() - collectionIds.set(options, acquisitionId) - acquisition = { - collectionId, - acquisitionId, - options: transportedOptions, - localOptions, - acquiredLeaderId: null, - inFlight: null, - forceReplay: false, - retryTimer: null, - retryAttempts: 0, - } - this.outboundRemoteSubsetAcquisitions.set( - remoteSubsetAcquisitionKey(collectionId, acquisitionId), - acquisition, - ) - } - - await this.acquireRemoteSubset(acquisition) - } - - async requestReleaseRemoteSubset( - collectionId: string, - options: LoadSubsetOptions, - ): Promise { - const collectionIds = this.remoteSubsetIds.get(collectionId) - const acquisitionId = collectionIds?.get(options) - if (!acquisitionId) return - const key = remoteSubsetAcquisitionKey(collectionId, acquisitionId) - const acquisition = this.outboundRemoteSubsetAcquisitions.get(key) - if (!acquisition) return - this.outboundRemoteSubsetAcquisitions.delete(key) - this.cancelRemoteSubsetReplayRetry(acquisition) - collectionIds!.delete(options) - - const request: Extract< - RPCRequest, - { type: `rpc:releaseRemoteSubset:req` } - > = { - type: `rpc:releaseRemoteSubset:req`, - rpcId: safeRandomUUID(), - acquisitionId, - } - const response = this.isLeader(collectionId) - ? await this.handleReleaseRemoteSubset(collectionId, request, this.nodeId) - : await this.sendRPC(collectionId, request) - - if (!response.ok) { - throw new Error(`releaseRemoteSubset failed: ${response.error}`) - } - } - - private async acquireRemoteSubset( - acquisition: OutboundRemoteSubsetAcquisition, - ): Promise { - if (acquisition.inFlight) return acquisition.inFlight - - const route = { localOwner: false } - let resolveWork!: () => void - let rejectWork!: (error: unknown) => void - const work = new Promise((resolve, reject) => { - resolveWork = resolve - rejectWork = reject + super({ + ...options, + coordinatorName: `BrowserCollectionCoordinator`, }) - acquisition.inFlight = work - const run = async (): Promise => { - const request: Extract< - RPCRequest, - { type: `rpc:ensureRemoteSubset:req` } - > = { - type: `rpc:ensureRemoteSubset:req`, - rpcId: safeRandomUUID(), - acquisitionId: acquisition.acquisitionId, - options: acquisition.options, - } - route.localOwner = this.isLeader(acquisition.collectionId) - let response: EnsureRemoteSubsetResponse - try { - response = route.localOwner - ? await this.handleEnsureRemoteSubset( - acquisition.collectionId, - request, - this.nodeId, - acquisition.localOptions, - ) - : await this.sendRPC( - acquisition.collectionId, - request, - ) - } catch (error) { - if ( - route.localOwner || - error instanceof RetryableRemoteSubsetAcquisitionError - ) { - throw error - } - throw new RetryableRemoteSubsetAcquisitionError( - `Remote subset transport failed`, - error, - ) - } - - if (!response.ok) { - if (response.retryable) { - throw new RetryableRemoteSubsetAcquisitionError(response.error) - } - throw new Error(`ensureRemoteSubset failed: ${response.error}`) - } - acquisition.acquiredLeaderId = response.leaderId - } - void run().then(resolveWork, rejectWork) - let acquired = false - try { - await work - acquired = true - this.cancelRemoteSubsetReplayRetry(acquisition) - } finally { - if (acquisition.inFlight === work) acquisition.inFlight = null - const current = this.collections.get(acquisition.collectionId) - const currentLeaderId = current?.isLeader - ? this.nodeId - : (current?.leaderId ?? null) - const key = remoteSubsetAcquisitionKey( - acquisition.collectionId, - acquisition.acquisitionId, - ) - if ( - acquired && - this.outboundRemoteSubsetAcquisitions.get(key) === acquisition && - (acquisition.forceReplay || - (currentLeaderId !== null && - acquisition.acquiredLeaderId !== currentLeaderId)) - ) { - acquisition.forceReplay = false - void this.acquireRemoteSubset(acquisition).catch((error) => { - this.scheduleRemoteSubsetReplayRetry(acquisition, error) - }) - } - } - } - - async requestEnsurePersistedIndex( - collectionId: string, - signature: string, - spec: PersistedIndexSpec, - ): Promise { - if (this.isLeader(collectionId)) { - await this.requireAdapter(collectionId).ensureIndex( - collectionId, - signature, - spec, - ) - return - } - - const response = await this.sendRPC<{ - type: `rpc:ensurePersistedIndex:res` - rpcId: string - ok: boolean - error?: string - }>(collectionId, { - type: `rpc:ensurePersistedIndex:req`, - rpcId: safeRandomUUID(), - signature, - spec, - }) - - if (!response.ok) { - throw new Error( - `ensurePersistedIndex failed: ${response.error ?? `unknown error`}`, - ) - } - } - - async requestApplyLocalMutations( - collectionId: string, - mutations: Array, - ): Promise { - if (this.isLeader(collectionId)) { - return this.handleApplyLocalMutations(collectionId, { - type: `rpc:applyLocalMutations:req`, - rpcId: safeRandomUUID(), - envelopeId: safeRandomUUID(), - mutations, - }) - } - - return this.sendRPC(collectionId, { - type: `rpc:applyLocalMutations:req`, - rpcId: safeRandomUUID(), - envelopeId: safeRandomUUID(), - mutations, - }) - } - - async requestApplyCommittedTx( - collectionId: string, - tx: PersistedTx, - ): Promise { - const request: Extract = { - type: `rpc:applyCommittedTx:req`, - rpcId: safeRandomUUID(), - envelopeId: safeRandomUUID(), - tx, - } - if (this.isLeader(collectionId)) { - return this.handleApplyCommittedTx(collectionId, request) - } - - return this.sendRPC(collectionId, request) - } - - async pullSince( - collectionId: string, - fromRowVersion: number, - ): Promise { - if (this.isLeader(collectionId)) { - return this.handlePullSince(collectionId, { - type: `rpc:pullSince:req`, - rpcId: safeRandomUUID(), - fromRowVersion, - }) - } - - return this.sendRPC(collectionId, { - type: `rpc:pullSince:req`, - rpcId: safeRandomUUID(), - fromRowVersion, - }) - } - - // ----------------------------------------------------------------------- - // Lifecycle - // ----------------------------------------------------------------------- - - dispose(): void { - this.disposed = true - - for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { - this.cancelRemoteSubsetReplayRetry(acquisition) - this.postRemoteSubsetRelease(acquisition) - } - this.outboundRemoteSubsetAcquisitions.clear() - this.remoteSubsetIds.clear() - - for (const [collectionId, state] of this.collections) { - this.releaseLeadership(collectionId, state) - } - - for (const [, pending] of this.pendingRPCs) { - clearTimeout(pending.timer) - pending.reject(new Error(`coordinator disposed`)) - } - this.pendingRPCs.clear() - - this.channel.close() - this.collections.clear() - this.collectionAdapters.clear() - for (const collectionId of this.remoteSubsetOwners.keys()) { - this.releaseInboundRemoteSubsetAcquisitions(collectionId) - } - this.remoteSubsetOwners.clear() - this.inboundRemoteSubsetAcquisitions.clear() - this.releasedRemoteSubsetAcquisitionTimes.clear() - this.appliedEnvelopes.clear() - this.inFlightEnvelopes.clear() - } - - // ----------------------------------------------------------------------- - // Leadership via Web Locks - // ----------------------------------------------------------------------- - - private ensureCollectionState(collectionId: string): CollectionState { - let state = this.collections.get(collectionId) - if (!state) { - state = { - isLeader: false, - leaderId: null, - lockAbortController: null, - heartbeatTimer: null, - latestTerm: 0, - latestSeq: 0, - latestRowVersion: 0, - subscribers: new Set(), - } - this.collections.set(collectionId, state) - void this.acquireLeadership(collectionId, state) - } - return state - } - - private async acquireLeadership( - collectionId: string, - state: CollectionState, - ): Promise { - if (this.disposed || state.isLeader) return - - const lockName = `tsdb:leader:${this.dbName}:${collectionId}` - const abortController = new AbortController() - state.lockAbortController = abortController - - try { - await navigator.locks.request( - lockName, - { signal: abortController.signal }, - async () => { - if (this.isDisposed()) return - - try { - // Restore stream position from DB before claiming leadership - const adapter = this.requireAdapter(collectionId) - if (adapter.getStreamPosition) { - const pos = await adapter.getStreamPosition(collectionId) - state.latestTerm = pos.latestTerm - state.latestSeq = pos.latestSeq - state.latestRowVersion = pos.latestRowVersion - } - - state.latestTerm++ - state.isLeader = true - state.leaderId = this.nodeId - - this.emitHeartbeat(collectionId, state) - void this.replayRemoteSubsetAcquisitions(collectionId) - state.heartbeatTimer = setInterval(() => { - this.emitHeartbeat(collectionId, state) - }, HEARTBEAT_INTERVAL_MS) - - // Hold the lock until disposed or aborted - await new Promise((resolve) => { - const onAbort = () => { - abortController.signal.removeEventListener(`abort`, onAbort) - resolve() - } - if (abortController.signal.aborted) { - resolve() - return - } - abortController.signal.addEventListener(`abort`, onAbort) - }) - } finally { - this.releaseInboundRemoteSubsetAcquisitions(collectionId) - state.isLeader = false - state.leaderId = null - if (state.heartbeatTimer) { - clearInterval(state.heartbeatTimer) - state.heartbeatTimer = null - } - } - }, - ) - } catch (error) { - if (error instanceof DOMException && error.name === `AbortError`) { - return - } - console.warn(`Failed to acquire leadership for ${collectionId}:`, error) - } - - // Re-acquire if not disposed (leadership was released by another means) - if (!this.isDisposed()) { - void this.acquireLeadership(collectionId, state) - } - } - - private releaseLeadership( - collectionId: string, - state: CollectionState, - ): void { - this.releaseInboundRemoteSubsetAcquisitions(collectionId) - if (state.lockAbortController) { - state.lockAbortController.abort() - state.lockAbortController = null - } - if (state.heartbeatTimer) { - clearInterval(state.heartbeatTimer) - state.heartbeatTimer = null - } - state.isLeader = false - state.leaderId = null - } - - private postRemoteSubsetRelease( - acquisition: OutboundRemoteSubsetAcquisition, - ): void { - const request: Extract< - RPCRequest, - { type: `rpc:releaseRemoteSubset:req` } - > = { - type: `rpc:releaseRemoteSubset:req`, - rpcId: safeRandomUUID(), - acquisitionId: acquisition.acquisitionId, - } - if (this.isLeader(acquisition.collectionId)) { - void this.handleReleaseRemoteSubset( - acquisition.collectionId, - request, - this.nodeId, - ).catch(() => { - // The owner already received the exact unload failure through onError. - }) - return - } - this.channel.postMessage({ - v: 1, - dbName: this.dbName, - collectionId: acquisition.collectionId, - senderId: this.nodeId, - ts: Date.now(), - payload: request, - } satisfies ProtocolEnvelope) - } - - private cancelRemoteSubsetReplayRetry( - acquisition: OutboundRemoteSubsetAcquisition, - ): void { - if (acquisition.retryTimer !== null) { - clearTimeout(acquisition.retryTimer) - acquisition.retryTimer = null - } - acquisition.retryAttempts = 0 - } - - private scheduleRemoteSubsetReplayRetry( - acquisition: OutboundRemoteSubsetAcquisition, - error: unknown, - ): void { - const key = remoteSubsetAcquisitionKey( - acquisition.collectionId, - acquisition.acquisitionId, - ) - if ( - !(error instanceof RetryableRemoteSubsetAcquisitionError) || - this.isDisposed() || - this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition || - acquisition.retryTimer !== null || - acquisition.retryAttempts >= REMOTE_SUBSET_REPLAY_RETRY_ATTEMPTS - ) { - return - } - - acquisition.retryAttempts++ - acquisition.retryTimer = setTimeout(() => { - acquisition.retryTimer = null - if ( - this.isDisposed() || - this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition - ) { - return - } - void this.acquireRemoteSubset(acquisition).catch((retryError) => { - this.scheduleRemoteSubsetReplayRetry(acquisition, retryError) - }) - }, RPC_RETRY_DELAY_MS) - } - - private async replayRemoteSubsetAcquisitions( - collectionId: string, - ): Promise { - if (this.isDisposed()) return - const state = this.collections.get(collectionId) - const leaderId = state?.isLeader ? this.nodeId : state?.leaderId - if (!leaderId) return - - const replays: Array> = [] - for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { - if ( - acquisition.collectionId !== collectionId || - (!acquisition.forceReplay && acquisition.acquiredLeaderId === leaderId) - ) { - continue - } - if (acquisition.inFlight) { - acquisition.forceReplay = true - continue - } - acquisition.forceReplay = false - replays.push( - this.acquireRemoteSubset(acquisition).catch((error) => { - this.scheduleRemoteSubsetReplayRetry(acquisition, error) - }), - ) - } - await Promise.all(replays) - } - - private emitHeartbeat(collectionId: string, state: CollectionState): void { - const envelope: ProtocolEnvelope = { - v: 1, - dbName: this.dbName, - collectionId, - senderId: this.nodeId, - ts: Date.now(), - payload: { - type: `leader:heartbeat`, - term: state.latestTerm, - leaderId: this.nodeId, - latestSeq: state.latestSeq, - latestRowVersion: state.latestRowVersion, - }, - } - this.channel.postMessage(envelope) - } - - // ----------------------------------------------------------------------- - // BroadcastChannel message handling - // ----------------------------------------------------------------------- - - private onChannelMessage(data: unknown): void { - if (!isProtocolEnvelope(data)) return - - const envelope = data - - // Ignore own messages - if (envelope.senderId === this.nodeId) return - - const payload = envelope.payload - if (!payload || typeof payload !== `object`) return - - const type = (payload as Record).type as string | undefined - - if (type === `leader:heartbeat`) { - const heartbeat = payload as { - leaderId?: unknown - term?: unknown - latestSeq?: unknown - latestRowVersion?: unknown - } - if ( - typeof heartbeat.leaderId === `string` && - typeof heartbeat.term === `number` && - typeof heartbeat.latestSeq === `number` && - typeof heartbeat.latestRowVersion === `number` - ) { - const state = this.ensureCollectionState(envelope.collectionId) - if (heartbeat.term < state.latestTerm) return - const changedLeader = state.leaderId !== heartbeat.leaderId - state.leaderId = heartbeat.leaderId - state.latestTerm = Math.max(state.latestTerm, heartbeat.term) - state.latestSeq = Math.max(state.latestSeq, heartbeat.latestSeq) - state.latestRowVersion = Math.max( - state.latestRowVersion, - heartbeat.latestRowVersion, - ) - if (changedLeader) { - void this.replayRemoteSubsetAcquisitions(envelope.collectionId) - } - } - } - - // Handle RPC responses (for pending outbound RPCs) - if (type && type.endsWith(`:res`)) { - const rpcId = (payload as { rpcId?: string }).rpcId - if (rpcId && this.pendingRPCs.has(rpcId)) { - const pending = this.pendingRPCs.get(rpcId)! - this.pendingRPCs.delete(rpcId) - clearTimeout(pending.timer) - pending.resolve(payload as RPCResponse) - return - } - } - - // Handle RPC requests (leader only) - if (type && type.endsWith(`:req`)) { - if (!isRPCRequest(payload)) return - const collectionId = envelope.collectionId - if (this.isLeader(collectionId)) { - void this.handleRPCRequest(collectionId, payload, envelope.senderId) - } - return - } - - // Forward protocol messages to subscribers - const state = this.collections.get(envelope.collectionId) - if (state) { - for (const subscriber of state.subscribers) { - subscriber(envelope) - } - } - } - - // ----------------------------------------------------------------------- - // RPC - Outbound (follower side) - // ----------------------------------------------------------------------- - - private async sendRPC( - collectionId: string, - request: RPCRequest, - ): Promise { - let lastError: Error | undefined - let firstTransportCause: unknown - const mutationRequestType = isMutatingRPCRequest(request) - ? request.type - : undefined - const mutationRoute = mutationRequestType - ? this.captureMutationRoute(collectionId) - : undefined - - for (let attempt = 0; attempt <= RPC_RETRY_ATTEMPTS; attempt++) { - if (attempt > 0) { - await sleep(RPC_RETRY_DELAY_MS * attempt) - } - - if ( - mutationRoute && - mutationRequestType && - firstTransportCause !== undefined - ) { - this.assertMutationRouteUnchanged( - collectionId, - mutationRequestType, - mutationRoute, - firstTransportCause, - ) - } - - if (this.isLeader(collectionId)) { - return (await this.dispatchRPCRequest( - collectionId, - request, - this.nodeId, - )) as T - } - - try { - return await this.sendRPCOnce(collectionId, request) - } catch (error) { - if (this.isDisposed()) throw error - firstTransportCause ??= error - if (mutationRoute && mutationRequestType) { - this.assertMutationRouteUnchanged( - collectionId, - mutationRequestType, - mutationRoute, - firstTransportCause, - ) - } - lastError = error instanceof Error ? error : new Error(String(error)) - } - } - - throw lastError ?? new Error(`RPC failed after retries`) - } - - private captureMutationRoute(collectionId: string): { - leaderId: string | null - term: number | null - } { - const state = this.collections.get(collectionId) - return { - leaderId: state?.isLeader ? this.nodeId : (state?.leaderId ?? null), - term: state?.latestTerm ?? null, - } - } - - private assertMutationRouteUnchanged( - collectionId: string, - requestType: IndeterminateCommitRequestType, - previous: { leaderId: string | null; term: number | null }, - cause: unknown, - ): void { - const current = this.captureMutationRoute(collectionId) - if ( - previous.leaderId !== null && - previous.term !== null && - current.leaderId === previous.leaderId && - current.term === previous.term - ) { - return - } - throw new IndeterminateCommitError({ - collectionId, - requestType, - previousLeaderId: previous.leaderId, - previousTerm: previous.term, - currentLeaderId: current.leaderId, - currentTerm: current.term, - cause, - }) - } - - private sendRPCOnce( - collectionId: string, - request: RPCRequest, - ): Promise { - return new Promise((resolve, reject) => { - const rpcId = request.rpcId - - const timer = setTimeout(() => { - this.pendingRPCs.delete(rpcId) - reject( - new Error(`RPC ${request.type} timed out after ${RPC_TIMEOUT_MS}ms`), - ) - }, RPC_TIMEOUT_MS) - - this.pendingRPCs.set(rpcId, { - resolve: resolve as (response: RPCResponse) => void, - reject, - timer, - }) - - const envelope: ProtocolEnvelope = { - v: 1, - dbName: this.dbName, - collectionId, - senderId: this.nodeId, - ts: Date.now(), - payload: request, - } - this.channel.postMessage(envelope) - }) - } - - // ----------------------------------------------------------------------- - // RPC - Inbound (leader side) - // ----------------------------------------------------------------------- - - private async handleRPCRequest( - collectionId: string, - request: RPCRequest, - requesterId: string, - ): Promise { - let response: RPCResponse - - try { - response = await this.dispatchRPCRequest( - collectionId, - request, - requesterId, - ) - } catch (error) { - response = createRPCErrorResponse(request, error) - } - - if (this.isDisposed()) { - return - } - - const envelope: ProtocolEnvelope = { - v: 1, - dbName: this.dbName, - collectionId, - senderId: this.nodeId, - ts: Date.now(), - payload: response, - } - this.channel.postMessage(envelope) - } - - private dispatchRPCRequest( - collectionId: string, - request: RPCRequest, - requesterId: string, - ): Promise { - switch (request.type) { - case `rpc:ensureRemoteSubset:req`: - return this.handleEnsureRemoteSubset(collectionId, request, requesterId) - case `rpc:releaseRemoteSubset:req`: - return this.handleReleaseRemoteSubset( - collectionId, - request, - requesterId, - ) - case `rpc:ensurePersistedIndex:req`: - return this.handleEnsurePersistedIndex(collectionId, request) - case `rpc:applyLocalMutations:req`: - return this.handleApplyLocalMutations(collectionId, request) - case `rpc:applyCommittedTx:req`: - return this.handleApplyCommittedTx(collectionId, request) - case `rpc:pullSince:req`: - return this.handlePullSince(collectionId, request) - } - } - - private async handleEnsureRemoteSubset( - collectionId: string, - request: Extract, - requesterId: string, - localOptions?: TransportedLoadSubsetOptions, - ): Promise { - this.pruneReleasedRemoteSubsetAcquisitions() - const key = inboundRemoteSubsetAcquisitionKey( - collectionId, - requesterId, - request.acquisitionId, - ) - const existing = this.inboundRemoteSubsetAcquisitions.get(key) - const awaitingOwner = - existing && `awaitingOwner` in existing ? existing : undefined - if (existing) { - if (`owner` in existing) { - if (!existing.released) { - await existing.load - return { - type: `rpc:ensureRemoteSubset:res`, - rpcId: request.rpcId, - ok: true, - leaderId: this.nodeId, - } - } - await existing.release - if (existing.terminalRelease) { - return { - type: `rpc:ensureRemoteSubset:res`, - rpcId: request.rpcId, - ok: true, - leaderId: this.nodeId, - } - } - if (this.inboundRemoteSubsetAcquisitions.get(key) === existing) { - this.inboundRemoteSubsetAcquisitions.delete(key) - } - } else if (!(`awaitingOwner` in existing)) { - return { - type: `rpc:ensureRemoteSubset:res`, - rpcId: request.rpcId, - ok: true, - leaderId: this.nodeId, - } - } - } - - const owner = this.remoteSubsetOwners.get(collectionId) - if (!owner) { - throw new RetryableRemoteSubsetAcquisitionError( - `BrowserCollectionCoordinator: no remote subset owner registered for collection "${collectionId}"`, - ) - } - - const acquisition: ActiveRemoteSubsetAcquisition = { - collectionId, - requesterId, - acquisitionId: request.acquisitionId, - owner, - options: awaitingOwner?.options ?? localOptions ?? request.options, - load: Promise.resolve(), - transferred: false, - released: false, - terminalRelease: false, - release: null, - } - this.releasedRemoteSubsetAcquisitionTimes.delete(key) - this.inboundRemoteSubsetAcquisitions.set(key, acquisition) - let resolveLoad!: () => void - let rejectLoad!: (error: unknown) => void - acquisition.load = new Promise((resolve, reject) => { - resolveLoad = resolve - rejectLoad = reject - }) - try { - const load = owner(acquisition.options) - acquisition.transferred = true - void Promise.resolve(load).then(resolveLoad, rejectLoad) - } catch (error) { - rejectLoad(error) - } - try { - await acquisition.load - } catch (error) { - if ( - !acquisition.transferred && - this.inboundRemoteSubsetAcquisitions.get(key) === acquisition - ) { - if (awaitingOwner) { - this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) - } else { - this.inboundRemoteSubsetAcquisitions.delete(key) - } - } - reportRemoteSubsetOwnerError(owner, error) - throw error - } - return { - type: `rpc:ensureRemoteSubset:res`, - rpcId: request.rpcId, - ok: true, - leaderId: this.nodeId, - } - } - - private async handleReleaseRemoteSubset( - collectionId: string, - request: Extract, - requesterId: string, - ): Promise { - this.pruneReleasedRemoteSubsetAcquisitions() - const key = inboundRemoteSubsetAcquisitionKey( - collectionId, - requesterId, - request.acquisitionId, - ) - const acquisition = this.inboundRemoteSubsetAcquisitions.get(key) - if (!acquisition) { - this.setReleasedRemoteSubsetAcquisition(key, { - collectionId, - requesterId, - acquisitionId: request.acquisitionId, - released: true, - }) - } else if (`owner` in acquisition) { - acquisition.terminalRelease = true - await this.releaseRemoteSubsetAcquisition(acquisition) - if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { - this.setReleasedRemoteSubsetAcquisition(key, { - collectionId, - requesterId, - acquisitionId: request.acquisitionId, - released: true, - }) - } - } else if (`awaitingOwner` in acquisition) { - this.setReleasedRemoteSubsetAcquisition(key, { - collectionId, - requesterId, - acquisitionId: request.acquisitionId, - released: true, - }) - } - return { - type: `rpc:releaseRemoteSubset:res`, - rpcId: request.rpcId, - ok: true, - } - } - - private releaseRemoteSubsetAcquisition( - acquisition: ActiveRemoteSubsetAcquisition, - ): Promise { - if (acquisition.release) return acquisition.release - acquisition.released = true - acquisition.release = (async () => { - try { - await acquisition.load - } catch { - // A returned promise transfers the lease even when initial loading fails. - } - try { - if (acquisition.transferred) { - await unloadRemoteSubsetOwner(acquisition.owner, acquisition.options) - } - } finally { - if (!acquisition.terminalRelease) { - const key = inboundRemoteSubsetAcquisitionKey( - acquisition.collectionId, - acquisition.requesterId, - acquisition.acquisitionId, - ) - if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { - this.inboundRemoteSubsetAcquisitions.set(key, { - collectionId: acquisition.collectionId, - requesterId: acquisition.requesterId, - acquisitionId: acquisition.acquisitionId, - options: acquisition.options, - released: true, - awaitingOwner: true, - }) - } - } - } - })() - return acquisition.release - } - - private releaseInboundRemoteSubsetAcquisitions( - collectionId: string, - owner?: RemoteSubsetOwner, - ): void { - for (const acquisition of this.inboundRemoteSubsetAcquisitions.values()) { - if ( - !(`owner` in acquisition) || - acquisition.collectionId !== collectionId || - (owner && acquisition.owner !== owner) - ) { - continue - } - void this.releaseRemoteSubsetAcquisition(acquisition).catch( - () => undefined, - ) - } - } - - private rebindRemoteInboundSubsetAcquisitions( - collectionId: string, - owner: RemoteSubsetOwner, - ): void { - for (const acquisition of this.inboundRemoteSubsetAcquisitions.values()) { - if (`awaitingOwner` in acquisition) { - if ( - acquisition.collectionId === collectionId && - acquisition.requesterId !== this.nodeId - ) { - void this.bindAwaitingRemoteSubsetAcquisition( - acquisition, - owner, - ).catch(() => { - // The owner receives the exact load failure through onError. - }) - } - continue - } - if ( - !(`owner` in acquisition) || - acquisition.collectionId !== collectionId || - acquisition.requesterId === this.nodeId || - !acquisition.released || - acquisition.terminalRelease - ) { - continue - } - - void this.rebindRemoteInboundSubsetAcquisition(acquisition, owner).catch( - () => undefined, - ) - } - } - - private async rebindRemoteInboundSubsetAcquisition( - previous: ActiveRemoteSubsetAcquisition, - owner: RemoteSubsetOwner, - ): Promise { - await previous.release - if ( - previous.terminalRelease || - this.remoteSubsetOwners.get(previous.collectionId) !== owner - ) { - return - } - - const key = inboundRemoteSubsetAcquisitionKey( - previous.collectionId, - previous.requesterId, - previous.acquisitionId, - ) - const current = this.inboundRemoteSubsetAcquisitions.get(key) - if (current && current !== previous) { - if (`awaitingOwner` in current) { - await this.bindAwaitingRemoteSubsetAcquisition(current, owner) - } - return - } - - const awaitingOwner: AwaitingRemoteSubsetOwnerAcquisition = { - collectionId: previous.collectionId, - requesterId: previous.requesterId, - acquisitionId: previous.acquisitionId, - options: previous.options, - released: true, - awaitingOwner: true, - } - this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) - await this.bindAwaitingRemoteSubsetAcquisition(awaitingOwner, owner) - } - - private async bindAwaitingRemoteSubsetAcquisition( - awaitingOwner: AwaitingRemoteSubsetOwnerAcquisition, - owner: RemoteSubsetOwner, - ): Promise { - if (this.remoteSubsetOwners.get(awaitingOwner.collectionId) !== owner) { - return - } - const key = inboundRemoteSubsetAcquisitionKey( - awaitingOwner.collectionId, - awaitingOwner.requesterId, - awaitingOwner.acquisitionId, - ) - if (this.inboundRemoteSubsetAcquisitions.get(key) !== awaitingOwner) return - - const acquisition: ActiveRemoteSubsetAcquisition = { - collectionId: awaitingOwner.collectionId, - requesterId: awaitingOwner.requesterId, - acquisitionId: awaitingOwner.acquisitionId, - owner, - options: awaitingOwner.options, - load: Promise.resolve(), - transferred: false, - released: false, - terminalRelease: false, - release: null, - } - this.inboundRemoteSubsetAcquisitions.set(key, acquisition) - let resolveLoad!: () => void - let rejectLoad!: (error: unknown) => void - acquisition.load = new Promise((resolve, reject) => { - resolveLoad = resolve - rejectLoad = reject - }) - try { - const load = owner(acquisition.options) - acquisition.transferred = true - void Promise.resolve(load).then(resolveLoad, rejectLoad) - } catch (error) { - rejectLoad(error) - } - try { - await acquisition.load - } catch (error) { - if ( - !acquisition.transferred && - this.inboundRemoteSubsetAcquisitions.get(key) === acquisition - ) { - this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) - } - reportRemoteSubsetOwnerError(owner, error) - throw error - } - } - - private async handleEnsurePersistedIndex( - collectionId: string, - request: { - type: `rpc:ensurePersistedIndex:req` - rpcId: string - signature: string - spec: PersistedIndexSpec - }, - ): Promise { - await this.withWriterLock(() => - this.requireAdapter(collectionId).ensureIndex( - collectionId, - request.signature, - request.spec, - ), - ) - return { - type: `rpc:ensurePersistedIndex:res`, - rpcId: request.rpcId, - ok: true, - } - } - - private async handleApplyLocalMutations( - collectionId: string, - request: { - type: `rpc:applyLocalMutations:req` - rpcId: string - envelopeId: string - mutations: Array - }, - ): Promise { - const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) - const appliedEnvelope = this.appliedEnvelopes.get(envelopeKey) - if (appliedEnvelope?.requestType === `rpc:applyLocalMutations:req`) { - return { ...appliedEnvelope.response, rpcId: request.rpcId } - } - if (appliedEnvelope) { - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: false, - code: `CONFLICT`, - error: `envelope ${request.envelopeId} already applied`, - } - } - - const inFlightEnvelope = this.inFlightEnvelopes.get(envelopeKey) - if (inFlightEnvelope?.requestType === `rpc:applyLocalMutations:req`) { - const response = await inFlightEnvelope.response - return { ...response, rpcId: request.rpcId } - } - if (inFlightEnvelope) { - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: false, - code: `CONFLICT`, - error: `envelope ${request.envelopeId} is already in flight`, - } - } - - const response = this.applyLocalMutationsOnce(collectionId, request) - const pendingEnvelope: InFlightEnvelope = { - requestType: request.type, - response, - } - this.inFlightEnvelopes.set(envelopeKey, pendingEnvelope) - try { - return await response - } finally { - if (this.inFlightEnvelopes.get(envelopeKey) === pendingEnvelope) { - this.inFlightEnvelopes.delete(envelopeKey) - } - } - } - - private async applyLocalMutationsOnce( - collectionId: string, - request: Extract, - ): Promise { - const state = this.collections.get(collectionId) - if (!state || !state.isLeader) { - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: false, - code: `NOT_LEADER`, - error: `not the leader for ${collectionId}`, - } - } - - // Assign stream position - state.latestSeq++ - state.latestRowVersion++ - - const term = state.latestTerm - const seq = state.latestSeq - const rowVersion = state.latestRowVersion - - // Build and apply the persisted transaction - const rowMetadataMutations: Array = [] - for (const mutation of request.mutations) { - if (!(`metadataChanged` in mutation) || !mutation.metadataChanged) { - continue - } - rowMetadataMutations.push( - mutation.metadata === undefined - ? { type: `delete`, key: mutation.key } - : { - type: `set`, - key: mutation.key, - value: mutation.metadata, - }, - ) - } - const tx = { - txId: safeRandomUUID(), - term, - seq, - rowVersion, - mutations: request.mutations.map((m) => ({ - type: m.type, - key: m.key, - value: m.value, - ...(`metadataChanged` in m - ? { metadata: m.metadata, metadataChanged: m.metadataChanged } - : {}), - })), - rowMetadataMutations, - } - - try { - await this.withWriterLock(() => - this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), - ) - } catch (error) { - throw toPersistedCollectionDurabilityError(collectionId, error) - } - - const response: ApplyLocalMutationsResponse = { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: true, - term, - seq, - latestRowVersion: rowVersion, - acceptedMutationIds: request.mutations.map((m) => m.mutationId), - } - if (this.isDisposed()) { - return response - } - this.appliedEnvelopes.set( - appliedEnvelopeKey(collectionId, request.envelopeId), - { - appliedAt: Date.now(), - requestType: request.type, - response, - }, - ) - this.pruneAppliedEnvelopes() - - // Broadcast tx:committed to all tabs - const changedRows = request.mutations - .filter((m) => m.type !== `delete`) - .map((m) => ({ key: m.key, value: m.value })) - const deletedKeys = request.mutations - .filter((m) => m.type === `delete`) - .map((m) => m.key) - - const txCommitted: ProtocolEnvelope = { - v: 1, - dbName: this.dbName, - collectionId, - senderId: this.nodeId, - ts: Date.now(), - payload: { - type: `tx:committed`, - term, - seq, - txId: tx.txId, - latestRowVersion: rowVersion, - requiresFullReload: false, - changedRows, - deletedKeys, - rowMetadataMutations, - }, - } - this.channel.postMessage(txCommitted) - - // Deliver to local subscribers too - for (const subscriber of state.subscribers) { - subscriber(txCommitted) - } - - return response - } - - private async handleApplyCommittedTx( - collectionId: string, - request: Extract, - ): Promise { - const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) - const appliedEnvelope = this.appliedEnvelopes.get(envelopeKey) - if (appliedEnvelope?.requestType === `rpc:applyCommittedTx:req`) { - return { ...appliedEnvelope.response, rpcId: request.rpcId } - } - if (appliedEnvelope) { - return { - type: `rpc:applyCommittedTx:res`, - rpcId: request.rpcId, - ok: false, - code: `CONFLICT`, - error: `envelope ${request.envelopeId} already applied`, - } - } - - const inFlightEnvelope = this.inFlightEnvelopes.get(envelopeKey) - if (inFlightEnvelope?.requestType === `rpc:applyCommittedTx:req`) { - const response = await inFlightEnvelope.response - return { ...response, rpcId: request.rpcId } - } - if (inFlightEnvelope) { - return { - type: `rpc:applyCommittedTx:res`, - rpcId: request.rpcId, - ok: false, - code: `CONFLICT`, - error: `envelope ${request.envelopeId} is already in flight`, - } - } - - const response = this.applyCommittedTxOnce(collectionId, request) - const pendingEnvelope: InFlightEnvelope = { - requestType: request.type, - response, - } - this.inFlightEnvelopes.set(envelopeKey, pendingEnvelope) - try { - return await response - } finally { - if (this.inFlightEnvelopes.get(envelopeKey) === pendingEnvelope) { - this.inFlightEnvelopes.delete(envelopeKey) - } - } - } - - private async applyCommittedTxOnce( - collectionId: string, - request: Extract, - ): Promise { - const state = this.collections.get(collectionId) - if (!state || !state.isLeader) { - return { - type: `rpc:applyCommittedTx:res`, - rpcId: request.rpcId, - ok: false, - code: `NOT_LEADER`, - error: `not the leader for ${collectionId}`, - } - } - - state.latestSeq++ - state.latestRowVersion++ - const tx: PersistedTx = { - ...request.tx, - term: state.latestTerm, - seq: state.latestSeq, - rowVersion: state.latestRowVersion, - } - - try { - await this.withWriterLock(() => - this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), - ) - } catch (error) { - throw toPersistedCollectionDurabilityError(collectionId, error) - } - const response: ApplyCommittedTxResponse = { - type: `rpc:applyCommittedTx:res`, - rpcId: request.rpcId, - ok: true, - term: tx.term, - seq: tx.seq, - latestRowVersion: tx.rowVersion, - } - if (this.isDisposed()) { - return response - } - this.appliedEnvelopes.set( - appliedEnvelopeKey(collectionId, request.envelopeId), - { - appliedAt: Date.now(), - requestType: request.type, - response, - }, - ) - this.pruneAppliedEnvelopes() - - const committedBase = { - type: `tx:committed` as const, - term: tx.term, - seq: tx.seq, - txId: tx.txId, - latestRowVersion: tx.rowVersion, - } - const committedPayload: TxCommitted = tx.truncate - ? { - ...committedBase, - requiresFullReload: true, - } - : { - ...committedBase, - requiresFullReload: false, - changedRows: tx.mutations - .filter((mutation) => mutation.type !== `delete`) - .map((mutation) => ({ - key: mutation.key, - value: mutation.value, - })), - deletedKeys: tx.mutations - .filter((mutation) => mutation.type === `delete`) - .map((mutation) => mutation.key), - rowMetadataMutations: tx.rowMetadataMutations, - collectionMetadataMutations: tx.collectionMetadataMutations, - } - const committed: ProtocolEnvelope = { - v: 1, - dbName: this.dbName, - collectionId, - senderId: this.nodeId, - ts: Date.now(), - payload: committedPayload, - } - this.channel.postMessage(committed) - for (const subscriber of state.subscribers) { - subscriber(committed) - } - - return response - } - - private async handlePullSince( - collectionId: string, - request: { - type: `rpc:pullSince:req` - rpcId: string - fromRowVersion: number - }, - ): Promise { - const state = this.collections.get(collectionId) - - const adapter = this.requireAdapter(collectionId) - if (!adapter.pullSince) { - return { - type: `rpc:pullSince:res`, - rpcId: request.rpcId, - ok: true, - latestTerm: state?.latestTerm ?? 0, - latestSeq: state?.latestSeq ?? 0, - latestRowVersion: state?.latestRowVersion ?? 0, - requiresFullReload: true, - } - } - - const result = await adapter.pullSince(collectionId, request.fromRowVersion) - - if (result.requiresFullReload) { - return { - type: `rpc:pullSince:res`, - rpcId: request.rpcId, - ok: true, - latestTerm: state?.latestTerm ?? 0, - latestSeq: state?.latestSeq ?? 0, - latestRowVersion: result.latestRowVersion, - requiresFullReload: true, - } - } - - return { - type: `rpc:pullSince:res`, - rpcId: request.rpcId, - ok: true, - latestTerm: state?.latestTerm ?? 0, - latestSeq: state?.latestSeq ?? 0, - latestRowVersion: result.latestRowVersion, - requiresFullReload: false, - changedKeys: result.changedKeys, - deletedKeys: result.deletedKeys, - } - } - - // ----------------------------------------------------------------------- - // DB Writer Lock (Workstream E) - // ----------------------------------------------------------------------- - - private async withWriterLock(fn: () => Promise): Promise { - const lockName = `tsdb:writer:${this.dbName}` - - for (let attempt = 0; attempt <= WRITER_LOCK_MAX_RETRIES; attempt++) { - const callbackState = { entered: false } - try { - return await navigator.locks.request(lockName, async () => { - callbackState.entered = true - return fn() - }) - } catch (error) { - if (callbackState.entered) { - throw error - } - if (error instanceof DOMException && error.name === `AbortError`) { - throw error - } - - if (attempt < WRITER_LOCK_MAX_RETRIES) { - await sleep(WRITER_LOCK_BUSY_RETRY_MS * Math.min(attempt + 1, 5)) - continue - } - - throw error - } - } - - // Unreachable but satisfies TypeScript - throw new Error(`writer lock acquisition failed`) - } - - // ----------------------------------------------------------------------- - // Helpers - // ----------------------------------------------------------------------- - - private pruneAppliedEnvelopes(): void { - // Keep envelopes for 60 seconds for dedup - const cutoff = Date.now() - RPC_DEDUPE_RETENTION_MS - for (const [key, envelope] of this.appliedEnvelopes) { - if (envelope.appliedAt < cutoff) { - this.appliedEnvelopes.delete(key) - } - } - } - - private setReleasedRemoteSubsetAcquisition( - key: string, - acquisition: RemoteSubsetAcquisition, - ): void { - this.inboundRemoteSubsetAcquisitions.set(key, acquisition) - this.releasedRemoteSubsetAcquisitionTimes.set(key, Date.now()) - } - - private pruneReleasedRemoteSubsetAcquisitions(): void { - const cutoff = Date.now() - RPC_DEDUPE_RETENTION_MS - for (const [key, releasedAt] of this.releasedRemoteSubsetAcquisitionTimes) { - if (releasedAt < cutoff) { - this.releasedRemoteSubsetAcquisitionTimes.delete(key) - this.inboundRemoteSubsetAcquisitions.delete(key) - } - } - } -} - -// --------------------------------------------------------------------------- -// Utilities -// --------------------------------------------------------------------------- - -function isProtocolEnvelope(data: unknown): data is ProtocolEnvelope { - if (!data || typeof data !== `object`) return false - const record = data as Record - return ( - record.v === 1 && - typeof record.dbName === `string` && - typeof record.collectionId === `string` && - typeof record.senderId === `string` && - typeof record.ts === `number` - ) -} - -function isRPCRequest(payload: unknown): payload is RPCRequest { - if (!payload || typeof payload !== `object`) return false - switch ((payload as { type?: unknown }).type) { - case `rpc:ensureRemoteSubset:req`: - case `rpc:releaseRemoteSubset:req`: - case `rpc:ensurePersistedIndex:req`: - case `rpc:applyLocalMutations:req`: - case `rpc:applyCommittedTx:req`: - case `rpc:pullSince:req`: - return true - default: - return false - } -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)) -} - -async function unloadRemoteSubsetOwner( - owner: RemoteSubsetOwner, - options: TransportedLoadSubsetOptions, -): Promise { - try { - const result = ( - owner.unloadSubset as unknown as ( - options: TransportedLoadSubsetOptions, - ) => unknown - )(options) - await Promise.resolve(result) - } catch (error) { - reportRemoteSubsetOwnerError(owner, error) - throw error - } -} - -function reportRemoteSubsetOwnerError( - owner: RemoteSubsetOwner, - error: unknown, -): void { - try { - owner.onError(error) - } catch { - // Reporting must not replace the original owner failure. - } -} - -function appliedEnvelopeKey(collectionId: string, envelopeId: string): string { - return JSON.stringify([collectionId, envelopeId]) -} - -function remoteSubsetAcquisitionKey( - collectionId: string, - acquisitionId: string, -): string { - return JSON.stringify([collectionId, acquisitionId]) -} - -function inboundRemoteSubsetAcquisitionKey( - collectionId: string, - requesterId: string, - acquisitionId: string, -): string { - return JSON.stringify([collectionId, requesterId, acquisitionId]) -} - -function createRPCErrorResponse( - request: RPCRequest, - cause: unknown, -): RPCResponse { - const error = cause instanceof Error ? cause.message : String(cause) - switch (request.type) { - case `rpc:ensureRemoteSubset:req`: - return { - type: `rpc:ensureRemoteSubset:res`, - rpcId: request.rpcId, - ok: false, - error, - ...(cause instanceof RetryableRemoteSubsetAcquisitionError - ? { retryable: true as const } - : {}), - } - case `rpc:releaseRemoteSubset:req`: - return { - type: `rpc:releaseRemoteSubset:res`, - rpcId: request.rpcId, - ok: false, - error, - } - case `rpc:ensurePersistedIndex:req`: - return { - type: `rpc:ensurePersistedIndex:res`, - rpcId: request.rpcId, - ok: false, - error, - } - case `rpc:applyLocalMutations:req`: - if (cause instanceof PersistedCollectionDurabilityError) { - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: false, - code: `PERSISTENCE_ERROR`, - error, - ...toSafeDurabilityDetails(cause), - } - } - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: false, - code: `CONFLICT`, - error, - } - case `rpc:applyCommittedTx:req`: - if (cause instanceof PersistedCollectionDurabilityError) { - return { - type: `rpc:applyCommittedTx:res`, - rpcId: request.rpcId, - ok: false, - code: `PERSISTENCE_ERROR`, - error, - ...toSafeDurabilityDetails(cause), - } - } - return { - type: `rpc:applyCommittedTx:res`, - rpcId: request.rpcId, - ok: false, - code: `CONFLICT`, - error, - } - case `rpc:pullSince:req`: - return { - type: `rpc:pullSince:res`, - rpcId: request.rpcId, - ok: false, - error, - } - } -} - -function isMutatingRPCRequest(request: RPCRequest): request is Extract< - RPCRequest, - { - type: IndeterminateCommitRequestType - } -> { - return ( - request.type === `rpc:applyLocalMutations:req` || - request.type === `rpc:applyCommittedTx:req` - ) -} - -function toSafeDurabilityDetails(error: PersistedCollectionDurabilityError): { - sourceCode?: string | number - path?: string | ReadonlyArray -} { - const sourceCode = - typeof error.code === `string` || typeof error.code === `number` - ? error.code - : undefined - const path = - typeof error.path === `string` || - (Array.isArray(error.path) && - error.path.every( - (part) => typeof part === `string` || typeof part === `number`, - )) - ? (error.path as string | ReadonlyArray) - : undefined - return { - ...(sourceCode === undefined ? {} : { sourceCode }), - ...(path === undefined ? {} : { path }), } } diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 7b3b78580c..0f9e46b811 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -21,14 +21,16 @@ import type { BrowserCollectionCoordinatorOptions } from '../src/browser-coordin * RFC #1659 requires every adapter operation to use the elected owner for its * collection. Complete committed transactions must cross that boundary * without losing metadata. A mutating RPC may replay only through the same - * known leader and term. Remote subset request data must be clone-safe, and - * each accepted physical acquisition creates one exact acquisition lease. + * known leader and term, and one envelope id cannot identify two mutation + * request types. Remote subset request data must be clone-safe, and each + * accepted physical acquisition creates one exact acquisition lease. * * The adapter call logs, transport controls, owner callbacks, and internal-map * snapshots are focused reference ledgers. Histories vary local and follower * routes, response loss, leadership change, owner replacement, duplicate * delivery, release, failure, and disposal. The production driver is the real - * `BrowserCollectionCoordinator`; only BroadcastChannel and Web Locks are + * `BrowserCollectionCoordinator`, which names the shared broadcast + * coordination engine for this host; only BroadcastChannel and Web Locks are * replaced with deterministic seams. * * Checkpoints sit at adapter entry, RPC response delivery, acquisition diff --git a/packages/browser-db-sqlite-persistence/tsconfig.json b/packages/browser-db-sqlite-persistence/tsconfig.json index 5b14f299c7..67f70f2efe 100644 --- a/packages/browser-db-sqlite-persistence/tsconfig.json +++ b/packages/browser-db-sqlite-persistence/tsconfig.json @@ -16,6 +16,9 @@ "@tanstack/db-ivm": ["../db-ivm/src"], "@tanstack/db-sqlite-persistence-core": [ "../db-sqlite-persistence-core/src" + ], + "@tanstack/db-sqlite-persistence-core/*": [ + "../db-sqlite-persistence-core/src/*" ] } }, diff --git a/packages/db-sqlite-persistence-core/package.json b/packages/db-sqlite-persistence-core/package.json index fb731bf1e4..c30503d4be 100644 --- a/packages/db-sqlite-persistence-core/package.json +++ b/packages/db-sqlite-persistence-core/package.json @@ -37,6 +37,16 @@ "default": "./dist/cjs/index.cjs" } }, + "./broadcast-coordinator": { + "import": { + "types": "./dist/esm/broadcast-coordinator.d.ts", + "default": "./dist/esm/broadcast-coordinator.js" + }, + "require": { + "types": "./dist/cjs/broadcast-coordinator.d.cts", + "default": "./dist/cjs/broadcast-coordinator.cjs" + } + }, "./package.json": "./package.json" }, "sideEffects": false, diff --git a/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts b/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts new file mode 100644 index 0000000000..80ab162eda --- /dev/null +++ b/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts @@ -0,0 +1,2096 @@ +import { safeRandomUUID } from '@tanstack/db' +import { + DuplicateRemoteSubsetOwnerError, + IndeterminateCommitError, + PersistedCollectionDurabilityError, + RetryableRemoteSubsetAcquisitionError, + toPersistedCollectionDurabilityError, +} from './errors' +import { + toProcessLocalLoadSubsetOptions, + toTransportedLoadSubsetOptions, +} from './remote-subset-wire' +import { + reportRemoteSubsetOwnerError, + unloadRemoteSubsetOwner, +} from './remote-subset-owner' +import type { LoadSubsetOptions } from '@tanstack/db' +import type { IndeterminateCommitRequestType } from './errors' +import type { TransportedLoadSubsetOptions } from './remote-subset-wire' +import type { RemoteSubsetOwner } from './remote-subset-owner' +import type { + ApplyCommittedTxResponse, + ApplyLocalMutationsResponse, + EnsureRemoteSubsetRequest, + EnsureRemoteSubsetResponse, + PersistedCollectionCoordinator, + PersistedIndexSpec, + PersistedMutationEnvelope, + PersistedRowMetadataMutation, + PersistedTx, + PersistenceAdapter, + ProtocolEnvelope, + PullSinceResponse, + ReleaseRemoteSubsetRequest, + ReleaseRemoteSubsetResponse, + TxCommitted, +} from './persisted' + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const HEARTBEAT_INTERVAL_MS = 3_000 +const RPC_TIMEOUT_MS = 10_000 +const RPC_RETRY_ATTEMPTS = 2 +const RPC_RETRY_DELAY_MS = 200 +const REMOTE_SUBSET_REPLAY_RETRY_ATTEMPTS = 2 +const RPC_DEDUPE_RETENTION_MS = 60_000 +const WRITER_LOCK_BUSY_RETRY_MS = 50 +const WRITER_LOCK_MAX_RETRIES = 20 + +// --------------------------------------------------------------------------- +// Internal types +// --------------------------------------------------------------------------- + +type RPCRequest = + | EnsureRemoteSubsetRequest + | ReleaseRemoteSubsetRequest + | { + type: `rpc:ensurePersistedIndex:req` + rpcId: string + signature: string + spec: PersistedIndexSpec + } + | { + type: `rpc:applyLocalMutations:req` + rpcId: string + envelopeId: string + mutations: Array + } + | { + type: `rpc:applyCommittedTx:req` + rpcId: string + envelopeId: string + tx: PersistedTx + } + | { + type: `rpc:pullSince:req` + rpcId: string + fromRowVersion: number + } + +type RPCResponse = + | EnsureRemoteSubsetResponse + | ReleaseRemoteSubsetResponse + | { + type: `rpc:ensurePersistedIndex:res` + rpcId: string + ok: boolean + error?: string + } + | ApplyLocalMutationsResponse + | ApplyCommittedTxResponse + | PullSinceResponse + +type PendingRPC = { + resolve: (response: RPCResponse) => void + reject: (error: Error) => void + timer: ReturnType +} + +type CollectionState = { + isLeader: boolean + leaderId: string | null + lockAbortController: AbortController | null + heartbeatTimer: ReturnType | null + latestTerm: number + latestSeq: number + latestRowVersion: number + subscribers: Set<(message: ProtocolEnvelope) => void> +} + +// Adapter capabilities used by coordinator-side operations +type CoordinatorAdapter = PersistenceAdapter & { + pullSince?: ( + collectionId: string, + fromRowVersion: number, + ) => Promise< + | { + latestRowVersion: number + requiresFullReload: true + } + | { + latestRowVersion: number + requiresFullReload: false + changedKeys: Array + deletedKeys: Array + } + > + getStreamPosition?: (collectionId: string) => Promise<{ + latestTerm: number + latestSeq: number + latestRowVersion: number + }> +} + +type ActiveRemoteSubsetAcquisition = { + collectionId: string + requesterId: string + acquisitionId: string + owner: RemoteSubsetOwner + options: TransportedLoadSubsetOptions + load: Promise + transferred: boolean + released: boolean + terminalRelease: boolean + release: Promise | null +} + +type AwaitingRemoteSubsetOwnerAcquisition = { + collectionId: string + requesterId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + released: true + awaitingOwner: true +} + +type RemoteSubsetAcquisition = + | ActiveRemoteSubsetAcquisition + | AwaitingRemoteSubsetOwnerAcquisition + | { + collectionId: string + requesterId: string + acquisitionId: string + released: true + } + +type OutboundRemoteSubsetAcquisition = { + collectionId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + localOptions: TransportedLoadSubsetOptions + acquiredLeaderId: string | null + inFlight: Promise | null + forceReplay: boolean + retryTimer: ReturnType | null + retryAttempts: number +} + +type AppliedEnvelope = + | { + appliedAt: number + requestType: `rpc:applyLocalMutations:req` + response: ApplyLocalMutationsResponse + } + | { + appliedAt: number + requestType: `rpc:applyCommittedTx:req` + response: ApplyCommittedTxResponse + } + +type InFlightEnvelope = + | { + requestType: `rpc:applyLocalMutations:req` + response: Promise + } + | { + requestType: `rpc:applyCommittedTx:req` + response: Promise + } + +// --------------------------------------------------------------------------- +// Options +// --------------------------------------------------------------------------- + +/** @internal Shared by the Browser and Electron persistence hosts. */ +export type BroadcastCollectionCoordinatorOptions = { + dbName: string + adapter?: CoordinatorAdapter + coordinatorName: string +} + +// --------------------------------------------------------------------------- +// BroadcastCollectionCoordinator +// --------------------------------------------------------------------------- + +/** @internal Shared coordination engine for BroadcastChannel/Web Locks hosts. */ +export class BroadcastCollectionCoordinator implements PersistedCollectionCoordinator { + private readonly nodeId = safeRandomUUID() + private readonly dbName: string + private readonly coordinatorName: string + private defaultAdapter: CoordinatorAdapter | null + private readonly collectionAdapters = new Map() + private readonly remoteSubsetOwners = new Map() + private readonly remoteSubsetIds = new Map< + string, + WeakMap + >() + private readonly outboundRemoteSubsetAcquisitions = new Map< + string, + OutboundRemoteSubsetAcquisition + >() + private readonly inboundRemoteSubsetAcquisitions = new Map< + string, + RemoteSubsetAcquisition + >() + private readonly releasedRemoteSubsetAcquisitionTimes = new Map< + string, + number + >() + private readonly channel: BroadcastChannel + private readonly collections = new Map() + private readonly pendingRPCs = new Map() + private readonly appliedEnvelopes = new Map() + private readonly inFlightEnvelopes = new Map() + private disposed = false + + /** Method indirection to prevent TypeScript from narrowing `disposed` across awaits */ + private isDisposed(): boolean { + return this.disposed + } + + private requireAdapter(collectionId: string): CoordinatorAdapter { + const adapter = + this.collectionAdapters.get(collectionId) ?? this.defaultAdapter + if (!adapter) { + throw new Error( + `${this.coordinatorName}: adapter not set for collection "${collectionId}". Call setAdapterForCollection() before using leader-side operations.`, + ) + } + return adapter + } + + constructor(options: BroadcastCollectionCoordinatorOptions) { + this.dbName = options.dbName + this.coordinatorName = options.coordinatorName + this.defaultAdapter = options.adapter ?? null + this.channel = new BroadcastChannel(`tsdb:coord:${this.dbName}`) + this.channel.onmessage = (event: MessageEvent) => { + this.onChannelMessage(event.data) + } + } + + /** + * Set or replace the persistence adapter used for leader-side RPC handling. + * Called by a host persistence factory to wire the internally-created adapter. + */ + setAdapter(adapter: CoordinatorAdapter): void { + this.defaultAdapter = adapter + } + + /** Register the persistence adapter that owns one collection. */ + setAdapterForCollection( + collectionId: string, + adapter: CoordinatorAdapter, + ): void { + this.collectionAdapters.set(collectionId, adapter) + } + + registerRemoteSubsetOwner( + collectionId: string, + owner: RemoteSubsetOwner, + ): () => void { + if (this.remoteSubsetOwners.has(collectionId)) { + throw new DuplicateRemoteSubsetOwnerError(collectionId) + } + this.remoteSubsetOwners.set(collectionId, owner) + for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + if (acquisition.collectionId !== collectionId) continue + acquisition.acquiredLeaderId = null + acquisition.forceReplay = true + } + void this.replayRemoteSubsetAcquisitions(collectionId) + this.rebindRemoteInboundSubsetAcquisitions(collectionId, owner) + return () => { + if (this.remoteSubsetOwners.get(collectionId) !== owner) return + this.remoteSubsetOwners.delete(collectionId) + this.releaseInboundRemoteSubsetAcquisitions(collectionId, owner) + } + } + + // ----------------------------------------------------------------------- + // PersistedCollectionCoordinator interface + // ----------------------------------------------------------------------- + + getNodeId(): string { + return this.nodeId + } + + subscribe( + collectionId: string, + onMessage: (message: ProtocolEnvelope) => void, + ): () => void { + const state = this.ensureCollectionState(collectionId) + state.subscribers.add(onMessage) + return () => { + state.subscribers.delete(onMessage) + } + } + + publish(_collectionId: string, message: ProtocolEnvelope): void { + this.channel.postMessage(message) + } + + isLeader(collectionId: string): boolean { + return this.collections.get(collectionId)?.isLeader ?? false + } + + async ensureLeadership(collectionId: string): Promise { + const state = this.ensureCollectionState(collectionId) + if (state.isLeader) return + await this.acquireLeadership(collectionId, state) + } + + async requestEnsureRemoteSubset( + collectionId: string, + options: LoadSubsetOptions, + ): Promise { + const transportedOptions = toTransportedLoadSubsetOptions(options) + const localOptions = toProcessLocalLoadSubsetOptions( + options, + transportedOptions, + ) + let collectionIds = this.remoteSubsetIds.get(collectionId) + if (!collectionIds) { + collectionIds = new WeakMap() + this.remoteSubsetIds.set(collectionId, collectionIds) + } + let acquisitionId = collectionIds.get(options) + let acquisition = acquisitionId + ? this.outboundRemoteSubsetAcquisitions.get( + remoteSubsetAcquisitionKey(collectionId, acquisitionId), + ) + : undefined + if (!acquisition) { + acquisitionId = safeRandomUUID() + collectionIds.set(options, acquisitionId) + acquisition = { + collectionId, + acquisitionId, + options: transportedOptions, + localOptions, + acquiredLeaderId: null, + inFlight: null, + forceReplay: false, + retryTimer: null, + retryAttempts: 0, + } + this.outboundRemoteSubsetAcquisitions.set( + remoteSubsetAcquisitionKey(collectionId, acquisitionId), + acquisition, + ) + } + + await this.acquireRemoteSubset(acquisition) + } + + async requestReleaseRemoteSubset( + collectionId: string, + options: LoadSubsetOptions, + ): Promise { + const collectionIds = this.remoteSubsetIds.get(collectionId) + const acquisitionId = collectionIds?.get(options) + if (!acquisitionId) return + const key = remoteSubsetAcquisitionKey(collectionId, acquisitionId) + const acquisition = this.outboundRemoteSubsetAcquisitions.get(key) + if (!acquisition) return + this.outboundRemoteSubsetAcquisitions.delete(key) + this.cancelRemoteSubsetReplayRetry(acquisition) + collectionIds!.delete(options) + + const request: Extract< + RPCRequest, + { type: `rpc:releaseRemoteSubset:req` } + > = { + type: `rpc:releaseRemoteSubset:req`, + rpcId: safeRandomUUID(), + acquisitionId, + } + const response = this.isLeader(collectionId) + ? await this.handleReleaseRemoteSubset(collectionId, request, this.nodeId) + : await this.sendRPC(collectionId, request) + + if (!response.ok) { + throw new Error(`releaseRemoteSubset failed: ${response.error}`) + } + } + + private async acquireRemoteSubset( + acquisition: OutboundRemoteSubsetAcquisition, + ): Promise { + if (acquisition.inFlight) return acquisition.inFlight + + const route = { localOwner: false } + let resolveWork!: () => void + let rejectWork!: (error: unknown) => void + const work = new Promise((resolve, reject) => { + resolveWork = resolve + rejectWork = reject + }) + acquisition.inFlight = work + const run = async (): Promise => { + const request: Extract< + RPCRequest, + { type: `rpc:ensureRemoteSubset:req` } + > = { + type: `rpc:ensureRemoteSubset:req`, + rpcId: safeRandomUUID(), + acquisitionId: acquisition.acquisitionId, + options: acquisition.options, + } + route.localOwner = this.isLeader(acquisition.collectionId) + let response: EnsureRemoteSubsetResponse + try { + response = route.localOwner + ? await this.handleEnsureRemoteSubset( + acquisition.collectionId, + request, + this.nodeId, + acquisition.localOptions, + ) + : await this.sendRPC( + acquisition.collectionId, + request, + ) + } catch (error) { + if ( + route.localOwner || + error instanceof RetryableRemoteSubsetAcquisitionError + ) { + throw error + } + throw new RetryableRemoteSubsetAcquisitionError( + `Remote subset transport failed`, + error, + ) + } + + if (!response.ok) { + if (response.retryable) { + throw new RetryableRemoteSubsetAcquisitionError(response.error) + } + throw new Error(`ensureRemoteSubset failed: ${response.error}`) + } + acquisition.acquiredLeaderId = response.leaderId + } + void run().then(resolveWork, rejectWork) + let acquired = false + try { + await work + acquired = true + this.cancelRemoteSubsetReplayRetry(acquisition) + } finally { + if (acquisition.inFlight === work) acquisition.inFlight = null + const current = this.collections.get(acquisition.collectionId) + const currentLeaderId = current?.isLeader + ? this.nodeId + : (current?.leaderId ?? null) + const key = remoteSubsetAcquisitionKey( + acquisition.collectionId, + acquisition.acquisitionId, + ) + if ( + acquired && + this.outboundRemoteSubsetAcquisitions.get(key) === acquisition && + (acquisition.forceReplay || + (currentLeaderId !== null && + acquisition.acquiredLeaderId !== currentLeaderId)) + ) { + acquisition.forceReplay = false + void this.acquireRemoteSubset(acquisition).catch((error) => { + this.scheduleRemoteSubsetReplayRetry(acquisition, error) + }) + } + } + } + + async requestEnsurePersistedIndex( + collectionId: string, + signature: string, + spec: PersistedIndexSpec, + ): Promise { + if (this.isLeader(collectionId)) { + await this.requireAdapter(collectionId).ensureIndex( + collectionId, + signature, + spec, + ) + return + } + + const response = await this.sendRPC<{ + type: `rpc:ensurePersistedIndex:res` + rpcId: string + ok: boolean + error?: string + }>(collectionId, { + type: `rpc:ensurePersistedIndex:req`, + rpcId: safeRandomUUID(), + signature, + spec, + }) + + if (!response.ok) { + throw new Error( + `ensurePersistedIndex failed: ${response.error ?? `unknown error`}`, + ) + } + } + + async requestApplyLocalMutations( + collectionId: string, + mutations: Array, + ): Promise { + if (this.isLeader(collectionId)) { + return this.handleApplyLocalMutations(collectionId, { + type: `rpc:applyLocalMutations:req`, + rpcId: safeRandomUUID(), + envelopeId: safeRandomUUID(), + mutations, + }) + } + + return this.sendRPC(collectionId, { + type: `rpc:applyLocalMutations:req`, + rpcId: safeRandomUUID(), + envelopeId: safeRandomUUID(), + mutations, + }) + } + + async requestApplyCommittedTx( + collectionId: string, + tx: PersistedTx, + ): Promise { + const request: Extract = { + type: `rpc:applyCommittedTx:req`, + rpcId: safeRandomUUID(), + envelopeId: safeRandomUUID(), + tx, + } + if (this.isLeader(collectionId)) { + return this.handleApplyCommittedTx(collectionId, request) + } + + return this.sendRPC(collectionId, request) + } + + async pullSince( + collectionId: string, + fromRowVersion: number, + ): Promise { + if (this.isLeader(collectionId)) { + return this.handlePullSince(collectionId, { + type: `rpc:pullSince:req`, + rpcId: safeRandomUUID(), + fromRowVersion, + }) + } + + return this.sendRPC(collectionId, { + type: `rpc:pullSince:req`, + rpcId: safeRandomUUID(), + fromRowVersion, + }) + } + + // ----------------------------------------------------------------------- + // Lifecycle + // ----------------------------------------------------------------------- + + dispose(): void { + this.disposed = true + + for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + this.cancelRemoteSubsetReplayRetry(acquisition) + this.postRemoteSubsetRelease(acquisition) + } + this.outboundRemoteSubsetAcquisitions.clear() + this.remoteSubsetIds.clear() + + for (const [collectionId, state] of this.collections) { + this.releaseLeadership(collectionId, state) + } + + for (const [, pending] of this.pendingRPCs) { + clearTimeout(pending.timer) + pending.reject(new Error(`coordinator disposed`)) + } + this.pendingRPCs.clear() + + this.channel.close() + this.collections.clear() + this.collectionAdapters.clear() + for (const collectionId of this.remoteSubsetOwners.keys()) { + this.releaseInboundRemoteSubsetAcquisitions(collectionId) + } + this.remoteSubsetOwners.clear() + this.inboundRemoteSubsetAcquisitions.clear() + this.releasedRemoteSubsetAcquisitionTimes.clear() + this.appliedEnvelopes.clear() + this.inFlightEnvelopes.clear() + } + + // ----------------------------------------------------------------------- + // Leadership via Web Locks + // ----------------------------------------------------------------------- + + private ensureCollectionState(collectionId: string): CollectionState { + let state = this.collections.get(collectionId) + if (!state) { + state = { + isLeader: false, + leaderId: null, + lockAbortController: null, + heartbeatTimer: null, + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + subscribers: new Set(), + } + this.collections.set(collectionId, state) + void this.acquireLeadership(collectionId, state) + } + return state + } + + private async acquireLeadership( + collectionId: string, + state: CollectionState, + ): Promise { + if (this.disposed || state.isLeader) return + + const lockName = `tsdb:leader:${this.dbName}:${collectionId}` + const abortController = new AbortController() + state.lockAbortController = abortController + + try { + await navigator.locks.request( + lockName, + { signal: abortController.signal }, + async () => { + if (this.isDisposed()) return + + try { + // Restore stream position from DB before claiming leadership + const adapter = this.requireAdapter(collectionId) + if (adapter.getStreamPosition) { + const pos = await adapter.getStreamPosition(collectionId) + state.latestTerm = pos.latestTerm + state.latestSeq = pos.latestSeq + state.latestRowVersion = pos.latestRowVersion + } + + state.latestTerm++ + state.isLeader = true + state.leaderId = this.nodeId + + this.emitHeartbeat(collectionId, state) + void this.replayRemoteSubsetAcquisitions(collectionId) + state.heartbeatTimer = setInterval(() => { + this.emitHeartbeat(collectionId, state) + }, HEARTBEAT_INTERVAL_MS) + + // Hold the lock until disposed or aborted + await new Promise((resolve) => { + const onAbort = () => { + abortController.signal.removeEventListener(`abort`, onAbort) + resolve() + } + if (abortController.signal.aborted) { + resolve() + return + } + abortController.signal.addEventListener(`abort`, onAbort) + }) + } finally { + this.releaseInboundRemoteSubsetAcquisitions(collectionId) + state.isLeader = false + state.leaderId = null + if (state.heartbeatTimer) { + clearInterval(state.heartbeatTimer) + state.heartbeatTimer = null + } + } + }, + ) + } catch (error) { + if (error instanceof DOMException && error.name === `AbortError`) { + return + } + console.warn(`Failed to acquire leadership for ${collectionId}:`, error) + } + + // Re-acquire if not disposed (leadership was released by another means) + if (!this.isDisposed()) { + void this.acquireLeadership(collectionId, state) + } + } + + private releaseLeadership( + collectionId: string, + state: CollectionState, + ): void { + this.releaseInboundRemoteSubsetAcquisitions(collectionId) + if (state.lockAbortController) { + state.lockAbortController.abort() + state.lockAbortController = null + } + if (state.heartbeatTimer) { + clearInterval(state.heartbeatTimer) + state.heartbeatTimer = null + } + state.isLeader = false + state.leaderId = null + } + + private postRemoteSubsetRelease( + acquisition: OutboundRemoteSubsetAcquisition, + ): void { + const request: Extract< + RPCRequest, + { type: `rpc:releaseRemoteSubset:req` } + > = { + type: `rpc:releaseRemoteSubset:req`, + rpcId: safeRandomUUID(), + acquisitionId: acquisition.acquisitionId, + } + if (this.isLeader(acquisition.collectionId)) { + void this.handleReleaseRemoteSubset( + acquisition.collectionId, + request, + this.nodeId, + ).catch(() => { + // The owner already received the exact unload failure through onError. + }) + return + } + this.channel.postMessage({ + v: 1, + dbName: this.dbName, + collectionId: acquisition.collectionId, + senderId: this.nodeId, + ts: Date.now(), + payload: request, + } satisfies ProtocolEnvelope) + } + + private cancelRemoteSubsetReplayRetry( + acquisition: OutboundRemoteSubsetAcquisition, + ): void { + if (acquisition.retryTimer !== null) { + clearTimeout(acquisition.retryTimer) + acquisition.retryTimer = null + } + acquisition.retryAttempts = 0 + } + + private scheduleRemoteSubsetReplayRetry( + acquisition: OutboundRemoteSubsetAcquisition, + error: unknown, + ): void { + const key = remoteSubsetAcquisitionKey( + acquisition.collectionId, + acquisition.acquisitionId, + ) + if ( + !(error instanceof RetryableRemoteSubsetAcquisitionError) || + this.isDisposed() || + this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition || + acquisition.retryTimer !== null || + acquisition.retryAttempts >= REMOTE_SUBSET_REPLAY_RETRY_ATTEMPTS + ) { + return + } + + acquisition.retryAttempts++ + acquisition.retryTimer = setTimeout(() => { + acquisition.retryTimer = null + if ( + this.isDisposed() || + this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition + ) { + return + } + void this.acquireRemoteSubset(acquisition).catch((retryError) => { + this.scheduleRemoteSubsetReplayRetry(acquisition, retryError) + }) + }, RPC_RETRY_DELAY_MS) + } + + private async replayRemoteSubsetAcquisitions( + collectionId: string, + ): Promise { + if (this.isDisposed()) return + const state = this.collections.get(collectionId) + const leaderId = state?.isLeader ? this.nodeId : state?.leaderId + if (!leaderId) return + + const replays: Array> = [] + for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + if ( + acquisition.collectionId !== collectionId || + (!acquisition.forceReplay && acquisition.acquiredLeaderId === leaderId) + ) { + continue + } + if (acquisition.inFlight) { + acquisition.forceReplay = true + continue + } + acquisition.forceReplay = false + replays.push( + this.acquireRemoteSubset(acquisition).catch((error) => { + this.scheduleRemoteSubsetReplayRetry(acquisition, error) + }), + ) + } + await Promise.all(replays) + } + + private emitHeartbeat(collectionId: string, state: CollectionState): void { + const envelope: ProtocolEnvelope = { + v: 1, + dbName: this.dbName, + collectionId, + senderId: this.nodeId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term: state.latestTerm, + leaderId: this.nodeId, + latestSeq: state.latestSeq, + latestRowVersion: state.latestRowVersion, + }, + } + this.channel.postMessage(envelope) + } + + // ----------------------------------------------------------------------- + // BroadcastChannel message handling + // ----------------------------------------------------------------------- + + private onChannelMessage(data: unknown): void { + if (!isProtocolEnvelope(data)) return + + const envelope = data + + // Ignore own messages + if (envelope.senderId === this.nodeId) return + + const payload = envelope.payload + if (!payload || typeof payload !== `object`) return + + const type = (payload as Record).type as string | undefined + + if (type === `leader:heartbeat`) { + const heartbeat = payload as { + leaderId?: unknown + term?: unknown + latestSeq?: unknown + latestRowVersion?: unknown + } + if ( + typeof heartbeat.leaderId === `string` && + typeof heartbeat.term === `number` && + typeof heartbeat.latestSeq === `number` && + typeof heartbeat.latestRowVersion === `number` + ) { + const state = this.ensureCollectionState(envelope.collectionId) + if (heartbeat.term < state.latestTerm) return + const changedLeader = state.leaderId !== heartbeat.leaderId + state.leaderId = heartbeat.leaderId + state.latestTerm = Math.max(state.latestTerm, heartbeat.term) + state.latestSeq = Math.max(state.latestSeq, heartbeat.latestSeq) + state.latestRowVersion = Math.max( + state.latestRowVersion, + heartbeat.latestRowVersion, + ) + if (changedLeader) { + void this.replayRemoteSubsetAcquisitions(envelope.collectionId) + } + } + } + + // Handle RPC responses (for pending outbound RPCs) + if (type && type.endsWith(`:res`)) { + const rpcId = (payload as { rpcId?: string }).rpcId + if (rpcId && this.pendingRPCs.has(rpcId)) { + const pending = this.pendingRPCs.get(rpcId)! + this.pendingRPCs.delete(rpcId) + clearTimeout(pending.timer) + pending.resolve(payload as RPCResponse) + return + } + } + + // Handle RPC requests (leader only) + if (type && type.endsWith(`:req`)) { + if (!isRPCRequest(payload)) return + const collectionId = envelope.collectionId + if (this.isLeader(collectionId)) { + void this.handleRPCRequest(collectionId, payload, envelope.senderId) + } + return + } + + // Forward protocol messages to subscribers + const state = this.collections.get(envelope.collectionId) + if (state) { + for (const subscriber of state.subscribers) { + subscriber(envelope) + } + } + } + + // ----------------------------------------------------------------------- + // RPC - Outbound (follower side) + // ----------------------------------------------------------------------- + + private async sendRPC( + collectionId: string, + request: RPCRequest, + ): Promise { + let lastError: Error | undefined + let firstTransportCause: unknown + const mutationRequestType = isMutatingRPCRequest(request) + ? request.type + : undefined + const mutationRoute = mutationRequestType + ? this.captureMutationRoute(collectionId) + : undefined + + for (let attempt = 0; attempt <= RPC_RETRY_ATTEMPTS; attempt++) { + if (attempt > 0) { + await sleep(RPC_RETRY_DELAY_MS * attempt) + } + + if ( + mutationRoute && + mutationRequestType && + firstTransportCause !== undefined + ) { + this.assertMutationRouteUnchanged( + collectionId, + mutationRequestType, + mutationRoute, + firstTransportCause, + ) + } + + if (this.isLeader(collectionId)) { + return (await this.dispatchRPCRequest( + collectionId, + request, + this.nodeId, + )) as T + } + + try { + return await this.sendRPCOnce(collectionId, request) + } catch (error) { + if (this.isDisposed()) throw error + firstTransportCause ??= error + if (mutationRoute && mutationRequestType) { + this.assertMutationRouteUnchanged( + collectionId, + mutationRequestType, + mutationRoute, + firstTransportCause, + ) + } + lastError = error instanceof Error ? error : new Error(String(error)) + } + } + + throw lastError ?? new Error(`RPC failed after retries`) + } + + private captureMutationRoute(collectionId: string): { + leaderId: string | null + term: number | null + } { + const state = this.collections.get(collectionId) + return { + leaderId: state?.isLeader ? this.nodeId : (state?.leaderId ?? null), + term: state?.latestTerm ?? null, + } + } + + private assertMutationRouteUnchanged( + collectionId: string, + requestType: IndeterminateCommitRequestType, + previous: { leaderId: string | null; term: number | null }, + cause: unknown, + ): void { + const current = this.captureMutationRoute(collectionId) + if ( + previous.leaderId !== null && + previous.term !== null && + current.leaderId === previous.leaderId && + current.term === previous.term + ) { + return + } + throw new IndeterminateCommitError({ + collectionId, + requestType, + previousLeaderId: previous.leaderId, + previousTerm: previous.term, + currentLeaderId: current.leaderId, + currentTerm: current.term, + cause, + }) + } + + private sendRPCOnce( + collectionId: string, + request: RPCRequest, + ): Promise { + return new Promise((resolve, reject) => { + const rpcId = request.rpcId + + const timer = setTimeout(() => { + this.pendingRPCs.delete(rpcId) + reject( + new Error(`RPC ${request.type} timed out after ${RPC_TIMEOUT_MS}ms`), + ) + }, RPC_TIMEOUT_MS) + + this.pendingRPCs.set(rpcId, { + resolve: resolve as (response: RPCResponse) => void, + reject, + timer, + }) + + const envelope: ProtocolEnvelope = { + v: 1, + dbName: this.dbName, + collectionId, + senderId: this.nodeId, + ts: Date.now(), + payload: request, + } + this.channel.postMessage(envelope) + }) + } + + // ----------------------------------------------------------------------- + // RPC - Inbound (leader side) + // ----------------------------------------------------------------------- + + private async handleRPCRequest( + collectionId: string, + request: RPCRequest, + requesterId: string, + ): Promise { + let response: RPCResponse + + try { + response = await this.dispatchRPCRequest( + collectionId, + request, + requesterId, + ) + } catch (error) { + response = createRPCErrorResponse(request, error) + } + + if (this.isDisposed()) { + return + } + + const envelope: ProtocolEnvelope = { + v: 1, + dbName: this.dbName, + collectionId, + senderId: this.nodeId, + ts: Date.now(), + payload: response, + } + this.channel.postMessage(envelope) + } + + private dispatchRPCRequest( + collectionId: string, + request: RPCRequest, + requesterId: string, + ): Promise { + switch (request.type) { + case `rpc:ensureRemoteSubset:req`: + return this.handleEnsureRemoteSubset(collectionId, request, requesterId) + case `rpc:releaseRemoteSubset:req`: + return this.handleReleaseRemoteSubset( + collectionId, + request, + requesterId, + ) + case `rpc:ensurePersistedIndex:req`: + return this.handleEnsurePersistedIndex(collectionId, request) + case `rpc:applyLocalMutations:req`: + return this.handleApplyLocalMutations(collectionId, request) + case `rpc:applyCommittedTx:req`: + return this.handleApplyCommittedTx(collectionId, request) + case `rpc:pullSince:req`: + return this.handlePullSince(collectionId, request) + } + } + + private async handleEnsureRemoteSubset( + collectionId: string, + request: Extract, + requesterId: string, + localOptions?: TransportedLoadSubsetOptions, + ): Promise { + this.pruneReleasedRemoteSubsetAcquisitions() + const key = inboundRemoteSubsetAcquisitionKey( + collectionId, + requesterId, + request.acquisitionId, + ) + const existing = this.inboundRemoteSubsetAcquisitions.get(key) + const awaitingOwner = + existing && `awaitingOwner` in existing ? existing : undefined + if (existing) { + if (`owner` in existing) { + if (!existing.released) { + await existing.load + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + await existing.release + if (existing.terminalRelease) { + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + if (this.inboundRemoteSubsetAcquisitions.get(key) === existing) { + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } else if (!(`awaitingOwner` in existing)) { + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + } + + const owner = this.remoteSubsetOwners.get(collectionId) + if (!owner) { + throw new RetryableRemoteSubsetAcquisitionError( + `${this.coordinatorName}: no remote subset owner registered for collection "${collectionId}"`, + ) + } + + const acquisition: ActiveRemoteSubsetAcquisition = { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + owner, + options: awaitingOwner?.options ?? localOptions ?? request.options, + load: Promise.resolve(), + transferred: false, + released: false, + terminalRelease: false, + release: null, + } + this.releasedRemoteSubsetAcquisitionTimes.delete(key) + this.inboundRemoteSubsetAcquisitions.set(key, acquisition) + let resolveLoad!: () => void + let rejectLoad!: (error: unknown) => void + acquisition.load = new Promise((resolve, reject) => { + resolveLoad = resolve + rejectLoad = reject + }) + try { + const load = owner(acquisition.options) + acquisition.transferred = true + void Promise.resolve(load).then(resolveLoad, rejectLoad) + } catch (error) { + rejectLoad(error) + } + try { + await acquisition.load + } catch (error) { + if ( + !acquisition.transferred && + this.inboundRemoteSubsetAcquisitions.get(key) === acquisition + ) { + if (awaitingOwner) { + this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + } else { + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } + reportRemoteSubsetOwnerError(owner, error) + throw error + } + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + + private async handleReleaseRemoteSubset( + collectionId: string, + request: Extract, + requesterId: string, + ): Promise { + this.pruneReleasedRemoteSubsetAcquisitions() + const key = inboundRemoteSubsetAcquisitionKey( + collectionId, + requesterId, + request.acquisitionId, + ) + const acquisition = this.inboundRemoteSubsetAcquisitions.get(key) + if (!acquisition) { + this.setReleasedRemoteSubsetAcquisition(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } else if (`owner` in acquisition) { + acquisition.terminalRelease = true + await this.releaseRemoteSubsetAcquisition(acquisition) + if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { + this.setReleasedRemoteSubsetAcquisition(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } + } else if (`awaitingOwner` in acquisition) { + this.setReleasedRemoteSubsetAcquisition(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } + return { + type: `rpc:releaseRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + } + } + + private releaseRemoteSubsetAcquisition( + acquisition: ActiveRemoteSubsetAcquisition, + ): Promise { + if (acquisition.release) return acquisition.release + acquisition.released = true + acquisition.release = (async () => { + try { + await acquisition.load + } catch { + // A returned promise transfers the lease even when initial loading fails. + } + try { + if (acquisition.transferred) { + await unloadRemoteSubsetOwner(acquisition.owner, acquisition.options) + } + } finally { + if (!acquisition.terminalRelease) { + const key = inboundRemoteSubsetAcquisitionKey( + acquisition.collectionId, + acquisition.requesterId, + acquisition.acquisitionId, + ) + if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId: acquisition.collectionId, + requesterId: acquisition.requesterId, + acquisitionId: acquisition.acquisitionId, + options: acquisition.options, + released: true, + awaitingOwner: true, + }) + } + } + } + })() + return acquisition.release + } + + private releaseInboundRemoteSubsetAcquisitions( + collectionId: string, + owner?: RemoteSubsetOwner, + ): void { + for (const acquisition of this.inboundRemoteSubsetAcquisitions.values()) { + if ( + !(`owner` in acquisition) || + acquisition.collectionId !== collectionId || + (owner && acquisition.owner !== owner) + ) { + continue + } + void this.releaseRemoteSubsetAcquisition(acquisition).catch( + () => undefined, + ) + } + } + + private rebindRemoteInboundSubsetAcquisitions( + collectionId: string, + owner: RemoteSubsetOwner, + ): void { + for (const acquisition of this.inboundRemoteSubsetAcquisitions.values()) { + if (`awaitingOwner` in acquisition) { + if ( + acquisition.collectionId === collectionId && + acquisition.requesterId !== this.nodeId + ) { + void this.bindAwaitingRemoteSubsetAcquisition( + acquisition, + owner, + ).catch(() => { + // The owner receives the exact load failure through onError. + }) + } + continue + } + if ( + !(`owner` in acquisition) || + acquisition.collectionId !== collectionId || + acquisition.requesterId === this.nodeId || + !acquisition.released || + acquisition.terminalRelease + ) { + continue + } + + void this.rebindRemoteInboundSubsetAcquisition(acquisition, owner).catch( + () => undefined, + ) + } + } + + private async rebindRemoteInboundSubsetAcquisition( + previous: ActiveRemoteSubsetAcquisition, + owner: RemoteSubsetOwner, + ): Promise { + await previous.release + if ( + previous.terminalRelease || + this.remoteSubsetOwners.get(previous.collectionId) !== owner + ) { + return + } + + const key = inboundRemoteSubsetAcquisitionKey( + previous.collectionId, + previous.requesterId, + previous.acquisitionId, + ) + const current = this.inboundRemoteSubsetAcquisitions.get(key) + if (current && current !== previous) { + if (`awaitingOwner` in current) { + await this.bindAwaitingRemoteSubsetAcquisition(current, owner) + } + return + } + + const awaitingOwner: AwaitingRemoteSubsetOwnerAcquisition = { + collectionId: previous.collectionId, + requesterId: previous.requesterId, + acquisitionId: previous.acquisitionId, + options: previous.options, + released: true, + awaitingOwner: true, + } + this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + await this.bindAwaitingRemoteSubsetAcquisition(awaitingOwner, owner) + } + + private async bindAwaitingRemoteSubsetAcquisition( + awaitingOwner: AwaitingRemoteSubsetOwnerAcquisition, + owner: RemoteSubsetOwner, + ): Promise { + if (this.remoteSubsetOwners.get(awaitingOwner.collectionId) !== owner) { + return + } + const key = inboundRemoteSubsetAcquisitionKey( + awaitingOwner.collectionId, + awaitingOwner.requesterId, + awaitingOwner.acquisitionId, + ) + if (this.inboundRemoteSubsetAcquisitions.get(key) !== awaitingOwner) return + + const acquisition: ActiveRemoteSubsetAcquisition = { + collectionId: awaitingOwner.collectionId, + requesterId: awaitingOwner.requesterId, + acquisitionId: awaitingOwner.acquisitionId, + owner, + options: awaitingOwner.options, + load: Promise.resolve(), + transferred: false, + released: false, + terminalRelease: false, + release: null, + } + this.inboundRemoteSubsetAcquisitions.set(key, acquisition) + let resolveLoad!: () => void + let rejectLoad!: (error: unknown) => void + acquisition.load = new Promise((resolve, reject) => { + resolveLoad = resolve + rejectLoad = reject + }) + try { + const load = owner(acquisition.options) + acquisition.transferred = true + void Promise.resolve(load).then(resolveLoad, rejectLoad) + } catch (error) { + rejectLoad(error) + } + try { + await acquisition.load + } catch (error) { + if ( + !acquisition.transferred && + this.inboundRemoteSubsetAcquisitions.get(key) === acquisition + ) { + this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + } + reportRemoteSubsetOwnerError(owner, error) + throw error + } + } + + private async handleEnsurePersistedIndex( + collectionId: string, + request: { + type: `rpc:ensurePersistedIndex:req` + rpcId: string + signature: string + spec: PersistedIndexSpec + }, + ): Promise { + await this.withWriterLock(() => + this.requireAdapter(collectionId).ensureIndex( + collectionId, + request.signature, + request.spec, + ), + ) + return { + type: `rpc:ensurePersistedIndex:res`, + rpcId: request.rpcId, + ok: true, + } + } + + private async handleApplyLocalMutations( + collectionId: string, + request: { + type: `rpc:applyLocalMutations:req` + rpcId: string + envelopeId: string + mutations: Array + }, + ): Promise { + const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) + const appliedEnvelope = this.appliedEnvelopes.get(envelopeKey) + if (appliedEnvelope?.requestType === `rpc:applyLocalMutations:req`) { + return { ...appliedEnvelope.response, rpcId: request.rpcId } + } + if (appliedEnvelope) { + return { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error: `envelope ${request.envelopeId} already applied`, + } + } + + const inFlightEnvelope = this.inFlightEnvelopes.get(envelopeKey) + if (inFlightEnvelope?.requestType === `rpc:applyLocalMutations:req`) { + const response = await inFlightEnvelope.response + return { ...response, rpcId: request.rpcId } + } + if (inFlightEnvelope) { + return { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error: `envelope ${request.envelopeId} is already in flight`, + } + } + + const response = this.applyLocalMutationsOnce(collectionId, request) + const pendingEnvelope: InFlightEnvelope = { + requestType: request.type, + response, + } + this.inFlightEnvelopes.set(envelopeKey, pendingEnvelope) + try { + return await response + } finally { + if (this.inFlightEnvelopes.get(envelopeKey) === pendingEnvelope) { + this.inFlightEnvelopes.delete(envelopeKey) + } + } + } + + private async applyLocalMutationsOnce( + collectionId: string, + request: Extract, + ): Promise { + const state = this.collections.get(collectionId) + if (!state || !state.isLeader) { + return { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: false, + code: `NOT_LEADER`, + error: `not the leader for ${collectionId}`, + } + } + + // Assign stream position + state.latestSeq++ + state.latestRowVersion++ + + const term = state.latestTerm + const seq = state.latestSeq + const rowVersion = state.latestRowVersion + + // Build and apply the persisted transaction + const rowMetadataMutations: Array = [] + for (const mutation of request.mutations) { + if (!(`metadataChanged` in mutation) || !mutation.metadataChanged) { + continue + } + rowMetadataMutations.push( + mutation.metadata === undefined + ? { type: `delete`, key: mutation.key } + : { + type: `set`, + key: mutation.key, + value: mutation.metadata, + }, + ) + } + const tx = { + txId: safeRandomUUID(), + term, + seq, + rowVersion, + mutations: request.mutations.map((m) => ({ + type: m.type, + key: m.key, + value: m.value, + ...(`metadataChanged` in m + ? { metadata: m.metadata, metadataChanged: m.metadataChanged } + : {}), + })), + rowMetadataMutations, + } + + try { + await this.withWriterLock(() => + this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), + ) + } catch (error) { + throw toPersistedCollectionDurabilityError(collectionId, error) + } + + const response: ApplyLocalMutationsResponse = { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: true, + term, + seq, + latestRowVersion: rowVersion, + acceptedMutationIds: request.mutations.map((m) => m.mutationId), + } + if (this.isDisposed()) { + return response + } + this.appliedEnvelopes.set( + appliedEnvelopeKey(collectionId, request.envelopeId), + { + appliedAt: Date.now(), + requestType: request.type, + response, + }, + ) + this.pruneAppliedEnvelopes() + + // Broadcast tx:committed to all tabs + const changedRows = request.mutations + .filter((m) => m.type !== `delete`) + .map((m) => ({ key: m.key, value: m.value })) + const deletedKeys = request.mutations + .filter((m) => m.type === `delete`) + .map((m) => m.key) + + const txCommitted: ProtocolEnvelope = { + v: 1, + dbName: this.dbName, + collectionId, + senderId: this.nodeId, + ts: Date.now(), + payload: { + type: `tx:committed`, + term, + seq, + txId: tx.txId, + latestRowVersion: rowVersion, + requiresFullReload: false, + changedRows, + deletedKeys, + rowMetadataMutations, + }, + } + this.channel.postMessage(txCommitted) + + // Deliver to local subscribers too + for (const subscriber of state.subscribers) { + subscriber(txCommitted) + } + + return response + } + + private async handleApplyCommittedTx( + collectionId: string, + request: Extract, + ): Promise { + const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) + const appliedEnvelope = this.appliedEnvelopes.get(envelopeKey) + if (appliedEnvelope?.requestType === `rpc:applyCommittedTx:req`) { + return { ...appliedEnvelope.response, rpcId: request.rpcId } + } + if (appliedEnvelope) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error: `envelope ${request.envelopeId} already applied`, + } + } + + const inFlightEnvelope = this.inFlightEnvelopes.get(envelopeKey) + if (inFlightEnvelope?.requestType === `rpc:applyCommittedTx:req`) { + const response = await inFlightEnvelope.response + return { ...response, rpcId: request.rpcId } + } + if (inFlightEnvelope) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error: `envelope ${request.envelopeId} is already in flight`, + } + } + + const response = this.applyCommittedTxOnce(collectionId, request) + const pendingEnvelope: InFlightEnvelope = { + requestType: request.type, + response, + } + this.inFlightEnvelopes.set(envelopeKey, pendingEnvelope) + try { + return await response + } finally { + if (this.inFlightEnvelopes.get(envelopeKey) === pendingEnvelope) { + this.inFlightEnvelopes.delete(envelopeKey) + } + } + } + + private async applyCommittedTxOnce( + collectionId: string, + request: Extract, + ): Promise { + const state = this.collections.get(collectionId) + if (!state || !state.isLeader) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `NOT_LEADER`, + error: `not the leader for ${collectionId}`, + } + } + + state.latestSeq++ + state.latestRowVersion++ + const tx: PersistedTx = { + ...request.tx, + term: state.latestTerm, + seq: state.latestSeq, + rowVersion: state.latestRowVersion, + } + + try { + await this.withWriterLock(() => + this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), + ) + } catch (error) { + throw toPersistedCollectionDurabilityError(collectionId, error) + } + const response: ApplyCommittedTxResponse = { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + } + if (this.isDisposed()) { + return response + } + this.appliedEnvelopes.set( + appliedEnvelopeKey(collectionId, request.envelopeId), + { + appliedAt: Date.now(), + requestType: request.type, + response, + }, + ) + this.pruneAppliedEnvelopes() + + const committedBase = { + type: `tx:committed` as const, + term: tx.term, + seq: tx.seq, + txId: tx.txId, + latestRowVersion: tx.rowVersion, + } + const committedPayload: TxCommitted = tx.truncate + ? { + ...committedBase, + requiresFullReload: true, + } + : { + ...committedBase, + requiresFullReload: false, + changedRows: tx.mutations + .filter((mutation) => mutation.type !== `delete`) + .map((mutation) => ({ + key: mutation.key, + value: mutation.value, + })), + deletedKeys: tx.mutations + .filter((mutation) => mutation.type === `delete`) + .map((mutation) => mutation.key), + rowMetadataMutations: tx.rowMetadataMutations, + collectionMetadataMutations: tx.collectionMetadataMutations, + } + const committed: ProtocolEnvelope = { + v: 1, + dbName: this.dbName, + collectionId, + senderId: this.nodeId, + ts: Date.now(), + payload: committedPayload, + } + this.channel.postMessage(committed) + for (const subscriber of state.subscribers) { + subscriber(committed) + } + + return response + } + + private async handlePullSince( + collectionId: string, + request: { + type: `rpc:pullSince:req` + rpcId: string + fromRowVersion: number + }, + ): Promise { + const state = this.collections.get(collectionId) + + const adapter = this.requireAdapter(collectionId) + if (!adapter.pullSince) { + return { + type: `rpc:pullSince:res`, + rpcId: request.rpcId, + ok: true, + latestTerm: state?.latestTerm ?? 0, + latestSeq: state?.latestSeq ?? 0, + latestRowVersion: state?.latestRowVersion ?? 0, + requiresFullReload: true, + } + } + + const result = await adapter.pullSince(collectionId, request.fromRowVersion) + + if (result.requiresFullReload) { + return { + type: `rpc:pullSince:res`, + rpcId: request.rpcId, + ok: true, + latestTerm: state?.latestTerm ?? 0, + latestSeq: state?.latestSeq ?? 0, + latestRowVersion: result.latestRowVersion, + requiresFullReload: true, + } + } + + return { + type: `rpc:pullSince:res`, + rpcId: request.rpcId, + ok: true, + latestTerm: state?.latestTerm ?? 0, + latestSeq: state?.latestSeq ?? 0, + latestRowVersion: result.latestRowVersion, + requiresFullReload: false, + changedKeys: result.changedKeys, + deletedKeys: result.deletedKeys, + } + } + + // ----------------------------------------------------------------------- + // DB Writer Lock + // ----------------------------------------------------------------------- + + private async withWriterLock(fn: () => Promise): Promise { + const lockName = `tsdb:writer:${this.dbName}` + + for (let attempt = 0; attempt <= WRITER_LOCK_MAX_RETRIES; attempt++) { + const callbackState = { entered: false } + try { + return await navigator.locks.request(lockName, async () => { + callbackState.entered = true + return fn() + }) + } catch (error) { + if (callbackState.entered) { + throw error + } + if (error instanceof DOMException && error.name === `AbortError`) { + throw error + } + + if (attempt < WRITER_LOCK_MAX_RETRIES) { + await sleep(WRITER_LOCK_BUSY_RETRY_MS * Math.min(attempt + 1, 5)) + continue + } + + throw error + } + } + + // Unreachable but satisfies TypeScript + throw new Error(`writer lock acquisition failed`) + } + + // ----------------------------------------------------------------------- + // Helpers + // ----------------------------------------------------------------------- + + private pruneAppliedEnvelopes(): void { + // Keep envelopes for 60 seconds for dedup + const cutoff = Date.now() - RPC_DEDUPE_RETENTION_MS + for (const [key, envelope] of this.appliedEnvelopes) { + if (envelope.appliedAt < cutoff) { + this.appliedEnvelopes.delete(key) + } + } + } + + private setReleasedRemoteSubsetAcquisition( + key: string, + acquisition: RemoteSubsetAcquisition, + ): void { + this.inboundRemoteSubsetAcquisitions.set(key, acquisition) + this.releasedRemoteSubsetAcquisitionTimes.set(key, Date.now()) + } + + private pruneReleasedRemoteSubsetAcquisitions(): void { + const cutoff = Date.now() - RPC_DEDUPE_RETENTION_MS + for (const [key, releasedAt] of this.releasedRemoteSubsetAcquisitionTimes) { + if (releasedAt < cutoff) { + this.releasedRemoteSubsetAcquisitionTimes.delete(key) + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } + } +} + +// --------------------------------------------------------------------------- +// Utilities +// --------------------------------------------------------------------------- + +function isProtocolEnvelope(data: unknown): data is ProtocolEnvelope { + if (!data || typeof data !== `object`) return false + const record = data as Record + return ( + record.v === 1 && + typeof record.dbName === `string` && + typeof record.collectionId === `string` && + typeof record.senderId === `string` && + typeof record.ts === `number` + ) +} + +function isRPCRequest(payload: unknown): payload is RPCRequest { + if (!payload || typeof payload !== `object`) return false + switch ((payload as { type?: unknown }).type) { + case `rpc:ensureRemoteSubset:req`: + case `rpc:releaseRemoteSubset:req`: + case `rpc:ensurePersistedIndex:req`: + case `rpc:applyLocalMutations:req`: + case `rpc:applyCommittedTx:req`: + case `rpc:pullSince:req`: + return true + default: + return false + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +function appliedEnvelopeKey(collectionId: string, envelopeId: string): string { + return JSON.stringify([collectionId, envelopeId]) +} + +function remoteSubsetAcquisitionKey( + collectionId: string, + acquisitionId: string, +): string { + return JSON.stringify([collectionId, acquisitionId]) +} + +function inboundRemoteSubsetAcquisitionKey( + collectionId: string, + requesterId: string, + acquisitionId: string, +): string { + return JSON.stringify([collectionId, requesterId, acquisitionId]) +} + +function createRPCErrorResponse( + request: RPCRequest, + cause: unknown, +): RPCResponse { + const error = cause instanceof Error ? cause.message : String(cause) + switch (request.type) { + case `rpc:ensureRemoteSubset:req`: + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: false, + error, + ...(cause instanceof RetryableRemoteSubsetAcquisitionError + ? { retryable: true as const } + : {}), + } + case `rpc:releaseRemoteSubset:req`: + return { + type: `rpc:releaseRemoteSubset:res`, + rpcId: request.rpcId, + ok: false, + error, + } + case `rpc:ensurePersistedIndex:req`: + return { + type: `rpc:ensurePersistedIndex:res`, + rpcId: request.rpcId, + ok: false, + error, + } + case `rpc:applyLocalMutations:req`: + if (cause instanceof PersistedCollectionDurabilityError) { + return { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: false, + code: `PERSISTENCE_ERROR`, + error, + ...toSafeDurabilityDetails(cause), + } + } + return { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error, + } + case `rpc:applyCommittedTx:req`: + if (cause instanceof PersistedCollectionDurabilityError) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `PERSISTENCE_ERROR`, + error, + ...toSafeDurabilityDetails(cause), + } + } + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error, + } + case `rpc:pullSince:req`: + return { + type: `rpc:pullSince:res`, + rpcId: request.rpcId, + ok: false, + error, + } + } +} + +function isMutatingRPCRequest(request: RPCRequest): request is Extract< + RPCRequest, + { + type: IndeterminateCommitRequestType + } +> { + return ( + request.type === `rpc:applyLocalMutations:req` || + request.type === `rpc:applyCommittedTx:req` + ) +} + +function toSafeDurabilityDetails(error: PersistedCollectionDurabilityError): { + sourceCode?: string | number + path?: string | ReadonlyArray +} { + const sourceCode = + typeof error.code === `string` || typeof error.code === `number` + ? error.code + : undefined + const path = + typeof error.path === `string` || + (Array.isArray(error.path) && + error.path.every( + (part) => typeof part === `string` || typeof part === `number`, + )) + ? (error.path as string | ReadonlyArray) + : undefined + return { + ...(sourceCode === undefined ? {} : { sourceCode }), + ...(path === undefined ? {} : { path }), + } +} diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 934fcc7469..848d4fca37 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -20,7 +20,12 @@ import { toProcessLocalLoadSubsetOptions, toTransportedLoadSubsetOptions, } from './remote-subset-wire' +import { + reportRemoteSubsetOwnerError, + unloadRemoteSubsetOwner, +} from './remote-subset-owner' import type { TransportedLoadSubsetOptions } from './remote-subset-wire' +import type { RemoteSubsetOwner } from './remote-subset-owner' import type { StandardSchemaV1 } from '@standard-schema/spec' import type { ChangeMessageOrDeleteKeyMessage, @@ -358,40 +363,7 @@ export interface PersistenceAdapter { }> } -export type RemoteSubsetOwner = (( - options: TransportedLoadSubsetOptions, -) => Promise | void) & { - unloadSubset: (options: TransportedLoadSubsetOptions) => void - onError: (error: unknown) => void -} - -function reportRemoteSubsetOwnerError( - owner: RemoteSubsetOwner, - error: unknown, -): void { - try { - owner.onError(error) - } catch { - // Reporting must not replace the original owner failure. - } -} - -async function unloadRemoteSubsetOwner( - owner: RemoteSubsetOwner, - options: TransportedLoadSubsetOptions, -): Promise { - try { - const result = ( - owner.unloadSubset as unknown as ( - options: TransportedLoadSubsetOptions, - ) => unknown - )(options) - await Promise.resolve(result) - } catch (error) { - reportRemoteSubsetOwnerError(owner, error) - throw error - } -} +export type { RemoteSubsetOwner } from './remote-subset-owner' type SingleProcessRemoteSubsetAcquisition = { owner: RemoteSubsetOwner diff --git a/packages/db-sqlite-persistence-core/src/remote-subset-owner.ts b/packages/db-sqlite-persistence-core/src/remote-subset-owner.ts new file mode 100644 index 0000000000..5803aeff6f --- /dev/null +++ b/packages/db-sqlite-persistence-core/src/remote-subset-owner.ts @@ -0,0 +1,36 @@ +import type { TransportedLoadSubsetOptions } from './remote-subset-wire' + +export type RemoteSubsetOwner = (( + options: TransportedLoadSubsetOptions, +) => Promise | void) & { + unloadSubset: (options: TransportedLoadSubsetOptions) => void + onError: (error: unknown) => void +} + +export function reportRemoteSubsetOwnerError( + owner: RemoteSubsetOwner, + error: unknown, +): void { + try { + owner.onError(error) + } catch { + // Reporting must not replace the original owner failure. + } +} + +export async function unloadRemoteSubsetOwner( + owner: RemoteSubsetOwner, + options: TransportedLoadSubsetOptions, +): Promise { + try { + const result = ( + owner.unloadSubset as unknown as ( + options: TransportedLoadSubsetOptions, + ) => unknown + )(options) + await Promise.resolve(result) + } catch (error) { + reportRemoteSubsetOwnerError(owner, error) + throw error + } +} diff --git a/packages/db-sqlite-persistence-core/vite.config.ts b/packages/db-sqlite-persistence-core/vite.config.ts index 2928c6fdfe..52bf45a0f0 100644 --- a/packages/db-sqlite-persistence-core/vite.config.ts +++ b/packages/db-sqlite-persistence-core/vite.config.ts @@ -19,7 +19,7 @@ const config = defineConfig({ export default mergeConfig( config, tanstackViteConfig({ - entry: `./src/index.ts`, + entry: [`./src/index.ts`, `./src/broadcast-coordinator.ts`], srcDir: `./src`, }), ) diff --git a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts index fd5f3b4769..951cf0d963 100644 --- a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts +++ b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts @@ -1,2060 +1,16 @@ -import { - DuplicateRemoteSubsetOwnerError, - IndeterminateCommitError, - PersistedCollectionDurabilityError, - RetryableRemoteSubsetAcquisitionError, - safeRandomUUID, - toPersistedCollectionDurabilityError, - toProcessLocalLoadSubsetOptions, - toTransportedLoadSubsetOptions, -} from '@tanstack/db-sqlite-persistence-core' -import type { - ApplyCommittedTxRequest, - ApplyCommittedTxResponse, - ApplyLocalMutationsResponse, - EnsureRemoteSubsetRequest, - EnsureRemoteSubsetResponse, - IndeterminateCommitRequestType, - PersistedCollectionCoordinator, - PersistedIndexSpec, - PersistedMutationEnvelope, - PersistedRowMetadataMutation, - PersistedTx, - PersistenceAdapter, - ProtocolEnvelope, - PullSinceResponse, - ReleaseRemoteSubsetRequest, - ReleaseRemoteSubsetResponse, - RemoteSubsetOwner, - TransportedLoadSubsetOptions, - TxCommitted, -} from '@tanstack/db-sqlite-persistence-core' -import type { LoadSubsetOptions } from '@tanstack/db' +import { BroadcastCollectionCoordinator } from '@tanstack/db-sqlite-persistence-core/broadcast-coordinator' +import type { BroadcastCollectionCoordinatorOptions } from '@tanstack/db-sqlite-persistence-core/broadcast-coordinator' -// --------------------------------------------------------------------------- -// Constants -// --------------------------------------------------------------------------- - -const HEARTBEAT_INTERVAL_MS = 3_000 -const RPC_TIMEOUT_MS = 10_000 -const RPC_RETRY_ATTEMPTS = 2 -const RPC_RETRY_DELAY_MS = 200 -const REMOTE_SUBSET_REPLAY_RETRY_ATTEMPTS = 2 -const RPC_DEDUPE_RETENTION_MS = 60_000 -const WRITER_LOCK_BUSY_RETRY_MS = 50 -const WRITER_LOCK_MAX_RETRIES = 20 - -// --------------------------------------------------------------------------- -// Internal types -// --------------------------------------------------------------------------- - -type RPCRequest = - | EnsureRemoteSubsetRequest - | ReleaseRemoteSubsetRequest - | { - type: `rpc:ensurePersistedIndex:req` - rpcId: string - signature: string - spec: PersistedIndexSpec - } - | { - type: `rpc:applyLocalMutations:req` - rpcId: string - envelopeId: string - mutations: Array - } - | ApplyCommittedTxRequest - | { - type: `rpc:pullSince:req` - rpcId: string - fromRowVersion: number - } - -type RPCResponse = - | EnsureRemoteSubsetResponse - | ReleaseRemoteSubsetResponse - | { - type: `rpc:ensurePersistedIndex:res` - rpcId: string - ok: boolean - error?: string - } - | ApplyLocalMutationsResponse - | ApplyCommittedTxResponse - | PullSinceResponse - -type PendingRPC = { - resolve: (response: RPCResponse) => void - reject: (error: Error) => void - timer: ReturnType -} - -type CollectionState = { - isLeader: boolean - leaderId: string | null - lockAbortController: AbortController | null - heartbeatTimer: ReturnType | null - latestTerm: number - latestSeq: number - latestRowVersion: number - subscribers: Set<(message: ProtocolEnvelope) => void> -} - -type ActiveRemoteSubsetAcquisition = { - collectionId: string - requesterId: string - acquisitionId: string - owner: RemoteSubsetOwner - options: TransportedLoadSubsetOptions - load: Promise - transferred: boolean - released: boolean - terminalRelease: boolean - release: Promise | null -} - -type AwaitingRemoteSubsetOwnerAcquisition = { - collectionId: string - requesterId: string - acquisitionId: string - options: TransportedLoadSubsetOptions - released: true - awaitingOwner: true -} - -type RemoteSubsetAcquisition = - | ActiveRemoteSubsetAcquisition - | AwaitingRemoteSubsetOwnerAcquisition - | { - collectionId: string - requesterId: string - acquisitionId: string - released: true - } - -type OutboundRemoteSubsetAcquisition = { - collectionId: string - acquisitionId: string - options: TransportedLoadSubsetOptions - localOptions: TransportedLoadSubsetOptions - acquiredLeaderId: string | null - inFlight: Promise | null - forceReplay: boolean - retryTimer: ReturnType | null - retryAttempts: number -} - -// Adapter with pullSince support -type AdapterWithPullSince = PersistenceAdapter & { - pullSince?: ( - collectionId: string, - fromRowVersion: number, - ) => Promise< - | { - latestRowVersion: number - requiresFullReload: true - } - | { - latestRowVersion: number - requiresFullReload: false - changedKeys: Array - deletedKeys: Array - } - > - getStreamPosition?: (collectionId: string) => Promise<{ - latestTerm: number - latestSeq: number - latestRowVersion: number - }> -} - -// --------------------------------------------------------------------------- -// Options -// --------------------------------------------------------------------------- - -export type ElectronCollectionCoordinatorOptions = { - dbName: string - adapter?: AdapterWithPullSince -} - -// --------------------------------------------------------------------------- -// ElectronCollectionCoordinator -// --------------------------------------------------------------------------- - -export class ElectronCollectionCoordinator implements PersistedCollectionCoordinator { - private readonly nodeId = safeRandomUUID() - private readonly dbName: string - private defaultAdapter: AdapterWithPullSince | null - private readonly collectionAdapters = new Map() - private readonly remoteSubsetOwners = new Map() - private readonly remoteSubsetIds = new Map< - string, - WeakMap - >() - private readonly outboundRemoteSubsetAcquisitions = new Map< - string, - OutboundRemoteSubsetAcquisition - >() - private readonly inboundRemoteSubsetAcquisitions = new Map< - string, - RemoteSubsetAcquisition - >() - private readonly releasedRemoteSubsetAcquisitionTimes = new Map< - string, - number - >() - private readonly channel: BroadcastChannel - private readonly collections = new Map() - private readonly pendingRPCs = new Map() - private readonly appliedEnvelopeIds = new Map< - string, - { appliedAt: number; response: ApplyLocalMutationsResponse } - >() - private readonly inFlightLocalMutationEnvelopes = new Map< - string, - Promise - >() - private readonly appliedCommittedTxEnvelopes = new Map< - string, - { appliedAt: number; response: ApplyCommittedTxResponse } - >() - private readonly inFlightCommittedTxEnvelopes = new Map< - string, - Promise - >() - private disposed = false - - /** Method indirection to prevent TypeScript from narrowing `disposed` across awaits */ - private isDisposed(): boolean { - return this.disposed - } - - private requireAdapter(collectionId: string): AdapterWithPullSince { - const adapter = - this.collectionAdapters.get(collectionId) ?? this.defaultAdapter - if (!adapter) { - throw new Error( - `ElectronCollectionCoordinator: adapter not set for collection "${collectionId}". Call setAdapterForCollection() before using leader-side operations.`, - ) - } - return adapter - } +export type ElectronCollectionCoordinatorOptions = Omit< + BroadcastCollectionCoordinatorOptions, + `coordinatorName` +> +export class ElectronCollectionCoordinator extends BroadcastCollectionCoordinator { constructor(options: ElectronCollectionCoordinatorOptions) { - this.dbName = options.dbName - this.defaultAdapter = options.adapter ?? null - this.channel = new BroadcastChannel(`tsdb:coord:${this.dbName}`) - this.channel.onmessage = (event: MessageEvent) => { - this.onChannelMessage(event.data) - } - } - - /** - * Set or replace the persistence adapter used for leader-side RPC handling. - * Called by `createElectronSQLitePersistence` to wire the internally-created - * adapter into the coordinator. - */ - setAdapter(adapter: AdapterWithPullSince): void { - this.defaultAdapter = adapter - } - - setAdapterForCollection( - collectionId: string, - adapter: AdapterWithPullSince, - ): void { - this.collectionAdapters.set(collectionId, adapter) - } - - registerRemoteSubsetOwner( - collectionId: string, - owner: RemoteSubsetOwner, - ): () => void { - if (this.remoteSubsetOwners.has(collectionId)) { - throw new DuplicateRemoteSubsetOwnerError(collectionId) - } - this.remoteSubsetOwners.set(collectionId, owner) - for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { - if (acquisition.collectionId !== collectionId) continue - acquisition.acquiredLeaderId = null - acquisition.forceReplay = true - } - void this.replayRemoteSubsetAcquisitions(collectionId) - this.rebindRemoteInboundSubsetAcquisitions(collectionId, owner) - return () => { - if (this.remoteSubsetOwners.get(collectionId) !== owner) return - this.remoteSubsetOwners.delete(collectionId) - this.releaseInboundRemoteSubsetAcquisitions(collectionId, owner) - } - } - - // ----------------------------------------------------------------------- - // PersistedCollectionCoordinator interface - // ----------------------------------------------------------------------- - - getNodeId(): string { - return this.nodeId - } - - subscribe( - collectionId: string, - onMessage: (message: ProtocolEnvelope) => void, - ): () => void { - const state = this.ensureCollectionState(collectionId) - state.subscribers.add(onMessage) - return () => { - state.subscribers.delete(onMessage) - } - } - - publish(_collectionId: string, message: ProtocolEnvelope): void { - this.channel.postMessage(message) - } - - isLeader(collectionId: string): boolean { - return this.collections.get(collectionId)?.isLeader ?? false - } - - async ensureLeadership(collectionId: string): Promise { - const state = this.ensureCollectionState(collectionId) - if (state.isLeader) return - await this.acquireLeadership(collectionId, state) - } - - async requestEnsureRemoteSubset( - collectionId: string, - options: LoadSubsetOptions, - ): Promise { - const transportedOptions = toTransportedLoadSubsetOptions(options) - const localOptions = toProcessLocalLoadSubsetOptions( - options, - transportedOptions, - ) - let collectionIds = this.remoteSubsetIds.get(collectionId) - if (!collectionIds) { - collectionIds = new WeakMap() - this.remoteSubsetIds.set(collectionId, collectionIds) - } - let acquisitionId = collectionIds.get(options) - let acquisition = acquisitionId - ? this.outboundRemoteSubsetAcquisitions.get( - remoteSubsetAcquisitionKey(collectionId, acquisitionId), - ) - : undefined - if (!acquisition) { - acquisitionId = safeRandomUUID() - collectionIds.set(options, acquisitionId) - acquisition = { - collectionId, - acquisitionId, - options: transportedOptions, - localOptions, - acquiredLeaderId: null, - inFlight: null, - forceReplay: false, - retryTimer: null, - retryAttempts: 0, - } - this.outboundRemoteSubsetAcquisitions.set( - remoteSubsetAcquisitionKey(collectionId, acquisitionId), - acquisition, - ) - } - - await this.acquireRemoteSubset(acquisition) - } - - async requestReleaseRemoteSubset( - collectionId: string, - options: LoadSubsetOptions, - ): Promise { - const collectionIds = this.remoteSubsetIds.get(collectionId) - const acquisitionId = collectionIds?.get(options) - if (!acquisitionId) return - const key = remoteSubsetAcquisitionKey(collectionId, acquisitionId) - const acquisition = this.outboundRemoteSubsetAcquisitions.get(key) - if (!acquisition) return - this.outboundRemoteSubsetAcquisitions.delete(key) - this.cancelRemoteSubsetReplayRetry(acquisition) - collectionIds!.delete(options) - - const request: Extract< - RPCRequest, - { type: `rpc:releaseRemoteSubset:req` } - > = { - type: `rpc:releaseRemoteSubset:req`, - rpcId: safeRandomUUID(), - acquisitionId, - } - const response = this.isLeader(collectionId) - ? await this.handleReleaseRemoteSubset(collectionId, request, this.nodeId) - : await this.sendRPC(collectionId, request) - - if (!response.ok) { - throw new Error(`releaseRemoteSubset failed: ${response.error}`) - } - } - - private async acquireRemoteSubset( - acquisition: OutboundRemoteSubsetAcquisition, - ): Promise { - if (acquisition.inFlight) return acquisition.inFlight - - const routedToLocalOwner = this.isLeader(acquisition.collectionId) - let resolveWork!: () => void - let rejectWork!: (error: unknown) => void - const work = new Promise((resolve, reject) => { - resolveWork = resolve - rejectWork = reject + super({ + ...options, + coordinatorName: `ElectronCollectionCoordinator`, }) - acquisition.inFlight = work - const run = async (): Promise => { - const request: Extract< - RPCRequest, - { type: `rpc:ensureRemoteSubset:req` } - > = { - type: `rpc:ensureRemoteSubset:req`, - rpcId: safeRandomUUID(), - acquisitionId: acquisition.acquisitionId, - options: acquisition.options, - } - let response: EnsureRemoteSubsetResponse - try { - response = routedToLocalOwner - ? await this.handleEnsureRemoteSubset( - acquisition.collectionId, - request, - this.nodeId, - acquisition.localOptions, - ) - : await this.sendRPC( - acquisition.collectionId, - request, - ) - } catch (error) { - if ( - routedToLocalOwner || - error instanceof RetryableRemoteSubsetAcquisitionError - ) { - throw error - } - throw new RetryableRemoteSubsetAcquisitionError( - `Remote subset transport failed`, - error, - ) - } - - if (!response.ok) { - if (response.retryable) { - throw new RetryableRemoteSubsetAcquisitionError(response.error) - } - throw new Error(`ensureRemoteSubset failed: ${response.error}`) - } - acquisition.acquiredLeaderId = response.leaderId - } - void run().then(resolveWork, rejectWork) - let acquired = false - try { - await work - acquired = true - this.cancelRemoteSubsetReplayRetry(acquisition) - } finally { - if (acquisition.inFlight === work) acquisition.inFlight = null - const current = this.collections.get(acquisition.collectionId) - const currentLeaderId = current?.isLeader - ? this.nodeId - : (current?.leaderId ?? null) - const key = remoteSubsetAcquisitionKey( - acquisition.collectionId, - acquisition.acquisitionId, - ) - if ( - acquired && - this.outboundRemoteSubsetAcquisitions.get(key) === acquisition && - (acquisition.forceReplay || - (currentLeaderId !== null && - acquisition.acquiredLeaderId !== currentLeaderId)) - ) { - acquisition.forceReplay = false - void this.acquireRemoteSubset(acquisition).catch((error) => { - this.scheduleRemoteSubsetReplayRetry(acquisition, error) - }) - } - } - } - - async requestEnsurePersistedIndex( - collectionId: string, - signature: string, - spec: PersistedIndexSpec, - ): Promise { - if (this.isLeader(collectionId)) { - await this.requireAdapter(collectionId).ensureIndex( - collectionId, - signature, - spec, - ) - return - } - - const response = await this.sendRPC<{ - type: `rpc:ensurePersistedIndex:res` - rpcId: string - ok: boolean - error?: string - }>(collectionId, { - type: `rpc:ensurePersistedIndex:req`, - rpcId: safeRandomUUID(), - signature, - spec, - }) - - if (!response.ok) { - throw new Error( - `ensurePersistedIndex failed: ${response.error ?? `unknown error`}`, - ) - } - } - - async requestApplyLocalMutations( - collectionId: string, - mutations: Array, - ): Promise { - if (this.isLeader(collectionId)) { - return this.handleApplyLocalMutations(collectionId, { - type: `rpc:applyLocalMutations:req`, - rpcId: safeRandomUUID(), - envelopeId: safeRandomUUID(), - mutations, - }) - } - - return this.sendRPC(collectionId, { - type: `rpc:applyLocalMutations:req`, - rpcId: safeRandomUUID(), - envelopeId: safeRandomUUID(), - mutations, - }) - } - - async requestApplyCommittedTx( - collectionId: string, - tx: PersistedTx, - ): Promise { - const request: ApplyCommittedTxRequest = { - type: `rpc:applyCommittedTx:req`, - rpcId: safeRandomUUID(), - envelopeId: safeRandomUUID(), - tx, - } - if (this.isLeader(collectionId)) { - return this.handleApplyCommittedTx(collectionId, request) - } - - return this.sendRPC(collectionId, request) - } - - async pullSince( - collectionId: string, - fromRowVersion: number, - ): Promise { - if (this.isLeader(collectionId)) { - return this.handlePullSince(collectionId, { - type: `rpc:pullSince:req`, - rpcId: safeRandomUUID(), - fromRowVersion, - }) - } - - return this.sendRPC(collectionId, { - type: `rpc:pullSince:req`, - rpcId: safeRandomUUID(), - fromRowVersion, - }) - } - - // ----------------------------------------------------------------------- - // Lifecycle - // ----------------------------------------------------------------------- - - dispose(): void { - this.disposed = true - - for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { - this.cancelRemoteSubsetReplayRetry(acquisition) - this.postRemoteSubsetRelease(acquisition) - } - this.outboundRemoteSubsetAcquisitions.clear() - this.remoteSubsetIds.clear() - - for (const [collectionId, state] of this.collections) { - this.releaseLeadership(collectionId, state) - } - - for (const [, pending] of this.pendingRPCs) { - clearTimeout(pending.timer) - pending.reject(new Error(`coordinator disposed`)) - } - this.pendingRPCs.clear() - - this.channel.close() - this.collections.clear() - this.collectionAdapters.clear() - for (const collectionId of this.remoteSubsetOwners.keys()) { - this.releaseInboundRemoteSubsetAcquisitions(collectionId) - } - this.remoteSubsetOwners.clear() - this.inboundRemoteSubsetAcquisitions.clear() - this.releasedRemoteSubsetAcquisitionTimes.clear() - this.appliedEnvelopeIds.clear() - this.inFlightLocalMutationEnvelopes.clear() - this.appliedCommittedTxEnvelopes.clear() - this.inFlightCommittedTxEnvelopes.clear() - } - - // ----------------------------------------------------------------------- - // Leadership via Web Locks - // ----------------------------------------------------------------------- - - private ensureCollectionState(collectionId: string): CollectionState { - let state = this.collections.get(collectionId) - if (!state) { - state = { - isLeader: false, - leaderId: null, - lockAbortController: null, - heartbeatTimer: null, - latestTerm: 0, - latestSeq: 0, - latestRowVersion: 0, - subscribers: new Set(), - } - this.collections.set(collectionId, state) - void this.acquireLeadership(collectionId, state) - } - return state - } - - private async acquireLeadership( - collectionId: string, - state: CollectionState, - ): Promise { - if (this.disposed || state.isLeader) return - - const lockName = `tsdb:leader:${this.dbName}:${collectionId}` - const abortController = new AbortController() - state.lockAbortController = abortController - - try { - await navigator.locks.request( - lockName, - { signal: abortController.signal }, - async () => { - if (this.isDisposed()) return - - try { - // Restore stream position from DB before claiming leadership - const adapter = this.requireAdapter(collectionId) - if (adapter.getStreamPosition) { - const pos = await adapter.getStreamPosition(collectionId) - state.latestTerm = pos.latestTerm - state.latestSeq = pos.latestSeq - state.latestRowVersion = pos.latestRowVersion - } - - state.latestTerm++ - state.isLeader = true - state.leaderId = this.nodeId - - this.emitHeartbeat(collectionId, state) - void this.replayRemoteSubsetAcquisitions(collectionId) - state.heartbeatTimer = setInterval(() => { - this.emitHeartbeat(collectionId, state) - }, HEARTBEAT_INTERVAL_MS) - - // Hold the lock until disposed or aborted - await new Promise((resolve) => { - const onAbort = () => { - abortController.signal.removeEventListener(`abort`, onAbort) - resolve() - } - if (abortController.signal.aborted) { - resolve() - return - } - abortController.signal.addEventListener(`abort`, onAbort) - }) - } finally { - this.releaseInboundRemoteSubsetAcquisitions(collectionId) - state.isLeader = false - state.leaderId = null - if (state.heartbeatTimer) { - clearInterval(state.heartbeatTimer) - state.heartbeatTimer = null - } - } - }, - ) - } catch (error) { - if (error instanceof DOMException && error.name === `AbortError`) { - return - } - console.warn(`Failed to acquire leadership for ${collectionId}:`, error) - } - - // Re-acquire if not disposed (leadership was released by another means) - if (!this.isDisposed()) { - void this.acquireLeadership(collectionId, state) - } - } - - private releaseLeadership( - collectionId: string, - state: CollectionState, - ): void { - this.releaseInboundRemoteSubsetAcquisitions(collectionId) - if (state.lockAbortController) { - state.lockAbortController.abort() - state.lockAbortController = null - } - if (state.heartbeatTimer) { - clearInterval(state.heartbeatTimer) - state.heartbeatTimer = null - } - state.isLeader = false - state.leaderId = null - } - - private postRemoteSubsetRelease( - acquisition: OutboundRemoteSubsetAcquisition, - ): void { - const request: Extract< - RPCRequest, - { type: `rpc:releaseRemoteSubset:req` } - > = { - type: `rpc:releaseRemoteSubset:req`, - rpcId: safeRandomUUID(), - acquisitionId: acquisition.acquisitionId, - } - if (this.isLeader(acquisition.collectionId)) { - void this.handleReleaseRemoteSubset( - acquisition.collectionId, - request, - this.nodeId, - ).catch(() => { - // The owner already received the exact unload failure through onError. - }) - return - } - this.channel.postMessage({ - v: 1, - dbName: this.dbName, - collectionId: acquisition.collectionId, - senderId: this.nodeId, - ts: Date.now(), - payload: request, - } satisfies ProtocolEnvelope) - } - - private cancelRemoteSubsetReplayRetry( - acquisition: OutboundRemoteSubsetAcquisition, - ): void { - if (acquisition.retryTimer !== null) { - clearTimeout(acquisition.retryTimer) - acquisition.retryTimer = null - } - acquisition.retryAttempts = 0 - } - - private scheduleRemoteSubsetReplayRetry( - acquisition: OutboundRemoteSubsetAcquisition, - error: unknown, - ): void { - const key = remoteSubsetAcquisitionKey( - acquisition.collectionId, - acquisition.acquisitionId, - ) - if ( - !(error instanceof RetryableRemoteSubsetAcquisitionError) || - this.isDisposed() || - this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition || - acquisition.retryTimer !== null || - acquisition.retryAttempts >= REMOTE_SUBSET_REPLAY_RETRY_ATTEMPTS - ) { - return - } - - acquisition.retryAttempts++ - acquisition.retryTimer = setTimeout(() => { - acquisition.retryTimer = null - if ( - this.isDisposed() || - this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition - ) { - return - } - void this.acquireRemoteSubset(acquisition).catch((retryError) => { - this.scheduleRemoteSubsetReplayRetry(acquisition, retryError) - }) - }, RPC_RETRY_DELAY_MS) - } - - private async replayRemoteSubsetAcquisitions( - collectionId: string, - ): Promise { - if (this.isDisposed()) return - const state = this.collections.get(collectionId) - const leaderId = state?.isLeader ? this.nodeId : state?.leaderId - if (!leaderId) return - - const replays: Array> = [] - for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { - if ( - acquisition.collectionId !== collectionId || - (!acquisition.forceReplay && acquisition.acquiredLeaderId === leaderId) - ) { - continue - } - if (acquisition.inFlight) { - acquisition.forceReplay = true - continue - } - acquisition.forceReplay = false - replays.push( - this.acquireRemoteSubset(acquisition).catch((error) => { - this.scheduleRemoteSubsetReplayRetry(acquisition, error) - }), - ) - } - await Promise.all(replays) - } - - private emitHeartbeat(collectionId: string, state: CollectionState): void { - const envelope: ProtocolEnvelope = { - v: 1, - dbName: this.dbName, - collectionId, - senderId: this.nodeId, - ts: Date.now(), - payload: { - type: `leader:heartbeat`, - term: state.latestTerm, - leaderId: this.nodeId, - latestSeq: state.latestSeq, - latestRowVersion: state.latestRowVersion, - }, - } - this.channel.postMessage(envelope) - } - - // ----------------------------------------------------------------------- - // BroadcastChannel message handling - // ----------------------------------------------------------------------- - - private onChannelMessage(data: unknown): void { - if (!isProtocolEnvelope(data)) return - - const envelope = data - - // Ignore own messages - if (envelope.senderId === this.nodeId) return - - const payload = envelope.payload - if (!payload || typeof payload !== `object`) return - - const type = (payload as Record).type as string | undefined - - if (type === `leader:heartbeat`) { - const heartbeat = payload as { - leaderId?: unknown - term?: unknown - latestSeq?: unknown - latestRowVersion?: unknown - } - if ( - typeof heartbeat.leaderId === `string` && - typeof heartbeat.term === `number` && - typeof heartbeat.latestSeq === `number` && - typeof heartbeat.latestRowVersion === `number` - ) { - const state = this.ensureCollectionState(envelope.collectionId) - if (heartbeat.term < state.latestTerm) return - const changedLeader = state.leaderId !== heartbeat.leaderId - state.leaderId = heartbeat.leaderId - state.latestTerm = Math.max(state.latestTerm, heartbeat.term) - state.latestSeq = Math.max(state.latestSeq, heartbeat.latestSeq) - state.latestRowVersion = Math.max( - state.latestRowVersion, - heartbeat.latestRowVersion, - ) - if (changedLeader) { - void this.replayRemoteSubsetAcquisitions(envelope.collectionId) - } - } - } - - // Handle RPC responses (for pending outbound RPCs) - if (type && type.endsWith(`:res`)) { - const rpcId = (payload as { rpcId?: string }).rpcId - if (rpcId && this.pendingRPCs.has(rpcId)) { - const pending = this.pendingRPCs.get(rpcId)! - this.pendingRPCs.delete(rpcId) - clearTimeout(pending.timer) - pending.resolve(payload as RPCResponse) - return - } - } - - // Handle RPC requests (leader only) - if (type && type.endsWith(`:req`)) { - if (!isRPCRequest(payload)) return - const collectionId = envelope.collectionId - if (this.isLeader(collectionId)) { - void this.handleRPCRequest(collectionId, payload, envelope.senderId) - } - return - } - - // Forward protocol messages to subscribers - const state = this.collections.get(envelope.collectionId) - if (state) { - for (const subscriber of state.subscribers) { - subscriber(envelope) - } - } - } - - // ----------------------------------------------------------------------- - // RPC - Outbound (follower side) - // ----------------------------------------------------------------------- - - private async sendRPC( - collectionId: string, - request: RPCRequest, - ): Promise { - let lastError: Error | undefined - let firstTransportCause: unknown - const mutationRequestType = isMutatingRPCRequest(request) - ? request.type - : undefined - const mutationRoute = mutationRequestType - ? this.captureMutationRoute(collectionId) - : undefined - - for (let attempt = 0; attempt <= RPC_RETRY_ATTEMPTS; attempt++) { - if (attempt > 0) { - await sleep(RPC_RETRY_DELAY_MS * attempt) - } - - if ( - mutationRoute && - mutationRequestType && - firstTransportCause !== undefined - ) { - this.assertMutationRouteUnchanged( - collectionId, - mutationRequestType, - mutationRoute, - firstTransportCause, - ) - } - - if (this.isLeader(collectionId)) { - return (await this.dispatchRPCRequest( - collectionId, - request, - this.nodeId, - )) as T - } - - try { - return await this.sendRPCOnce(collectionId, request) - } catch (error) { - if (this.isDisposed()) throw error - firstTransportCause ??= error - if (mutationRoute && mutationRequestType) { - this.assertMutationRouteUnchanged( - collectionId, - mutationRequestType, - mutationRoute, - firstTransportCause, - ) - } - lastError = error instanceof Error ? error : new Error(String(error)) - } - } - - throw lastError ?? new Error(`RPC failed after retries`) - } - - private captureMutationRoute(collectionId: string): { - leaderId: string | null - term: number | null - } { - const state = this.collections.get(collectionId) - return { - leaderId: state?.isLeader ? this.nodeId : (state?.leaderId ?? null), - term: state?.latestTerm ?? null, - } - } - - private assertMutationRouteUnchanged( - collectionId: string, - requestType: IndeterminateCommitRequestType, - previous: { leaderId: string | null; term: number | null }, - cause: unknown, - ): void { - const current = this.captureMutationRoute(collectionId) - if ( - previous.leaderId !== null && - previous.term !== null && - current.leaderId === previous.leaderId && - current.term === previous.term - ) { - return - } - throw new IndeterminateCommitError({ - collectionId, - requestType, - previousLeaderId: previous.leaderId, - previousTerm: previous.term, - currentLeaderId: current.leaderId, - currentTerm: current.term, - cause, - }) - } - - private sendRPCOnce( - collectionId: string, - request: RPCRequest, - ): Promise { - return new Promise((resolve, reject) => { - const rpcId = request.rpcId - - const timer = setTimeout(() => { - this.pendingRPCs.delete(rpcId) - reject( - new Error(`RPC ${request.type} timed out after ${RPC_TIMEOUT_MS}ms`), - ) - }, RPC_TIMEOUT_MS) - - this.pendingRPCs.set(rpcId, { - resolve: resolve as (response: RPCResponse) => void, - reject, - timer, - }) - - const envelope: ProtocolEnvelope = { - v: 1, - dbName: this.dbName, - collectionId, - senderId: this.nodeId, - ts: Date.now(), - payload: request, - } - this.channel.postMessage(envelope) - }) - } - - // ----------------------------------------------------------------------- - // RPC - Inbound (leader side) - // ----------------------------------------------------------------------- - - private async handleRPCRequest( - collectionId: string, - request: RPCRequest, - requesterId: string, - ): Promise { - let response: RPCResponse - - try { - response = await this.dispatchRPCRequest( - collectionId, - request, - requesterId, - ) - } catch (error) { - response = createRPCErrorResponse(request, error) - } - - if (this.isDisposed()) { - return - } - - const envelope: ProtocolEnvelope = { - v: 1, - dbName: this.dbName, - collectionId, - senderId: this.nodeId, - ts: Date.now(), - payload: response, - } - this.channel.postMessage(envelope) - } - - private dispatchRPCRequest( - collectionId: string, - request: RPCRequest, - requesterId: string, - ): Promise { - switch (request.type) { - case `rpc:ensureRemoteSubset:req`: - return this.handleEnsureRemoteSubset(collectionId, request, requesterId) - case `rpc:releaseRemoteSubset:req`: - return this.handleReleaseRemoteSubset( - collectionId, - request, - requesterId, - ) - case `rpc:ensurePersistedIndex:req`: - return this.handleEnsurePersistedIndex(collectionId, request) - case `rpc:applyLocalMutations:req`: - return this.handleApplyLocalMutations(collectionId, request) - case `rpc:applyCommittedTx:req`: - return this.handleApplyCommittedTx(collectionId, request) - case `rpc:pullSince:req`: - return this.handlePullSince(collectionId, request) - } - } - - private async handleEnsureRemoteSubset( - collectionId: string, - request: Extract, - requesterId: string, - localOptions?: TransportedLoadSubsetOptions, - ): Promise { - this.pruneReleasedRemoteSubsetAcquisitions() - const key = inboundRemoteSubsetAcquisitionKey( - collectionId, - requesterId, - request.acquisitionId, - ) - const existing = this.inboundRemoteSubsetAcquisitions.get(key) - const awaitingOwner = - existing && `awaitingOwner` in existing ? existing : undefined - if (existing) { - if (`owner` in existing) { - if (!existing.released) { - await existing.load - return { - type: `rpc:ensureRemoteSubset:res`, - rpcId: request.rpcId, - ok: true, - leaderId: this.nodeId, - } - } - await existing.release - if (existing.terminalRelease) { - return { - type: `rpc:ensureRemoteSubset:res`, - rpcId: request.rpcId, - ok: true, - leaderId: this.nodeId, - } - } - if (this.inboundRemoteSubsetAcquisitions.get(key) === existing) { - this.inboundRemoteSubsetAcquisitions.delete(key) - } - } else if (!(`awaitingOwner` in existing)) { - return { - type: `rpc:ensureRemoteSubset:res`, - rpcId: request.rpcId, - ok: true, - leaderId: this.nodeId, - } - } - } - - const owner = this.remoteSubsetOwners.get(collectionId) - if (!owner) { - throw new RetryableRemoteSubsetAcquisitionError( - `ElectronCollectionCoordinator: no remote subset owner registered for collection "${collectionId}"`, - ) - } - - const acquisition: ActiveRemoteSubsetAcquisition = { - collectionId, - requesterId, - acquisitionId: request.acquisitionId, - owner, - options: awaitingOwner?.options ?? localOptions ?? request.options, - load: Promise.resolve(), - transferred: false, - released: false, - terminalRelease: false, - release: null, - } - this.releasedRemoteSubsetAcquisitionTimes.delete(key) - this.inboundRemoteSubsetAcquisitions.set(key, acquisition) - let resolveLoad!: () => void - let rejectLoad!: (error: unknown) => void - acquisition.load = new Promise((resolve, reject) => { - resolveLoad = resolve - rejectLoad = reject - }) - try { - const load = owner(acquisition.options) - acquisition.transferred = true - void Promise.resolve(load).then(resolveLoad, rejectLoad) - } catch (error) { - rejectLoad(error) - } - try { - await acquisition.load - } catch (error) { - if ( - !acquisition.transferred && - this.inboundRemoteSubsetAcquisitions.get(key) === acquisition - ) { - if (awaitingOwner) { - this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) - } else { - this.inboundRemoteSubsetAcquisitions.delete(key) - } - } - reportRemoteSubsetOwnerError(owner, error) - throw error - } - return { - type: `rpc:ensureRemoteSubset:res`, - rpcId: request.rpcId, - ok: true, - leaderId: this.nodeId, - } - } - - private async handleReleaseRemoteSubset( - collectionId: string, - request: Extract, - requesterId: string, - ): Promise { - this.pruneReleasedRemoteSubsetAcquisitions() - const key = inboundRemoteSubsetAcquisitionKey( - collectionId, - requesterId, - request.acquisitionId, - ) - const acquisition = this.inboundRemoteSubsetAcquisitions.get(key) - if (!acquisition) { - this.setReleasedRemoteSubsetAcquisition(key, { - collectionId, - requesterId, - acquisitionId: request.acquisitionId, - released: true, - }) - } else if (`owner` in acquisition) { - acquisition.terminalRelease = true - await this.releaseRemoteSubsetAcquisition(acquisition) - if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { - this.setReleasedRemoteSubsetAcquisition(key, { - collectionId, - requesterId, - acquisitionId: request.acquisitionId, - released: true, - }) - } - } else if (`awaitingOwner` in acquisition) { - this.setReleasedRemoteSubsetAcquisition(key, { - collectionId, - requesterId, - acquisitionId: request.acquisitionId, - released: true, - }) - } - return { - type: `rpc:releaseRemoteSubset:res`, - rpcId: request.rpcId, - ok: true, - } - } - - private releaseRemoteSubsetAcquisition( - acquisition: ActiveRemoteSubsetAcquisition, - ): Promise { - if (acquisition.release) return acquisition.release - acquisition.released = true - acquisition.release = (async () => { - try { - await acquisition.load - } catch { - // A returned promise transfers the lease even when initial loading fails. - } - try { - if (acquisition.transferred) { - await unloadRemoteSubsetOwner(acquisition.owner, acquisition.options) - } - } finally { - if (!acquisition.terminalRelease) { - const key = inboundRemoteSubsetAcquisitionKey( - acquisition.collectionId, - acquisition.requesterId, - acquisition.acquisitionId, - ) - if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { - this.inboundRemoteSubsetAcquisitions.set(key, { - collectionId: acquisition.collectionId, - requesterId: acquisition.requesterId, - acquisitionId: acquisition.acquisitionId, - options: acquisition.options, - released: true, - awaitingOwner: true, - }) - } - } - } - })() - return acquisition.release - } - - private releaseInboundRemoteSubsetAcquisitions( - collectionId: string, - owner?: RemoteSubsetOwner, - ): void { - for (const acquisition of this.inboundRemoteSubsetAcquisitions.values()) { - if ( - !(`owner` in acquisition) || - acquisition.collectionId !== collectionId || - (owner && acquisition.owner !== owner) - ) { - continue - } - void this.releaseRemoteSubsetAcquisition(acquisition).catch( - () => undefined, - ) - } - } - - private rebindRemoteInboundSubsetAcquisitions( - collectionId: string, - owner: RemoteSubsetOwner, - ): void { - for (const acquisition of this.inboundRemoteSubsetAcquisitions.values()) { - if (`awaitingOwner` in acquisition) { - if ( - acquisition.collectionId === collectionId && - acquisition.requesterId !== this.nodeId - ) { - void this.bindAwaitingRemoteSubsetAcquisition( - acquisition, - owner, - ).catch(() => { - // The owner receives the exact load failure through onError. - }) - } - continue - } - if ( - !(`owner` in acquisition) || - acquisition.collectionId !== collectionId || - acquisition.requesterId === this.nodeId || - !acquisition.released || - acquisition.terminalRelease - ) { - continue - } - - void this.rebindRemoteInboundSubsetAcquisition(acquisition, owner).catch( - () => undefined, - ) - } - } - - private async rebindRemoteInboundSubsetAcquisition( - previous: ActiveRemoteSubsetAcquisition, - owner: RemoteSubsetOwner, - ): Promise { - await previous.release - if ( - previous.terminalRelease || - this.remoteSubsetOwners.get(previous.collectionId) !== owner - ) { - return - } - - const key = inboundRemoteSubsetAcquisitionKey( - previous.collectionId, - previous.requesterId, - previous.acquisitionId, - ) - const current = this.inboundRemoteSubsetAcquisitions.get(key) - if (current && current !== previous) { - if (`awaitingOwner` in current) { - await this.bindAwaitingRemoteSubsetAcquisition(current, owner) - } - return - } - - const awaitingOwner: AwaitingRemoteSubsetOwnerAcquisition = { - collectionId: previous.collectionId, - requesterId: previous.requesterId, - acquisitionId: previous.acquisitionId, - options: previous.options, - released: true, - awaitingOwner: true, - } - this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) - await this.bindAwaitingRemoteSubsetAcquisition(awaitingOwner, owner) - } - - private async bindAwaitingRemoteSubsetAcquisition( - awaitingOwner: AwaitingRemoteSubsetOwnerAcquisition, - owner: RemoteSubsetOwner, - ): Promise { - if (this.remoteSubsetOwners.get(awaitingOwner.collectionId) !== owner) { - return - } - const key = inboundRemoteSubsetAcquisitionKey( - awaitingOwner.collectionId, - awaitingOwner.requesterId, - awaitingOwner.acquisitionId, - ) - if (this.inboundRemoteSubsetAcquisitions.get(key) !== awaitingOwner) return - - const acquisition: ActiveRemoteSubsetAcquisition = { - collectionId: awaitingOwner.collectionId, - requesterId: awaitingOwner.requesterId, - acquisitionId: awaitingOwner.acquisitionId, - owner, - options: awaitingOwner.options, - load: Promise.resolve(), - transferred: false, - released: false, - terminalRelease: false, - release: null, - } - this.inboundRemoteSubsetAcquisitions.set(key, acquisition) - let resolveLoad!: () => void - let rejectLoad!: (error: unknown) => void - acquisition.load = new Promise((resolve, reject) => { - resolveLoad = resolve - rejectLoad = reject - }) - try { - const load = owner(acquisition.options) - acquisition.transferred = true - void Promise.resolve(load).then(resolveLoad, rejectLoad) - } catch (error) { - rejectLoad(error) - } - try { - await acquisition.load - } catch (error) { - if ( - !acquisition.transferred && - this.inboundRemoteSubsetAcquisitions.get(key) === acquisition - ) { - this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) - } - reportRemoteSubsetOwnerError(owner, error) - throw error - } - } - - private async handleEnsurePersistedIndex( - collectionId: string, - request: { - type: `rpc:ensurePersistedIndex:req` - rpcId: string - signature: string - spec: PersistedIndexSpec - }, - ): Promise { - await this.withWriterLock(() => - this.requireAdapter(collectionId).ensureIndex( - collectionId, - request.signature, - request.spec, - ), - ) - return { - type: `rpc:ensurePersistedIndex:res`, - rpcId: request.rpcId, - ok: true, - } - } - - private async handleApplyLocalMutations( - collectionId: string, - request: { - type: `rpc:applyLocalMutations:req` - rpcId: string - envelopeId: string - mutations: Array - }, - ): Promise { - const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) - const appliedEnvelope = this.appliedEnvelopeIds.get(envelopeKey) - if (appliedEnvelope) { - return { ...appliedEnvelope.response, rpcId: request.rpcId } - } - - const inFlightEnvelope = - this.inFlightLocalMutationEnvelopes.get(envelopeKey) - if (inFlightEnvelope) { - const response = await inFlightEnvelope - return { ...response, rpcId: request.rpcId } - } - - const response = this.applyLocalMutationsOnce(collectionId, request) - this.inFlightLocalMutationEnvelopes.set(envelopeKey, response) - try { - return await response - } finally { - if (this.inFlightLocalMutationEnvelopes.get(envelopeKey) === response) { - this.inFlightLocalMutationEnvelopes.delete(envelopeKey) - } - } - } - - private async applyLocalMutationsOnce( - collectionId: string, - request: { - type: `rpc:applyLocalMutations:req` - rpcId: string - envelopeId: string - mutations: Array - }, - ): Promise { - const state = this.collections.get(collectionId) - if (!state || !state.isLeader) { - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: false, - code: `NOT_LEADER`, - error: `not the leader for ${collectionId}`, - } - } - - // Assign stream position - state.latestSeq++ - state.latestRowVersion++ - - const term = state.latestTerm - const seq = state.latestSeq - const rowVersion = state.latestRowVersion - - // Build and apply the persisted transaction - const rowMetadataMutations: Array = [] - for (const mutation of request.mutations) { - if (!(`metadataChanged` in mutation) || !mutation.metadataChanged) { - continue - } - rowMetadataMutations.push( - mutation.metadata === undefined - ? { type: `delete`, key: mutation.key } - : { - type: `set`, - key: mutation.key, - value: mutation.metadata, - }, - ) - } - const tx = { - txId: safeRandomUUID(), - term, - seq, - rowVersion, - mutations: request.mutations.map((m) => ({ - type: m.type, - key: m.key, - value: m.value, - ...(`metadataChanged` in m - ? { metadata: m.metadata, metadataChanged: m.metadataChanged } - : {}), - })), - rowMetadataMutations, - } - - try { - await this.withWriterLock(() => - this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), - ) - } catch (error) { - throw toPersistedCollectionDurabilityError(collectionId, error) - } - - const response: ApplyLocalMutationsResponse = { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: true, - term, - seq, - latestRowVersion: rowVersion, - acceptedMutationIds: request.mutations.map((m) => m.mutationId), - } - if (this.isDisposed()) { - return response - } - - this.appliedEnvelopeIds.set( - appliedEnvelopeKey(collectionId, request.envelopeId), - { appliedAt: Date.now(), response }, - ) - this.pruneAppliedEnvelopeIds() - - // Broadcast tx:committed to all tabs - const changedRows = request.mutations - .filter((m) => m.type !== `delete`) - .map((m) => ({ key: m.key, value: m.value })) - const deletedKeys = request.mutations - .filter((m) => m.type === `delete`) - .map((m) => m.key) - - const txCommitted: ProtocolEnvelope = { - v: 1, - dbName: this.dbName, - collectionId, - senderId: this.nodeId, - ts: Date.now(), - payload: { - type: `tx:committed`, - term, - seq, - txId: tx.txId, - latestRowVersion: rowVersion, - requiresFullReload: false, - changedRows, - deletedKeys, - rowMetadataMutations, - }, - } - this.channel.postMessage(txCommitted) - - // Deliver to local subscribers too - for (const subscriber of state.subscribers) { - subscriber(txCommitted) - } - - return response - } - - private async handleApplyCommittedTx( - collectionId: string, - request: ApplyCommittedTxRequest, - ): Promise { - const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) - const appliedEnvelope = this.appliedCommittedTxEnvelopes.get(envelopeKey) - if (appliedEnvelope) { - return { ...appliedEnvelope.response, rpcId: request.rpcId } - } - - const inFlightEnvelope = this.inFlightCommittedTxEnvelopes.get(envelopeKey) - if (inFlightEnvelope) { - const response = await inFlightEnvelope - return { ...response, rpcId: request.rpcId } - } - - const response = this.applyCommittedTxOnce(collectionId, request) - this.inFlightCommittedTxEnvelopes.set(envelopeKey, response) - try { - return await response - } finally { - if (this.inFlightCommittedTxEnvelopes.get(envelopeKey) === response) { - this.inFlightCommittedTxEnvelopes.delete(envelopeKey) - } - } - } - - private async applyCommittedTxOnce( - collectionId: string, - request: ApplyCommittedTxRequest, - ): Promise { - const state = this.collections.get(collectionId) - if (!state || !state.isLeader) { - return { - type: `rpc:applyCommittedTx:res`, - rpcId: request.rpcId, - ok: false, - code: `NOT_LEADER`, - error: `not the leader for ${collectionId}`, - } - } - - state.latestSeq++ - state.latestRowVersion++ - const tx: PersistedTx = { - ...request.tx, - term: state.latestTerm, - seq: state.latestSeq, - rowVersion: state.latestRowVersion, - } - - try { - await this.withWriterLock(() => - this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), - ) - } catch (error) { - throw toPersistedCollectionDurabilityError(collectionId, error) - } - - const response: ApplyCommittedTxResponse = { - type: `rpc:applyCommittedTx:res`, - rpcId: request.rpcId, - ok: true, - term: tx.term, - seq: tx.seq, - latestRowVersion: tx.rowVersion, - } - if (this.isDisposed()) { - return response - } - this.appliedCommittedTxEnvelopes.set( - appliedEnvelopeKey(collectionId, request.envelopeId), - { appliedAt: Date.now(), response }, - ) - this.pruneAppliedEnvelopeIds() - - const committedBase = { - type: `tx:committed` as const, - term: tx.term, - seq: tx.seq, - txId: tx.txId, - latestRowVersion: tx.rowVersion, - } - const committedPayload: TxCommitted = tx.truncate - ? { ...committedBase, requiresFullReload: true } - : { - ...committedBase, - requiresFullReload: false, - changedRows: tx.mutations - .filter((mutation) => mutation.type !== `delete`) - .map((mutation) => ({ - key: mutation.key, - value: mutation.value, - })), - deletedKeys: tx.mutations - .filter((mutation) => mutation.type === `delete`) - .map((mutation) => mutation.key), - rowMetadataMutations: tx.rowMetadataMutations, - collectionMetadataMutations: tx.collectionMetadataMutations, - } - const committed: ProtocolEnvelope = { - v: 1, - dbName: this.dbName, - collectionId, - senderId: this.nodeId, - ts: Date.now(), - payload: committedPayload, - } - this.channel.postMessage(committed) - for (const subscriber of state.subscribers) { - subscriber(committed) - } - - return response - } - - private async handlePullSince( - collectionId: string, - request: { - type: `rpc:pullSince:req` - rpcId: string - fromRowVersion: number - }, - ): Promise { - const state = this.collections.get(collectionId) - - const adapter = this.requireAdapter(collectionId) - if (!adapter.pullSince) { - return { - type: `rpc:pullSince:res`, - rpcId: request.rpcId, - ok: true, - latestTerm: state?.latestTerm ?? 0, - latestSeq: state?.latestSeq ?? 0, - latestRowVersion: state?.latestRowVersion ?? 0, - requiresFullReload: true, - } - } - - const result = await adapter.pullSince(collectionId, request.fromRowVersion) - - if (result.requiresFullReload) { - return { - type: `rpc:pullSince:res`, - rpcId: request.rpcId, - ok: true, - latestTerm: state?.latestTerm ?? 0, - latestSeq: state?.latestSeq ?? 0, - latestRowVersion: result.latestRowVersion, - requiresFullReload: true, - } - } - - return { - type: `rpc:pullSince:res`, - rpcId: request.rpcId, - ok: true, - latestTerm: state?.latestTerm ?? 0, - latestSeq: state?.latestSeq ?? 0, - latestRowVersion: result.latestRowVersion, - requiresFullReload: false, - changedKeys: result.changedKeys, - deletedKeys: result.deletedKeys, - } - } - - // ----------------------------------------------------------------------- - // DB Writer Lock - // ----------------------------------------------------------------------- - - private async withWriterLock(fn: () => Promise): Promise { - const lockName = `tsdb:writer:${this.dbName}` - - for (let attempt = 0; attempt <= WRITER_LOCK_MAX_RETRIES; attempt++) { - const callbackState = { entered: false } - try { - return await navigator.locks.request(lockName, async () => { - callbackState.entered = true - return fn() - }) - } catch (error) { - if (callbackState.entered) { - throw error - } - if (error instanceof DOMException && error.name === `AbortError`) { - throw error - } - - if (attempt < WRITER_LOCK_MAX_RETRIES) { - await sleep(WRITER_LOCK_BUSY_RETRY_MS * Math.min(attempt + 1, 5)) - continue - } - - throw error - } - } - - // Unreachable but satisfies TypeScript - throw new Error(`writer lock acquisition failed`) - } - - // ----------------------------------------------------------------------- - // Helpers - // ----------------------------------------------------------------------- - - private pruneAppliedEnvelopeIds(): void { - // Keep envelopes for 60 seconds for dedup - const cutoff = Date.now() - RPC_DEDUPE_RETENTION_MS - for (const [id, envelope] of this.appliedEnvelopeIds) { - if (envelope.appliedAt < cutoff) { - this.appliedEnvelopeIds.delete(id) - } - } - for (const [key, envelope] of this.appliedCommittedTxEnvelopes) { - if (envelope.appliedAt < cutoff) { - this.appliedCommittedTxEnvelopes.delete(key) - } - } - } - - private setReleasedRemoteSubsetAcquisition( - key: string, - acquisition: RemoteSubsetAcquisition, - ): void { - this.inboundRemoteSubsetAcquisitions.set(key, acquisition) - this.releasedRemoteSubsetAcquisitionTimes.set(key, Date.now()) - } - - private pruneReleasedRemoteSubsetAcquisitions(): void { - const cutoff = Date.now() - RPC_DEDUPE_RETENTION_MS - for (const [key, releasedAt] of this.releasedRemoteSubsetAcquisitionTimes) { - if (releasedAt < cutoff) { - this.releasedRemoteSubsetAcquisitionTimes.delete(key) - this.inboundRemoteSubsetAcquisitions.delete(key) - } - } - } -} - -// --------------------------------------------------------------------------- -// Utilities -// --------------------------------------------------------------------------- - -function isProtocolEnvelope(data: unknown): data is ProtocolEnvelope { - if (!data || typeof data !== `object`) return false - const record = data as Record - return ( - record.v === 1 && - typeof record.dbName === `string` && - typeof record.collectionId === `string` && - typeof record.senderId === `string` && - typeof record.ts === `number` - ) -} - -function isRPCRequest(payload: unknown): payload is RPCRequest { - if (!payload || typeof payload !== `object`) return false - switch ((payload as { type?: unknown }).type) { - case `rpc:ensureRemoteSubset:req`: - case `rpc:releaseRemoteSubset:req`: - case `rpc:ensurePersistedIndex:req`: - case `rpc:applyLocalMutations:req`: - case `rpc:applyCommittedTx:req`: - case `rpc:pullSince:req`: - return true - default: - return false - } -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)) -} - -async function unloadRemoteSubsetOwner( - owner: RemoteSubsetOwner, - options: TransportedLoadSubsetOptions, -): Promise { - try { - const result = ( - owner.unloadSubset as unknown as ( - options: TransportedLoadSubsetOptions, - ) => unknown - )(options) - await Promise.resolve(result) - } catch (error) { - reportRemoteSubsetOwnerError(owner, error) - throw error - } -} - -function reportRemoteSubsetOwnerError( - owner: RemoteSubsetOwner, - error: unknown, -): void { - try { - owner.onError(error) - } catch { - // Reporting must not replace the original owner failure. - } -} - -function appliedEnvelopeKey(collectionId: string, envelopeId: string): string { - return JSON.stringify([collectionId, envelopeId]) -} - -function remoteSubsetAcquisitionKey( - collectionId: string, - acquisitionId: string, -): string { - return JSON.stringify([collectionId, acquisitionId]) -} - -function inboundRemoteSubsetAcquisitionKey( - collectionId: string, - requesterId: string, - acquisitionId: string, -): string { - return JSON.stringify([collectionId, requesterId, acquisitionId]) -} - -function createRPCErrorResponse( - request: RPCRequest, - cause: unknown, -): RPCResponse { - const error = cause instanceof Error ? cause.message : String(cause) - switch (request.type) { - case `rpc:ensureRemoteSubset:req`: - return { - type: `rpc:ensureRemoteSubset:res`, - rpcId: request.rpcId, - ok: false, - error, - ...(cause instanceof RetryableRemoteSubsetAcquisitionError - ? { retryable: true as const } - : {}), - } - case `rpc:releaseRemoteSubset:req`: - return { - type: `rpc:releaseRemoteSubset:res`, - rpcId: request.rpcId, - ok: false, - error, - } - case `rpc:ensurePersistedIndex:req`: - return { - type: `rpc:ensurePersistedIndex:res`, - rpcId: request.rpcId, - ok: false, - error, - } - case `rpc:applyLocalMutations:req`: - if (cause instanceof PersistedCollectionDurabilityError) { - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: false, - code: `PERSISTENCE_ERROR`, - error, - ...toSafeDurabilityDetails(cause), - } - } - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: false, - code: `CONFLICT`, - error, - } - case `rpc:applyCommittedTx:req`: - if (cause instanceof PersistedCollectionDurabilityError) { - return { - type: `rpc:applyCommittedTx:res`, - rpcId: request.rpcId, - ok: false, - code: `PERSISTENCE_ERROR`, - error, - ...toSafeDurabilityDetails(cause), - } - } - return { - type: `rpc:applyCommittedTx:res`, - rpcId: request.rpcId, - ok: false, - code: `CONFLICT`, - error, - } - case `rpc:pullSince:req`: - return { - type: `rpc:pullSince:res`, - rpcId: request.rpcId, - ok: false, - error, - } - } -} - -function isMutatingRPCRequest(request: RPCRequest): request is Extract< - RPCRequest, - { - type: IndeterminateCommitRequestType - } -> { - return ( - request.type === `rpc:applyLocalMutations:req` || - request.type === `rpc:applyCommittedTx:req` - ) -} - -function toSafeDurabilityDetails(error: PersistedCollectionDurabilityError): { - sourceCode?: string | number - path?: string | ReadonlyArray -} { - const sourceCode = - typeof error.code === `string` || typeof error.code === `number` - ? error.code - : undefined - const path = - typeof error.path === `string` || - (Array.isArray(error.path) && - error.path.every( - (part) => typeof part === `string` || typeof part === `number`, - )) - ? (error.path as string | ReadonlyArray) - : undefined - return { - ...(sourceCode === undefined ? {} : { sourceCode }), - ...(path === undefined ? {} : { path }), } } diff --git a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts index 1276608eb9..8db1f436b5 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts @@ -47,15 +47,17 @@ import type { * RFC #1659 requires renderer work to reach the exact collection adapter in * the elected main-process owner. Complete committed transactions retain row * and collection metadata. Mutating RPC replay is limited to the same known - * leader and term. Remote subset request data stays inside the clone-safe wire - * domain, and each accepted physical acquisition has one acquisition lease. + * leader and term, and one envelope id cannot identify two mutation request + * types. Remote subset request data stays inside the clone-safe wire domain, + * and each accepted physical acquisition has one acquisition lease. * * Expected transactions, adapter call logs, SQLite rows, metadata, owner * callbacks, and coordinator snapshots form the reference observations. * Histories vary response loss, leadership change, owner replacement, * duplicate delivery, acquisition release, durability failure, cleanup, and - * reopen. The driver crosses the real Electron coordinator and IPC persistence - * adapter; the durable witness reopens a real SQLite database. + * reopen. The driver crosses the real Electron coordinator, its shared + * broadcast coordination engine, and the IPC persistence adapter; the durable + * witness reopens a real SQLite database. * * Checkpoints sit at adapter entry, RPC settlement, acquisition acceptance and * release, lifecycle error, disposal, and durable reopen. Hostile wire values, @@ -1436,6 +1438,68 @@ describe(`electron sqlite persistence bridge`, () => { expect(applyCalls).toBe(1) }) + it(`rejects a different mutation operation that reuses a completed envelope`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + let applyCalls = 0 + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-cross-operation-envelope`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: () => { + applyCalls++ + return Promise.resolve() + }, + ensureIndex: () => Promise.resolve(), + }, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.subscribe(`todos`, () => {}) + await waitForLeadership(coordinator, `todos`) + + const internals = coordinator as unknown as { + handleApplyLocalMutations: ( + collectionId: string, + request: ApplyLocalMutationsRequest, + ) => Promise + handleApplyCommittedTx: ( + collectionId: string, + request: ApplyCommittedTxRequest, + ) => Promise + } + const envelopeId = `cross-operation-envelope` + + await expect( + internals.handleApplyLocalMutations(`todos`, { + type: `rpc:applyLocalMutations:req`, + rpcId: `local-rpc`, + envelopeId, + mutations: [], + }), + ).resolves.toMatchObject({ ok: true }) + + await expect( + internals.handleApplyCommittedTx(`todos`, { + type: `rpc:applyCommittedTx:req`, + rpcId: `committed-rpc`, + envelopeId, + tx: { + txId: `committed-tx`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }, + }), + ).resolves.toMatchObject({ + type: `rpc:applyCommittedTx:res`, + rpcId: `committed-rpc`, + ok: false, + code: `CONFLICT`, + error: expect.stringContaining(`already applied`), + }) + expect(applyCalls).toBe(1) + }) + it(`classifies Electron durability failures on local and follower routes`, async () => { registerCleanup(installImmediatelyGrantedWebLocks()) const persistenceError = Object.assign(new Error(`electron disk failed`), { @@ -1567,8 +1631,8 @@ describe(`electron sqlite persistence bridge`, () => { collectionId: string, request: ApplyCommittedTxRequest, ) => Promise - appliedCommittedTxEnvelopes: Map - inFlightCommittedTxEnvelopes: Map + appliedEnvelopes: Map + inFlightEnvelopes: Map } const outcomePromise = internals .handleApplyCommittedTx(`todos`, { @@ -1607,8 +1671,8 @@ describe(`electron sqlite persistence bridge`, () => { expect(durableTransactions).toHaveLength(1) expect(durableTransactions[0]?.txId).toBe(`held-committed-tx`) expect({ - completed: internals.appliedCommittedTxEnvelopes.size, - inFlight: internals.inFlightCommittedTxEnvelopes.size, + completed: internals.appliedEnvelopes.size, + inFlight: internals.inFlightEnvelopes.size, }).toEqual({ completed: 0, inFlight: 0 }) } finally { releaseApply() @@ -1645,7 +1709,7 @@ describe(`electron sqlite persistence bridge`, () => { collectionId: string, request: ApplyLocalMutationsRequest, ) => Promise - appliedEnvelopeIds: Map + appliedEnvelopes: Map } const outcomePromise = internals .handleApplyLocalMutations(`todos`, { @@ -1691,7 +1755,7 @@ describe(`electron sqlite persistence bridge`, () => { value: { id: `held-local-row` }, }, ]) - expect(internals.appliedEnvelopeIds.size).toBe(0) + expect(internals.appliedEnvelopes.size).toBe(0) } finally { releaseApply() coordinator.dispose() @@ -3483,8 +3547,8 @@ describe(`electron sqlite persistence bridge`, () => { const leaderInternals = leader as unknown as { channel: BroadcastChannel - appliedCommittedTxEnvelopes: Map - inFlightCommittedTxEnvelopes: Map + appliedEnvelopes: Map + inFlightEnvelopes: Map } const followerInternals = follower as unknown as { sendRPCOnce: ( @@ -3530,8 +3594,8 @@ describe(`electron sqlite persistence bridge`, () => { expect({ postAfterDispose, - completed: leaderInternals.appliedCommittedTxEnvelopes.size, - inFlight: leaderInternals.inFlightCommittedTxEnvelopes.size, + completed: leaderInternals.appliedEnvelopes.size, + inFlight: leaderInternals.inFlightEnvelopes.size, }).toEqual({ postAfterDispose: 0, completed: 0, inFlight: 0 }) follower.dispose() diff --git a/packages/electron-db-sqlite-persistence/tsconfig.json b/packages/electron-db-sqlite-persistence/tsconfig.json index d2f1d836ab..b302f24c78 100644 --- a/packages/electron-db-sqlite-persistence/tsconfig.json +++ b/packages/electron-db-sqlite-persistence/tsconfig.json @@ -19,6 +19,9 @@ ], "@tanstack/db-sqlite-persistence-core": [ "../db-sqlite-persistence-core/src" + ], + "@tanstack/db-sqlite-persistence-core/*": [ + "../db-sqlite-persistence-core/src/*" ] } }, From 9bb7eb8671a1af68138648213ae2e3d9b2f5bdce Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 18:30:09 +0100 Subject: [PATCH 14/18] fix(sqlite): pace leadership recovery --- .../tests/browser-coordinator.test.ts | 93 ++++++++++++++++++- .../src/broadcast-coordinator.ts | 4 + 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 0f9e46b811..e1bc41ffe4 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -28,10 +28,10 @@ import type { BrowserCollectionCoordinatorOptions } from '../src/browser-coordin * The adapter call logs, transport controls, owner callbacks, and internal-map * snapshots are focused reference ledgers. Histories vary local and follower * routes, response loss, leadership change, owner replacement, duplicate - * delivery, release, failure, and disposal. The production driver is the real - * `BrowserCollectionCoordinator`, which names the shared broadcast - * coordination engine for this host; only BroadcastChannel and Web Locks are - * replaced with deterministic seams. + * delivery, paced leadership retry, release, failure, and disposal. The + * production driver is the real `BrowserCollectionCoordinator`, which names + * the shared broadcast coordination engine for this host; only + * BroadcastChannel and Web Locks are replaced with deterministic seams. * * Checkpoints sit at adapter entry, RPC response delivery, acquisition * acceptance, acquisition release, lifecycle failure, and disposal. Fault @@ -570,6 +570,91 @@ describe(`BrowserCollectionCoordinator`, () => { coord.dispose() }) + it(`paces leadership retry after stream-position failure`, async () => { + vi.useFakeTimers() + const streamError = new Error(`stream position unavailable`) + const adapter = createStubAdapter() + const getStreamPosition = vi + .fn() + .mockRejectedValueOnce(streamError) + .mockResolvedValue({ + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + }) + adapter.getStreamPosition = getStreamPosition + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + const coordinator = createCoordinator(adapter) + + try { + coordinator.subscribe(`todos`, () => {}) + await vi.advanceTimersByTimeAsync(0) + + expect({ + streamPositionReads: getStreamPosition.mock.calls.length, + isLeader: coordinator.isLeader(`todos`), + warnings: warning.mock.calls.length, + }).toEqual({ + streamPositionReads: 1, + isLeader: false, + warnings: 1, + }) + + await vi.advanceTimersByTimeAsync(1_000) + + expect({ + streamPositionReads: getStreamPosition.mock.calls.length, + isLeader: coordinator.isLeader(`todos`), + warnings: warning.mock.calls.length, + }).toEqual({ + streamPositionReads: 2, + isLeader: true, + warnings: 1, + }) + } finally { + coordinator.dispose() + warning.mockRestore() + vi.useRealTimers() + } + }) + + it(`cancels a delayed leadership retry on disposal`, async () => { + vi.useFakeTimers() + const adapter = createStubAdapter() + const getStreamPosition = vi + .fn() + .mockRejectedValueOnce(new Error(`stream position unavailable`)) + .mockResolvedValue({ + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + }) + adapter.getStreamPosition = getStreamPosition + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + const coordinator = createCoordinator(adapter) + + try { + coordinator.subscribe(`todos`, () => {}) + await vi.advanceTimersByTimeAsync(0) + coordinator.dispose() + await vi.advanceTimersByTimeAsync(1_000) + + expect({ + streamPositionReads: getStreamPosition.mock.calls.length, + isLeader: coordinator.isLeader(`todos`), + warnings: warning.mock.calls.length, + }).toEqual({ + streamPositionReads: 1, + isLeader: false, + warnings: 1, + }) + } finally { + coordinator.dispose() + warning.mockRestore() + vi.useRealTimers() + } + }) + it(`second coordinator waits for leadership`, async () => { const coord1 = createCoordinator() coord1.subscribe(`todos`, () => {}) diff --git a/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts b/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts index 80ab162eda..7c28ca089d 100644 --- a/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts +++ b/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts @@ -41,6 +41,7 @@ import type { // --------------------------------------------------------------------------- const HEARTBEAT_INTERVAL_MS = 3_000 +const LEADERSHIP_RETRY_DELAY_MS = 200 const RPC_TIMEOUT_MS = 10_000 const RPC_RETRY_ATTEMPTS = 2 const RPC_RETRY_DELAY_MS = 200 @@ -721,6 +722,9 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi return } console.warn(`Failed to acquire leadership for ${collectionId}:`, error) + if (!this.isDisposed()) { + await sleep(LEADERSHIP_RETRY_DELAY_MS) + } } // Re-acquire if not disposed (leadership was released by another means) From 93c924ad42471c114f06da3eb012f09ce4e6e0fb Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 19:09:29 +0100 Subject: [PATCH 15/18] fix(sqlite): preserve coordinator failure boundaries --- .../tests/browser-coordinator.test.ts | 231 +++++++++++++++++- .../src/broadcast-coordinator.ts | 132 ++++++---- .../src/persisted.ts | 6 + 3 files changed, 320 insertions(+), 49 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index e1bc41ffe4..b98a8aef30 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -23,7 +23,11 @@ import type { BrowserCollectionCoordinatorOptions } from '../src/browser-coordin * without losing metadata. A mutating RPC may replay only through the same * known leader and term, and one envelope id cannot identify two mutation * request types. Remote subset request data must be clone-safe, and each - * accepted physical acquisition creates one exact acquisition lease. + * accepted physical acquisition creates one exact acquisition lease. A + * passive heartbeat can update a route, but only a local participant may join + * that collection's leadership. Durable stream positions advance after the + * adapter accepts the write, and failed release transport retains its retry + * route. * * The adapter call logs, transport controls, owner callbacks, and internal-map * snapshots are focused reference ledgers. Histories vary local and follower @@ -704,6 +708,52 @@ describe(`BrowserCollectionCoordinator`, () => { coord2.dispose() }) + it(`records an unrelated heartbeat without joining that collection's leadership`, async () => { + const adapter = createStubAdapter() + const getStreamPosition = vi.spyOn(adapter, `getStreamPosition`) + const coordinator = createCoordinator(adapter) + coordinator.subscribe(`todos`, () => {}) + await flush(50) + + injectBroadcastMessage(`tsdb:coord:test-db`, { + v: 1, + dbName: `test-db`, + collectionId: `notes`, + senderId: `notes-owner`, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term: 4, + leaderId: `notes-owner`, + latestSeq: 3, + latestRowVersion: 8, + }, + }) + await flush(0) + + const state = ( + coordinator as unknown as { + collections: Map< + string, + { leaderId: string | null; latestTerm: number } + > + } + ).collections.get(`notes`) + expect({ + leaderId: state?.leaderId, + latestTerm: state?.latestTerm, + joinedLeadership: coordinator.isLeader(`notes`), + streamPositionCollections: getStreamPosition.mock.calls.map( + ([collectionId]) => collectionId, + ), + }).toEqual({ + leaderId: `notes-owner`, + latestTerm: 4, + joinedLeadership: false, + streamPositionCollections: [`todos`], + }) + }) + it(`returns unique node ids`, () => { const coord1 = createCoordinator() const coord2 = createCoordinator() @@ -1253,6 +1303,61 @@ describe(`BrowserCollectionCoordinator`, () => { follower.dispose() } }) + + it(`reuses the durable stream position after local mutation persistence fails`, async () => { + const adapter = createStubAdapter() + const persistenceError = new Error(`local disk full`) + const attemptedPositions: Array<{ seq: number; rowVersion: number }> = [] + adapter.applyCommittedTx = vi.fn((_collectionId, tx) => { + attemptedPositions.push({ seq: tx.seq, rowVersion: tx.rowVersion }) + return attemptedPositions.length === 1 + ? Promise.reject(persistenceError) + : Promise.resolve() + }) + const coordinator = createCoordinator(adapter) + coordinator.subscribe(`todos`, () => {}) + await flush(50) + + try { + await expect( + coordinator.requestApplyLocalMutations(`todos`, [ + { + mutationId: `failed-local-mutation`, + type: `insert`, + key: `failed-local-mutation`, + value: { id: `failed-local-mutation` }, + }, + ]), + ).rejects.toMatchObject({ cause: persistenceError }) + + const response = await coordinator.requestApplyLocalMutations(`todos`, [ + { + mutationId: `successful-local-mutation`, + type: `insert`, + key: `successful-local-mutation`, + value: { id: `successful-local-mutation` }, + }, + ]) + + expect({ attemptedPositions, response }).toEqual({ + attemptedPositions: [ + { seq: 1, rowVersion: 1 }, + { seq: 1, rowVersion: 1 }, + ], + response: { + type: `rpc:applyLocalMutations:res`, + rpcId: expect.any(String), + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: [`successful-local-mutation`], + }, + }) + } finally { + coordinator.dispose() + } + }) }) describe(`RPC - applyCommittedTx`, () => { @@ -1444,6 +1549,66 @@ describe(`BrowserCollectionCoordinator`, () => { } }) + it(`reuses the durable stream position after a committed transaction fails`, async () => { + const adapter = createStubAdapter() + const persistenceError = new Error(`disk full`) + const attemptedPositions: Array<{ + term: number + seq: number + rowVersion: number + }> = [] + adapter.applyCommittedTx = vi.fn((_collectionId, tx) => { + attemptedPositions.push({ + term: tx.term, + seq: tx.seq, + rowVersion: tx.rowVersion, + }) + return attemptedPositions.length === 1 + ? Promise.reject(persistenceError) + : Promise.resolve() + }) + const coordinator = createCoordinator(adapter) + coordinator.subscribe(`todos`, () => {}) + await flush(50) + + try { + await expect( + coordinator.requestApplyCommittedTx(`todos`, { + txId: `failed-source-tx`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }), + ).rejects.toMatchObject({ cause: persistenceError }) + + const response = await coordinator.requestApplyCommittedTx(`todos`, { + txId: `successful-source-tx`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }) + + expect({ attemptedPositions, response }).toEqual({ + attemptedPositions: [ + { term: 1, seq: 1, rowVersion: 1 }, + { term: 1, seq: 1, rowVersion: 1 }, + ], + response: { + type: `rpc:applyCommittedTx:res`, + rpcId: expect.any(String), + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + }, + }) + } finally { + coordinator.dispose() + } + }) + it(`replays the successful response when only that response is lost`, async () => { const leaderAdapter = createStubAdapter() const followerAdapter = createStubAdapter() @@ -2402,6 +2567,70 @@ describe(`BrowserCollectionCoordinator`, () => { } }) + it(`retains a follower acquisition when its release transport fails`, async () => { + const leader = createCoordinator() + const follower = createCoordinator() + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + const owner = Object.assign( + vi.fn(() => Promise.resolve()), + { + unloadSubset: vi.fn(() => Promise.resolve()), + onError: vi.fn(), + }, + ) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + const options: LoadSubsetOptions = { limit: 1 } + const followerInternals = follower as unknown as { + sendRPC: (collectionId: string, request: unknown) => Promise + outboundRemoteSubsetAcquisitions: Map + } + + try { + await follower.requestEnsureRemoteSubset(`todos`, options) + const sendRPC = followerInternals.sendRPC.bind(follower) + let releaseAttempts = 0 + followerInternals.sendRPC = async (collectionId, request) => { + if ( + (request as { type?: string }).type === + `rpc:releaseRemoteSubset:req` + ) { + releaseAttempts++ + if (releaseAttempts === 1) { + return { + type: `rpc:releaseRemoteSubset:res`, + rpcId: (request as { rpcId: string }).rpcId, + ok: false, + error: `transient release transport failure`, + } + } + } + return sendRPC(collectionId, request) + } + + await expect( + follower.requestReleaseRemoteSubset(`todos`, options), + ).rejects.toThrow(`transient release transport failure`) + expect({ + releaseAttempts, + retained: followerInternals.outboundRemoteSubsetAcquisitions.size, + unloads: owner.unloadSubset.mock.calls.length, + }).toEqual({ releaseAttempts: 1, retained: 1, unloads: 0 }) + + await follower.requestReleaseRemoteSubset(`todos`, options) + expect({ + releaseAttempts, + retained: followerInternals.outboundRemoteSubsetAcquisitions.size, + unloads: owner.unloadSubset.mock.calls.length, + }).toEqual({ releaseAttempts: 2, retained: 0, unloads: 1 }) + } finally { + unregisterOwner() + leader.dispose() + follower.dispose() + } + }) + it(`keeps a Browser release tombstone when a transferred load rejects concurrently`, async () => { const coordinator = createCoordinator() coordinator.subscribe(`todos`, () => {}) diff --git a/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts b/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts index 7c28ca089d..33774b6e82 100644 --- a/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts +++ b/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts @@ -101,6 +101,7 @@ type PendingRPC = { } type CollectionState = { + participatesInLeadership: boolean isLeader: boolean leaderId: string | null lockAbortController: AbortController | null @@ -174,6 +175,7 @@ type OutboundRemoteSubsetAcquisition = { localOptions: TransportedLoadSubsetOptions acquiredLeaderId: string | null inFlight: Promise | null + release: Promise | null forceReplay: boolean retryTimer: ReturnType | null retryAttempts: number @@ -374,6 +376,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi localOptions, acquiredLeaderId: null, inFlight: null, + release: null, forceReplay: false, retryTimer: null, retryAttempts: 0, @@ -397,24 +400,41 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi const key = remoteSubsetAcquisitionKey(collectionId, acquisitionId) const acquisition = this.outboundRemoteSubsetAcquisitions.get(key) if (!acquisition) return - this.outboundRemoteSubsetAcquisitions.delete(key) this.cancelRemoteSubsetReplayRetry(acquisition) - collectionIds!.delete(options) + if (!acquisition.release) { + acquisition.release = (async () => { + const request: Extract< + RPCRequest, + { type: `rpc:releaseRemoteSubset:req` } + > = { + type: `rpc:releaseRemoteSubset:req`, + rpcId: safeRandomUUID(), + acquisitionId, + } + const response = this.isLeader(collectionId) + ? await this.handleReleaseRemoteSubset( + collectionId, + request, + this.nodeId, + ) + : await this.sendRPC( + collectionId, + request, + ) - const request: Extract< - RPCRequest, - { type: `rpc:releaseRemoteSubset:req` } - > = { - type: `rpc:releaseRemoteSubset:req`, - rpcId: safeRandomUUID(), - acquisitionId, + if (!response.ok) { + throw new Error(`releaseRemoteSubset failed: ${response.error}`) + } + if (this.outboundRemoteSubsetAcquisitions.get(key) === acquisition) { + this.outboundRemoteSubsetAcquisitions.delete(key) + collectionIds!.delete(options) + } + })() } - const response = this.isLeader(collectionId) - ? await this.handleReleaseRemoteSubset(collectionId, request, this.nodeId) - : await this.sendRPC(collectionId, request) - - if (!response.ok) { - throw new Error(`releaseRemoteSubset failed: ${response.error}`) + try { + await acquisition.release + } finally { + acquisition.release = null } } @@ -638,10 +658,11 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi // Leadership via Web Locks // ----------------------------------------------------------------------- - private ensureCollectionState(collectionId: string): CollectionState { + private getOrCreateCollectionState(collectionId: string): CollectionState { let state = this.collections.get(collectionId) if (!state) { state = { + participatesInLeadership: false, isLeader: false, leaderId: null, lockAbortController: null, @@ -652,6 +673,14 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi subscribers: new Set(), } this.collections.set(collectionId, state) + } + return state + } + + private ensureCollectionState(collectionId: string): CollectionState { + const state = this.getOrCreateCollectionState(collectionId) + if (!state.participatesInLeadership) { + state.participatesInLeadership = true void this.acquireLeadership(collectionId, state) } return state @@ -753,6 +782,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi private postRemoteSubsetRelease( acquisition: OutboundRemoteSubsetAcquisition, ): void { + if (acquisition.release) return const request: Extract< RPCRequest, { type: `rpc:releaseRemoteSubset:req` } @@ -836,6 +866,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { if ( acquisition.collectionId !== collectionId || + acquisition.release || (!acquisition.forceReplay && acquisition.acquiredLeaderId === leaderId) ) { continue @@ -902,7 +933,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi typeof heartbeat.latestSeq === `number` && typeof heartbeat.latestRowVersion === `number` ) { - const state = this.ensureCollectionState(envelope.collectionId) + const state = this.getOrCreateCollectionState(envelope.collectionId) if (heartbeat.term < state.latestTerm) return const changedLeader = state.leaderId !== heartbeat.leaderId state.leaderId = heartbeat.leaderId @@ -1567,14 +1598,6 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi } } - // Assign stream position - state.latestSeq++ - state.latestRowVersion++ - - const term = state.latestTerm - const seq = state.latestSeq - const rowVersion = state.latestRowVersion - // Build and apply the persisted transaction const rowMetadataMutations: Array = [] for (const mutation of request.mutations) { @@ -1591,11 +1614,8 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi }, ) } - const tx = { + const pendingTx = { txId: safeRandomUUID(), - term, - seq, - rowVersion, mutations: request.mutations.map((m) => ({ type: m.type, key: m.key, @@ -1607,9 +1627,12 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi rowMetadataMutations, } + let tx: PersistedTx try { - await this.withWriterLock(() => - this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), + tx = await this.applyDurablyAtNextStreamPosition( + collectionId, + state, + pendingTx, ) } catch (error) { throw toPersistedCollectionDurabilityError(collectionId, error) @@ -1619,9 +1642,9 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi type: `rpc:applyLocalMutations:res`, rpcId: request.rpcId, ok: true, - term, - seq, - latestRowVersion: rowVersion, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, acceptedMutationIds: request.mutations.map((m) => m.mutationId), } if (this.isDisposed()) { @@ -1653,10 +1676,10 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi ts: Date.now(), payload: { type: `tx:committed`, - term, - seq, + term: tx.term, + seq: tx.seq, txId: tx.txId, - latestRowVersion: rowVersion, + latestRowVersion: tx.rowVersion, requiresFullReload: false, changedRows, deletedKeys, @@ -1737,18 +1760,12 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi } } - state.latestSeq++ - state.latestRowVersion++ - const tx: PersistedTx = { - ...request.tx, - term: state.latestTerm, - seq: state.latestSeq, - rowVersion: state.latestRowVersion, - } - + let tx: PersistedTx try { - await this.withWriterLock(() => - this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), + tx = await this.applyDurablyAtNextStreamPosition( + collectionId, + state, + request.tx, ) } catch (error) { throw toPersistedCollectionDurabilityError(collectionId, error) @@ -1817,6 +1834,25 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi return response } + private async applyDurablyAtNextStreamPosition( + collectionId: string, + state: CollectionState, + pendingTx: Omit, + ): Promise { + return this.withWriterLock(async () => { + const tx: PersistedTx = { + ...pendingTx, + term: state.latestTerm, + seq: state.latestSeq + 1, + rowVersion: state.latestRowVersion + 1, + } + await this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx) + state.latestSeq = tx.seq + state.latestRowVersion = tx.rowVersion + return tx + }) + } + private async handlePullSince( collectionId: string, request: { diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 848d4fca37..dc787b3196 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -51,7 +51,9 @@ export type PersistedMutationEnvelope = type: `insert` key: string | number value: Record + /** Persisted row metadata, not optimistic-transaction metadata. */ metadata?: unknown + /** Whether this envelope replaces or deletes the persisted row metadata. */ metadataChanged?: boolean } | { @@ -59,7 +61,9 @@ export type PersistedMutationEnvelope = type: `update` key: string | number value: Record + /** Persisted row metadata, not optimistic-transaction metadata. */ metadata?: unknown + /** Whether this envelope replaces or deletes the persisted row metadata. */ metadataChanged?: boolean } | { @@ -1019,6 +1023,8 @@ function toPersistedMutationEnvelope( ? (mutation.original as Record) : mutation.modified + // PendingMutation.metadata belongs to the optimistic transaction and is + // consumed by mutation handlers. It must not overwrite persisted row metadata. return { mutationId: mutation.mutationId, type: mutation.type, From e9085cf099d8f1937180392ce8d7d103f64d0f48 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 11:06:02 +0100 Subject: [PATCH 16/18] fix(sqlite): harden remote subset lifecycles --- .../tests/browser-coordinator.test.ts | 689 ++++++++++++++---- .../src/broadcast-coordinator.ts | 471 +++++++----- .../src/persisted.ts | 36 +- .../tests/persisted.test.ts | 205 +++++- .../tests/electron-ipc.test.ts | 285 ++++++-- 5 files changed, 1321 insertions(+), 365 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index b98a8aef30..3afb806480 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -1304,6 +1304,92 @@ describe(`BrowserCollectionCoordinator`, () => { } }) + it.each([ + { requestType: `committed transaction` as const }, + { requestType: `local mutations` as const }, + ])( + `keeps pre-adapter failures outside the durability boundary for $requestType`, + async ({ requestType }) => { + type LeaderState = { + participatesInLeadership: boolean + isLeader: boolean + leaderId: string + lockAbortController: null + heartbeatTimer: null + latestTerm: number + latestSeq: number + latestRowVersion: number + subscribers: Set<(message: unknown) => void> + } + type CoordinatorInternals = { + collections: Map + withWriterLock: (task: () => Promise) => Promise + } + const leaderState = (): LeaderState => ({ + participatesInLeadership: true, + isLeader: true, + leaderId: `forced-local-leader`, + lockAbortController: null, + heartbeatTimer: null, + latestTerm: 1, + latestSeq: 0, + latestRowVersion: 0, + subscribers: new Set(), + }) + const invoke = ( + coordinator: BrowserCollectionCoordinator, + suffix: string, + ) => + requestType === `committed transaction` + ? coordinator.requestApplyCommittedTx(`todos`, { + txId: `classification-boundary-${suffix}`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }) + : coordinator.requestApplyLocalMutations(`todos`, [ + { + mutationId: `classification-boundary-${suffix}`, + type: `insert`, + key: `classification-boundary-${suffix}`, + value: { id: `classification-boundary-${suffix}` }, + }, + ]) + const missingAdapter = new BrowserCollectionCoordinator({ + dbName: `missing-adapter`, + }) + const missingInternals = + missingAdapter as unknown as CoordinatorInternals + missingInternals.collections.set(`todos`, leaderState()) + const lockFailure = new Error(`writer lock unavailable`) + const lockCoordinator = createCoordinator() + const lockInternals = lockCoordinator as unknown as CoordinatorInternals + lockInternals.collections.set(`todos`, leaderState()) + lockInternals.withWriterLock = () => Promise.reject(lockFailure) + + try { + const missingError = await invoke(missingAdapter, `missing`).then( + () => undefined, + (error: unknown) => error, + ) + const lockError = await invoke(lockCoordinator, `writer-lock`).then( + () => undefined, + (error: unknown) => error, + ) + + expect(missingError).toMatchObject({ + name: `Error`, + message: expect.stringContaining(`adapter not set`), + }) + expect(lockError).toBe(lockFailure) + } finally { + missingAdapter.dispose() + lockCoordinator.dispose() + } + }, + ) + it(`reuses the durable stream position after local mutation persistence fails`, async () => { const adapter = createStubAdapter() const persistenceError = new Error(`local disk full`) @@ -2268,6 +2354,230 @@ describe(`BrowserCollectionCoordinator`, () => { }) describe(`RPC - ensureRemoteSubset`, () => { + /** + * Route-pending was absent from the earlier acquisition grammar: every + * generated request began after a leader was known. This law varies the + * terminal action while the same first acquisition is waiting for its + * route, and checks both transport exclusion and live local-only identity. + */ + it.each([ + { terminalAction: `complete` as const }, + { terminalAction: `abort` as const }, + ])( + `settles a route-pending first subset request after $terminalAction without premature transport or replay`, + async ({ terminalAction }) => { + const lockName = `tsdb:leader:test-db:todos` + let releaseInitialLock = (): void => {} + heldLocks.set(lockName, { + release: () => { + heldLocks.delete(lockName) + tryGrantNextLock(lockName) + }, + }) + releaseInitialLock = heldLocks.get(lockName)!.release + const coordinator = createCoordinator() + const owner = withUnusedUnloadSubset(vi.fn()) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + coordinator.subscribe(`todos`, () => {}) + const internals = coordinator as unknown as { + sendRPCOnce: ( + collectionId: string, + request: unknown, + ) => Promise + } + const originalSendRPCOnce = internals.sendRPCOnce.bind(coordinator) + const prematureTransport = vi.fn(() => + Promise.reject(new Error(`no leader route yet`)), + ) + internals.sendRPCOnce = prematureTransport + const abortController = new AbortController() + const signal = abortController.signal + const subscription = { on: () => () => {} } as unknown as Subscription + const options: LoadSubsetOptions = { limit: 1, signal, subscription } + const acquisitionInternals = coordinator as unknown as { + outboundRemoteSubsetAcquisitions: Map + } + + try { + const request = coordinator.requestEnsureRemoteSubset( + `todos`, + options, + ) + await flush(0) + const transportCallsBeforeLeadership = + prematureTransport.mock.calls.length + + if (terminalAction === `abort`) { + abortController.abort() + await expect(request).rejects.toMatchObject({ name: `AbortError` }) + } + releaseInitialLock() + await vi.waitFor(() => + expect(coordinator.isLeader(`todos`)).toBe(true), + ) + if (terminalAction === `complete`) await request + await flush(0) + + expect(transportCallsBeforeLeadership).toBe(0) + if (terminalAction === `complete`) { + expect(owner).toHaveBeenCalledTimes(1) + expect(owner.mock.calls[0]?.[0]).toMatchObject({ limit: 1 }) + expect(owner.mock.calls[0]?.[0].signal).toBe(signal) + expect(owner.mock.calls[0]?.[0].subscription).toBe(subscription) + } else { + expect(owner).not.toHaveBeenCalled() + expect( + acquisitionInternals.outboundRemoteSubsetAcquisitions.size, + ).toBe(0) + } + await coordinator.requestReleaseRemoteSubset(`todos`, options) + } finally { + internals.sendRPCOnce = originalSendRPCOnce + releaseInitialLock() + unregisterOwner() + coordinator.dispose() + } + }, + ) + + it.each([ + { cleanupOrder: `owner then subscriptions` as const, subscriberCount: 1 }, + { cleanupOrder: `owner then subscriptions` as const, subscriberCount: 2 }, + { cleanupOrder: `subscriptions then owner` as const, subscriberCount: 1 }, + { cleanupOrder: `subscriptions then owner` as const, subscriberCount: 2 }, + ])( + `relinquishes leadership after $subscriberCount runtime subscriber(s) clean up in $cleanupOrder order`, + async ({ cleanupOrder, subscriberCount }) => { + const first = createCoordinator() + const second = createCoordinator() + const unsubscribeFirst = Array.from({ length: subscriberCount }, () => + first.subscribe(`todos`, () => {}), + ) + second.subscribe(`todos`, () => {}) + const firstOwner = withUnusedUnloadSubset(vi.fn()) + const secondOwner = withUnusedUnloadSubset(vi.fn()) + const unregisterFirst = first.registerRemoteSubsetOwner( + `todos`, + firstOwner, + ) + const unregisterSecond = second.registerRemoteSubsetOwner( + `todos`, + secondOwner, + ) + + try { + await vi.waitFor(() => { + expect({ + first: first.isLeader(`todos`), + second: second.isLeader(`todos`), + }).toEqual({ first: true, second: false }) + }) + + if (cleanupOrder === `owner then subscriptions`) unregisterFirst() + for (const [index, unsubscribe] of unsubscribeFirst.entries()) { + unsubscribe() + if (index < unsubscribeFirst.length - 1) { + expect(first.isLeader(`todos`)).toBe(true) + } + } + if (cleanupOrder === `subscriptions then owner`) unregisterFirst() + + await vi.waitFor(() => { + expect({ + first: first.isLeader(`todos`), + second: second.isLeader(`todos`), + }).toEqual({ first: false, second: true }) + }) + + const options: LoadSubsetOptions = { limit: 1 } + await second.requestEnsureRemoteSubset(`todos`, options) + expect(secondOwner).toHaveBeenCalledTimes(1) + await second.requestReleaseRemoteSubset(`todos`, options) + } finally { + for (const unsubscribe of unsubscribeFirst) unsubscribe() + unregisterFirst() + unregisterSecond() + first.dispose() + second.dispose() + } + }, + ) + + it(`does not orphan an overlapping leadership attempt across rapid unsubscribe and resubscribe`, async () => { + // The earlier generated histories changed leaders only after a settled + // acquisition. This hostile history leaves getStreamPosition in flight + // while participation toggles twice, which can overlap lock loops. + const NativeAbortController = globalThis.AbortController + const leadershipControllers: Array = [] + class TrackingAbortController extends NativeAbortController { + constructor() { + super() + leadershipControllers.push(this) + } + } + Object.defineProperty(globalThis, `AbortController`, { + value: TrackingAbortController, + writable: true, + configurable: true, + }) + const adapter = createStubAdapter() + const positionResolvers: Array<() => void> = [] + adapter.getStreamPosition = vi.fn( + () => + new Promise<{ + latestTerm: number + latestSeq: number + latestRowVersion: number + }>((resolve) => { + positionResolvers.push(() => + resolve({ + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + }), + ) + }), + ) + const participant = createCoordinator(adapter) + let unsubscribe = participant.subscribe(`todos`, () => {}) + let contender: BrowserCollectionCoordinator | undefined + + try { + await vi.waitFor(() => expect(positionResolvers).toHaveLength(1)) + + unsubscribe() + unsubscribe = participant.subscribe(`todos`, () => {}) + positionResolvers[0]!() + await vi.waitFor(() => expect(positionResolvers).toHaveLength(2)) + await flush(0) + + contender = createCoordinator() + contender.subscribe(`todos`, () => {}) + unsubscribe() + positionResolvers[1]!() + + await vi.waitFor(() => { + expect({ + participant: participant.isLeader(`todos`), + contender: contender!.isLeader(`todos`), + }).toEqual({ participant: false, contender: true }) + }) + } finally { + unsubscribe() + for (const controller of leadershipControllers) controller.abort() + participant.dispose() + contender?.dispose() + Object.defineProperty(globalThis, `AbortController`, { + value: NativeAbortController, + writable: true, + configurable: true, + }) + } + }) + it(`rejects a second live remote subset owner instead of replacing the first`, () => { const coordinator = createCoordinator() const first = Object.assign( @@ -2499,137 +2809,208 @@ describe(`BrowserCollectionCoordinator`, () => { } }) - it(`releases a transferred Browser lease whose initial load rejected`, async () => { - const coordinator = createCoordinator() - coordinator.subscribe(`todos`, () => {}) - await flush(50) - expect(coordinator.isLeader(`todos`)).toBe(true) - const loadError = new Error(`browser transferred load failed`) - const ownerErrors: Array = [] - const owner = Object.assign( - vi.fn((_options: TransportedLoadSubsetOptions) => - Promise.reject(loadError), - ), - { - unloadSubset: vi.fn( - (_options: TransportedLoadSubsetOptions) => undefined, - ), - onError: (error: unknown) => ownerErrors.push(error), - }, - ) - const unregisterOwner = coordinator.registerRemoteSubsetOwner( - `todos`, - owner, - ) - const options: LoadSubsetOptions = { offset: 20 } - const unhandled: Array = [] - const onUnhandled = (error: unknown) => unhandled.push(error) - process.on(`unhandledRejection`, onUnhandled) - const internals = coordinator as unknown as { - inboundRemoteSubsetAcquisitions: Map> - } + it.each([ + { failureMode: `before lease transfer` as const, expectedUnloads: 1 }, + { failureMode: `after lease transfer` as const, expectedUnloads: 2 }, + ])( + `retires a rejected owner load $failureMode and permits a fresh retry`, + async ({ failureMode, expectedUnloads }) => { + // The previous grammar generated only fulfilled owner loads. Both sides + // of the promise-return lease-transfer boundary must permit a fresh + // attempt without caching the rejected acquisition. + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + const loadError = new Error(`browser transferred load failed`) + const ownerErrors: Array = [] + let loadCalls = 0 + const owner = Object.assign( + vi.fn((_options: TransportedLoadSubsetOptions) => { + loadCalls++ + if (loadCalls !== 1) return Promise.resolve() + if (failureMode === `before lease transfer`) throw loadError + return Promise.reject(loadError) + }), + { + unloadSubset: vi.fn( + (_options: TransportedLoadSubsetOptions) => undefined, + ), + onError: (error: unknown) => ownerErrors.push(error), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { offset: 20 } + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + inboundRemoteSubsetAcquisitions: Map> + } - try { - const ensureError = await coordinator - .requestEnsureRemoteSubset(`todos`, options) - .then( - () => undefined, - (error: unknown) => error, + try { + const ensureError = await coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then( + () => undefined, + (error: unknown) => error, + ) + await coordinator.requestEnsureRemoteSubset(`todos`, options) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + await flush(0) + const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() + + expect(ensureError).toBe(loadError) + expect(owner).toHaveBeenCalledTimes(2) + expect(owner.unloadSubset).toHaveBeenCalledTimes(expectedUnloads) + if (failureMode === `after lease transfer`) { + expect(owner.unloadSubset.mock.calls[0]?.[0]).toBe( + owner.mock.calls[0]?.[0], + ) + } + expect(owner.unloadSubset.mock.calls.at(-1)?.[0]).toBe( + owner.mock.calls[1]?.[0], ) - await coordinator.requestReleaseRemoteSubset(`todos`, options) - await flush(0) - const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() + expect(ownerErrors).toEqual([]) + expect(unhandled).toEqual([]) + expect({ + inbound: internals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }, + ) - expect(ensureError).toBe(loadError) - expect(owner).toHaveBeenCalledTimes(1) - expect(owner.unloadSubset).toHaveBeenCalledTimes(1) - expect(owner.unloadSubset.mock.calls[0]?.[0]).toBe( - owner.mock.calls[0]?.[0], + it.each([ + { + transportFailure: `delivered negative response` as const, + retainedAfterFailure: 0, + }, + { + transportFailure: `undelivered rejection` as const, + retainedAfterFailure: 1, + }, + ])( + `resolves a $transportFailure by delivery boundary without takeover replay`, + async ({ transportFailure, retainedAfterFailure }) => { + // Earlier takeover histories replayed only live demand; release failure + // was tested without a real leader change. No-response transport keeps + // retry debt; a leader response proves delivery and retires it. Both + // terminal paths must suppress takeover replay. + const leader = createCoordinator() + const follower = createCoordinator() + const unsubscribeLeader = leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + const firstOwner = Object.assign( + vi.fn(() => Promise.resolve()), + { + unloadSubset: vi.fn(() => Promise.resolve()), + onError: vi.fn(), + }, ) - expect(ownerErrors).toEqual([loadError]) - expect(unhandled).toEqual([]) - expect({ - inbound: internals.inboundRemoteSubsetAcquisitions.size, - terminalKeys: Object.keys(terminal ?? {}).sort(), - }).toEqual({ - inbound: 1, - terminalKeys: [ - `acquisitionId`, - `collectionId`, - `released`, - `requesterId`, - ], - }) - } finally { - process.off(`unhandledRejection`, onUnhandled) - unregisterOwner() - coordinator.dispose() - } - }) - - it(`retains a follower acquisition when its release transport fails`, async () => { - const leader = createCoordinator() - const follower = createCoordinator() - leader.subscribe(`todos`, () => {}) - follower.subscribe(`todos`, () => {}) - await flush(50) - const owner = Object.assign( - vi.fn(() => Promise.resolve()), - { - unloadSubset: vi.fn(() => Promise.resolve()), - onError: vi.fn(), - }, - ) - const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) - const options: LoadSubsetOptions = { limit: 1 } - const followerInternals = follower as unknown as { - sendRPC: (collectionId: string, request: unknown) => Promise - outboundRemoteSubsetAcquisitions: Map - } + const secondOwner = Object.assign( + vi.fn(() => Promise.resolve()), + { + unloadSubset: vi.fn(() => Promise.resolve()), + onError: vi.fn(), + }, + ) + const unregisterFirst = leader.registerRemoteSubsetOwner( + `todos`, + firstOwner, + ) + const unregisterSecond = follower.registerRemoteSubsetOwner( + `todos`, + secondOwner, + ) + const options: LoadSubsetOptions = { limit: 1 } + const followerInternals = follower as unknown as { + sendRPC: (collectionId: string, request: unknown) => Promise + outboundRemoteSubsetAcquisitions: Map + } - try { - await follower.requestEnsureRemoteSubset(`todos`, options) - const sendRPC = followerInternals.sendRPC.bind(follower) - let releaseAttempts = 0 - followerInternals.sendRPC = async (collectionId, request) => { - if ( - (request as { type?: string }).type === - `rpc:releaseRemoteSubset:req` - ) { - releaseAttempts++ - if (releaseAttempts === 1) { - return { - type: `rpc:releaseRemoteSubset:res`, - rpcId: (request as { rpcId: string }).rpcId, - ok: false, - error: `transient release transport failure`, + try { + await follower.requestEnsureRemoteSubset(`todos`, options) + const sendRPC = followerInternals.sendRPC.bind(follower) + let releaseAttempts = 0 + followerInternals.sendRPC = async (collectionId, request) => { + if ( + (request as { type?: string }).type === + `rpc:releaseRemoteSubset:req` + ) { + releaseAttempts++ + if (releaseAttempts === 1) { + if (transportFailure === `undelivered rejection`) { + throw new Error(`transient release transport rejection`) + } + return { + type: `rpc:releaseRemoteSubset:res`, + rpcId: (request as { rpcId: string }).rpcId, + ok: false, + error: `transient release transport failure`, + } } } + return sendRPC(collectionId, request) } - return sendRPC(collectionId, request) - } - await expect( - follower.requestReleaseRemoteSubset(`todos`, options), - ).rejects.toThrow(`transient release transport failure`) - expect({ - releaseAttempts, - retained: followerInternals.outboundRemoteSubsetAcquisitions.size, - unloads: owner.unloadSubset.mock.calls.length, - }).toEqual({ releaseAttempts: 1, retained: 1, unloads: 0 }) + await expect( + follower.requestReleaseRemoteSubset(`todos`, options), + ).rejects.toThrow(/transient release transport/) + expect({ + releaseAttempts, + retained: followerInternals.outboundRemoteSubsetAcquisitions.size, + unloads: firstOwner.unloadSubset.mock.calls.length, + }).toEqual({ + releaseAttempts: 1, + retained: retainedAfterFailure, + unloads: 0, + }) - await follower.requestReleaseRemoteSubset(`todos`, options) - expect({ - releaseAttempts, - retained: followerInternals.outboundRemoteSubsetAcquisitions.size, - unloads: owner.unloadSubset.mock.calls.length, - }).toEqual({ releaseAttempts: 2, retained: 0, unloads: 1 }) - } finally { - unregisterOwner() - leader.dispose() - follower.dispose() - } - }) + unsubscribeLeader() + await vi.waitFor(() => expect(follower.isLeader(`todos`)).toBe(true)) + await vi.waitFor(() => + expect(firstOwner.unloadSubset).toHaveBeenCalledTimes(1), + ) + expect(secondOwner).not.toHaveBeenCalled() + + await follower.requestReleaseRemoteSubset(`todos`, options) + expect({ + releaseAttempts, + retained: followerInternals.outboundRemoteSubsetAcquisitions.size, + firstUnloads: firstOwner.unloadSubset.mock.calls.length, + takeoverLoads: secondOwner.mock.calls.length, + }).toEqual({ + releaseAttempts: 1, + retained: 0, + firstUnloads: 1, + takeoverLoads: 0, + }) + } finally { + unsubscribeLeader() + unregisterFirst() + unregisterSecond() + leader.dispose() + follower.dispose() + } + }, + ) it(`keeps a Browser release tombstone when a transferred load rejects concurrently`, async () => { const coordinator = createCoordinator() @@ -2720,7 +3101,7 @@ describe(`BrowserCollectionCoordinator`, () => { }) expect(owner).toHaveBeenCalledTimes(1) expect(owner.unloadSubset).toHaveBeenCalledTimes(1) - expect(ownerErrors).toEqual([loadError]) + expect(ownerErrors).toEqual([]) expect(unhandled).toEqual([]) expect({ inbound: internals.inboundRemoteSubsetAcquisitions.size, @@ -3637,7 +4018,7 @@ describe(`BrowserCollectionCoordinator`, () => { } }) - it(`reports a failed Browser replay once without self-retrying`, async () => { + it(`contains a failed Browser replay without fail-stopping the owner`, async () => { const coordinator = createCoordinator() coordinator.subscribe(`todos`, () => {}) await flush(50) @@ -3689,7 +4070,7 @@ describe(`BrowserCollectionCoordinator`, () => { unhandled, }).toEqual({ attempts: 1, - ownerErrors: [replayError], + ownerErrors: [], acquiredLeaderId: `retired-browser-leader`, inFlight: null, unhandled: [], @@ -4353,6 +4734,66 @@ describe(`BrowserCollectionCoordinator`, () => { } }) + it.each([ + { failureMode: `synchronous throw` as const }, + { failureMode: `asynchronous rejection` as const }, + ])( + `retires a terminal owner release after reporting one $failureMode`, + async ({ failureMode }) => { + // Previous cleanup laws observed and reported unload rejection, but + // never repeated the same terminal caller release. The owner receives + // one release attempt; after it reports failure the logical demand is + // terminal and must not replay or accrue retry debt here. + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + const unloadError = new Error(`transient Browser owner unload failure`) + const ownerErrors: Array = [] + let unloadAttempts = 0 + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(() => { + unloadAttempts++ + if (unloadAttempts !== 1) return undefined + if (failureMode === `synchronous throw`) throw unloadError + return Promise.reject(unloadError) + }), + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { offset: 41 } + const internals = coordinator as unknown as { + outboundRemoteSubsetAcquisitions: Map + replayRemoteSubsetAcquisitions: ( + collectionId: string, + ) => Promise + } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, options) + await expect( + coordinator.requestReleaseRemoteSubset(`todos`, options), + ).rejects.toBe(unloadError) + + await expect( + coordinator.requestReleaseRemoteSubset(`todos`, options), + ).resolves.toBeUndefined() + await internals.replayRemoteSubsetAcquisitions(`todos`) + expect({ unloadAttempts, ownerErrors }).toEqual({ + unloadAttempts: 1, + ownerErrors: [unloadError], + }) + expect(internals.outboundRemoteSubsetAcquisitions.size).toBe(0) + expect(owner).toHaveBeenCalledTimes(1) + } finally { + unregisterOwner() + coordinator.dispose() + } + }, + ) + it(`reports local Browser disposal unload rejection once and completes cleanup`, async () => { const coordinator = createCoordinator() coordinator.subscribe(`todos`, () => {}) diff --git a/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts b/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts index 33774b6e82..737425f1d1 100644 --- a/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts +++ b/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts @@ -10,10 +10,7 @@ import { toProcessLocalLoadSubsetOptions, toTransportedLoadSubsetOptions, } from './remote-subset-wire' -import { - reportRemoteSubsetOwnerError, - unloadRemoteSubsetOwner, -} from './remote-subset-owner' +import { unloadRemoteSubsetOwner } from './remote-subset-owner' import type { LoadSubsetOptions } from '@tanstack/db' import type { IndeterminateCommitRequestType } from './errors' import type { TransportedLoadSubsetOptions } from './remote-subset-wire' @@ -110,6 +107,7 @@ type CollectionState = { latestSeq: number latestRowVersion: number subscribers: Set<(message: ProtocolEnvelope) => void> + routeWaiters?: Set<() => void> } // Adapter capabilities used by coordinator-side operations @@ -172,10 +170,12 @@ type OutboundRemoteSubsetAcquisition = { collectionId: string acquisitionId: string options: TransportedLoadSubsetOptions - localOptions: TransportedLoadSubsetOptions + localOptions: TransportedLoadSubsetOptions & + Pick acquiredLeaderId: string | null inFlight: Promise | null release: Promise | null + releaseRequested?: boolean forceReplay: boolean retryTimer: ReturnType | null retryAttempts: number @@ -254,6 +254,11 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi return this.disposed } + /** Method indirection to re-read leadership participation across awaits. */ + private participatesInLeadership(state: CollectionState): boolean { + return state.participatesInLeadership + } + private requireAdapter(collectionId: string): CoordinatorAdapter { const adapter = this.collectionAdapters.get(collectionId) ?? this.defaultAdapter @@ -329,6 +334,10 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi state.subscribers.add(onMessage) return () => { state.subscribers.delete(onMessage) + if (state.subscribers.size === 0 && state.participatesInLeadership) { + state.participatesInLeadership = false + this.releaseLeadership(collectionId, state) + } } } @@ -377,6 +386,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi acquiredLeaderId: null, inFlight: null, release: null, + releaseRequested: false, forceReplay: false, retryTimer: null, retryAttempts: 0, @@ -387,7 +397,27 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi ) } - await this.acquireRemoteSubset(acquisition) + try { + await this.acquireRemoteSubset(acquisition) + } catch (error) { + if (options.signal?.aborted) { + acquisition.releaseRequested = true + this.cancelRemoteSubsetReplayRetry(acquisition) + // The route-pending case owns no lease, while a concurrent response + // loss may have reached an owner. A best-effort terminal release is + // harmless for the former and prevents an orphan in the latter. + this.postRemoteSubsetRelease(acquisition) + const key = remoteSubsetAcquisitionKey( + collectionId, + acquisition.acquisitionId, + ) + if (this.outboundRemoteSubsetAcquisitions.get(key) === acquisition) { + this.outboundRemoteSubsetAcquisitions.delete(key) + collectionIds.delete(options) + } + } + throw error + } } async requestReleaseRemoteSubset( @@ -400,6 +430,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi const key = remoteSubsetAcquisitionKey(collectionId, acquisitionId) const acquisition = this.outboundRemoteSubsetAcquisitions.get(key) if (!acquisition) return + acquisition.releaseRequested = true this.cancelRemoteSubsetReplayRetry(acquisition) if (!acquisition.release) { acquisition.release = (async () => { @@ -411,23 +442,35 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi rpcId: safeRandomUUID(), acquisitionId, } - const response = this.isLeader(collectionId) - ? await this.handleReleaseRemoteSubset( - collectionId, - request, - this.nodeId, - ) - : await this.sendRPC( - collectionId, - request, - ) - - if (!response.ok) { - throw new Error(`releaseRemoteSubset failed: ${response.error}`) - } - if (this.outboundRemoteSubsetAcquisitions.get(key) === acquisition) { - this.outboundRemoteSubsetAcquisitions.delete(key) - collectionIds!.delete(options) + const localLeader = this.isLeader(collectionId) + let releaseReachedLeader = localLeader + try { + const response = localLeader + ? await this.handleReleaseRemoteSubset( + collectionId, + request, + this.nodeId, + ) + : await this.sendRPC( + collectionId, + request, + ) + releaseReachedLeader = true + if (!response.ok) { + throw new Error(`releaseRemoteSubset failed: ${response.error}`) + } + } finally { + // A response (including an owner unload failure) proves that the + // release reached the leader. Logical demand then retires exactly + // once. Only a transport failure with no response retains retry + // debt because delivery remains unknown. + if ( + releaseReachedLeader && + this.outboundRemoteSubsetAcquisitions.get(key) === acquisition + ) { + this.outboundRemoteSubsetAcquisitions.delete(key) + collectionIds!.delete(options) + } } })() } @@ -461,6 +504,18 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi acquisitionId: acquisition.acquisitionId, options: acquisition.options, } + const state = this.collections.get(acquisition.collectionId) + if ( + state?.participatesInLeadership && + !state.isLeader && + state.leaderId === null + ) { + await this.waitForLeadershipRoute( + acquisition.collectionId, + state, + acquisition.localOptions.signal, + ) + } route.localOwner = this.isLeader(acquisition.collectionId) let response: EnsureRemoteSubsetResponse try { @@ -514,6 +569,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi ) if ( acquired && + !acquisition.releaseRequested && this.outboundRemoteSubsetAcquisitions.get(key) === acquisition && (acquisition.forceReplay || (currentLeaderId !== null && @@ -671,6 +727,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi latestSeq: 0, latestRowVersion: 0, subscribers: new Set(), + routeWaiters: new Set(), } this.collections.set(collectionId, state) } @@ -690,10 +747,13 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi collectionId: string, state: CollectionState, ): Promise { - if (this.disposed || state.isLeader) return + if (this.disposed || !state.participatesInLeadership || state.isLeader) { + return + } const lockName = `tsdb:leader:${this.dbName}:${collectionId}` const abortController = new AbortController() + state.lockAbortController?.abort() state.lockAbortController = abortController try { @@ -713,9 +773,19 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi state.latestRowVersion = pos.latestRowVersion } + if ( + this.isDisposed() || + !this.participatesInLeadership(state) || + abortController.signal.aborted || + state.lockAbortController !== abortController + ) { + return + } + state.latestTerm++ state.isLeader = true state.leaderId = this.nodeId + this.notifyRouteWaiters(state) this.emitHeartbeat(collectionId, state) void this.replayRemoteSubsetAcquisitions(collectionId) @@ -751,17 +821,73 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi return } console.warn(`Failed to acquire leadership for ${collectionId}:`, error) - if (!this.isDisposed()) { + if (!this.isDisposed() && this.participatesInLeadership(state)) { await sleep(LEADERSHIP_RETRY_DELAY_MS) } + } finally { + if (state.lockAbortController === abortController) { + state.lockAbortController = null + } } // Re-acquire if not disposed (leadership was released by another means) - if (!this.isDisposed()) { + if ( + !this.isDisposed() && + this.participatesInLeadership(state) && + state.lockAbortController === null + ) { void this.acquireLeadership(collectionId, state) } } + private waitForLeadershipRoute( + collectionId: string, + state: CollectionState, + signal?: AbortSignal, + ): Promise { + if (state.isLeader || state.leaderId !== null) return Promise.resolve() + + return new Promise((resolve, reject) => { + const finish = () => { + state.routeWaiters?.delete(onRouteChange) + signal?.removeEventListener(`abort`, onAbort) + } + const onAbort = () => { + finish() + reject( + new DOMException(`Remote subset acquisition aborted`, `AbortError`), + ) + } + const onRouteChange = () => { + if (this.isDisposed() || !state.participatesInLeadership) { + finish() + reject( + new Error( + `${this.coordinatorName}: leadership route closed for collection "${collectionId}"`, + ), + ) + return + } + if (state.isLeader || state.leaderId !== null) { + finish() + resolve() + } + } + + ;(state.routeWaiters ??= new Set()).add(onRouteChange) + signal?.addEventListener(`abort`, onAbort, { once: true }) + if (signal?.aborted) { + onAbort() + } else { + onRouteChange() + } + }) + } + + private notifyRouteWaiters(state: CollectionState): void { + for (const notify of Array.from(state.routeWaiters ?? [])) notify() + } + private releaseLeadership( collectionId: string, state: CollectionState, @@ -777,6 +903,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi } state.isLeader = false state.leaderId = null + this.notifyRouteWaiters(state) } private postRemoteSubsetRelease( @@ -833,6 +960,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi !(error instanceof RetryableRemoteSubsetAcquisitionError) || this.isDisposed() || this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition || + acquisition.releaseRequested || acquisition.retryTimer !== null || acquisition.retryAttempts >= REMOTE_SUBSET_REPLAY_RETRY_ATTEMPTS ) { @@ -844,7 +972,8 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi acquisition.retryTimer = null if ( this.isDisposed() || - this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition + this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition || + acquisition.releaseRequested ) { return } @@ -866,7 +995,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { if ( acquisition.collectionId !== collectionId || - acquisition.release || + acquisition.releaseRequested || (!acquisition.forceReplay && acquisition.acquiredLeaderId === leaderId) ) { continue @@ -943,6 +1072,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi state.latestRowVersion, heartbeat.latestRowVersion, ) + this.notifyRouteWaiters(state) if (changedLeader) { void this.replayRemoteSubsetAcquisitions(envelope.collectionId) } @@ -1228,49 +1358,14 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi ) } - const acquisition: ActiveRemoteSubsetAcquisition = { + await this.startRemoteSubsetOwnerAcquisition({ collectionId, requesterId, acquisitionId: request.acquisitionId, owner, options: awaitingOwner?.options ?? localOptions ?? request.options, - load: Promise.resolve(), - transferred: false, - released: false, - terminalRelease: false, - release: null, - } - this.releasedRemoteSubsetAcquisitionTimes.delete(key) - this.inboundRemoteSubsetAcquisitions.set(key, acquisition) - let resolveLoad!: () => void - let rejectLoad!: (error: unknown) => void - acquisition.load = new Promise((resolve, reject) => { - resolveLoad = resolve - rejectLoad = reject + restoreOnSynchronousFailure: awaitingOwner, }) - try { - const load = owner(acquisition.options) - acquisition.transferred = true - void Promise.resolve(load).then(resolveLoad, rejectLoad) - } catch (error) { - rejectLoad(error) - } - try { - await acquisition.load - } catch (error) { - if ( - !acquisition.transferred && - this.inboundRemoteSubsetAcquisitions.get(key) === acquisition - ) { - if (awaitingOwner) { - this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) - } else { - this.inboundRemoteSubsetAcquisitions.delete(key) - } - } - reportRemoteSubsetOwnerError(owner, error) - throw error - } return { type: `rpc:ensureRemoteSubset:res`, rpcId: request.rpcId, @@ -1300,14 +1395,17 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi }) } else if (`owner` in acquisition) { acquisition.terminalRelease = true - await this.releaseRemoteSubsetAcquisition(acquisition) - if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { - this.setReleasedRemoteSubsetAcquisition(key, { - collectionId, - requesterId, - acquisitionId: request.acquisitionId, - released: true, - }) + try { + await this.releaseRemoteSubsetAcquisition(acquisition) + } finally { + if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { + this.setReleasedRemoteSubsetAcquisition(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } } } else if (`awaitingOwner` in acquisition) { this.setReleasedRemoteSubsetAcquisition(key, { @@ -1466,18 +1564,42 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi ) if (this.inboundRemoteSubsetAcquisitions.get(key) !== awaitingOwner) return - const acquisition: ActiveRemoteSubsetAcquisition = { + await this.startRemoteSubsetOwnerAcquisition({ collectionId: awaitingOwner.collectionId, requesterId: awaitingOwner.requesterId, acquisitionId: awaitingOwner.acquisitionId, owner, options: awaitingOwner.options, + restoreOnSynchronousFailure: awaitingOwner, + }) + } + + private async startRemoteSubsetOwnerAcquisition(options: { + collectionId: string + requesterId: string + acquisitionId: string + owner: RemoteSubsetOwner + options: TransportedLoadSubsetOptions + restoreOnSynchronousFailure?: AwaitingRemoteSubsetOwnerAcquisition + }): Promise { + const key = inboundRemoteSubsetAcquisitionKey( + options.collectionId, + options.requesterId, + options.acquisitionId, + ) + const acquisition: ActiveRemoteSubsetAcquisition = { + collectionId: options.collectionId, + requesterId: options.requesterId, + acquisitionId: options.acquisitionId, + owner: options.owner, + options: options.options, load: Promise.resolve(), transferred: false, released: false, terminalRelease: false, release: null, } + this.releasedRemoteSubsetAcquisitionTimes.delete(key) this.inboundRemoteSubsetAcquisitions.set(key, acquisition) let resolveLoad!: () => void let rejectLoad!: (error: unknown) => void @@ -1486,22 +1608,35 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi rejectLoad = reject }) try { - const load = owner(acquisition.options) + const load = options.owner(acquisition.options) acquisition.transferred = true void Promise.resolve(load).then(resolveLoad, rejectLoad) } catch (error) { rejectLoad(error) } + try { await acquisition.load } catch (error) { - if ( - !acquisition.transferred && - this.inboundRemoteSubsetAcquisitions.get(key) === acquisition - ) { - this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + if (!acquisition.transferred) { + if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { + if (options.restoreOnSynchronousFailure) { + this.inboundRemoteSubsetAcquisitions.set( + key, + options.restoreOnSynchronousFailure, + ) + } else { + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } + } else { + // A returned promise transfers a physical lease even when it rejects. + // Retire that failed lease before allowing the same logical demand to + // retry against the current owner. + await this.releaseRemoteSubsetAcquisition(acquisition).catch( + () => undefined, + ) } - reportRemoteSubsetOwnerError(owner, error) throw error } } @@ -1539,48 +1674,30 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi }, ): Promise { const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) - const appliedEnvelope = this.appliedEnvelopes.get(envelopeKey) - if (appliedEnvelope?.requestType === `rpc:applyLocalMutations:req`) { - return { ...appliedEnvelope.response, rpcId: request.rpcId } - } - if (appliedEnvelope) { - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: false, - code: `CONFLICT`, - error: `envelope ${request.envelopeId} already applied`, - } - } - - const inFlightEnvelope = this.inFlightEnvelopes.get(envelopeKey) - if (inFlightEnvelope?.requestType === `rpc:applyLocalMutations:req`) { - const response = await inFlightEnvelope.response - return { ...response, rpcId: request.rpcId } - } - if (inFlightEnvelope) { - return { + return this.runDeduplicatedEnvelope({ + envelopeKey, + rpcId: request.rpcId, + selectApplied: (envelope) => + envelope.requestType === `rpc:applyLocalMutations:req` + ? envelope.response + : undefined, + selectInFlight: (envelope) => + envelope.requestType === `rpc:applyLocalMutations:req` + ? envelope.response + : undefined, + conflict: (phase) => ({ type: `rpc:applyLocalMutations:res`, rpcId: request.rpcId, ok: false, code: `CONFLICT`, - error: `envelope ${request.envelopeId} is already in flight`, - } - } - - const response = this.applyLocalMutationsOnce(collectionId, request) - const pendingEnvelope: InFlightEnvelope = { - requestType: request.type, - response, - } - this.inFlightEnvelopes.set(envelopeKey, pendingEnvelope) - try { - return await response - } finally { - if (this.inFlightEnvelopes.get(envelopeKey) === pendingEnvelope) { - this.inFlightEnvelopes.delete(envelopeKey) - } - } + error: `envelope ${request.envelopeId} ${phase}`, + }), + apply: () => this.applyLocalMutationsOnce(collectionId, request), + createPending: (response) => ({ + requestType: request.type, + response, + }), + }) } private async applyLocalMutationsOnce( @@ -1627,16 +1744,11 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi rowMetadataMutations, } - let tx: PersistedTx - try { - tx = await this.applyDurablyAtNextStreamPosition( - collectionId, - state, - pendingTx, - ) - } catch (error) { - throw toPersistedCollectionDurabilityError(collectionId, error) - } + const tx = await this.applyDurablyAtNextStreamPosition( + collectionId, + state, + pendingTx, + ) const response: ApplyLocalMutationsResponse = { type: `rpc:applyLocalMutations:res`, @@ -1701,46 +1813,69 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi request: Extract, ): Promise { const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) - const appliedEnvelope = this.appliedEnvelopes.get(envelopeKey) - if (appliedEnvelope?.requestType === `rpc:applyCommittedTx:req`) { - return { ...appliedEnvelope.response, rpcId: request.rpcId } - } - if (appliedEnvelope) { - return { + return this.runDeduplicatedEnvelope({ + envelopeKey, + rpcId: request.rpcId, + selectApplied: (envelope) => + envelope.requestType === `rpc:applyCommittedTx:req` + ? envelope.response + : undefined, + selectInFlight: (envelope) => + envelope.requestType === `rpc:applyCommittedTx:req` + ? envelope.response + : undefined, + conflict: (phase) => ({ type: `rpc:applyCommittedTx:res`, rpcId: request.rpcId, ok: false, code: `CONFLICT`, - error: `envelope ${request.envelopeId} already applied`, - } - } + error: `envelope ${request.envelopeId} ${phase}`, + }), + apply: () => this.applyCommittedTxOnce(collectionId, request), + createPending: (response) => ({ + requestType: request.type, + response, + }), + }) + } - const inFlightEnvelope = this.inFlightEnvelopes.get(envelopeKey) - if (inFlightEnvelope?.requestType === `rpc:applyCommittedTx:req`) { - const response = await inFlightEnvelope.response - return { ...response, rpcId: request.rpcId } + private async runDeduplicatedEnvelope< + TResponse extends { rpcId: string }, + >(options: { + envelopeKey: string + rpcId: string + selectApplied: (envelope: AppliedEnvelope) => TResponse | undefined + selectInFlight: ( + envelope: InFlightEnvelope, + ) => Promise | undefined + conflict: (phase: `already applied` | `is already in flight`) => TResponse + apply: () => Promise + createPending: (response: Promise) => InFlightEnvelope + }): Promise { + const appliedEnvelope = this.appliedEnvelopes.get(options.envelopeKey) + if (appliedEnvelope) { + const response = options.selectApplied(appliedEnvelope) + return response + ? { ...response, rpcId: options.rpcId } + : options.conflict(`already applied`) } + + const inFlightEnvelope = this.inFlightEnvelopes.get(options.envelopeKey) if (inFlightEnvelope) { - return { - type: `rpc:applyCommittedTx:res`, - rpcId: request.rpcId, - ok: false, - code: `CONFLICT`, - error: `envelope ${request.envelopeId} is already in flight`, - } + const pending = options.selectInFlight(inFlightEnvelope) + if (!pending) return options.conflict(`is already in flight`) + const response = await pending + return { ...response, rpcId: options.rpcId } } - const response = this.applyCommittedTxOnce(collectionId, request) - const pendingEnvelope: InFlightEnvelope = { - requestType: request.type, - response, - } - this.inFlightEnvelopes.set(envelopeKey, pendingEnvelope) + const response = options.apply() + const pendingEnvelope = options.createPending(response) + this.inFlightEnvelopes.set(options.envelopeKey, pendingEnvelope) try { return await response } finally { - if (this.inFlightEnvelopes.get(envelopeKey) === pendingEnvelope) { - this.inFlightEnvelopes.delete(envelopeKey) + if (this.inFlightEnvelopes.get(options.envelopeKey) === pendingEnvelope) { + this.inFlightEnvelopes.delete(options.envelopeKey) } } } @@ -1760,16 +1895,11 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi } } - let tx: PersistedTx - try { - tx = await this.applyDurablyAtNextStreamPosition( - collectionId, - state, - request.tx, - ) - } catch (error) { - throw toPersistedCollectionDurabilityError(collectionId, error) - } + const tx = await this.applyDurablyAtNextStreamPosition( + collectionId, + state, + request.tx, + ) const response: ApplyCommittedTxResponse = { type: `rpc:applyCommittedTx:res`, rpcId: request.rpcId, @@ -1846,7 +1976,12 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi seq: state.latestSeq + 1, rowVersion: state.latestRowVersion + 1, } - await this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx) + const adapter = this.requireAdapter(collectionId) + try { + await adapter.applyCommittedTx(collectionId, tx) + } catch (error) { + throw toPersistedCollectionDurabilityError(collectionId, error) + } state.latestSeq = tx.seq state.latestRowVersion = tx.rowVersion return tx diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index dc787b3196..4f714fd29c 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -1342,6 +1342,13 @@ class PersistedCollectionRuntime< if (lifecycleGeneration !== this.lifecycleGeneration) return await this.waitForAppliedReceiptsAfter(appliedCursor) + if ( + options.signal?.aborted || + this.activeSubsets.get(this.getSubsetKey(options)) !== options + ) { + return + } + if (this.canRouteRemoteDemandThroughCoordinator()) { try { await this.persistence.coordinator.requestEnsureRemoteSubset( @@ -2030,6 +2037,18 @@ class PersistedCollectionRuntime< ) if (!response.ok) { + if (response.code === `PERSISTENCE_ERROR`) { + const error = new PersistedCollectionDurabilityError( + `Failed to durably persist collection "${this.collectionId}": ${response.error}`, + { + cause: response, + code: response.sourceCode ?? response.code, + path: response.path, + }, + ) + this.reportSyncError(error) + throw error + } throw new Error( `failed to apply local mutations through coordinator: ${response.error}`, ) @@ -2227,6 +2246,7 @@ class PersistedCollectionRuntime< private queueRemoteSubsetEnsure(options: LoadSubsetOptions): void { if ( + options.signal?.aborted || !this.canRouteRemoteDemandThroughCoordinator() || this.activeSubsets.get(this.getSubsetKey(options)) !== options ) { @@ -2272,6 +2292,13 @@ class PersistedCollectionRuntime< } for (const [subsetKey, options] of this.pendingRemoteSubsetEnsures) { + if ( + options.signal?.aborted || + this.activeSubsets.get(subsetKey) !== options + ) { + this.pendingRemoteSubsetEnsures.delete(subsetKey) + continue + } try { await this.persistence.coordinator.requestEnsureRemoteSubset( this.collectionId, @@ -2279,7 +2306,14 @@ class PersistedCollectionRuntime< ) this.pendingRemoteSubsetEnsures.delete(subsetKey) } catch (error) { - console.warn(`Failed to ensure remote subset:`, error) + if ( + options.signal?.aborted || + this.activeSubsets.get(subsetKey) !== options + ) { + this.pendingRemoteSubsetEnsures.delete(subsetKey) + } else { + console.warn(`Failed to ensure remote subset:`, error) + } } } diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 40223ca225..1c50bfb5b1 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -3179,6 +3179,93 @@ describe(`persistedCollectionOptions`, () => { } }) + it.each([ + { + action: `retain` as const, + expectedEnsures: 1, + expectedReleases: 0, + }, + { + action: `unload` as const, + expectedEnsures: 0, + expectedReleases: 1, + }, + { + action: `abort` as const, + expectedEnsures: 0, + expectedReleases: 0, + }, + ])( + `acquires coordinator demand after hydration iff it remains active: $action`, + async ({ action, expectedEnsures, expectedReleases }) => { + const adapter = createRecordingAdapter() + const hydrate = adapter.loadSubset + let enterHydration!: () => void + let finishHydration!: () => void + const entered = new Promise((resolve) => { + enterHydration = resolve + }) + const gate = new Promise((resolve) => { + finishHydration = resolve + }) + adapter.loadSubset = async (...args) => { + enterHydration() + await gate + return hydrate(...args) + } + const ensure = vi.fn(async () => {}) + const release = vi.fn(async () => {}) + const coordinator = createCoordinatorHarness() + coordinator.isLeader = () => false + coordinator.requestEnsureRemoteSubset = ensure + coordinator.requestReleaseRemoteSubset = release + const collection = createCollection( + persistedCollectionOptions({ + id: `coordinator-hydration-demand-${action}`, + syncMode: `on-demand`, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + markReady() + return {} + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const controller = new AbortController() + const options: LoadSubsetOptions = { + limit: 1, + signal: controller.signal, + } + + try { + collection.startSyncImmediate() + const pending = collection._sync.loadSubset(options) + await entered + if (action === `unload`) collection._sync.unloadSubset(options) + if (action === `abort`) controller.abort() + finishHydration() + await pending + + // The earlier history law cancelled only after remote acquisition had + // begun. Holding the real adapter hydration seam exposes the distinct + // post-hydration transfer boundary. `retain` is the hostile control: + // the cancellation guard must not suppress live demand. + expect({ + ensures: ensure.mock.calls.length, + releases: release.mock.calls.length, + }).toEqual({ + ensures: expectedEnsures, + releases: expectedReleases, + }) + } finally { + finishHydration() + await collection.cleanup() + } + }, + ) + it(`hydrates a multiprocess follower locally while only the elected owner acquires remote demand`, async () => { const adapter = createRecordingAdapter([ { id: `persisted`, title: `Persisted follower row` }, @@ -3280,7 +3367,7 @@ describe(`persistedCollectionOptions`, () => { } }) - it.each([`abort`, `release`, `offline`] as const)( + it.each([`abort`, `signal-abort`, `release`, `offline`] as const)( `handles remote ensure after %s without resurrecting cancelled demand`, async (action) => { vi.useFakeTimers() @@ -3331,7 +3418,11 @@ describe(`persistedCollectionOptions`, () => { persistence: { adapter: createRecordingAdapter(), coordinator }, }), ) - const options = { limit: 1 } + const controller = new AbortController() + const options: LoadSubsetOptions = { + limit: 1, + ...(action === `signal-abort` ? { signal: controller.signal } : {}), + } try { collection.startSyncImmediate() const result = await Promise.resolve( @@ -3341,8 +3432,12 @@ describe(`persistedCollectionOptions`, () => { (error: unknown) => error, ) if (action === `release`) collection._sync.unloadSubset(options) + if (action === `signal-abort`) controller.abort() const callsBeforeRetry = ensure.mock.calls.length await vi.advanceTimersByTimeAsync(200) + // The old matrix supplied an AbortError or explicitly unloaded demand, + // but never aborted an already-queued retry. The signal row owns that + // distinct boundary; offline is its hostile liveness control. expect(result).toBe(failure) expect(followerUpstreamLoad).not.toHaveBeenCalled() if (action === `offline`) { @@ -3663,6 +3758,112 @@ describe(`persistedCollectionOptions`, () => { expect(collection.get(`ack-mismatch`)).toBeUndefined() }) + it.each([ + { + boundary: `follower` as const, + responseCode: `PERSISTENCE_ERROR` as const, + expectedDurabilityError: true, + }, + { + boundary: `elected-leader` as const, + responseCode: `PERSISTENCE_ERROR` as const, + expectedDurabilityError: true, + }, + { + boundary: `follower` as const, + responseCode: `NOT_LEADER` as const, + expectedDurabilityError: false, + }, + { + boundary: `follower` as const, + responseCode: `VALIDATION_ERROR` as const, + expectedDurabilityError: false, + }, + { + boundary: `follower` as const, + responseCode: `CONFLICT` as const, + expectedDurabilityError: false, + }, + { + boundary: `follower` as const, + responseCode: `TIMEOUT` as const, + expectedDurabilityError: false, + }, + ])( + `classifies public local-mutation failure at the $boundary boundary: $responseCode`, + async ({ boundary, responseCode, expectedDurabilityError }) => { + const persistenceResponse = + responseCode === `PERSISTENCE_ERROR` + ? { + type: `rpc:applyLocalMutations:res` as const, + rpcId: `local-${boundary}-persistence-response`, + ok: false as const, + code: `PERSISTENCE_ERROR` as const, + error: `disk write failed`, + sourceCode: `SQLITE_IOERR_FSYNC`, + path: [`database`, `wal`] as const, + } + : { + type: `rpc:applyLocalMutations:res` as const, + rpcId: `local-${boundary}-failure-response`, + ok: false as const, + code: responseCode, + error: `mutation route changed`, + } + const coordinator = createCoordinatorHarness() + coordinator.isLeader = () => boundary === `elected-leader` + coordinator.requestApplyLocalMutations = () => + Promise.resolve(persistenceResponse) + const collection = createCollection( + persistedCollectionOptions({ + id: `local-mutation-${boundary}-${responseCode}`, + getKey: (item) => item.id, + persistence: { + adapter: createRecordingAdapter(), + coordinator, + }, + }), + ) + + try { + const tx = collection.insert({ + id: `local-failure`, + title: `Must not persist`, + }) + const rejection = await tx.isPersisted.promise.then( + () => undefined, + (error: unknown) => error, + ) + + // Earlier laws covered external-sync commit classification and a + // successful follower mutation acknowledgement. They never varied a + // serialized applyLocalMutations failure by route. The elected-leader + // and non-durability rows are hostile controls for topology parity and + // for avoiding an over-broad fail-stop classifier. + if (expectedDurabilityError) { + expect(rejection).toMatchObject({ + name: `PersistedCollectionDurabilityError`, + code: `SQLITE_IOERR_FSYNC`, + path: [`database`, `wal`], + cause: persistenceResponse, + }) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(rejection) + } else { + expect(rejection).toMatchObject({ + name: `Error`, + message: + `failed to apply local mutations through coordinator: ` + + `mutation route changed`, + }) + expect(collection._lifecycle.getSyncError()).toBeUndefined() + } + } finally { + await collection.cleanup() + } + }, + ) + it(`targeted update avoids full loadSubset call`, async () => { const adapter = createRecordingAdapter([{ id: `1`, title: `Original` }]) const coordinator = createCoordinatorHarness() diff --git a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts index 8db1f436b5..95fde9b1d0 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts @@ -1854,6 +1854,72 @@ describe(`electron sqlite persistence bridge`, () => { } }) + it(`retires an aborted route-pending Electron subset request before transport or replay`, async () => { + // Route-pending was absent from the earlier Electron acquisition grammar: + // every generated request began after a leader was known. Exercise the + // shared engine through the Electron wrapper and prove abort retires the + // logical demand without transport, owner work, or later replay debt. + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-route-pending-abort`, + }) + registerCleanup(() => coordinator.dispose()) + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const internals = coordinator as unknown as { + acquireLeadership: () => Promise + sendRPCOnce: (collectionId: string, request: unknown) => Promise + collections: Map void> }> + outboundRemoteSubsetAcquisitions: Map + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + internals.acquireLeadership = vi.fn(() => Promise.resolve()) + const prematureTransport = vi.fn(() => + Promise.reject(new Error(`no Electron leader route yet`)), + ) + internals.sendRPCOnce = prematureTransport + const unsubscribe = coordinator.subscribe(`todos`, () => {}) + const abortController = new AbortController() + const options: LoadSubsetOptions = { + limit: 1, + signal: abortController.signal, + } + + try { + const request = coordinator.requestEnsureRemoteSubset(`todos`, options) + await Promise.resolve() + abortController.abort() + + await expect(request).rejects.toMatchObject({ name: `AbortError` }) + await internals.replayRemoteSubsetAcquisitions(`todos`) + expect({ + ownerCalls: owner.mock.calls.length, + transportCalls: prematureTransport.mock.calls.length, + outbound: internals.outboundRemoteSubsetAcquisitions.size, + routeWaiters: + internals.collections.get(`todos`)?.routeWaiters?.size ?? 0, + }).toEqual({ + ownerCalls: 0, + transportCalls: 0, + outbound: 0, + routeWaiters: 0, + }) + await expect( + coordinator.requestReleaseRemoteSubset(`todos`, options), + ).resolves.toBeUndefined() + } finally { + abortController.abort() + unsubscribe() + unregisterOwner() + coordinator.dispose() + } + }) + it(`holds same-stack Electron subset reentry behind the original owner load`, async () => { const coordinator = new ElectronCollectionCoordinator({ dbName: `electron-subset-same-stack`, @@ -1990,75 +2056,94 @@ describe(`electron sqlite persistence bridge`, () => { } }) - it(`releases a transferred Electron lease whose initial load rejected`, async () => { - const coordinator = new ElectronCollectionCoordinator({ - dbName: `electron-subset-rejected-transfer`, - adapter: createElectronCoordinatorTestAdapter(), - }) - registerCleanup(() => coordinator.dispose()) - coordinator.isLeader = () => true - const loadError = new Error(`electron transferred load failed`) - const ownerErrors: Array = [] - const owner = Object.assign( - vi.fn((_options: TransportedLoadSubsetOptions) => - Promise.reject(loadError), - ), - { - unloadSubset: vi.fn( - (_options: TransportedLoadSubsetOptions) => undefined, - ), - onError: (error: unknown) => ownerErrors.push(error), - }, - ) - const unregisterOwner = coordinator.registerRemoteSubsetOwner( - `todos`, - owner, - ) - const options: LoadSubsetOptions = { offset: 20 } - const unhandled: Array = [] - const onUnhandled = (error: unknown) => unhandled.push(error) - process.on(`unhandledRejection`, onUnhandled) - const internals = coordinator as unknown as { - inboundRemoteSubsetAcquisitions: Map> - } + it.each([ + { failureMode: `before lease transfer` as const, expectedUnloads: 1 }, + { failureMode: `after lease transfer` as const, expectedUnloads: 2 }, + ])( + `retires a rejected Electron owner load $failureMode and permits a fresh retry`, + async ({ failureMode, expectedUnloads }) => { + // The previous grammar generated only fulfilled owner loads. Both sides + // of the promise-return lease-transfer boundary must permit a fresh + // attempt without caching the rejected acquisition. + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-rejected-transfer-${failureMode}`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const loadError = new Error(`electron transferred load failed`) + const ownerErrors: Array = [] + let loadCalls = 0 + const owner = Object.assign( + vi.fn((_options: TransportedLoadSubsetOptions) => { + loadCalls++ + if (loadCalls !== 1) return Promise.resolve() + if (failureMode === `before lease transfer`) throw loadError + return Promise.reject(loadError) + }), + { + unloadSubset: vi.fn( + (_options: TransportedLoadSubsetOptions) => undefined, + ), + onError: (error: unknown) => ownerErrors.push(error), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { offset: 20 } + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + inboundRemoteSubsetAcquisitions: Map> + } - try { - const ensureError = await coordinator - .requestEnsureRemoteSubset(`todos`, options) - .then( - () => undefined, - (error: unknown) => error, + try { + const ensureError = await coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then( + () => undefined, + (error: unknown) => error, + ) + await coordinator.requestEnsureRemoteSubset(`todos`, options) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + await new Promise((resolve) => setTimeout(resolve, 0)) + const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() + + expect(ensureError).toBe(loadError) + expect(owner).toHaveBeenCalledTimes(2) + expect(owner.unloadSubset).toHaveBeenCalledTimes(expectedUnloads) + if (failureMode === `after lease transfer`) { + expect(owner.unloadSubset.mock.calls[0]?.[0]).toBe( + owner.mock.calls[0]?.[0], + ) + } + expect(owner.unloadSubset.mock.calls.at(-1)?.[0]).toBe( + owner.mock.calls[1]?.[0], ) - await coordinator.requestReleaseRemoteSubset(`todos`, options) - await new Promise((resolve) => setTimeout(resolve, 0)) - const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() - - expect(ensureError).toBe(loadError) - expect(owner).toHaveBeenCalledTimes(1) - expect(owner.unloadSubset).toHaveBeenCalledTimes(1) - expect(owner.unloadSubset.mock.calls[0]?.[0]).toBe( - owner.mock.calls[0]?.[0], - ) - expect(ownerErrors).toEqual([loadError]) - expect(unhandled).toEqual([]) - expect({ - inbound: internals.inboundRemoteSubsetAcquisitions.size, - terminalKeys: Object.keys(terminal ?? {}).sort(), - }).toEqual({ - inbound: 1, - terminalKeys: [ - `acquisitionId`, - `collectionId`, - `released`, - `requesterId`, - ], - }) - } finally { - process.off(`unhandledRejection`, onUnhandled) - unregisterOwner() - coordinator.dispose() - } - }) + expect(ownerErrors).toEqual([]) + expect(unhandled).toEqual([]) + expect({ + inbound: internals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }, + ) it(`keeps an Electron release tombstone when a transferred load rejects concurrently`, async () => { const coordinator = new ElectronCollectionCoordinator({ @@ -2151,7 +2236,7 @@ describe(`electron sqlite persistence bridge`, () => { }) expect(owner).toHaveBeenCalledTimes(1) expect(owner.unloadSubset).toHaveBeenCalledTimes(1) - expect(ownerErrors).toEqual([loadError]) + expect(ownerErrors).toEqual([]) expect(unhandled).toEqual([]) expect({ inbound: internals.inboundRemoteSubsetAcquisitions.size, @@ -3102,7 +3187,7 @@ describe(`electron sqlite persistence bridge`, () => { } }) - it(`reports a failed Electron replay once without self-retrying`, async () => { + it(`contains a failed Electron replay without fail-stopping the owner`, async () => { const coordinator = new ElectronCollectionCoordinator({ dbName: `electron-subset-replay-error`, }) @@ -3160,7 +3245,7 @@ describe(`electron sqlite persistence bridge`, () => { unhandled, }).toEqual({ attempts: 1, - ownerErrors: [replayError], + ownerErrors: [], acquiredLeaderId: `retired-electron-leader`, inFlight: null, unhandled: [], @@ -3351,6 +3436,66 @@ describe(`electron sqlite persistence bridge`, () => { }) }) + it.each([ + { failureMode: `synchronous throw` as const }, + { failureMode: `asynchronous rejection` as const }, + ])( + `retires a terminal Electron owner release after reporting one $failureMode`, + async ({ failureMode }) => { + // Previous cleanup laws observed unload rejection, but never repeated the + // same caller release. A delivered terminal release retires the logical + // demand after one owner unload attempt, even when that attempt fails. + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-terminal-unload-${failureMode}`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const unloadError = new Error(`transient Electron owner unload failure`) + const ownerErrors: Array = [] + let unloadAttempts = 0 + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(() => { + unloadAttempts++ + if (unloadAttempts !== 1) return undefined + if (failureMode === `synchronous throw`) throw unloadError + return Promise.reject(unloadError) + }), + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { offset: 41 } + const internals = coordinator as unknown as { + outboundRemoteSubsetAcquisitions: Map + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, options) + await expect( + coordinator.requestReleaseRemoteSubset(`todos`, options), + ).rejects.toBe(unloadError) + + await expect( + coordinator.requestReleaseRemoteSubset(`todos`, options), + ).resolves.toBeUndefined() + await internals.replayRemoteSubsetAcquisitions(`todos`) + expect({ unloadAttempts, ownerErrors }).toEqual({ + unloadAttempts: 1, + ownerErrors: [unloadError], + }) + expect(internals.outboundRemoteSubsetAcquisitions.size).toBe(0) + expect(owner).toHaveBeenCalledTimes(1) + } finally { + unregisterOwner() + coordinator.dispose() + } + }, + ) + it(`reports owner unload rejection while completing sibling Electron cleanup`, async () => { const coordinator = new ElectronCollectionCoordinator({ dbName: `electron-subset-owner-unload-error`, From 365ee24282df562d14ec248112a5d3cf41c2755d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 15:15:01 +0100 Subject: [PATCH 17/18] test(sqlite): strengthen coordinator oracle campaigns --- .../per-collection-coordinator-oracle.test.ts | 466 +++++++++++++++--- 1 file changed, 404 insertions(+), 62 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts index 5e6bd28a6a..2157618c4a 100644 --- a/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts @@ -21,22 +21,90 @@ import type { import type { LoadSubsetOptions, Subscription, SyncConfig } from '@tanstack/db' import type { BrowserWASQLiteDatabase } from '../src' -const seedText = process.env.TANSTACK_DB_COORDINATOR_ORACLE_SEED ?? `165902` -const runText = process.env.TANSTACK_DB_COORDINATOR_ORACLE_RUNS ?? `12` -const SEED = Number(seedText) -const RUNS = Number(runText) -const PATH = process.env.TANSTACK_DB_COORDINATOR_ORACLE_PATH - -if (!/^-?\d+$/.test(seedText) || !Number.isSafeInteger(SEED)) { - throw new Error(`Invalid TANSTACK_DB_COORDINATOR_ORACLE_SEED`) +const FIXED_SEED = 165902 +const DEFAULT_RUNS = 12 + +type CoordinatorOracleProperty = `remote-lease` | `routing` +type CoordinatorOracleEnvironment = Record +type CoordinatorOracleReplay = { + property: CoordinatorOracleProperty + seed: number + path: string +} +type CoordinatorOracleCampaign = { + mode: `fixed` | `random` | `replay` + parameters: { + numRuns: number + seed?: number + path?: string + } } -if (!/^\d+$/.test(runText) || !Number.isSafeInteger(RUNS) || RUNS < 1) { - throw new Error(`Invalid TANSTACK_DB_COORDINATOR_ORACLE_RUNS`) + +function readCoordinatorOracleConfig( + environment: CoordinatorOracleEnvironment = process.env, +): { runs: number; replay?: CoordinatorOracleReplay } { + const runText = + environment.TANSTACK_DB_COORDINATOR_ORACLE_RUNS ?? String(DEFAULT_RUNS) + const runs = Number(runText) + if (!/^\d+$/.test(runText) || !Number.isSafeInteger(runs) || runs < 1) { + throw new Error(`Invalid TANSTACK_DB_COORDINATOR_ORACLE_RUNS`) + } + + const seedText = environment.TANSTACK_DB_COORDINATOR_ORACLE_SEED + const path = environment.TANSTACK_DB_COORDINATOR_ORACLE_PATH + const property = environment.TANSTACK_DB_COORDINATOR_ORACLE_PROPERTY + if (seedText === undefined && path === undefined && property === undefined) { + return { runs } + } + if (seedText === undefined || path === undefined || property === undefined) { + throw new Error( + `Coordinator oracle replay requires SEED, PATH, and PROPERTY together`, + ) + } + + const seed = Number(seedText) + if (!/^-?\d+$/.test(seedText) || !Number.isSafeInteger(seed)) { + throw new Error(`Invalid TANSTACK_DB_COORDINATOR_ORACLE_SEED`) + } + if (!/^\d+(?::\d+)*$/.test(path)) { + throw new Error( + `TANSTACK_DB_COORDINATOR_ORACLE_PATH must be a numeric shrink path`, + ) + } + if (property !== `remote-lease` && property !== `routing`) { + throw new Error( + `TANSTACK_DB_COORDINATOR_ORACLE_PROPERTY must be remote-lease or routing`, + ) + } + return { runs, replay: { property, seed, path } } } -if (PATH !== undefined && !/^\d+(?::\d+)*$/.test(PATH)) { - throw new Error( - `TANSTACK_DB_COORDINATOR_ORACLE_PATH must be a numeric shrink path`, - ) + +const { runs: RUNS, replay: REPLAY } = readCoordinatorOracleConfig() + +function coordinatorOracleCampaigns( + property: CoordinatorOracleProperty, +): Array { + if (REPLAY) { + return REPLAY.property === property + ? [ + { + mode: `replay`, + parameters: { + numRuns: RUNS, + seed: REPLAY.seed, + path: REPLAY.path, + }, + }, + ] + : [] + } + return [ + { + mode: `fixed`, + parameters: { numRuns: RUNS, seed: FIXED_SEED }, + }, + { mode: `random`, parameters: { numRuns: RUNS } }, + ] } /* @@ -107,9 +175,12 @@ Reach, challenge, replay, cleanup, and limits: the leader upstream fixture can really enter/complete, and the ownership checker rejects an unowned apply. Every production-path test records a reached checkpoint before comparing the independent ledger. -- The default is one fixed fast-check campaign. Replay with - TANSTACK_DB_COORDINATOR_ORACLE_{SEED,PATH,RUNS}; the thrown report retains the - first failing trace and final shrunk candidate. +- Each generated property uses the same generator, driver, observations, and + run budget in a retained fixed-seed lane and a seedless random lane. Replay + registers only the requested property when + TANSTACK_DB_COORDINATOR_ORACLE_{PROPERTY,SEED,PATH} are supplied together; + RUNS may adjust the shared budget. The thrown report retains the first + failing trace and final shrunk candidate. - Coordinators, Collections, and databases use failure-preserving cleanup. A final lifecycle test proves no channel, held lock, queued lock, or delayed delivery remains and that a fresh database name elects normally. @@ -1529,6 +1600,43 @@ const remoteLeaseHistoryArbitrary = fc.record({ reverseFinalRelease: fc.boolean(), }) +type RemoteLeaseGrammarPlan = { + demands: Array<{ limit: number; offset?: number }> + releaseCount: number + prefixReleaseIndexes: Array + finalReleaseIndexes: Array + releasePasses: number +} + +function remoteLeaseGrammarPlan( + history: RemoteLeaseHistory, +): RemoteLeaseGrammarPlan { + const demands = Array.from( + { length: history.siblingCount }, + (_unused, index) => ({ + limit: 1, + ...(history.identical ? {} : { offset: index }), + }), + ) + const releaseCount = Math.min(history.releasePrefix, history.siblingCount - 1) + const prefixReleaseIndexes = Array.from( + { length: releaseCount }, + (_unused, index) => index, + ) + const finalReleaseIndexes = Array.from( + { length: history.siblingCount - releaseCount }, + (_unused, index) => releaseCount + index, + ) + if (history.reverseFinalRelease) finalReleaseIndexes.reverse() + return { + demands, + releaseCount, + prefixReleaseIndexes, + finalReleaseIndexes, + releasePasses: history.duplicateRelease ? 2 : 1, + } +} + // Every accepted physical acquisition remains active until its exact // acquisition lease is released. Leadership transfer releases retired-owner // acquisitions and establishes replacements only for surviving demand. @@ -1536,7 +1644,7 @@ function remoteLeaseHistoryViolations( history: RemoteLeaseHistory, actual: RemoteLeaseHistoryTrace, ): Array { - const releaseCount = Math.min(history.releasePrefix, history.siblingCount - 1) + const { releaseCount } = remoteLeaseGrammarPlan(history) const remaining = history.siblingCount - releaseCount const expected: RemoteLeaseHistoryTrace = { firstAfterAcquire: { @@ -1913,7 +2021,71 @@ describe(`remote subset ownership lease oracle`, () => { ]) }) - it(`preserves independent sibling leases through release, retry-safe takeover, and cleanup histories`, async () => { + it(`reconstructs the remote-lease grammar and ablates every declared axis`, () => { + const witness: RemoteLeaseHistory = { + siblingCount: 3, + releasePrefix: 1, + identical: false, + duplicateRelease: true, + reverseFinalRelease: true, + } + const reconstructed = remoteLeaseGrammarPlan(witness) + expect(reconstructed).toEqual({ + demands: [ + { limit: 1, offset: 0 }, + { limit: 1, offset: 1 }, + { limit: 1, offset: 2 }, + ], + releaseCount: 1, + prefixReleaseIndexes: [0], + finalReleaseIndexes: [2, 1], + releasePasses: 2, + }) + + const ablations: Array<{ + axis: keyof RemoteLeaseHistory + history: RemoteLeaseHistory + observe: (plan: RemoteLeaseGrammarPlan) => unknown + }> = [ + { + axis: `siblingCount`, + history: { ...witness, siblingCount: 1 }, + observe: ({ demands }) => demands.length, + }, + { + axis: `releasePrefix`, + history: { ...witness, releasePrefix: 0 }, + observe: ({ prefixReleaseIndexes }) => prefixReleaseIndexes, + }, + { + axis: `identical`, + history: { ...witness, identical: true }, + observe: ({ demands }) => demands, + }, + { + axis: `duplicateRelease`, + history: { ...witness, duplicateRelease: false }, + observe: ({ releasePasses }) => releasePasses, + }, + { + axis: `reverseFinalRelease`, + history: { ...witness, reverseFinalRelease: false }, + observe: ({ finalReleaseIndexes }) => finalReleaseIndexes, + }, + ] + expect(ablations.map(({ axis }) => axis).sort()).toEqual( + (Object.keys(witness) as Array).sort(), + ) + for (const { history, observe } of ablations) { + expect(observe(remoteLeaseGrammarPlan(history))).not.toEqual( + observe(reconstructed), + ) + } + }) + + async function runRemoteLeaseCampaign( + campaign: CoordinatorOracleCampaign, + ): Promise { let originalHistory: RemoteLeaseHistory | undefined let originalViolation: RemoteLeaseTakeoverViolation | undefined let targetDiscriminant: string | undefined @@ -1995,13 +2167,8 @@ describe(`remote subset ownership lease oracle`, () => { ) cleanupCollection = () => collection.cleanup() collection.startSyncImmediate() - const demands = Array.from( - { length: history.siblingCount }, - (_, index): LoadSubsetOptions => ({ - limit: 1, - ...(history.identical ? {} : { offset: index }), - }), - ) + const plan = remoteLeaseGrammarPlan(history) + const demands: Array = plan.demands checkpoint = `initial acquisitions` for (const demand of demands) { @@ -2014,13 +2181,9 @@ describe(`remote subset ownership lease oracle`, () => { const firstAfterAcquire = firstOwner.snapshot() checkpoint = `prefix releases` - const releaseCount = Math.min( - history.releasePrefix, - history.siblingCount - 1, - ) - for (const demand of demands.slice(0, releaseCount)) { - collection._sync.unloadSubset(demand) - if (history.duplicateRelease) { + for (const index of plan.prefixReleaseIndexes) { + const demand = demands[index]! + for (let pass = 0; pass < plan.releasePasses; pass++) { collection._sync.unloadSubset(demand) } } @@ -2040,11 +2203,9 @@ describe(`remote subset ownership lease oracle`, () => { const secondAfterTakeover = secondOwner.snapshot() checkpoint = `final releases` - const finalDemands = demands.slice(releaseCount) - if (history.reverseFinalRelease) finalDemands.reverse() - for (const demand of finalDemands) { - collection._sync.unloadSubset(demand) - if (history.duplicateRelease) { + for (const index of plan.finalReleaseIndexes) { + const demand = demands[index]! + for (let pass = 0; pass < plan.releasePasses; pass++) { collection._sync.unloadSubset(demand) } } @@ -2140,9 +2301,7 @@ describe(`remote subset ownership lease oracle`, () => { if (semanticFailure !== NO_PRIMARY_FAILURE) throw semanticFailure }), { - seed: SEED, - numRuns: RUNS, - ...(PATH === undefined ? {} : { path: PATH }), + ...campaign.parameters, examples: [ [ { @@ -2163,7 +2322,12 @@ describe(`remote subset ownership lease oracle`, () => { expect.soft(cleanupDiagnostics).toEqual([]) expect.soft(executionDiagnostics).toEqual([]) if (propertyFailure !== NO_PRIMARY_FAILURE) throw propertyFailure - }) + } + + for (const campaign of coordinatorOracleCampaigns(`remote-lease`)) { + it(`preserves independent sibling leases through release, retry-safe takeover, and cleanup histories (${campaign.mode})`, () => + runRemoteLeaseCampaign(campaign)) + } }) type RawCollectionSnapshot = { @@ -2622,6 +2786,22 @@ const orderArbitrary = fc forward ? [`alpha`, `beta`] : [`beta`, `alpha`], ) +type RoutingHistoryAxes = Omit & { + versionDelta: number +} + +function routingHistoryFromAxes({ + alphaVersion, + versionDelta, + ...history +}: RoutingHistoryAxes): RoutingHistory { + return { + ...history, + alphaVersion, + betaVersion: alphaVersion + versionDelta, + } +} + const routingHistoryArbitrary: fc.Arbitrary = fc .record({ alphaVersion: fc.integer({ min: 1, max: 50 }), @@ -2630,11 +2810,34 @@ const routingHistoryArbitrary: fc.Arbitrary = fc firstDeliveryOrder: orderArbitrary, secondDeliveryOrder: orderArbitrary, }) - .map(({ alphaVersion, versionDelta, ...history }) => ({ - ...history, - alphaVersion, - betaVersion: alphaVersion + versionDelta, - })) + .map(routingHistoryFromAxes) + +type RoutingGrammarPlan = { + initialCollection: CollectionName + registrationOrder: [CollectionName, CollectionName] + firstDeliveryOrder: [CollectionName, CollectionName] + secondDeliveryOrder: [CollectionName, CollectionName] + adapterIds: Record> +} + +function routingGrammarPlan(history: RoutingHistory): RoutingGrammarPlan { + return { + initialCollection: history.registrationOrder[0], + registrationOrder: history.registrationOrder, + firstDeliveryOrder: history.firstDeliveryOrder, + secondDeliveryOrder: history.secondDeliveryOrder, + adapterIds: { + leader: { + alpha: `leader-alpha-v${history.alphaVersion}-reset`, + beta: `leader-beta-v${history.betaVersion}-error`, + }, + follower: { + alpha: `follower-alpha-v${history.alphaVersion}-reset`, + beta: `follower-beta-v${history.betaVersion}-error`, + }, + }, + } +} function mutation( collectionId: CollectionName, @@ -2695,6 +2898,48 @@ async function requestBothCollections( } describe(`generated collection-route histories`, () => { + it(`requires a complete property, seed, and shrink path for replay`, () => { + expect(readCoordinatorOracleConfig({})).toEqual({ runs: DEFAULT_RUNS }) + for (const environment of [ + { TANSTACK_DB_COORDINATOR_ORACLE_SEED: `123` }, + { TANSTACK_DB_COORDINATOR_ORACLE_PATH: `0:1` }, + { TANSTACK_DB_COORDINATOR_ORACLE_PROPERTY: `routing` }, + { + TANSTACK_DB_COORDINATOR_ORACLE_SEED: `123`, + TANSTACK_DB_COORDINATOR_ORACLE_PATH: `0:1`, + }, + ]) { + expect(() => readCoordinatorOracleConfig(environment)).toThrow( + /requires SEED, PATH, and PROPERTY together/, + ) + } + expect(() => + readCoordinatorOracleConfig({ + TANSTACK_DB_COORDINATOR_ORACLE_SEED: `123`, + TANSTACK_DB_COORDINATOR_ORACLE_PATH: `not-a-path`, + TANSTACK_DB_COORDINATOR_ORACLE_PROPERTY: `routing`, + }), + ).toThrow(/numeric shrink path/) + expect(() => + readCoordinatorOracleConfig({ + TANSTACK_DB_COORDINATOR_ORACLE_SEED: `123`, + TANSTACK_DB_COORDINATOR_ORACLE_PATH: `0:1`, + TANSTACK_DB_COORDINATOR_ORACLE_PROPERTY: `unknown`, + }), + ).toThrow(/remote-lease or routing/) + expect( + readCoordinatorOracleConfig({ + TANSTACK_DB_COORDINATOR_ORACLE_RUNS: `7`, + TANSTACK_DB_COORDINATOR_ORACLE_SEED: `123`, + TANSTACK_DB_COORDINATOR_ORACLE_PATH: `0:1`, + TANSTACK_DB_COORDINATOR_ORACLE_PROPERTY: `routing`, + }), + ).toEqual({ + runs: 7, + replay: { property: `routing`, seed: 123, path: `0:1` }, + }) + }) + it(`calibrates collection-key comparison against one peer-routed apply`, () => { const expected: Array = [ { @@ -2760,6 +3005,99 @@ describe(`generated collection-route histories`, () => { ]) }) + it(`reconstructs the routing grammar and ablates every declared axis`, () => { + const witnessAxes: RoutingHistoryAxes = { + alphaVersion: 11, + versionDelta: 18, + registrationOrder: [`beta`, `alpha`], + firstDeliveryOrder: [`alpha`, `beta`], + secondDeliveryOrder: [`beta`, `alpha`], + } + const witness = routingHistoryFromAxes(witnessAxes) + expect(witness).toEqual({ + alphaVersion: 11, + betaVersion: 29, + registrationOrder: [`beta`, `alpha`], + firstDeliveryOrder: [`alpha`, `beta`], + secondDeliveryOrder: [`beta`, `alpha`], + }) + const reconstructed = routingGrammarPlan(witness) + expect(reconstructed).toEqual({ + initialCollection: `beta`, + registrationOrder: [`beta`, `alpha`], + firstDeliveryOrder: [`alpha`, `beta`], + secondDeliveryOrder: [`beta`, `alpha`], + adapterIds: { + leader: { + alpha: `leader-alpha-v11-reset`, + beta: `leader-beta-v29-error`, + }, + follower: { + alpha: `follower-alpha-v11-reset`, + beta: `follower-beta-v29-error`, + }, + }, + }) + + const ablations: Array<{ + axis: keyof RoutingHistoryAxes + history: RoutingHistory + observe: (plan: RoutingGrammarPlan) => unknown + }> = [ + { + axis: `alphaVersion`, + history: routingHistoryFromAxes({ + ...witnessAxes, + alphaVersion: 12, + }), + observe: ({ adapterIds }) => adapterIds.leader.alpha, + }, + { + axis: `versionDelta`, + history: routingHistoryFromAxes({ + ...witnessAxes, + versionDelta: 19, + }), + observe: ({ adapterIds }) => adapterIds.leader.beta, + }, + { + axis: `registrationOrder`, + history: routingHistoryFromAxes({ + ...witnessAxes, + registrationOrder: [`alpha`, `beta`], + }), + observe: ({ initialCollection, registrationOrder }) => ({ + initialCollection, + registrationOrder, + }), + }, + { + axis: `firstDeliveryOrder`, + history: routingHistoryFromAxes({ + ...witnessAxes, + firstDeliveryOrder: [`beta`, `alpha`], + }), + observe: ({ firstDeliveryOrder }) => firstDeliveryOrder, + }, + { + axis: `secondDeliveryOrder`, + history: routingHistoryFromAxes({ + ...witnessAxes, + secondDeliveryOrder: [`alpha`, `beta`], + }), + observe: ({ secondDeliveryOrder }) => secondDeliveryOrder, + }, + ] + expect(ablations.map(({ axis }) => axis).sort()).toEqual( + (Object.keys(witnessAxes) as Array).sort(), + ) + for (const { history, observe } of ablations) { + expect(observe(routingGrammarPlan(history))).not.toEqual( + observe(reconstructed), + ) + } + }) + it(`routes every adapter-bound RPC by collection before and after leadership transfer`, async () => { const makePullResult = ( marker: string, @@ -3008,7 +3346,9 @@ describe(`generated collection-route histories`, () => { ]) }) - it(`uses the collection's registered adapter before and after leadership transfer`, async () => { + async function runRoutingCampaign( + campaign: CoordinatorOracleCampaign, + ): Promise { let originalFailingTrace: RoutingHistory | undefined let originalViolation: RouteViolation | undefined let targetDiscriminant: string | undefined @@ -3024,33 +3364,34 @@ describe(`generated collection-route histories`, () => { try { await fc.assert( fc.asyncProperty(routingHistoryArbitrary, async (history) => { + const plan = routingGrammarPlan(history) const oracle = new PerCollectionRouteOracle() const leaderAdapters = { alpha: createRecordingAdapter({ - id: `leader-alpha-v${history.alphaVersion}-reset`, + id: plan.adapterIds.leader.alpha, schemaVersion: history.alphaVersion, policy: `sync-present-reset`, }), beta: createRecordingAdapter({ - id: `leader-beta-v${history.betaVersion}-error`, + id: plan.adapterIds.leader.beta, schemaVersion: history.betaVersion, policy: `sync-absent-error`, }), } const followerAdapters = { alpha: createRecordingAdapter({ - id: `follower-alpha-v${history.alphaVersion}-reset`, + id: plan.adapterIds.follower.alpha, schemaVersion: history.alphaVersion, policy: `sync-present-reset`, }), beta: createRecordingAdapter({ - id: `follower-beta-v${history.betaVersion}-error`, + id: plan.adapterIds.follower.beta, schemaVersion: history.betaVersion, policy: `sync-absent-error`, }), } const dbName = `generated-route-${history.alphaVersion}-${history.betaVersion}` - const initialCollection = history.registrationOrder[0] + const initialCollection = plan.initialCollection let leader: BrowserCollectionCoordinator | undefined let follower: BrowserCollectionCoordinator | undefined const observed: Array = [] @@ -3069,7 +3410,7 @@ describe(`generated collection-route histories`, () => { followerAdapters[initialCollection], ) - for (const collectionId of history.registrationOrder) { + for (const collectionId of plan.registrationOrder) { registerCollectionAdapter( leader, collectionId, @@ -3111,7 +3452,7 @@ describe(`generated collection-route histories`, () => { await requestBothCollections( follower, `initial-owner`, - history.firstDeliveryOrder, + plan.firstDeliveryOrder, ) observed.push( ...applyObservation(`initial-owner`, [ @@ -3137,7 +3478,7 @@ describe(`generated collection-route histories`, () => { checkpoint = `post-takeover routed applies` // The new leader's direct calls still exercise the same production // handler and must look up adapters per collection. - for (const collectionId of history.secondDeliveryOrder) { + for (const collectionId of plan.secondDeliveryOrder) { const response = await follower.requestApplyLocalMutations( collectionId, mutation(collectionId, `after-takeover`), @@ -3230,11 +3571,7 @@ describe(`generated collection-route histories`, () => { if (semanticFailure !== NO_PRIMARY_FAILURE) throw semanticFailure }), - { - seed: SEED, - numRuns: RUNS, - ...(PATH === undefined ? {} : { path: PATH }), - }, + campaign.parameters, ) } catch (error) { propertyFailure = error @@ -3247,7 +3584,12 @@ describe(`generated collection-route histories`, () => { // only the locked semantic discriminant is eligible for shrinking. expect.soft(executionDiagnostics).toEqual([]) if (propertyFailure !== NO_PRIMARY_FAILURE) throw propertyFailure - }) + } + + for (const campaign of coordinatorOracleCampaigns(`routing`)) { + it(`uses the collection's registered adapter before and after leadership transfer (${campaign.mode})`, () => + runRoutingCampaign(campaign)) + } }) type OwnershipObservation = { From 3ee18e5ee7c414ee611a69c3ba72f1ab05dc73b9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 16:52:39 +0100 Subject: [PATCH 18/18] test(browser-sqlite): preserve delivered mutation success --- .../tests/browser-coordinator.test.ts | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 3afb806480..5e402565c2 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -1093,6 +1093,62 @@ describe(`BrowserCollectionCoordinator`, () => { } }) + it(`accepts a delivered mutation success after the observed route changes`, async () => { + const coordinator = createCoordinator() + const internals = coordinator as unknown as { + collections: Map< + string, + { isLeader: boolean; leaderId: string | null; latestTerm: number } + > + } + internals.collections.set(`todos`, { + isLeader: false, + leaderId: `leader-before`, + latestTerm: 1, + }) + const deliveredSuccess = vi.fn( + (_collectionId: string, request: ApplyLocalMutationsRequest) => { + const state = internals.collections.get(`todos`)! + state.leaderId = `leader-after` + state.latestTerm = 2 + return Promise.resolve({ + type: `rpc:applyLocalMutations:res` as const, + rpcId: request.rpcId, + ok: true as const, + term: 1, + seq: 7, + latestRowVersion: 7, + acceptedMutationIds: request.mutations.map( + (mutation) => mutation.mutationId, + ), + }) + }, + ) + Object.defineProperty(coordinator, `sendRPCOnce`, { + value: deliveredSuccess, + configurable: true, + }) + + try { + await expect( + coordinator.requestApplyLocalMutations(`todos`, [ + { + mutationId: `delivered-before-route-change`, + type: `insert`, + key: `delivered-before-route-change`, + value: { id: `delivered-before-route-change` }, + }, + ]), + ).resolves.toMatchObject({ + ok: true, + acceptedMutationIds: [`delivered-before-route-change`], + }) + expect(deliveredSuccess).toHaveBeenCalledOnce() + } finally { + coordinator.dispose() + } + }) + it(`fails indeterminate when a local-mutation success is lost across leader change`, async () => { const retiredLeaderAdapter = createStubAdapter() const requesterAdapter = createStubAdapter()