From 07f8d91a2b2ed3ef5faeca6c7229984e5b7dbd86 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 17 Sep 2026 08:02:23 -0600 Subject: [PATCH 01/23] fix: preserve persisted resume baseline integrity --- docs/contributing/oracle-coverage.md | 21 +- packages/db-sqlite-persistence-core/README.md | 24 + .../src/persisted.ts | 156 +++- .../src/sqlite-core-adapter.ts | 426 +++++++++- .../tests/persisted.test-d.ts | 46 +- .../tests/sqlite-core-adapter.test.ts | 684 ++++++++++++++++ .../tests/sqlite-resume-snapshot.test.ts | 744 ++++++++++++++++++ .../electric-db-collection/src/electric.ts | 45 +- .../tests/electric-recovery-oracle.test.ts | 562 ++++++++++++- .../electric-resume-snapshot-races.test.ts | 711 +++++++++++++++++ 10 files changed, 3367 insertions(+), 52 deletions(-) create mode 100644 packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts create mode 100644 packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 1b2aa2f157..055ad3b46e 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -35,15 +35,32 @@ comment and the current API/architecture contract before extending its model. | Drafts and native values | [proxy](../../packages/db/tests/proxy.test.ts), [detachment](../../packages/db/tests/proxy-detachment-contract.test.ts), [iteration](../../packages/db/tests/proxy-iteration-contract.test.ts) | Native-operation controls, exact patches and actual stored rows; alias/cycle/adversarial-key histories. General native-mutator and symbol-write support is not established by a plain-object oracle. | | Query DB and observer | [ownership](../../packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts), [load lifecycle](../../packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts), [observer histories](../../packages/db/tests/live-query-observer-history.property.test.ts) | Real QueryClient boundary and a per-listener eligibility ledger, not a duplicate dispatch queue. Check reentry, peer survival, FIFO and disposal independently of final rows. | | Ordered acquisition | [pagination](../../packages/db/tests/query/pagination-oracle.property.test.ts), [ordered work](../../packages/db/tests/query/ordered-work-oracle.property.test.ts), [ordered lifecycle](../../packages/db/tests/query/ordered-lifecycle-oracle.property.test.ts) | Complete finite provider results, pending windows, ties/nulls, ownership and documented repair timing. Request completion is not proof of unrequested source extent. | -| Electric and TrailBase | [Electric histories](../../packages/electric-db-collection/tests/electric-oracle.property.test.ts), [PostgreSQL semantics](../../packages/electric-db-collection/e2e/sql-predicate-semantics.e2e.test.ts), [TrailBase contract](../../packages/trailbase-db-collection/tests/ORACLE.md) | Installed SDK delivery/framing, independent predicates, exact subscription arguments and late errors. SDK fixtures and a real service test earn different credit. | +| Opaque backend pagination | [window oracle](../../packages/query-db-collection/tests/cursor-pagination.oracle.test.ts), [cache histories](../../packages/query-db-collection/tests/cursor-pagination.cache-oracle.test.ts), [cache publication](../../packages/query-db-collection/tests/cursor-pagination.publication-oracle.test.ts), [browser acquisition boundaries](../../packages/query-db-collection/tests/cursor-pagination.boundary-oracle.test.ts), [QueryCollection integration](../../packages/query-db-collection/tests/cursor-pagination.integration.test.ts) | Full filter/sort/slice reference, opaque token transport, actual Query cache expiry/invalidation/GC, forced refresh during growth, protocol failure publication/recovery, bounded slice work, nested cancellation/replacement, reader abort, browser retry defaults, manual-write cache isolation, and production window publications. Stable backend sequences; not snapshot guarantees for changing endpoints. Peek-ahead remains enabled. | +| Electric and TrailBase | [Electric histories](../../packages/electric-db-collection/tests/electric-oracle.property.test.ts), [recovery histories](../../packages/electric-db-collection/tests/electric-recovery-oracle.test.ts), [held resume snapshots](../../packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts), [PostgreSQL semantics](../../packages/electric-db-collection/e2e/sql-predicate-semantics.e2e.test.ts), [TrailBase contract](../../packages/trailbase-db-collection/tests/ORACLE.md) | Installed SDK delivery/framing, independent predicates, exact subscription arguments, restart/reset lineage, held certification races, and late errors. The recovery fixtures use a mocked ShapeStream; they do not establish live Electric-service framing or native persistence-host behavior. | | PowerSync | [tests](../../packages/powersync-db-collection/tests) | Applied receipt positions crossed with held peers, native SQLite/SDK and cleanup evidence. A timeout mutant proves a progress failure, not every value assertion. | -| SQLite persistence and native hosts | [persisted histories](../../packages/db-sqlite-persistence-core/tests/persisted.test.ts), [driver contracts](../../packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [113-law manifest](../../packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Cache/remote rejection/peer/reopen histories and exact driver results. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | +| SQLite persistence and native hosts | [persisted histories](../../packages/db-sqlite-persistence-core/tests/persisted.test.ts), [reset/resume histories](../../packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts), [dual-adapter resume snapshots](../../packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts), [driver contracts](../../packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [113-law manifest](../../packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Cache/remote rejection/peer/reopen histories, atomic reset/resume lineage, key-set evidence, dual-adapter races, and exact driver results. The reset/resume owners use sqlite3 CLI and in-memory node:sqlite seams; they do not prove multi-process WAL, mobile/Tauri, or other native-device execution. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | | Offline execution | [scheduler](../../packages/offline-transactions/tests/KeyScheduler.property.test.ts), [leadership](../../packages/offline-transactions/tests/leadership-replay.property.test.ts), [settlement](../../packages/offline-transactions/tests/transaction-settlement.property.test.ts), [serialization](../../packages/offline-transactions/tests/transaction-serializer.property.test.ts) | Declarative FIFO eligibility, per-transaction outcomes, durable state and typed wire trees. Issued work may finish after ownership loss, but new work must not start. Exactly-once network execution is not promised. | | Frameworks | [React conformance](../../packages/react-db/tests/conformance.test.tsx), [React pagination](../../packages/react-db/tests/infinite-query-conformance.test.tsx), [shared suites](../../packages/db-collection-e2e/src/suites) | Exact exposed rows/pages and each framework's own lifecycle cuts. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. | | Small structures and test mechanics | [SortedMap](../../packages/db/tests/SortedMap.test.ts), [cleanup queue](../../packages/db/tests/cleanup-queue.property.test.ts), [guarded replay](../../packages/db/tests/oracle-replay.test.ts) | Map/full-sort and appointment-list models; executed target/seed/path checks. Callback-reentrant scheduling is outside the initial cleanup-queue domain. | ## Acceptance map +The post-merge review added three missing domains to existing owners: + +- [Top-K batch contracts](../../packages/db-ivm/tests/operators/topk-batch-contract.test.ts) + cross sparse-array length/holes and RegExp source/flags/position with equal + controls, replacement order, hash consolidation, and actual retained graph + output. Ordinary replacements also run without the global `File` constructor. +- [Leadership replay](../../packages/offline-transactions/tests/leadership-replay.property.test.ts) + holds real storage-read delivery across successful and permanently rejected + durable removals, with bounded scans, concurrent loads, and unfinished peers. + This is distinct from exactly-once execution across independent owners. +- [Accepted-snapshot retention](../../packages/db/tests/collection-state-retention-oracle.property.test.ts) + varies truncate before/during/after an optimistic delete, rejection versus + rollback, post-capture direct insertion, and later ordinary sync/key reuse. A + hidden accepted insert returns after rollback; an uncaptured insert retires, + and neither snapshot is rebased onto synced fields. + | Issue obligation | Implemented evidence | Limit | | --- | --- | --- | | Metamorphic laws | Includes cross-formulation/partition, D2 independent-key commutation, DBSP incremental/full recomputation, pagination provider/UI boundaries, optimistic snapshot stability | Equivalence premises are explicit; not arbitrary query rewrites. | diff --git a/packages/db-sqlite-persistence-core/README.md b/packages/db-sqlite-persistence-core/README.md index 1ed225957f..235df60776 100644 --- a/packages/db-sqlite-persistence-core/README.md +++ b/packages/db-sqlite-persistence-core/README.md @@ -31,6 +31,7 @@ binding. Provide a runtime `SQLiteDriver` implementation from a wrapper package. - `PullSinceResponse` - `CollectionReset` - `PersistedIndexSpec` +- `PersistedKeySetEvidence` - `PersistedTx` - `PersistenceAdapter` - `SQLiteDriver` @@ -64,6 +65,29 @@ and resolves persistence using: This lets runtime wrappers expose one shared persistence instance per database while still handling per-collection schema versions correctly. +### Atomic resume snapshots + +Persistence adapters may implement +`loadResumeSnapshot(collectionId, options)` to let a sync source certify a +persisted resume baseline. One call must read rows, collection metadata, stream +position, reset epoch, and key-set evidence from the same atomic database +snapshot. `includeRows: false` requests the same certification data without +materializing rows; `requiredIndexSignatures` carries the indexes needed by a +row-bearing snapshot. + +`PersistedKeySetEvidence.status` has three states: + +- `consistent`: the persisted rows match the adapter's durable expected-key + ledger. +- `incompatible`: row loss, substitution, or a reset-generation change makes + the saved resume baseline unsafe. +- `unknown`: the adapter has no authoritative pre-migration key set and does + not claim completeness. + +The method is optional so existing adapters remain assignable. Without it, the +wrapper retains the legacy stream-position and metadata path. Adapter methods +are invoked with their receiver and may rely on instance state through `this`. + ### SQLite core adapter APIs - `SQLiteCoreAdapterOptions` diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 0f1e4112ac..69bdaaf6de 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -219,6 +219,10 @@ export type PersistedRowScanOptions = { metadataOnly?: boolean } +export type PersistedKeySetEvidence = { + status: `unknown` | `consistent` | `incompatible` +} + export type PersistedTx< T extends object = Record, TKey extends string | number = string | number, @@ -261,6 +265,25 @@ export interface PersistenceAdapter { metadata?: unknown }> > + loadResumeSnapshot?: ( + collectionId: string, + ctx?: { + requiredIndexSignatures?: ReadonlyArray + includeRows?: boolean + }, + ) => Promise<{ + rows: Array<{ + key: string | number + value: Record + metadata?: unknown + }> + keySet?: PersistedKeySetEvidence + collectionMetadata: Array<{ key: string; value: unknown }> + latestTerm: number + latestSeq: number + latestRowVersion: number + resetEpoch: number + }> applyCommittedTx: (collectionId: string, tx: PersistedTx) => Promise loadCollectionMetadata?: ( collectionId: string, @@ -279,6 +302,7 @@ export interface PersistenceAdapter { latestTerm: number latestSeq: number latestRowVersion: number + keySet?: PersistedKeySetEvidence }> } @@ -806,6 +830,9 @@ class PersistedCollectionRuntime< private startupMetadataPromise: Promise | null = null private startPromise: Promise | null = null private resumeBaselinePromise: Promise | null = null + private resumeCertificationPromise: Promise | null = null + private persistedKeySetEvidence: PersistedKeySetEvidence | undefined + private persistedResumeGeneration: string | undefined private lifecycleGeneration = 0 private internalApplyDepth = 0 private appliedReceiptSequence = 0 @@ -925,6 +952,36 @@ class PersistedCollectionRuntime< return this.resumeBaselinePromise } + ensureResumeBaselineCertified(): Promise { + if (this.resumeCertificationPromise) { + return this.resumeCertificationPromise + } + + const lifecycleGeneration = this.lifecycleGeneration + this.resumeCertificationPromise = (async () => { + await this.ensureStarted() + if (lifecycleGeneration !== this.lifecycleGeneration) return + + const adapter = this.persistence.adapter + if (!adapter.loadResumeSnapshot) return + const snapshot = await adapter.loadResumeSnapshot(this.collectionId, { + requiredIndexSignatures: this.getRequiredIndexSignatures(), + includeRows: false, + }) + if (lifecycleGeneration !== this.lifecycleGeneration) return + this.bindResumeSnapshotEvidence(snapshot) + })() + return this.resumeCertificationPromise + } + + getPersistedKeySetEvidence(): PersistedKeySetEvidence | undefined { + return this.persistedKeySetEvidence + } + + supportsResumeSnapshot(): boolean { + return this.persistence.adapter.loadResumeSnapshot !== undefined + } + private async hydrateBaseline(lifecycleGeneration: number): Promise { if (lifecycleGeneration !== this.lifecycleGeneration) return @@ -936,6 +993,7 @@ class PersistedCollectionRuntime< await this.hydrateSubsetUnsafe(baseline, { requestRemoteEnsure: false, lifecycleGeneration, + bindKeySetEvidence: true, }) }) if (lifecycleGeneration !== this.lifecycleGeneration) return @@ -976,6 +1034,24 @@ class PersistedCollectionRuntime< private async loadStartupMetadataInternal( lifecycleGeneration: number, ): Promise { + if (this.persistence.adapter.loadResumeSnapshot) { + const snapshot = await this.persistence.adapter.loadResumeSnapshot( + this.collectionId, + { includeRows: false }, + ) + if (lifecycleGeneration !== this.lifecycleGeneration) return + this.persistedResumeGeneration = + this.getResumeSnapshotGeneration(snapshot) + this.persistedKeySetEvidence = snapshot.keySet + this.observeStreamPosition( + snapshot.latestTerm, + snapshot.latestSeq, + snapshot.latestRowVersion, + ) + this.replaceCollectionMetadataSnapshot(snapshot.collectionMetadata) + return + } + // Restore stream position from the database so that new mutations // don't collide with previously applied transactions. if (this.persistence.adapter.getStreamPosition) { @@ -983,6 +1059,10 @@ class PersistedCollectionRuntime< this.collectionId, ) if (lifecycleGeneration !== this.lifecycleGeneration) return + this.persistedKeySetEvidence = + position.keySet?.status === `consistent` + ? { status: `unknown` } + : position.keySet this.observeStreamPosition( position.latestTerm, position.latestSeq, @@ -1247,6 +1327,9 @@ class PersistedCollectionRuntime< this.startupMetadataPromise = null this.startPromise = null this.resumeBaselinePromise = null + this.resumeCertificationPromise = null + this.persistedKeySetEvidence = undefined + this.persistedResumeGeneration = undefined } private withInternalApply(task: () => TResult): TResult { @@ -1300,14 +1383,38 @@ class PersistedCollectionRuntime< config: { requestRemoteEnsure: boolean lifecycleGeneration: number + bindKeySetEvidence?: boolean }, ): Promise { this.hydratingGeneration = config.lifecycleGeneration try { - const rows = await this.loadSubsetRowsUnsafe(options) + let rows: Array<{ key: TKey; value: T; metadata?: unknown }> + if ( + config.bindKeySetEvidence && + this.persistence.adapter.loadResumeSnapshot + ) { + const snapshot = await this.persistence.adapter.loadResumeSnapshot( + this.collectionId, + { + requiredIndexSignatures: this.getRequiredIndexSignatures(), + includeRows: true, + }, + ) + rows = snapshot.rows as Array<{ + key: TKey + value: T + metadata?: unknown + }> + if (config.lifecycleGeneration !== this.lifecycleGeneration) return + this.bindResumeSnapshotEvidence(snapshot) + } else { + rows = await this.loadSubsetRowsUnsafe(options) + } if (config.lifecycleGeneration !== this.lifecycleGeneration) return - this.applyRowsToCollection(rows) + if (this.persistedKeySetEvidence?.status !== `incompatible`) { + this.applyRowsToCollection(rows) + } } finally { if (this.hydratingGeneration === config.lifecycleGeneration) { this.hydratingGeneration = null @@ -1351,6 +1458,39 @@ class PersistedCollectionRuntime< }) } + private getResumeSnapshotGeneration(snapshot: { + latestTerm: number + latestSeq: number + latestRowVersion: number + resetEpoch: number + }): string { + return `${snapshot.resetEpoch}:${snapshot.latestTerm}:${snapshot.latestSeq}:${snapshot.latestRowVersion}` + } + + private bindResumeSnapshotEvidence(snapshot: { + keySet?: PersistedKeySetEvidence + latestTerm: number + latestSeq: number + latestRowVersion: number + resetEpoch: number + }): void { + const generation = this.getResumeSnapshotGeneration(snapshot) + // Moving between equally uncertified snapshots cannot make either + // trustworthy; preserve that status so sync performs a fresh replacement. + const remainsUncertified = + this.persistedKeySetEvidence?.status === snapshot.keySet?.status && + snapshot.keySet?.status !== `consistent` + this.observeStreamPosition( + snapshot.latestTerm, + snapshot.latestSeq, + snapshot.latestRowVersion, + ) + this.persistedKeySetEvidence = + this.persistedResumeGeneration === generation || remainsUncertified + ? snapshot.keySet + : { status: `incompatible` } + } + private replaceCollectionSnapshot( rows: Array<{ key: TKey; value: T; metadata?: unknown }>, collectionMetadata: Array<{ key: string; value: unknown }>, @@ -2428,6 +2568,12 @@ function createWrappedSyncConfig< startupState.cleanedUp ? Promise.resolve() : runtime.ensureResumeBaselineHydrated(), + certifyPersistedResume: runtime.supportsResumeSnapshot() + ? () => + startupState.cleanedUp + ? Promise.resolve() + : runtime.ensureResumeBaselineCertified() + : undefined, get: (key: TKey) => { if (startupState.cleanedUp) return undefined const openTransaction = getOpenTransaction() @@ -2447,6 +2593,12 @@ function createWrappedSyncConfig< startupState.cleanedUp ? Promise.resolve([]) : runtime.scanPersistedRows(options), + getPersistedKeySetEvidence: runtime.supportsResumeSnapshot() + ? () => + startupState.cleanedUp + ? undefined + : runtime.getPersistedKeySetEvidence() + : undefined, set: (key: TKey, value: unknown) => { if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index 69f29fc604..de5eef9b53 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -15,6 +15,7 @@ import { import type { LoadSubsetOptions } from '@tanstack/db' import type { PersistedIndexSpec, + PersistedKeySetEvidence, PersistedRowScanOptions, PersistedScannedRow, PersistedTx, @@ -1159,36 +1160,150 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { })) } - async applyCommittedTx(collectionId: string, tx: PersistedTx): Promise { + async loadResumeSnapshot( + collectionId: string, + ctx?: { + requiredIndexSignatures?: ReadonlyArray + includeRows?: boolean + }, + ): Promise<{ + rows: Array<{ + key: string | number + value: Record + metadata?: unknown + }> + keySet: PersistedKeySetEvidence + collectionMetadata: Array<{ key: string; value: unknown }> + latestTerm: number + latestSeq: number + latestRowVersion: number + resetEpoch: number + }> { const tableMapping = await this.ensureCollectionReady(collectionId) - const collectionTableSql = quoteIdentifier(tableMapping.tableName) - const tombstoneTableSql = quoteIdentifier(tableMapping.tombstoneTableName) + const includeRows = ctx?.includeRows !== false + if (includeRows) { + await this.touchRequiredIndexes( + collectionId, + ctx?.requiredIndexSignatures, + ) + } - await this.runInTransaction(async (transactionDriver) => { - const alreadyApplied = await transactionDriver.query<{ applied: number }>( - `SELECT 1 AS applied + return this.runInTransaction(async (transactionDriver) => { + const rows = includeRows + ? await this.loadSubsetInternal(tableMapping, {}, transactionDriver) + : [] + const { latestRowVersion, keySet } = await this.readKeySetEvidence( + collectionId, + tableMapping, + transactionDriver, + ) + const collectionMetadataRows = await transactionDriver.query<{ + key: string + value: string + }>( + `SELECT key, value + FROM collection_metadata + WHERE collection_id = ?`, + [collectionId], + ) + const termRows = await transactionDriver.query<{ latest_term: number }>( + `SELECT latest_term + FROM leader_term + WHERE collection_id = ? + LIMIT 1`, + [collectionId], + ) + const seqRows = await transactionDriver.query<{ max_seq: number }>( + `SELECT MAX(seq) AS max_seq FROM applied_tx - WHERE collection_id = ? AND term = ? AND seq = ? + WHERE collection_id = ? AND term = ( + SELECT latest_term FROM leader_term WHERE collection_id = ? LIMIT 1 + )`, + [collectionId, collectionId], + ) + const resetRows = await transactionDriver.query<{ reset_epoch: number }>( + `SELECT reset_epoch + FROM collection_reset_epoch + WHERE collection_id = ? LIMIT 1`, - [collectionId, tx.term, tx.seq], + [collectionId], ) - if (alreadyApplied.length > 0) { - return + return { + rows: rows.map((row) => ({ + key: row.key, + value: row.value, + metadata: row.metadata, + })), + keySet, + collectionMetadata: collectionMetadataRows.map((row) => ({ + key: row.key, + value: deserializePersistedRowValue(row.value), + })), + latestTerm: termRows[0]?.latest_term ?? 0, + latestSeq: seqRows[0]?.max_seq ?? 0, + latestRowVersion, + resetEpoch: resetRows[0]?.reset_epoch ?? 0, } + }) + } + + async applyCommittedTx(collectionId: string, tx: PersistedTx): Promise { + const tableMapping = await this.ensureCollectionReady(collectionId) + const collectionTableSql = quoteIdentifier(tableMapping.tableName) + const tombstoneTableSql = quoteIdentifier(tableMapping.tombstoneTableName) + await this.runInTransaction(async (transactionDriver) => { const versionRows = await transactionDriver.query<{ latest_row_version: number + key_set_evidence_available: number + schema_version: number + already_applied: number }>( - `SELECT latest_row_version + `SELECT + latest_row_version, + key_set_evidence_available, + ( + SELECT schema_version + FROM collection_registry + WHERE collection_id = ? + LIMIT 1 + ) AS schema_version, + EXISTS ( + SELECT 1 + FROM applied_tx + WHERE collection_id = ? AND term = ? AND seq = ? + ) AS already_applied FROM collection_version WHERE collection_id = ? LIMIT 1`, - [collectionId], + [collectionId, collectionId, tx.term, tx.seq, collectionId], ) - const currentRowVersion = versionRows[0]?.latest_row_version ?? 0 + const version = versionRows[0] + + if (!version) { + throw new InvalidPersistedCollectionConfigError( + `Missing persisted version state for collection "${collectionId}"`, + ) + } + if (version.schema_version !== this.schemaVersion) { + throw new InvalidPersistedCollectionConfigError( + `Schema version mismatch for collection "${collectionId}": ` + + `found ${version.schema_version}, expected ${this.schemaVersion}. ` + + `Refusing to apply a committed transaction through a stale cached adapter.`, + ) + } + + if (version.already_applied === 1) { + return + } + + const currentRowVersion = version.latest_row_version const nextRowVersion = Math.max(currentRowVersion + 1, tx.rowVersion) - const replayDelta: ReplayableTxDelta | null = tx.truncate + const replacesPersistedBaseline = tx.truncate === true + const tracksPersistedKeySet = + version.key_set_evidence_available === 1 || replacesPersistedBaseline + const replayDelta: ReplayableTxDelta | null = replacesPersistedBaseline ? null : { txId: tx.txId, @@ -1206,7 +1321,12 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { collectionMetadataMutations: tx.collectionMetadataMutations ?? [], } - if (tx.truncate) { + if (replacesPersistedBaseline) { + await transactionDriver.run( + `DELETE FROM collection_expected_keys + WHERE collection_id = ?`, + [collectionId], + ) await transactionDriver.run(`DELETE FROM ${collectionTableSql}`) await transactionDriver.run(`DELETE FROM ${tombstoneTableSql}`) } @@ -1214,6 +1334,13 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { for (const mutation of tx.mutations) { const encodedKey = encodePersistedStorageKey(mutation.key) if (mutation.type === `delete`) { + if (tracksPersistedKeySet) { + await transactionDriver.run( + `DELETE FROM collection_expected_keys + WHERE collection_id = ? AND key = ?`, + [collectionId, encodedKey], + ) + } await transactionDriver.run( `DELETE FROM ${collectionTableSql} WHERE key = ?`, @@ -1262,6 +1389,14 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { ? mutation.metadata : existingMetadata + if (tracksPersistedKeySet) { + await transactionDriver.run( + `INSERT INTO collection_expected_keys (collection_id, key) + VALUES (?, ?) + ON CONFLICT(collection_id, key) DO NOTHING`, + [collectionId, encodedKey], + ) + } await transactionDriver.run( `INSERT INTO ${collectionTableSql} (key, value, metadata, row_version) VALUES (?, ?, ?, ?) @@ -1333,11 +1468,23 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { } await transactionDriver.run( - `INSERT INTO collection_version (collection_id, latest_row_version) - VALUES (?, ?) - ON CONFLICT(collection_id) DO UPDATE SET - latest_row_version = excluded.latest_row_version`, - [collectionId, nextRowVersion], + `UPDATE collection_version + SET latest_row_version = ?, + key_set_evidence_available = CASE + WHEN ? = 1 THEN 1 + ELSE key_set_evidence_available + END, + key_set_evidence_incompatible = CASE + WHEN ? = 1 THEN 0 + ELSE key_set_evidence_incompatible + END + WHERE collection_id = ?`, + [ + nextRowVersion, + replacesPersistedBaseline ? 1 : 0, + replacesPersistedBaseline ? 1 : 0, + collectionId, + ], ) await transactionDriver.run( @@ -1371,7 +1518,7 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { tx.txId, nextRowVersion, replayDelta ? stableStringify(replayDelta) : null, - tx.truncate ? 1 : 0, + replacesPersistedBaseline ? 1 : 0, ], ) @@ -1512,10 +1659,10 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { latestTerm: number latestSeq: number latestRowVersion: number + keySet?: PersistedKeySetEvidence }> { - await this.ensureCollectionReady(collectionId) - - const [termRows, versionRows, seqRows] = await Promise.all([ + const tableMapping = await this.ensureCollectionReady(collectionId) + const [termRows, version, seqRows] = await Promise.all([ this.driver.query<{ latest_term: number }>( `SELECT latest_term FROM leader_term @@ -1523,13 +1670,7 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { LIMIT 1`, [collectionId], ), - this.driver.query<{ latest_row_version: number }>( - `SELECT latest_row_version - FROM collection_version - WHERE collection_id = ? - LIMIT 1`, - [collectionId], - ), + this.readKeySetEvidence(collectionId, tableMapping, this.driver), this.driver.query<{ max_seq: number }>( `SELECT MAX(seq) AS max_seq FROM applied_tx @@ -1543,7 +1684,66 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { return { latestTerm: termRows[0]?.latest_term ?? 0, latestSeq: seqRows[0]?.max_seq ?? 0, - latestRowVersion: versionRows[0]?.latest_row_version ?? 0, + latestRowVersion: version.latestRowVersion, + keySet: version.keySet, + } + } + + private async readKeySetEvidence( + collectionId: string, + tableMapping: CollectionTableMapping, + driver: SQLiteDriver, + ): Promise<{ + latestRowVersion: number + keySet: PersistedKeySetEvidence + }> { + const collectionTableSql = quoteIdentifier(tableMapping.tableName) + const versionRows = await driver.query<{ + latest_row_version: number + key_set_evidence_available: number + key_set_incompatible: number + }>( + `SELECT + latest_row_version, + key_set_evidence_available, + CASE + WHEN key_set_evidence_available = 0 THEN 0 + WHEN key_set_evidence_incompatible = 1 THEN 1 + WHEN EXISTS ( + SELECT 1 + FROM ${collectionTableSql} AS actual + LEFT JOIN collection_expected_keys AS expected + ON expected.collection_id = ? + AND expected.key = actual.key + WHERE expected.key IS NULL + UNION ALL + SELECT 1 + FROM collection_expected_keys AS expected + LEFT JOIN ${collectionTableSql} AS actual + ON actual.key = expected.key + WHERE expected.collection_id = ? + AND actual.key IS NULL + LIMIT 1 + ) THEN 1 + ELSE 0 + END AS key_set_incompatible + FROM collection_version + WHERE collection_id = ? + LIMIT 1`, + [collectionId, collectionId, collectionId], + ) + const version = versionRows[0] + + return { + latestRowVersion: version?.latest_row_version ?? 0, + keySet: { + status: + version?.key_set_evidence_available !== 1 + ? `unknown` + : version.key_set_incompatible === 1 + ? `incompatible` + : `consistent`, + }, } } @@ -1685,6 +1885,7 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { private async loadSubsetInternal( tableMapping: CollectionTableMapping, options: LoadSubsetOptions, + driver: SQLiteDriver = this.driver, ): Promise>>> { const collectionTableSql = quoteIdentifier(tableMapping.tableName) const whereCompiled = options.where @@ -1705,10 +1906,7 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { queryParams.push(...orderByCompiled.params) } - const storedRows = await this.driver.query( - sql, - queryParams, - ) + const storedRows = await driver.query(sql, queryParams) const parsedRows = decodeStoredSqliteRows(storedRows) const filteredRows = this.applyInMemoryWhere(parsedRows, options.where) @@ -1959,8 +2157,13 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { ON ${tombstoneTableSql} (row_version)`, ) await this.driver.run( - `INSERT INTO collection_version (collection_id, latest_row_version) - VALUES (?, 0) + `INSERT INTO collection_version ( + collection_id, + latest_row_version, + key_set_evidence_available, + key_set_evidence_incompatible + ) + VALUES (?, 0, 1, 0) ON CONFLICT(collection_id) DO NOTHING`, [collectionId], ) @@ -1970,7 +2173,7 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { ON CONFLICT(collection_id) DO NOTHING`, [collectionId], ) - + await this.ensureCollectionKeyEvidenceTriggers(tableName) const mapping = { tableName, tombstoneTableName, @@ -1979,6 +2182,75 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { return mapping } + private async ensureCollectionKeyEvidenceTriggers( + tableName: string, + ): Promise { + const collectionTableSql = quoteIdentifier(tableName) + const tableNameLiteral = toSqliteLiteral(tableName) + const insertTriggerSql = quoteIdentifier(`${tableName}_key_evidence_insert`) + const deleteTriggerSql = quoteIdentifier(`${tableName}_key_evidence_delete`) + const updateTriggerSql = quoteIdentifier(`${tableName}_key_evidence_update`) + + await this.driver.exec( + `CREATE TRIGGER IF NOT EXISTS ${insertTriggerSql} + AFTER INSERT ON ${collectionTableSql} + WHEN NOT EXISTS ( + SELECT 1 + FROM collection_expected_keys + WHERE collection_id = ( + SELECT collection_id + FROM collection_registry + WHERE table_name = ${tableNameLiteral} + ) AND key = NEW.key + ) + BEGIN + UPDATE collection_version + SET key_set_evidence_incompatible = 1 + WHERE collection_id = ( + SELECT collection_id + FROM collection_registry + WHERE table_name = ${tableNameLiteral} + ); + END`, + ) + await this.driver.exec( + `CREATE TRIGGER IF NOT EXISTS ${deleteTriggerSql} + AFTER DELETE ON ${collectionTableSql} + WHEN EXISTS ( + SELECT 1 + FROM collection_expected_keys + WHERE collection_id = ( + SELECT collection_id + FROM collection_registry + WHERE table_name = ${tableNameLiteral} + ) AND key = OLD.key + ) + BEGIN + UPDATE collection_version + SET key_set_evidence_incompatible = 1 + WHERE collection_id = ( + SELECT collection_id + FROM collection_registry + WHERE table_name = ${tableNameLiteral} + ); + END`, + ) + await this.driver.exec( + `CREATE TRIGGER IF NOT EXISTS ${updateTriggerSql} + AFTER UPDATE OF key ON ${collectionTableSql} + WHEN OLD.key <> NEW.key + BEGIN + UPDATE collection_version + SET key_set_evidence_incompatible = 1 + WHERE collection_id = ( + SELECT collection_id + FROM collection_registry + WHERE table_name = ${tableNameLiteral} + ); + END`, + ) + } + private async handleSchemaMismatch( collectionId: string, previousSchemaVersion: number, @@ -1997,6 +2269,27 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { const tombstoneTableSql = quoteIdentifier(tombstoneTableName) await this.runInTransaction(async (transactionDriver) => { + const currentSchemaRows = await transactionDriver.query<{ + schema_version: number + }>( + `SELECT schema_version + FROM collection_registry + WHERE collection_id = ? + LIMIT 1`, + [collectionId], + ) + const currentSchemaVersion = currentSchemaRows[0]?.schema_version + if (currentSchemaVersion === nextSchemaVersion) { + return + } + if (currentSchemaVersion !== previousSchemaVersion) { + throw new InvalidPersistedCollectionConfigError( + `Schema version changed concurrently for collection "${collectionId}": ` + + `found ${currentSchemaVersion ?? `no registry entry`} after observing ${previousSchemaVersion}; ` + + `refusing to reset it to ${nextSchemaVersion}.`, + ) + } + const persistedIndexes = await transactionDriver.query<{ index_name: string }>( @@ -2011,6 +2304,11 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { ) } + await transactionDriver.run( + `DELETE FROM collection_expected_keys + WHERE collection_id = ?`, + [collectionId], + ) await transactionDriver.run(`DELETE FROM ${collectionTableSql}`) await transactionDriver.run(`DELETE FROM ${tombstoneTableSql}`) await transactionDriver.run( @@ -2023,6 +2321,11 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { WHERE collection_id = ?`, [collectionId], ) + await transactionDriver.run( + `DELETE FROM collection_metadata + WHERE collection_id = ?`, + [collectionId], + ) await transactionDriver.run( `UPDATE collection_registry SET schema_version = ?, @@ -2031,10 +2334,17 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { [nextSchemaVersion, collectionId], ) await transactionDriver.run( - `INSERT INTO collection_version (collection_id, latest_row_version) - VALUES (?, 0) + `INSERT INTO collection_version ( + collection_id, + latest_row_version, + key_set_evidence_available, + key_set_evidence_incompatible + ) + VALUES (?, 0, 1, 0) ON CONFLICT(collection_id) DO UPDATE SET - latest_row_version = 0`, + latest_row_version = 0, + key_set_evidence_available = 1, + key_set_evidence_incompatible = 0`, [collectionId], ) await transactionDriver.run( @@ -2110,7 +2420,37 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { await this.driver.exec( `CREATE TABLE IF NOT EXISTS collection_version ( collection_id TEXT PRIMARY KEY, - latest_row_version INTEGER NOT NULL + latest_row_version INTEGER NOT NULL, + key_set_evidence_available INTEGER NOT NULL DEFAULT 0, + key_set_evidence_incompatible INTEGER NOT NULL DEFAULT 0 + )`, + ) + const collectionVersionColumns = await this.driver.query<{ name: string }>( + `PRAGMA table_info(collection_version)`, + ) + const keyEvidenceColumns = [ + `key_set_evidence_available`, + `key_set_evidence_incompatible`, + ] as const + for (const columnName of keyEvidenceColumns) { + if (collectionVersionColumns.some(({ name }) => name === columnName)) { + continue + } + try { + await this.driver.exec( + `ALTER TABLE collection_version ADD COLUMN ${columnName} INTEGER NOT NULL DEFAULT 0`, + ) + } catch (error) { + if (!isDuplicateColumnAddError(error, columnName)) { + throw error + } + } + } + await this.driver.exec( + `CREATE TABLE IF NOT EXISTS collection_expected_keys ( + collection_id TEXT NOT NULL, + key TEXT NOT NULL, + PRIMARY KEY (collection_id, key) )`, ) await this.driver.exec( diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts b/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts index b07f9f787d..336c2ff676 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts @@ -1,7 +1,11 @@ import { describe, expectTypeOf, it } from 'vitest' import { createCollection } from '@tanstack/db' import { persistedCollectionOptions } from '../src' -import type { PersistedCollectionUtils, PersistenceAdapter } from '../src' +import type { + PersistedCollectionUtils, + PersistedKeySetEvidence, + PersistenceAdapter, +} from '../src' import type { SyncConfig, UtilsRecord } from '@tanstack/db' type Todo = { @@ -24,6 +28,46 @@ const adapter: PersistenceAdapter = { } describe(`persisted collection types`, () => { + it(`keeps the atomic resume snapshot extension optional and exact`, () => { + const legacyAdapter: PersistenceAdapter = adapter + const snapshotAdapter: PersistenceAdapter = { + ...adapter, + loadResumeSnapshot: (_collectionId, _options) => + Promise.resolve({ + rows: [], + keySet: { status: `consistent` }, + collectionMetadata: [], + latestTerm: 1, + latestSeq: 2, + latestRowVersion: 3, + resetEpoch: 4, + }), + } + type LoadResumeSnapshot = NonNullable< + PersistenceAdapter[`loadResumeSnapshot`] + > + type ResumeSnapshot = Awaited> + + expectTypeOf(legacyAdapter).toMatchTypeOf() + expectTypeOf(snapshotAdapter.loadResumeSnapshot).toMatchTypeOf< + LoadResumeSnapshot | undefined + >() + expectTypeOf[1]>().toEqualTypeOf< + | { + requiredIndexSignatures?: ReadonlyArray + includeRows?: boolean + } + | undefined + >() + expectTypeOf().toEqualTypeOf< + PersistedKeySetEvidence | undefined + >() + + // @ts-expect-error key-set evidence has exactly three supported states + const invalidEvidence: PersistedKeySetEvidence = { status: `verified` } + expectTypeOf(invalidEvidence).toEqualTypeOf() + }) + it(`adds persisted utils in sync-absent mode`, () => { const options = persistedCollectionOptions< Todo, diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts index de10c93e6c..86809a3eb6 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts @@ -4,6 +4,7 @@ import { copyFileSync, existsSync, mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { promisify } from 'node:util' +import { fc } from '@fast-check/vitest' import { afterEach, describe, expect, it } from 'vitest' import { IR } from '@tanstack/db' import { SQLiteCorePersistenceAdapter, createPersistedTableName } from '../src' @@ -211,6 +212,421 @@ function createHarness( } } +type ResetResumeHistory = { + fromSchemaVersion: number + rows: Array + resumeKind: `none` | `reset` | `resume` + transition: + | `compatible-reopen` + | `schema-reset` + | `partial-restore` + | `external-row-loss` + reopensBeforeTransition: number + reopensAfterTransition: number + unrelatedMetadataKeys: Array +} + +type ResetResumeObservation = { + checkpoint: `after-persistence-transition-restart` + resetEpoch: number + schemaVersion: number + durableRows: ReadonlyArray + tombstones: ReadonlyArray<{ key: string; rowVersion: number }> + appliedTransactions: ReadonlyArray<{ txId: string; rowVersion: number }> + latestRowVersion: number + metadataKeys: ReadonlyArray + resumeState: unknown +} + +type ResetResumeExpectation = { + metadataKeys: ReadonlyArray + resumeKind: unknown +} + +function destroysPersistedBaseline(history: ResetResumeHistory): boolean { + return ( + history.transition === `schema-reset` || + history.transition === `partial-restore` + ) +} + +function expectedResetResumeMetadata( + history: ResetResumeHistory, +): ResetResumeExpectation { + if (destroysPersistedBaseline(history)) { + return { metadataKeys: [], resumeKind: undefined } + } + + return { + metadataKeys: [ + ...(history.resumeKind === `none` ? [] : [`electric:resume`]), + ...history.unrelatedMetadataKeys.map((key) => `oracle:${key}`), + ].sort(), + resumeKind: history.resumeKind === `none` ? undefined : history.resumeKind, + } +} + +class ResetResumeOracleViolation extends Error { + readonly law: string = `reset-resume.baseline-lineage` + readonly discriminant: string + readonly history: ResetResumeHistory + readonly observation: ResetResumeObservation + readonly cleanupEvidence: string + readonly expected: ResetResumeExpectation + readonly actual: ResetResumeExpectation + + constructor( + history: ResetResumeHistory, + observation: ResetResumeObservation, + cleanupEvidence: string, + cause: unknown, + ) { + super( + `A persisted-baseline transition violated collection-metadata lineage at the ` + + `${observation.checkpoint}. ` + + `history=${JSON.stringify(history)} ` + + `observation=${JSON.stringify(observation)} ` + + `cleanup=${cleanupEvidence}`, + { cause }, + ) + this.name = `ResetResumeOracleViolation` + this.history = structuredClone(history) + this.observation = structuredClone(observation) + this.cleanupEvidence = cleanupEvidence + this.expected = expectedResetResumeMetadata(history) + this.actual = { + metadataKeys: [...observation.metadataKeys], + resumeKind: resumeKindOf(observation.resumeState), + } + this.discriminant = destroysPersistedBaseline(history) + ? `reset-retained-metadata` + : `non-reset-metadata-changed` + } +} + +function hasSameResetResumeFailure( + left: ResetResumeOracleViolation, + right: ResetResumeOracleViolation, +): boolean { + const signature = (failure: ResetResumeOracleViolation): string => + JSON.stringify({ + law: String(failure.law), + discriminant: String(failure.discriminant), + checkpoint: String(failure.observation.checkpoint), + expectedMetadataClass: + failure.expected.metadataKeys.length === 0 ? `empty` : `nonempty`, + actualMetadataClass: + failure.actual.metadataKeys.length === 0 ? `empty` : `nonempty`, + }) + return signature(left) === signature(right) +} + +function resumeKindOf(value: unknown): unknown { + return value && typeof value === `object` + ? (value as Record).kind + : undefined +} + +function expectResetResumeLaw( + history: ResetResumeHistory, + observation: ResetResumeObservation, +): void { + // The core adapter owns the reset transaction: it must clear every metadata + // record coupled to the destroyed baseline. Compatible reopen and raw + // external row loss do not give this generic layer authority to interpret an + // Electric cursor, so they preserve the metadata exactly at this checkpoint. + expect( + { + metadataKeys: observation.metadataKeys, + resumeKind: resumeKindOf(observation.resumeState), + }, + `reset clears all collection metadata; non-reset core transitions preserve it`, + ).toEqual(expectedResetResumeMetadata(history)) +} + +function attachResetResumeCleanupDiagnostics( + primary: unknown, + cleanupEvidence: string, + cleanupFailure: unknown, +): Error { + const error = + primary instanceof Error + ? primary + : new Error(`Reset/resume oracle failed with a non-Error value`, { + cause: primary, + }) + if (!(`cleanupEvidence` in error)) { + Object.defineProperty(error, `cleanupEvidence`, { + value: cleanupEvidence, + enumerable: true, + }) + } + if (cleanupFailure !== undefined) { + Object.defineProperty(error, `cleanupFailure`, { + value: cleanupFailure, + enumerable: true, + }) + } + return error +} + +async function observeResetResumeHistory( + history: ResetResumeHistory, + harnessFactory: SQLiteCoreAdapterHarnessFactory, +): Promise { + let harness: ReturnType | undefined + const collectionId = `reset-resume-oracle` + let observation!: ResetResumeObservation + let primaryFailure: unknown + let cleanupFailure: unknown + let failurePhase: `setup` | `reach` | `law` | undefined + let cleanupEvidence = `not-run` + const expectedRows = structuredClone(history.rows) + const seedRows = structuredClone(history.rows) + const restoreRows = structuredClone(history.rows) + + try { + harness = harnessFactory({ schemaVersion: history.fromSchemaVersion }) + await harness.adapter.applyCommittedTx(collectionId, { + txId: `seed-baseline`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + ...seedRows.map((row) => ({ + type: `insert` as const, + key: row.id, + value: structuredClone(row), + })), + { + type: `delete` as const, + key: `deleted-before-baseline`, + value: { + id: `deleted-before-baseline`, + title: `baseline tombstone`, + createdAt: `2026-01-01T00:00:00.000Z`, + score: -1, + }, + }, + ], + collectionMetadataMutations: [ + ...(history.resumeKind === `none` + ? [] + : [ + { + type: `set` as const, + key: `electric:resume`, + value: + history.resumeKind === `reset` + ? { kind: `reset`, updatedAt: 1 } + : { + kind: `resume`, + offset: `10_0`, + handle: `handle-before-reset`, + shapeId: `shape-before-reset`, + updatedAt: 1, + }, + }, + ]), + ...history.unrelatedMetadataKeys.map((key, index) => ({ + type: `set` as const, + key: `oracle:${key}`, + value: { index }, + })), + ], + }) + + for (let index = 0; index < history.reopensBeforeTransition; index++) { + const reopened = new SQLiteCorePersistenceAdapter({ + driver: harness.driver, + schemaVersion: history.fromSchemaVersion, + }) + await reopened.loadSubset(collectionId, {}) + await reopened.loadCollectionMetadata(collectionId) + } + + const usesSchemaReset = destroysPersistedBaseline(history) + const nextSchemaVersion = usesSchemaReset + ? history.fromSchemaVersion + 1 + : history.fromSchemaVersion + let restarted = new SQLiteCorePersistenceAdapter({ + driver: harness.driver, + schemaVersion: nextSchemaVersion, + schemaMismatchPolicy: `sync-present-reset`, + }) + await restarted.loadSubset(collectionId, {}) + + if (history.transition === `partial-restore`) { + await restarted.applyCommittedTx(collectionId, { + txId: `partial-restore`, + term: 2, + seq: 1, + rowVersion: 2, + mutations: restoreRows.slice(0, -1).map((row) => ({ + type: `insert` as const, + key: row.id, + value: structuredClone(row), + })), + }) + } else if (history.transition === `external-row-loss`) { + const collectionTable = createPersistedTableName(collectionId, `c`) + await harness.driver.run( + `DELETE FROM "${collectionTable}" WHERE json_extract(value, '$.id') = ?`, + [history.rows[0]!.id], + ) + } + + for (let index = 0; index < history.reopensAfterTransition; index++) { + restarted = new SQLiteCorePersistenceAdapter({ + driver: harness.driver, + schemaVersion: nextSchemaVersion, + schemaMismatchPolicy: `sync-present-reset`, + }) + await restarted.loadSubset(collectionId, {}) + } + + const metadata = await restarted.loadCollectionMetadata(collectionId) + const resetEpochRows = await harness.driver.query<{ reset_epoch: number }>( + `SELECT reset_epoch FROM collection_reset_epoch WHERE collection_id = ?`, + [collectionId], + ) + const registryRows = await harness.driver.query<{ schema_version: number }>( + `SELECT schema_version FROM collection_registry WHERE collection_id = ?`, + [collectionId], + ) + const tombstoneTable = createPersistedTableName(collectionId, `t`) + const tombstones = await harness.driver.query<{ + key: string + row_version: number + }>(`SELECT key, row_version FROM "${tombstoneTable}" ORDER BY key`) + const appliedTransactions = await harness.driver.query<{ + tx_id: string + row_version: number + }>( + `SELECT tx_id, row_version FROM applied_tx WHERE collection_id = ? ORDER BY term, seq`, + [collectionId], + ) + const versionRows = await harness.driver.query<{ + latest_row_version: number + }>( + `SELECT latest_row_version FROM collection_version WHERE collection_id = ?`, + [collectionId], + ) + observation = { + checkpoint: `after-persistence-transition-restart`, + resetEpoch: resetEpochRows[0]?.reset_epoch ?? -1, + schemaVersion: registryRows[0]?.schema_version ?? -1, + durableRows: (await restarted.loadSubset(collectionId, {})).sort((a, b) => + String(a.key).localeCompare(String(b.key)), + ), + tombstones: tombstones.map(({ key, row_version }) => ({ + key, + rowVersion: row_version, + })), + appliedTransactions: appliedTransactions.map( + ({ tx_id, row_version }) => ({ + txId: tx_id, + rowVersion: row_version, + }), + ), + latestRowVersion: versionRows[0]?.latest_row_version ?? -1, + metadataKeys: metadata.map(({ key }) => key).sort(), + resumeState: metadata.find(({ key }) => key === `electric:resume`)?.value, + } + + try { + // Positive reach evidence is separate from the semantic accusation. + expect(observation.schemaVersion).toBe(nextSchemaVersion) + expect(observation.resetEpoch).toBe(usesSchemaReset ? 1 : 0) + expect(observation.durableRows).toEqual( + (history.transition === `schema-reset` + ? [] + : history.transition === `partial-restore` + ? expectedRows.slice(0, -1) + : history.transition === `external-row-loss` + ? expectedRows.slice(1) + : expectedRows + ) + .map((value) => ({ key: value.id, value })) + .sort((a, b) => String(a.key).localeCompare(String(b.key))), + ) + expect(observation.tombstones).toEqual( + usesSchemaReset + ? [] + : [{ key: `s:deleted-before-baseline`, rowVersion: 1 }], + ) + expect(observation.appliedTransactions).toEqual( + history.transition === `schema-reset` + ? [] + : history.transition === `partial-restore` + ? [{ txId: `partial-restore`, rowVersion: 2 }] + : [{ txId: `seed-baseline`, rowVersion: 1 }], + ) + expect(observation.latestRowVersion).toBe( + history.transition === `schema-reset` + ? 0 + : history.transition === `partial-restore` + ? 2 + : 1, + ) + } catch (error) { + failurePhase = `reach` + primaryFailure = error + } + + if (primaryFailure === undefined) { + try { + expectResetResumeLaw(history, observation) + } catch (error) { + failurePhase = `law` + primaryFailure = error + } + } + } catch (error) { + if (primaryFailure === undefined) { + failurePhase = `setup` + primaryFailure = error + } + } finally { + if (harness) { + try { + await harness.cleanup() + cleanupEvidence = + `dbPath` in harness && typeof harness.dbPath === `string` + ? existsSync(harness.dbPath) + ? `failed: SQLite file still exists` + : `passed: SQLite file removed after captured checkpoint` + : `passed: registered harness cleanup completed after captured checkpoint` + } catch (cleanupError) { + cleanupEvidence = `failed: ${String(cleanupError)}` + cleanupFailure = cleanupError + } + } + } + + if (primaryFailure !== undefined) { + const failure = + failurePhase === `law` + ? new ResetResumeOracleViolation( + history, + observation, + cleanupEvidence, + primaryFailure, + ) + : primaryFailure + throw attachResetResumeCleanupDiagnostics( + failure, + cleanupEvidence, + cleanupFailure, + ) + } + if (cleanupFailure !== undefined) throw cleanupFailure + if (!cleanupEvidence.startsWith(`passed:`)) { + throw new Error(cleanupEvidence) + } + return observation +} + export type SQLiteCoreAdapterHarnessFactory = ( options?: Omit< ConstructorParameters[0], @@ -990,6 +1406,274 @@ export function runSQLiteCoreAdapterContractSuite( expect(resetRows).toEqual([]) }) + /** + * Reset/resume oracle card + * + * Law and source: a destructive schema reset creates a new persisted + * baseline and clears every collection-metadata record in that reset + * transaction. Same-schema restarts preserve metadata, including for a + * genuinely empty baseline. The production reset policy above and the + * independently reproduced history in + * https://github.com/TanStack/db/issues/1589 establish this narrow law. + * + * Domain and legal histories: zero-to-four committed rows; absent, reset, + * or non-initial resume metadata; same-schema restart, vN -> vN+1 + * sync-present-reset, a partial restore after reset, or out-of-band row + * loss; restarts on either side; unrelated metadata. + * + * Reference: a two-generation lineage relation. Same-schema reopen and raw + * external loss keep the core generation/metadata record; schema reset + * replaces the generation and starts with no metadata. This model does not + * inspect the adapter's SQL branches or infer compatibility from row + * cardinality. + * + * Production path and checkpoint: SQLiteCorePersistenceAdapter commits a + * real SQLite baseline, then a new adapter instance reaches + * ensureCollectionReady/handleSchemaMismatch. At the + * after-persistence-transition-restart checkpoint we inspect registry + * version, reset epoch/generation, complete durable rows, tombstones, + * applied transactions, and collection metadata. + * + * Observed result and known omissions: exact settled rows, exact metadata + * keys, and whether the durable Electric state is absent/reset/resume. The + * external-loss lane proves the generic adapter leaves metadata reachable; + * only the persisted+Electric suite judges whether that cursor is safe to + * consume. The injected loss is not a claim that arbitrary SQL is a + * supported public API. Partial resumed updates and SDK framing retain + * their executable owners in the Electric recovery and framing suites. + * + * Trust: reset_epoch and schema_version prove reach; retained metadata + * after reset is the whole-path fault control, while compatible and + * external-loss histories calibrate preservation. Every generated database + * is removed after evidence capture. The failure reports first and reduced + * histories plus a verified fast-check seed/path replay. + */ + it(`resets collection metadata with its persisted baseline across generated restart histories`, async () => { + const historyArbitrary = fc + .constantFrom< + ResetResumeHistory[`transition`] + >(`compatible-reopen`, `schema-reset`, `partial-restore`, `external-row-loss`) + .chain((transition) => + fc.record({ + fromSchemaVersion: fc.integer({ min: 1, max: 4 }), + rows: fc + .uniqueArray(fc.integer({ min: 0, max: 20 }), { + minLength: + transition === `partial-restore` + ? 2 + : transition === `external-row-loss` + ? 1 + : 0, + maxLength: 4, + }) + .map((ids) => + ids + .sort((left, right) => left - right) + .map((id) => ({ + id: String(id), + title: `row-${id}`, + createdAt: `2026-01-01T00:00:00.000Z`, + score: id, + })), + ), + resumeKind: fc.constantFrom(`none`, `reset`, `resume`), + transition: fc.constant(transition), + reopensBeforeTransition: fc.integer({ min: 0, max: 2 }), + reopensAfterTransition: fc.integer({ min: 0, max: 2 }), + unrelatedMetadataKeys: fc.uniqueArray( + fc.constantFrom(`gc`, `provider`, `custom`), + { maxLength: 3 }, + ), + }), + ) + + const seedText = + process.env.TANSTACK_DB_SQLITE_ORACLE_SEED ?? String(1659) + const seed = Number(seedText) + const path = process.env.TANSTACK_DB_SQLITE_ORACLE_PATH + const runsText = process.env.TANSTACK_DB_SQLITE_ORACLE_RUNS ?? String(24) + const numRuns = Number(runsText) + if (!Number.isSafeInteger(seed)) { + throw new Error(`TANSTACK_DB_SQLITE_ORACLE_SEED must be an integer`) + } + if (!Number.isSafeInteger(numRuns) || numRuns < 1) { + throw new Error(`TANSTACK_DB_SQLITE_ORACLE_RUNS must be positive`) + } + if (path !== undefined && !/^\d+(?::\d+)*$/.test(path)) { + throw new Error( + `TANSTACK_DB_SQLITE_ORACLE_PATH must be a numeric shrink path`, + ) + } + + let originalFailure: ResetResumeOracleViolation | undefined + const property = fc.asyncProperty(historyArbitrary, async (history) => { + try { + await observeResetResumeHistory(history, harnessFactory) + } catch (error) { + if ( + originalFailure === undefined && + error instanceof ResetResumeOracleViolation + ) { + originalFailure = error + } + throw error + } + }) + const failure = await fc.check(property, { + seed, + numRuns, + ...(path === undefined ? {} : { path, endOnFailure: true }), + }) + if (!failure.failed) return + if ( + !(failure.errorInstance instanceof ResetResumeOracleViolation) || + originalFailure === undefined || + failure.counterexamplePath === null + ) { + throw failure.errorInstance + } + if (!hasSameResetResumeFailure(originalFailure, failure.errorInstance)) { + throw new Error( + `Shrinking changed the reset/resume law or observation checkpoint`, + ) + } + + const replay = await fc.check( + fc.asyncProperty(historyArbitrary, async (history) => { + await observeResetResumeHistory(history, harnessFactory) + }), + { + seed: failure.seed, + path: failure.counterexamplePath, + endOnFailure: true, + }, + ) + if ( + !replay.failed || + !(replay.errorInstance instanceof ResetResumeOracleViolation) || + JSON.stringify(replay.counterexample) !== + JSON.stringify(failure.counterexample) || + !hasSameResetResumeFailure(failure.errorInstance, replay.errorInstance) + ) { + throw new Error( + `Reset/resume oracle replay did not reproduce the intended violation`, + ) + } + + const reducedFailure = failure.errorInstance + throw new Error( + `Reset/resume baseline-lineage violation. ` + + `seed=${failure.seed} path=${failure.counterexamplePath} ` + + `law=${reducedFailure.law} ` + + `discriminant=${reducedFailure.discriminant} ` + + `checkpoint=${reducedFailure.observation.checkpoint} ` + + `originalTrace=${JSON.stringify(originalFailure.history)} ` + + `reducedTrace=${JSON.stringify(reducedFailure.history)} ` + + `expected=${JSON.stringify(reducedFailure.expected)} ` + + `actual=${JSON.stringify(reducedFailure.actual)} ` + + `observation=${JSON.stringify(reducedFailure.observation)} ` + + `replay=verified ` + + `cleanup=${reducedFailure.cleanupEvidence}`, + { cause: reducedFailure }, + ) + }, 30_000) + + it(`leaves externally inconsistent metadata reachable for consumer validation`, async () => { + const observation = await observeResetResumeHistory( + { + fromSchemaVersion: 1, + rows: [ + { + id: `1`, + title: `lost externally`, + createdAt: `2026-01-01T00:00:00.000Z`, + score: 1, + }, + { + id: `2`, + title: `survives`, + createdAt: `2026-01-01T00:00:00.000Z`, + score: 2, + }, + ], + resumeKind: `resume`, + transition: `external-row-loss`, + reopensBeforeTransition: 1, + reopensAfterTransition: 1, + unrelatedMetadataKeys: [`provider`], + }, + harnessFactory, + ) + + expect(observation.metadataKeys).toEqual([ + `electric:resume`, + `oracle:provider`, + ]) + expect(resumeKindOf(observation.resumeState)).toBe(`resume`) + }) + + it(`requires complete metadata reset while preserving non-reset metadata`, () => { + const compatibleEmpty: ResetResumeHistory = { + fromSchemaVersion: 1, + rows: [], + resumeKind: `resume`, + transition: `compatible-reopen`, + reopensBeforeTransition: 0, + reopensAfterTransition: 1, + unrelatedMetadataKeys: [`provider`], + } + const observation: ResetResumeObservation = { + checkpoint: `after-persistence-transition-restart`, + resetEpoch: 0, + schemaVersion: 1, + durableRows: [], + tombstones: [], + appliedTransactions: [], + latestRowVersion: 1, + metadataKeys: [`electric:resume`, `oracle:provider`], + resumeState: { kind: `resume`, offset: `10_0` }, + } + expect(() => + expectResetResumeLaw(compatibleEmpty, observation), + ).not.toThrow() + expect(() => + expectResetResumeLaw( + { ...compatibleEmpty, transition: `external-row-loss` }, + observation, + ), + ).not.toThrow() + expect(() => + expectResetResumeLaw( + { ...compatibleEmpty, transition: `schema-reset` }, + { ...observation, resetEpoch: 1, schemaVersion: 2 }, + ), + ).toThrowError(expect.objectContaining({ name: `AssertionError` })) + expect(() => + expectResetResumeLaw( + { ...compatibleEmpty, transition: `schema-reset` }, + { + ...observation, + resetEpoch: 1, + schemaVersion: 2, + metadataKeys: [`oracle:provider`], + resumeState: undefined, + }, + ), + ).toThrowError(expect.objectContaining({ name: `AssertionError` })) + expect(() => + expectResetResumeLaw( + { ...compatibleEmpty, transition: `schema-reset` }, + { + ...observation, + resetEpoch: 1, + schemaVersion: 2, + metadataKeys: [], + resumeState: undefined, + }, + ), + ).not.toThrow() + }) + it(`returns pullSince deltas and requiresFullReload when threshold is exceeded`, async () => { const { adapter } = registerContractHarness({ pullSinceReloadThreshold: 1, diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts new file mode 100644 index 0000000000..0add417097 --- /dev/null +++ b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts @@ -0,0 +1,744 @@ +import { DatabaseSync } from 'node:sqlite' +import { describe, expect, it } from 'vitest' +import { SQLiteCorePersistenceAdapter, createPersistedTableName } from '../src' +import type { SQLiteDriver } from '../src' + +type CachedSchemaState = { + schemaVersion: number + resetEpoch: number + rows: Array<{ key: string | number; value: Record }> + metadata: Array<{ key: string; value: unknown }> + appliedTransactions: Array<{ + term: number + seq: number + txId: string + rowVersion: number + }> + keyEvidence: { + available: number + incompatible: number + expectedKeys: Array + } +} + +function toBinding(value: unknown): string | number | bigint | null { + if (value === null || value === undefined) return null + if (typeof value === `boolean`) return value ? 1 : 0 + if ( + typeof value === `string` || + typeof value === `number` || + typeof value === `bigint` + ) { + return value + } + return String(value) +} + +function createDriver( + database: DatabaseSync, + failTransactionRun?: (sql: string) => boolean, +): SQLiteDriver { + const driver: SQLiteDriver = { + exec: (sql) => { + database.exec(sql) + return Promise.resolve() + }, + query: (sql, params = []) => + Promise.resolve( + database + .prepare(sql) + .all(...params.map(toBinding)) + .map((row) => ({ ...row })) as Array, + ), + run: (sql, params = []) => { + database.prepare(sql).run(...params.map(toBinding)) + return Promise.resolve() + }, + transaction: async (transaction) => { + database.exec(`BEGIN IMMEDIATE`) + try { + const transactionDriver: SQLiteDriver = { + ...driver, + run: (sql, params) => + failTransactionRun?.(sql) + ? Promise.reject(new Error(`injected transaction failure`)) + : driver.run(sql, params), + } + const result = await transaction(transactionDriver) + database.exec(`COMMIT`) + return result + } catch (error) { + database.exec(`ROLLBACK`) + throw error + } + }, + } + return driver +} + +function deferred() { + let resolve!: () => void + const promise = new Promise((complete) => { + resolve = complete + }) + return { promise, resolve } +} + +async function reachCheckpoint( + promise: Promise, + checkpoint: string, +): Promise { + let timer: ReturnType | undefined + try { + await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`Did not reach checkpoint: ${checkpoint}`)), + 1_000, + ) + }), + ]) + } finally { + if (timer !== undefined) clearTimeout(timer) + } +} + +function closeDatabasePreservingPrimary( + database: DatabaseSync, + primaryFailure: unknown, +): never | void { + let cleanupFailure: unknown + try { + database.close() + } catch (error) { + cleanupFailure = error + } + + if (primaryFailure !== undefined) { + const failure = + primaryFailure instanceof Error + ? primaryFailure + : new Error(`SQLite resume snapshot failed`, { cause: primaryFailure }) + if (cleanupFailure !== undefined) { + Object.defineProperty(failure, `cleanupFailures`, { + value: [cleanupFailure], + enumerable: true, + }) + } + throw failure + } + if (cleanupFailure !== undefined) throw cleanupFailure +} + +async function observeCachedSchemaState( + adapter: SQLiteCorePersistenceAdapter, + driver: SQLiteDriver, + collectionId: string, +): Promise { + const snapshot = await adapter.loadResumeSnapshot(collectionId) + const registryRows = await driver.query<{ schema_version: number }>( + `SELECT schema_version FROM collection_registry WHERE collection_id = ?`, + [collectionId], + ) + const versionRows = await driver.query<{ + key_set_evidence_available: number + key_set_evidence_incompatible: number + }>( + `SELECT key_set_evidence_available, key_set_evidence_incompatible + FROM collection_version + WHERE collection_id = ?`, + [collectionId], + ) + const expectedKeys = await driver.query<{ key: string }>( + `SELECT key + FROM collection_expected_keys + WHERE collection_id = ? + ORDER BY key`, + [collectionId], + ) + const appliedTransactions = await driver.query<{ + term: number + seq: number + tx_id: string + row_version: number + }>( + `SELECT term, seq, tx_id, row_version + FROM applied_tx + WHERE collection_id = ? + ORDER BY term, seq`, + [collectionId], + ) + + return { + schemaVersion: registryRows[0]?.schema_version ?? -1, + resetEpoch: snapshot.resetEpoch, + rows: snapshot.rows + .map(({ key, value }) => ({ key, value })) + .sort((left, right) => String(left.key).localeCompare(String(right.key))), + metadata: snapshot.collectionMetadata.sort((left, right) => + left.key.localeCompare(right.key), + ), + appliedTransactions: appliedTransactions.map( + ({ term, seq, tx_id, row_version }) => ({ + term, + seq, + txId: tx_id, + rowVersion: row_version, + }), + ), + keyEvidence: { + available: versionRows[0]?.key_set_evidence_available ?? -1, + incompatible: versionRows[0]?.key_set_evidence_incompatible ?? -1, + expectedKeys: expectedKeys.map(({ key }) => key), + }, + } +} + +/** + * Narrow companion to sqlite-core-adapter.test.ts for atomic snapshot and DDL + * interleavings. That owner deliberately drives sqlite3 through a serialized + * copy-on-commit CLI harness, which cannot expose two adapters to the same + * in-flight connection state. This file uses node:sqlite only for that missing + * deterministic seam; expected key membership and reset lineage remain + * independent assertions, not a second production-shaped model. + */ +describe(`SQLite resume snapshots`, () => { + it(`keeps raw key loss sticky until a full replacement recertifies the baseline`, async () => { + const database = new DatabaseSync(`:memory:`) + let primaryFailure: unknown + try { + const collectionId = `resume-ledger` + const tableName = createPersistedTableName(collectionId, `c`) + let rejectCollectionInsert = false + const driver = createDriver( + database, + (sql) => + rejectCollectionInsert && sql.includes(`INSERT INTO "${tableName}"`), + ) + const adapter = new SQLiteCorePersistenceAdapter({ driver }) + await adapter.applyCommittedTx(collectionId, { + txId: `seed`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { type: `insert`, key: 1, value: { id: 1, name: `one` } }, + { type: `insert`, key: 2, value: { id: 2, name: `two` } }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `cursor`, value: `10_0` }, + ], + }) + + const initial = await adapter.loadResumeSnapshot(collectionId) + expect(initial.keySet).toEqual({ status: `consistent` }) + expect(initial.rows.map(({ key }) => key)).toEqual([1, 2]) + expect(initial.collectionMetadata).toEqual([ + { key: `cursor`, value: `10_0` }, + ]) + + await adapter.applyCommittedTx(collectionId, { + txId: `normal-update`, + term: 1, + seq: 2, + rowVersion: 2, + mutations: [ + { type: `update`, key: 1, value: { id: 1, name: `updated-one` } }, + ], + }) + const updated = await adapter.loadResumeSnapshot(collectionId) + expect(updated.rows.find(({ key }) => key === 1)?.value).toEqual({ + id: 1, + name: `updated-one`, + }) + expect(updated.keySet).toEqual({ status: `consistent` }) + + await adapter.applyCommittedTx(collectionId, { + txId: `normal-delete`, + term: 1, + seq: 3, + rowVersion: 3, + mutations: [{ type: `delete`, key: 2, value: { id: 2, name: `two` } }], + }) + await adapter.applyCommittedTx(collectionId, { + txId: `normal-delete`, + term: 1, + seq: 3, + rowVersion: 3, + mutations: [{ type: `delete`, key: 2, value: { id: 2, name: `two` } }], + }) + const deleted = await adapter.loadResumeSnapshot(collectionId) + expect(deleted.rows.map(({ key }) => key)).toEqual([1]) + expect(deleted.keySet).toEqual({ status: `consistent` }) + + rejectCollectionInsert = true + await expect( + adapter.applyCommittedTx(collectionId, { + txId: `rolled-back-insert`, + term: 1, + seq: 4, + rowVersion: 4, + mutations: [{ type: `insert`, key: 3, value: { id: 3 } }], + }), + ).rejects.toThrow(`injected transaction failure`) + rejectCollectionInsert = false + const rolledBack = await adapter.loadResumeSnapshot(collectionId) + expect(rolledBack.rows.map(({ key }) => key)).toEqual([1]) + expect(rolledBack.keySet).toEqual({ status: `consistent` }) + expect( + await driver.query<{ count: number }>( + `SELECT COUNT(*) AS count FROM collection_expected_keys WHERE collection_id = ?`, + [collectionId], + ), + ).toEqual([{ count: 1 }]) + expect( + await driver.query<{ count: number }>( + `SELECT COUNT(*) AS count FROM applied_tx WHERE collection_id = ? AND tx_id = ?`, + [collectionId, `normal-delete`], + ), + ).toEqual([{ count: 1 }]) + + await adapter.applyCommittedTx(collectionId, { + txId: `restore-second-row`, + term: 1, + seq: 5, + rowVersion: 5, + mutations: [{ type: `insert`, key: 2, value: { id: 2, name: `two` } }], + }) + + await driver.run(`DELETE FROM "${tableName}" WHERE key = ?`, [ + database + .prepare(`SELECT key FROM "${tableName}" ORDER BY key LIMIT 1`) + .get()!.key, + ]) + await adapter.applyCommittedTx(collectionId, { + txId: `metadata-after-loss`, + term: 1, + seq: 6, + rowVersion: 6, + mutations: [], + collectionMetadataMutations: [ + { type: `set`, key: `cursor`, value: `11_0` }, + ], + }) + expect((await adapter.loadResumeSnapshot(collectionId)).keySet).toEqual({ + status: `incompatible`, + }) + + await adapter.applyCommittedTx(collectionId, { + txId: `full-replacement`, + term: 1, + seq: 7, + rowVersion: 7, + truncate: true, + mutations: [ + { type: `insert`, key: 1, value: { id: 1, name: `one` } }, + { type: `insert`, key: 2, value: { id: 2, name: `two` } }, + ], + }) + expect((await adapter.loadResumeSnapshot(collectionId)).keySet).toEqual({ + status: `consistent`, + }) + + await driver.run( + `UPDATE "${tableName}" SET key = key || '-replacement' WHERE rowid = (SELECT MIN(rowid) FROM "${tableName}")`, + ) + const substituted = await adapter.loadResumeSnapshot(collectionId) + expect(substituted.rows).toHaveLength(2) + expect(substituted.keySet).toEqual({ status: `incompatible` }) + } catch (error) { + primaryFailure = error + } finally { + closeDatabasePreservingPrimary(database, primaryFailure) + } + }) + + it(`migrates concurrent legacy schemas and stays unknown until truncate`, async () => { + const database = new DatabaseSync(`:memory:`) + let releasePending = () => {} + let primaryFailure: unknown + try { + const driver = createDriver(database) + const collectionId = `legacy-ledger` + const tableName = createPersistedTableName(collectionId, `c`) + const tombstoneTableName = createPersistedTableName(collectionId, `t`) + await driver.exec( + `CREATE TABLE collection_registry ( + collection_id TEXT PRIMARY KEY, + table_name TEXT NOT NULL UNIQUE, + tombstone_table_name TEXT NOT NULL UNIQUE, + schema_version INTEGER NOT NULL, + updated_at INTEGER NOT NULL + )`, + ) + await driver.run( + `INSERT INTO collection_registry + (collection_id, table_name, tombstone_table_name, schema_version, updated_at) + VALUES (?, ?, ?, 1, 0)`, + [collectionId, tableName, tombstoneTableName], + ) + await driver.exec( + `CREATE TABLE "${tableName}" ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + metadata TEXT, + row_version INTEGER NOT NULL + )`, + ) + await driver.exec( + `CREATE TABLE "${tombstoneTableName}" ( + key TEXT PRIMARY KEY, + value TEXT, + row_version INTEGER NOT NULL, + deleted_at TEXT NOT NULL + )`, + ) + await driver.exec( + `CREATE TABLE collection_version ( + collection_id TEXT PRIMARY KEY, + latest_row_version INTEGER NOT NULL + )`, + ) + await driver.run( + `INSERT INTO collection_version (collection_id, latest_row_version) + VALUES (?, 0)`, + [collectionId], + ) + + const bothSawLegacyColumns = deferred() + let legacyColumnReaders = 0 + const migrationRelease = deferred() + releasePending = migrationRelease.resolve + const createMigrationDriver = (): SQLiteDriver => ({ + ...driver, + query: async (sql: string, params?: ReadonlyArray) => { + const rows = await driver.query(sql, params) + if (sql.includes(`PRAGMA table_info(collection_version)`)) { + legacyColumnReaders++ + if (legacyColumnReaders === 2) bothSawLegacyColumns.resolve() + await migrationRelease.promise + } + return rows + }, + }) + const migrated = new SQLiteCorePersistenceAdapter({ + driver: createMigrationDriver(), + }) + const concurrentMigrated = new SQLiteCorePersistenceAdapter({ + driver: createMigrationDriver(), + }) + const initialize = (adapter: SQLiteCorePersistenceAdapter) => + ( + adapter as unknown as { ensureInitialized: () => Promise } + ).ensureInitialized() + const migrations = Promise.all([ + initialize(migrated), + initialize(concurrentMigrated), + ]) + await reachCheckpoint( + bothSawLegacyColumns.promise, + `both adapters observed the legacy collection_version schema`, + ) + migrationRelease.resolve() + await migrations + const migratedColumns = await driver.query<{ name: string }>( + `PRAGMA table_info(collection_version)`, + ) + expect(migratedColumns.map(({ name }) => name)).toEqual( + expect.arrayContaining([ + `key_set_evidence_available`, + `key_set_evidence_incompatible`, + ]), + ) + + expect((await migrated.loadResumeSnapshot(collectionId)).keySet).toEqual({ + status: `unknown`, + }) + await migrated.applyCommittedTx(collectionId, { + txId: `legacy-insert`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [{ type: `insert`, key: 1, value: { id: 1, n: 1 } }], + }) + expect((await migrated.loadResumeSnapshot(collectionId)).keySet).toEqual({ + status: `unknown`, + }) + + await migrated.applyCommittedTx(collectionId, { + txId: `legacy-replacement`, + term: 1, + seq: 2, + rowVersion: 2, + truncate: true, + mutations: [{ type: `insert`, key: 1, value: { id: 1, n: 2 } }], + }) + expect((await migrated.loadResumeSnapshot(collectionId)).keySet).toEqual({ + status: `consistent`, + }) + } catch (error) { + primaryFailure = error + } finally { + releasePending() + closeDatabasePreservingPrimary(database, primaryFailure) + } + }) + + it(`does not repeat a schema reset observed through a stale adapter read`, async () => { + const database = new DatabaseSync(`:memory:`) + let releasePending = () => {} + let primaryFailure: unknown + try { + const driver = createDriver(database) + const collectionId = `concurrent-schema-reset` + const original = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 1, + }) + await original.applyCommittedTx(collectionId, { + txId: `seed`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [{ type: `insert`, key: 1, value: { id: 1 } }], + collectionMetadataMutations: [ + { type: `set`, key: `cursor`, value: `old` }, + ], + }) + + const staleRead = deferred() + const releaseStaleRead = deferred() + releasePending = releaseStaleRead.resolve + let intercepted = false + const gatedDriver: SQLiteDriver = { + ...driver, + query: async (sql: string, params?: ReadonlyArray) => { + const rows = await driver.query(sql, params) + if (!intercepted && sql.includes(`FROM collection_registry`)) { + intercepted = true + staleRead.resolve() + await releaseStaleRead.promise + } + return rows + }, + } + const staleAdapter = new SQLiteCorePersistenceAdapter({ + driver: gatedDriver, + schemaVersion: 2, + }) + const staleLoad = staleAdapter.loadSubset(collectionId, {}) + await reachCheckpoint( + staleRead.promise, + `stale schema-v1 registry read before competing reset`, + ) + + const winner = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 2, + }) + await winner.loadSubset(collectionId, {}) + await winner.applyCommittedTx(collectionId, { + txId: `recovery-write`, + term: 2, + seq: 1, + rowVersion: 1, + mutations: [{ type: `insert`, key: 2, value: { id: 2 } }], + collectionMetadataMutations: [ + { type: `set`, key: `cursor`, value: `new` }, + ], + }) + + releaseStaleRead.resolve() + await staleLoad + const snapshot = await winner.loadResumeSnapshot(collectionId) + expect(snapshot.resetEpoch).toBe(1) + expect(snapshot.rows.map(({ key }) => key)).toEqual([2]) + expect(snapshot.collectionMetadata).toEqual([ + { key: `cursor`, value: `new` }, + ]) + } catch (error) { + primaryFailure = error + } finally { + releasePending() + closeDatabasePreservingPrimary(database, primaryFailure) + } + }) + + it(`refuses to downgrade a newer schema observed after a stale read`, async () => { + const database = new DatabaseSync(`:memory:`) + let releasePending = () => {} + let primaryFailure: unknown + try { + const driver = createDriver(database) + const collectionId = `divergent-schema-reset` + const original = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 1, + }) + await original.applyCommittedTx(collectionId, { + txId: `seed`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [{ type: `insert`, key: 1, value: { id: 1 } }], + }) + + const staleRead = deferred() + const releaseStaleRead = deferred() + releasePending = releaseStaleRead.resolve + let intercepted = false + const staleDriver: SQLiteDriver = { + ...driver, + query: async (sql: string, params?: ReadonlyArray) => { + const rows = await driver.query(sql, params) + if (!intercepted && sql.includes(`FROM collection_registry`)) { + intercepted = true + staleRead.resolve() + await releaseStaleRead.promise + } + return rows + }, + } + const staleV2 = new SQLiteCorePersistenceAdapter({ + driver: staleDriver, + schemaVersion: 2, + }) + const staleLoad = staleV2.loadSubset(collectionId, {}) + await reachCheckpoint( + staleRead.promise, + `stale schema-v1 registry read before schema-v3 reset`, + ) + + const winnerV3 = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 3, + }) + await winnerV3.loadSubset(collectionId, {}) + await winnerV3.applyCommittedTx(collectionId, { + txId: `winner-write`, + term: 3, + seq: 1, + rowVersion: 1, + mutations: [{ type: `insert`, key: 3, value: { id: 3 } }], + collectionMetadataMutations: [ + { type: `set`, key: `cursor`, value: `v3` }, + ], + }) + + releaseStaleRead.resolve() + await expect(staleLoad).rejects.toThrow( + `Schema version changed concurrently`, + ) + const snapshot = await winnerV3.loadResumeSnapshot(collectionId) + expect(snapshot.resetEpoch).toBe(1) + expect(snapshot.rows.map(({ key }) => key)).toEqual([3]) + expect(snapshot.collectionMetadata).toEqual([ + { key: `cursor`, value: `v3` }, + ]) + expect( + await driver.query<{ schema_version: number }>( + `SELECT schema_version FROM collection_registry WHERE collection_id = ?`, + [collectionId], + ), + ).toEqual([{ schema_version: 3 }]) + } catch (error) { + primaryFailure = error + } finally { + releasePending() + closeDatabasePreservingPrimary(database, primaryFailure) + } + }) + + it(`rejects a committed transaction from a cached adapter after another adapter resets the schema`, async () => { + const database = new DatabaseSync(`:memory:`) + let primaryFailure: unknown + try { + const driver = createDriver(database) + const collectionId = `cached-schema-write` + const staleV1 = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 1, + }) + await staleV1.applyCommittedTx(collectionId, { + txId: `seed-v1`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [{ type: `insert`, key: 1, value: { id: 1 } }], + collectionMetadataMutations: [ + { type: `set`, key: `cursor`, value: `v1` }, + ], + }) + + const currentV2 = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 2, + }) + await currentV2.loadSubset(collectionId, {}) + await currentV2.applyCommittedTx(collectionId, { + txId: `seed-v2`, + term: 2, + seq: 1, + rowVersion: 1, + mutations: [{ type: `insert`, key: 2, value: { id: 2 } }], + collectionMetadataMutations: [ + { type: `set`, key: `cursor`, value: `v2` }, + ], + }) + const before = await observeCachedSchemaState( + currentV2, + driver, + collectionId, + ) + + let lateWriteError: unknown + try { + await staleV1.applyCommittedTx(collectionId, { + txId: `late-v1`, + term: 3, + seq: 1, + rowVersion: 2, + truncate: true, + mutations: [{ type: `insert`, key: 3, value: { id: 3 } }], + collectionMetadataMutations: [ + { type: `set`, key: `cursor`, value: `late-v1` }, + { type: `set`, key: `late`, value: true }, + ], + }) + } catch (error) { + lateWriteError = error + } + const after = await observeCachedSchemaState( + currentV2, + driver, + collectionId, + ) + + expect( + { + error: + lateWriteError instanceof Error + ? { name: lateWriteError.name, message: lateWriteError.message } + : lateWriteError, + before, + after, + }, + `a cached adapter must reject at its transaction boundary without changing any durable state`, + ).toEqual({ + error: { + name: `InvalidPersistedCollectionConfigError`, + message: + `Schema version mismatch for collection "cached-schema-write": ` + + `found 2, expected 1. Refusing to apply a committed transaction through a stale cached adapter.`, + }, + before, + after: before, + }) + } catch (error) { + primaryFailure = error + } finally { + closeDatabasePreservingPrimary(database, primaryFailure) + } + }) +}) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 8152fd0a54..b8ccf94fc8 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -72,6 +72,10 @@ type ElectricSyncMetadataWithHydration = SyncMetadataApi & { whenHydrated?: () => Promise // Capability marker for wrappers predating the hydration barrier. scanPersisted?: unknown + certifyPersistedResume?: () => Promise + getPersistedKeySetEvidence?: () => + | { status: `unknown` | `consistent` | `incompatible` } + | undefined } } @@ -1684,6 +1688,11 @@ function createElectricSync>( | undefined const scanPersisted = persistedMetadata?.row.scanPersisted const whenHydrated = persistedMetadata?.row.whenHydrated + const certifyPersistedResume = + persistedMetadata?.row.certifyPersistedResume + const getPersistedKeySetEvidence = + persistedMetadata?.row.getPersistedKeySetEvidence + const persistedKeySetEvidence = getPersistedKeySetEvidence?.() const persistedResumeState = getNewestElectricResumeState( readPersistedResumeState(), @@ -1702,6 +1711,12 @@ function createElectricSync>( persistedResumeState?.kind === `resume` && scanPersisted !== undefined && whenHydrated === undefined + // A pre-ledger `unknown` baseline cannot justify a non-initial cursor. + // One fresh replacement establishes consistent evidence for later resumes. + const lacksCompletePersistedKeySet = + persistedResumeState?.kind === `resume` && + getPersistedKeySetEvidence !== undefined && + persistedKeySetEvidence?.status !== `consistent` if (hasUnverifiablePersistedResume && !warnedUnverifiableResume) { warnedUnverifiableResume = true console.warn( @@ -1713,6 +1728,7 @@ function createElectricSync>( shapeOptions.handle === undefined && persistedResumeState !== undefined && (persistedResumeState.kind === `reset` || + lacksCompletePersistedKeySet || (!retainsTagState && persistedResumeState.requiresTagState !== false)) const canUsePersistedResume = shapeOptions.offset === undefined && @@ -1721,7 +1737,7 @@ function createElectricSync>( !hasIncompatiblePersistedResume && !hasUnverifiablePersistedResume && // Cached rows do not contain authoritative tag/active-condition state. - // Unknown (older) metadata is conservative; untagged shapes still resume. + // Only a complete adapter ledger can justify a persisted cursor. !needsFullSnapshot const hasExplicitResumeOffset = shapeOptions.offset !== undefined && shapeOptions.offset !== `-1` @@ -1729,6 +1745,8 @@ function createElectricSync>( clearTagTrackingState() } const receivesCompleteRows = shapeOptions.params?.replica === `full` + const requiresKeySetCertification = + canUsePersistedResume && certifyPersistedResume !== undefined // Eager and progressive streams that start after the initial offset can // only apply partial updates when the local materialization is complete. const requiresCompleteResume = @@ -1986,8 +2004,29 @@ function createElectricSync>( const resumeKeysPromise = requiresCompleteResume || freshSnapshotPending - ? whenHydrated?.() - : undefined + ? whenHydrated + ? (async () => { + await whenHydrated() + if ( + persistedKeySetEvidence?.status !== `incompatible` && + getPersistedKeySetEvidence?.()?.status === `incompatible` + ) { + throw new Error( + `Electric persisted resume baseline became incompatible during hydration`, + ) + } + })() + : undefined + : requiresKeySetCertification + ? (async () => { + await certifyPersistedResume() + if (getPersistedKeySetEvidence?.()?.status === `incompatible`) { + throw new Error( + `Electric persisted resume baseline became incompatible during certification`, + ) + } + })() + : undefined let areResumeKeysReady = !resumeKeysPromise const pendingResumeBatches: Array>> = [] let unsubscribeStream: () => void = () => {} diff --git a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts index 00b3fc09b4..ea723e6fc5 100644 --- a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts +++ b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts @@ -1,9 +1,14 @@ +import { DatabaseSync } from 'node:sqlite' import { isDeepStrictEqual } from 'node:util' import { fc, test as fcTest } from '@fast-check/vitest' import { beforeEach, describe, expect, it, vi } from 'vitest' import { createCollection } from '@tanstack/db' import { ShapeStream } from '@electric-sql/client' -import { persistedCollectionOptions } from '../../db-sqlite-persistence-core/src' +import { + SQLiteCorePersistenceAdapter, + createPersistedTableName, + persistedCollectionOptions, +} from '../../db-sqlite-persistence-core/src' import { electricCollectionOptions } from '../src/electric' import type { Message, Row } from '@electric-sql/client' import type { @@ -11,6 +16,7 @@ import type { PersistedTx, PersistenceAdapter, ProtocolEnvelope, + SQLiteDriver, } from '../../db-sqlite-persistence-core/src' import type { ElectricCollectionUtils, ElectricSyncMode } from '../src/electric' @@ -18,6 +24,57 @@ type Item = Row & { id: number; name: string; stable: string } type Subscriber = (messages: Array>) => void type Exposure = { cut: string; rows: Array } +function persistedResumeKind(value: unknown): unknown { + return value && typeof value === `object` + ? (value as Record).kind + : undefined +} + +function toSqliteBinding(value: unknown): string | number | bigint | null { + if (value === null || value === undefined) return null + if (typeof value === `boolean`) return value ? 1 : 0 + if ( + typeof value === `string` || + typeof value === `number` || + typeof value === `bigint` + ) { + return value + } + return String(value) +} + +function nodeSqliteDriver(database: DatabaseSync): SQLiteDriver { + const driver: SQLiteDriver = { + exec: (sql) => { + database.exec(sql) + return Promise.resolve() + }, + query: (sql, params = []) => + Promise.resolve( + database + .prepare(sql) + .all(...params.map(toSqliteBinding)) + .map((row) => ({ ...row })) as Array, + ), + run: (sql, params = []) => { + database.prepare(sql).run(...params.map(toSqliteBinding)) + return Promise.resolve() + }, + transaction: async (transaction) => { + database.exec(`BEGIN IMMEDIATE`) + try { + const result = await transaction(driver) + database.exec(`COMMIT`) + return result + } catch (error) { + database.exec(`ROLLBACK`) + throw error + } + }, + } + return driver +} + function expectWholeRecoveryTrace( entries: Array, allowed: Array>, @@ -62,6 +119,7 @@ function deferred() { } const oldRow: Item = { id: 1, name: `old`, stable: `stable-1` } +const otherOldRow: Item = { id: 3, name: `other-old`, stable: `stable-3` } const freshRow: Item = { id: 2, name: `fresh`, stable: `stable-2` } const upToDate: Message = { headers: { control: `up-to-date` } } @@ -196,12 +254,514 @@ const scenarios = ([`eager`, `progressive`] as const).flatMap((syncMode) => ), ) +type PersistedRestartScenario = { + rowState: `empty` | `nonempty` + resumeKind: `none` | `reset` | `resume` + transition: + | `compatible-reopen` + | `schema-reset` + | `partial-restore` + | `external-row-loss` + sourceHistory: `up-to-date-only` | `replayed-insert` +} + +const persistedRestartScenarios = ([`empty`, `nonempty`] as const).flatMap( + (rowState) => + ([`none`, `reset`, `resume`] as const).flatMap((resumeKind) => + ( + [ + `compatible-reopen`, + `schema-reset`, + ...(rowState === `nonempty` + ? ([`partial-restore`, `external-row-loss`] as const) + : []), + ] as const + ).flatMap((transition) => + ([`up-to-date-only`, `replayed-insert`] as const).map( + (sourceHistory): PersistedRestartScenario => ({ + rowState, + resumeKind, + transition, + sourceHistory, + }), + ), + ), + ), +) + +type PersistedRestartObservation = { + checkpoint: `post-restart-up-to-date` + requestedOffset: string | undefined + requestedHandle: string | undefined + visibleRows: Array + durableRows: Array + durableResumeKind: unknown + status: string +} + +function expectPersistedRestartLaw( + actual: PersistedRestartObservation, + expected: PersistedRestartObservation, +): void { + expect(actual).toEqual(expected) +} + +function cloneItem(item: Item): Item { + return { id: item.id, name: item.name, stable: item.stable } +} + +function attachPersistedRestartCleanupDiagnostics( + primary: unknown, + cleanupEvidence: string, + cleanupFailures: ReadonlyArray, +): Error { + const error = + primary instanceof Error + ? primary + : new Error(`Persisted restart oracle failed with a non-Error value`, { + cause: primary, + }) + Object.defineProperty(error, `cleanupEvidence`, { + value: cleanupEvidence, + enumerable: true, + }) + if (cleanupFailures.length > 0) { + Object.defineProperty(error, `cleanupFailures`, { + value: [...cleanupFailures], + enumerable: true, + }) + } + return error +} + +async function observePersistedRestart( + scenario: PersistedRestartScenario, +): Promise<{ + observation: PersistedRestartObservation + expected: PersistedRestartObservation + cleanupEvidence: string +}> { + subscribers.length = 0 + vi.clearAllMocks() + let database: DatabaseSync | undefined + let result: + | { + observation: PersistedRestartObservation + expected: PersistedRestartObservation + cleanupEvidence: string + } + | undefined + let deferredFailure: unknown + try { + database = new DatabaseSync(`:memory:`) + const driver = nodeSqliteDriver(database) + const collectionId = `persisted-schema-reset-electric` + const modelInitialRows = + scenario.rowState === `empty` + ? [] + : [cloneItem(oldRow), cloneItem(otherOldRow)] + const modelCanonicalRows = + scenario.sourceHistory === `replayed-insert` + ? [...modelInitialRows, cloneItem(freshRow)] + : modelInitialRows + modelCanonicalRows.sort((left, right) => left.id - right.id) + const productionSeedRows = + scenario.rowState === `empty` + ? [] + : [cloneItem(oldRow), cloneItem(otherOldRow)] + const canResume = + scenario.transition === `compatible-reopen` && + scenario.resumeKind === `resume` + const expected: PersistedRestartObservation = { + checkpoint: `post-restart-up-to-date`, + requestedOffset: canResume ? `10_0` : undefined, + requestedHandle: canResume ? `shape-old` : undefined, + visibleRows: modelCanonicalRows.map(cloneItem), + durableRows: modelCanonicalRows.map(cloneItem), + durableResumeKind: `resume`, + status: `ready`, + } + const resumeState = + scenario.resumeKind === `none` + ? [] + : [ + { + type: `set` as const, + key: `electric:resume`, + value: + scenario.resumeKind === `reset` + ? { kind: `reset`, updatedAt: 1 } + : { + kind: `resume`, + requiresTagState: false, + offset: `10_0`, + handle: `shape-old`, + shapeId: `{"params":{"table":"test_table"},"url":"http://test-url"}`, + updatedAt: 1, + }, + }, + ] + const originalAdapter = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 1, + }) + await originalAdapter.applyCommittedTx(collectionId, { + txId: `seed-electric-baseline`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: productionSeedRows.map((row) => ({ + type: `insert` as const, + key: row.id, + value: cloneItem(row), + })), + collectionMetadataMutations: resumeState, + }) + + const usesSchemaReset = + scenario.transition === `schema-reset` || + scenario.transition === `partial-restore` + const schemaVersion = usesSchemaReset ? 2 : 1 + const restartedAdapter = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion, + schemaMismatchPolicy: `sync-present-reset`, + }) + await restartedAdapter.loadSubset(collectionId, {}) + if (scenario.transition === `partial-restore`) { + await restartedAdapter.applyCommittedTx(collectionId, { + txId: `partial-electric-restore`, + term: 2, + seq: 1, + rowVersion: 2, + mutations: [ + { type: `insert`, key: oldRow.id, value: cloneItem(oldRow) }, + ], + }) + } else if (scenario.transition === `external-row-loss`) { + const collectionTable = createPersistedTableName(collectionId, `c`) + await driver.run( + `DELETE FROM "${collectionTable}" WHERE json_extract(value, '$.id') = ?`, + [oldRow.id], + ) + } + + const collection = createCollection( + persistedCollectionOptions< + Item, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id: collectionId, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `eager`, + getKey: (row) => row.id, + startSync: false, + }), + persistence: { adapter: restartedAdapter }, + }), + ) + + let streamSubscriber: Subscriber | undefined + let subscription: ReturnType | undefined + let observation!: PersistedRestartObservation + let semanticFailure: unknown + let processingFailure: unknown + const cleanupFailures: Array = [] + let cleanupEvidence = `not-run` + let publicationEvents = 0 + try { + collection.startSyncImmediate() + subscription = collection.subscribeChanges( + () => { + publicationEvents++ + }, + { includeInitialState: false }, + ) + await vi.waitFor(() => expect(subscribers).toHaveLength(1)) + streamSubscriber = subscribers[0] + const request = vi.mocked(ShapeStream).mock.calls[0]?.[0] as + | { offset?: string; handle?: string } + | undefined + if (!streamSubscriber || !request) { + throw new Error(`Persisted Electric stream did not reach subscription`) + } + + // The source chooses a legal response from the request production made. + // A resumed request receives only changes since its cursor; a fresh request + // receives the complete current snapshot. Both finish with up-to-date. + const sourceRows = + request.offset === undefined + ? modelCanonicalRows.map(cloneItem) + : scenario.sourceHistory === `replayed-insert` + ? [cloneItem(freshRow)] + : [] + streamSubscriber([ + ...sourceRows.map((row) => change(`insert`, cloneItem(row))), + structuredClone(upToDate), + ]) + await vi.waitFor(() => expect(collection.status).toBe(`ready`)) + await new Promise((resolve) => setTimeout(resolve, 0)) + await new Promise((resolve) => setTimeout(resolve, 0)) + + const durableRows = await restartedAdapter.loadSubset(collectionId, {}) + const durableMetadata = + await restartedAdapter.loadCollectionMetadata(collectionId) + observation = { + checkpoint: `post-restart-up-to-date`, + requestedOffset: request.offset, + requestedHandle: request.handle, + visibleRows: Array.from( + collection.values(), + ({ id, name, stable }) => ({ + id, + name, + stable, + }), + ).sort((left, right) => left.id - right.id), + durableRows: durableRows + .map(({ value }) => value as Item) + .map(({ id, name, stable }) => ({ id, name, stable })) + .sort((left, right) => left.id - right.id), + durableResumeKind: persistedResumeKind( + durableMetadata.find(({ key }) => key === `electric:resume`)?.value, + ), + status: collection.status, + } + try { + expectPersistedRestartLaw(observation, expected) + } catch (error) { + semanticFailure = error + } + } catch (error) { + processingFailure = error + } finally { + let collectionCleanupCompleted = false + try { + subscription?.unsubscribe() + } catch (error) { + cleanupFailures.push(error) + } + try { + await collection.cleanup() + collectionCleanupCompleted = true + } catch (error) { + cleanupFailures.push(error) + } + if (collectionCleanupCompleted && streamSubscriber) { + try { + const beforeLatePublicRows = Array.from( + collection.values(), + ({ id, name, stable }) => ({ id, name, stable }), + ).sort((left, right) => left.id - right.id) + const beforeLateStatus = collection.status + const beforeLateEvents = publicationEvents + const beforeLatePending = + collection._state.pendingSyncedTransactions.map( + ({ committed }) => committed, + ) + const beforeLateRows = await restartedAdapter.loadSubset( + collectionId, + {}, + ) + const beforeLateMetadata = + await restartedAdapter.loadCollectionMetadata(collectionId) + const beforeLateApplied = await driver.query<{ + term: number + seq: number + tx_id: string + row_version: number + }>( + `SELECT term, seq, tx_id, row_version FROM applied_tx WHERE collection_id = ? ORDER BY term, seq`, + [collectionId], + ) + // Deliver both data and the commit boundary. An uncommitted message + // would not challenge late application/publication after retirement. + streamSubscriber([ + change(`insert`, { id: 77, name: `late`, stable: `late` }), + structuredClone(upToDate), + ]) + await new Promise((resolve) => setTimeout(resolve, 0)) + await new Promise((resolve) => setTimeout(resolve, 0)) + const afterLatePublicRows = Array.from( + collection.values(), + ({ id, name, stable }) => ({ id, name, stable }), + ).sort((left, right) => left.id - right.id) + const afterLateRows = await restartedAdapter.loadSubset( + collectionId, + {}, + ) + const afterLateMetadata = + await restartedAdapter.loadCollectionMetadata(collectionId) + const afterLateApplied = await driver.query<{ + term: number + seq: number + tx_id: string + row_version: number + }>( + `SELECT term, seq, tx_id, row_version FROM applied_tx WHERE collection_id = ? ORDER BY term, seq`, + [collectionId], + ) + const afterLatePending = + collection._state.pendingSyncedTransactions.map( + ({ committed }) => committed, + ) + cleanupEvidence = + isDeepStrictEqual(afterLatePublicRows, beforeLatePublicRows) && + collection.status === beforeLateStatus && + publicationEvents === beforeLateEvents && + isDeepStrictEqual(beforeLatePending, []) && + isDeepStrictEqual(afterLatePending, beforeLatePending) && + isDeepStrictEqual(afterLateRows, beforeLateRows) && + isDeepStrictEqual(afterLateMetadata, beforeLateMetadata) && + isDeepStrictEqual(afterLateApplied, beforeLateApplied) + ? `passed: committed retired-stream delivery changed no public rows, events, status, pending transactions, durable rows, metadata, or applied effects` + : `failed: committed retired-stream delivery changed public or pending/durable effects` + } catch (error) { + cleanupFailures.push(error) + } + } else if (collectionCleanupCompleted) { + cleanupEvidence = `passed: collection cleanup completed before a retired stream became available` + } + if (cleanupFailures.length > 0) { + cleanupEvidence = + `failed: ${cleanupFailures.map((error) => String(error)).join(`; `)}; ` + + `retiredProbe=${cleanupEvidence}` + } + } + + if (semanticFailure !== undefined) { + deferredFailure = attachPersistedRestartCleanupDiagnostics( + new Error( + `Persisted Electric reset/resume violation. ` + + `checkpoint=${observation.checkpoint} ` + + `scenario=${JSON.stringify(scenario)} ` + + `expected=${JSON.stringify(expected)} ` + + `actual=${JSON.stringify(observation)} ` + + `cleanup=${cleanupEvidence}`, + { cause: semanticFailure }, + ), + cleanupEvidence, + cleanupFailures, + ) + } else if (processingFailure !== undefined) { + deferredFailure = attachPersistedRestartCleanupDiagnostics( + processingFailure, + cleanupEvidence, + cleanupFailures, + ) + } else if (cleanupFailures.length > 0) { + deferredFailure = new AggregateError(cleanupFailures, cleanupEvidence) + } else if (!cleanupEvidence.startsWith(`passed:`)) { + deferredFailure = new Error(cleanupEvidence) + } else { + result = { + observation, + expected, + cleanupEvidence, + } + } + } catch (error) { + deferredFailure ??= + error instanceof Error + ? error + : new Error(`Persisted restart setup failed with a non-Error value`, { + cause: error, + }) + } finally { + try { + database?.close() + } catch (closeError) { + if (deferredFailure instanceof Error) { + Object.defineProperty(deferredFailure, `databaseCloseFailure`, { + value: closeError, + enumerable: true, + }) + } else { + deferredFailure = closeError + } + } + } + + if (deferredFailure !== undefined) throw deferredFailure + if (!result) throw new Error(`Persisted restart result was not captured`) + return result +} + describe(`persisted Electric recovery laws`, () => { beforeEach(() => { subscribers.length = 0 vi.clearAllMocks() }) + /** + * This matrix is the reached consumer half of the SQLite reset/resume law. + * Its independent model carries only baseline lineage and canonical server + * rows. The real SQLite adapter performs the transition, the real persisted + * wrapper hydrates it, and electricCollectionOptions chooses the ShapeStream + * request. The mock boundary supplies the installed protocol's legal rule: + * resume gets only changes since its cursor; fresh sync gets a full snapshot; + * both end at the post-restart up-to-date checkpoint. + * + * We compare request offset/handle, complete public rows, durable rows, + * durable resume kind, and readiness after schema reset, partial restore, + * and one externally deleted-row fault. Callback multiplicity, real HTTP, + * and arbitrary external edit sequences are omitted. Electric's existing + * recovery properties own partial unseen updates and SDK framing. The + * nonempty + resume + schema-reset + up-to-date-only cell preserves the + * original report at https://github.com/TanStack/db/issues/1589. + */ + it.each(persistedRestartScenarios)( + `couples $resumeKind state to $transition with $rowState rows and $sourceHistory source history`, + async (scenario) => { + const { cleanupEvidence } = await observePersistedRestart(scenario) + expect(cleanupEvidence).toContain(`passed: committed retired-stream`) + }, + ) + + it(`accepts a compatible empty resume and rejects a stale reset request or missing canonical row`, () => { + const compatibleEmpty: PersistedRestartObservation = { + checkpoint: `post-restart-up-to-date`, + requestedOffset: `10_0`, + requestedHandle: `shape-old`, + visibleRows: [], + durableRows: [], + durableResumeKind: `resume`, + status: `ready`, + } + expect(() => + expectPersistedRestartLaw( + { ...compatibleEmpty, visibleRows: [], durableRows: [] }, + compatibleEmpty, + ), + ).not.toThrow() + + const freshNonempty: PersistedRestartObservation = { + ...compatibleEmpty, + requestedOffset: undefined, + requestedHandle: undefined, + visibleRows: [{ ...oldRow }, { ...otherOldRow }], + durableRows: [{ ...oldRow }, { ...otherOldRow }], + } + expect(() => + expectPersistedRestartLaw( + { + ...freshNonempty, + requestedOffset: `10_0`, + requestedHandle: `shape-old`, + visibleRows: [{ ...oldRow }], + durableRows: [{ ...oldRow }], + }, + freshNonempty, + ), + ).toThrowError(expect.objectContaining({ name: `AssertionError` })) + }) + it(`keeps repaired intermediate publications in the persisted recovery record`, async () => { const f = fixture(`eager`) try { diff --git a/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts new file mode 100644 index 0000000000..ff9c848d12 --- /dev/null +++ b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts @@ -0,0 +1,711 @@ +import { DatabaseSync } from 'node:sqlite' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { createCollection } from '@tanstack/db' +import { ShapeStream } from '@electric-sql/client' +import { + SQLiteCorePersistenceAdapter, + createPersistedTableName, + persistedCollectionOptions, +} from '../../db-sqlite-persistence-core/src' +import { electricCollectionOptions } from '../src/electric' +import type { Message, Row } from '@electric-sql/client' +import type { Collection } from '@tanstack/db' +import type { + PersistenceAdapter, + SQLiteDriver, +} from '../../db-sqlite-persistence-core/src' +import type { ElectricCollectionUtils } from '../src/electric' + +type Item = Row & { id: number; name: string } +type Subscriber = (messages: Array>) => void + +const subscribers: Array = [] +const mockSubscribe = vi.fn((subscriber: Subscriber) => { + subscribers.push(subscriber) + return vi.fn() +}) + +vi.mock(`@electric-sql/client`, async () => ({ + ...(await vi.importActual(`@electric-sql/client`)), + ShapeStream: vi.fn(() => ({ + subscribe: mockSubscribe, + requestSnapshot: vi.fn().mockResolvedValue(undefined), + fetchSnapshot: vi.fn().mockResolvedValue({ metadata: {}, data: [] }), + forceDisconnectAndRefresh: vi.fn().mockResolvedValue(undefined), + isUpToDate: false, + shapeHandle: `shape-current`, + lastOffset: `20_0`, + })), +})) + +function toBinding(value: unknown): string | number | bigint | null { + if (value === null || value === undefined) return null + if (typeof value === `boolean`) return value ? 1 : 0 + if ( + typeof value === `string` || + typeof value === `number` || + typeof value === `bigint` + ) { + return value + } + return String(value) +} + +function createDriver(database: DatabaseSync): SQLiteDriver { + const driver: SQLiteDriver = { + exec: (sql) => { + database.exec(sql) + return Promise.resolve() + }, + query: (sql, params = []) => + Promise.resolve( + database + .prepare(sql) + .all(...params.map(toBinding)) + .map((row) => ({ ...row })) as Array, + ), + run: (sql, params = []) => { + database.prepare(sql).run(...params.map(toBinding)) + return Promise.resolve() + }, + transaction: async (transaction) => { + database.exec(`BEGIN IMMEDIATE`) + try { + const result = await transaction(driver) + database.exec(`COMMIT`) + return result + } catch (error) { + database.exec(`ROLLBACK`) + throw error + } + }, + } + return driver +} + +function deferred() { + let resolve!: () => void + const promise = new Promise((complete) => { + resolve = complete + }) + return { promise, resolve } +} + +async function reachCheckpoint( + promise: Promise, + checkpoint: string, +): Promise { + let timer: ReturnType | undefined + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`Did not reach checkpoint: ${checkpoint}`)), + 1_000, + ) + }), + ]) + } finally { + if (timer !== undefined) clearTimeout(timer) + } +} + +function change( + operation: `insert` | `update` | `delete`, + value: Item, +): Message { + return { key: String(value.id), value, headers: { operation } } +} + +async function runRace( + transition: `none` | `external-row-loss` | `schema-reset` | `committed-write`, + syncMode: `eager` | `on-demand` = `eager`, + legacyUnknown = false, + missingKeySetEvidence = false, +): Promise { + const database = new DatabaseSync(`:memory:`) + const driver = createDriver(database) + const collectionId = + `resume-snapshot-${syncMode}-${transition}-` + + `${legacyUnknown}-${missingKeySetEvidence}` + const laterSnapshotEntered = deferred() + const releaseLaterSnapshot = deferred() + let collection: + | Collection> + | undefined + let unsubscribe: (() => void) | undefined + let primaryFailure: unknown + const cleanupFailures: Array = [] + try { + const seedAdapter = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 1, + }) + await seedAdapter.applyCommittedTx(collectionId, { + txId: `seed`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { type: `insert`, key: 1, value: { id: 1, name: `one` } }, + { type: `insert`, key: 2, value: { id: 2, name: `two` } }, + ], + collectionMetadataMutations: [ + { + type: `set`, + key: `electric:resume`, + value: { + kind: `resume`, + requiresTagState: false, + offset: `10_0`, + handle: `shape-old`, + shapeId: `{"params":{"table":"test_table"},"url":"http://test-url"}`, + updatedAt: 1, + }, + }, + ], + }) + if (legacyUnknown) { + if (syncMode !== `on-demand`) { + const tableName = createPersistedTableName(collectionId, `c`) + await driver.run( + `DELETE FROM "${tableName}" WHERE json_extract(value, '$.id') = ?`, + [1], + ) + } + await driver.run( + `UPDATE collection_version SET key_set_evidence_available = 0 WHERE collection_id = ?`, + [collectionId], + ) + await driver.run( + `DELETE FROM collection_expected_keys WHERE collection_id = ?`, + [collectionId], + ) + expect( + (await seedAdapter.loadResumeSnapshot(collectionId)).keySet, + ).toEqual({ status: `unknown` }) + } + + const restartedAdapter = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 1, + }) + let snapshotCalls = 0 + let laterSnapshotIncludedRows: boolean | undefined + let reportReceiverFailure!: (error: Error) => void + const receiverFailure = new Promise((resolve) => { + reportReceiverFailure = resolve + }) + const gatedAdapter = new Proxy(restartedAdapter, { + get(target, property) { + if (property === `loadResumeSnapshot`) { + return async function ( + this: PersistenceAdapter, + ...args: Parameters + ) { + if (this !== gatedAdapter) { + const error = new Error( + `Persistence adapter lost its receiver during resume certification`, + ) + reportReceiverFailure(error) + throw error + } + snapshotCalls++ + if (snapshotCalls > 1) { + laterSnapshotIncludedRows = args[1]?.includeRows + laterSnapshotEntered.resolve() + await releaseLaterSnapshot.promise + } + const snapshot = await target.loadResumeSnapshot(...args) + return missingKeySetEvidence + ? { ...snapshot, keySet: undefined } + : snapshot + } + } + const value = Reflect.get(target, property, target) as unknown + return typeof value === `function` ? value.bind(target) : value + }, + }) as unknown as PersistenceAdapter + + collection = createCollection( + persistedCollectionOptions< + Item, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id: collectionId, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode, + getKey: (row) => row.id, + startSync: false, + }), + persistence: { adapter: gatedAdapter }, + }), + ) + + let publications = 0 + collection.startSyncImmediate() + const subscription = collection.subscribeChanges( + () => { + publications++ + }, + { includeInitialState: false }, + ) + unsubscribe = () => subscription.unsubscribe() + await reachCheckpoint( + Promise.race([ + laterSnapshotEntered.promise, + receiverFailure.then((error) => { + throw error + }), + ]), + `${syncMode} resume reached its second atomic snapshot`, + ) + await vi.waitFor(() => expect(subscribers).toHaveLength(1)) + const subscriber = subscribers[0]! + const request = vi.mocked(ShapeStream).mock.calls[0]![0] as { + offset?: string + handle?: string + } + const replacesUncertifiedBaseline = legacyUnknown || missingKeySetEvidence + + subscriber( + request.offset === undefined + ? [ + change(`insert`, { id: 1, name: `one` }), + change(`insert`, { id: 2, name: `two` }), + { headers: { control: `up-to-date` } }, + ] + : transition === `none` + ? [{ headers: { control: `up-to-date` } }] + : [ + change(`update`, { id: 2, name: `new-two` }), + { headers: { control: `up-to-date` } }, + ], + ) + if (transition === `external-row-loss`) { + const tableName = createPersistedTableName(collectionId, `c`) + await driver.run( + `DELETE FROM "${tableName}" WHERE json_extract(value, '$.id') = ?`, + [1], + ) + } else if (transition === `schema-reset`) { + const resettingAdapter = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 2, + }) + await resettingAdapter.loadSubset(collectionId, {}) + } else if (transition === `committed-write`) { + await seedAdapter.applyCommittedTx(collectionId, { + txId: `concurrent-writer`, + term: 1, + seq: 2, + rowVersion: 2, + mutations: [ + { type: `insert`, key: 3, value: { id: 3, name: `three` } }, + ], + }) + } + releaseLaterSnapshot.resolve() + + if (transition === `none`) { + await vi.waitFor(() => expect(collection!.status).toBe(`ready`)) + expect( + Array.from(collection.values(), ({ id, name }) => ({ id, name })), + ).toEqual( + replacesUncertifiedBaseline + ? [ + { id: 1, name: `one` }, + { id: 2, name: `two` }, + ] + : [], + ) + expect( + (await restartedAdapter.loadSubset(collectionId, {})).map( + ({ value }) => value, + ), + ).toEqual([ + { id: 1, name: `one` }, + { id: 2, name: `two` }, + ]) + } else { + await vi.waitFor(() => expect(collection!.status).toBe(`error`)) + await vi.waitFor(async () => { + const metadata = + await restartedAdapter.loadCollectionMetadata(collectionId) + const resumeState = metadata.find( + ({ key }) => key === `electric:resume`, + )?.value + if (transition === `schema-reset`) { + // The atomic SQLite reset already removed the stale cursor. A stale + // adapter must not write another marker after the schema changed. + expect(resumeState).toBeUndefined() + } else { + expect(resumeState).toMatchObject({ kind: `reset` }) + } + }) + expect(Array.from(collection.values())).toEqual([]) + expect(collection.status).not.toBe(`ready`) + expect(publications).toBe(0) + const durableRowsBeforeLateDelivery = await restartedAdapter.loadSubset( + collectionId, + {}, + ) + expect(durableRowsBeforeLateDelivery.map(({ value }) => value)).toEqual( + transition === `external-row-loss` + ? [{ id: 2, name: `two` }] + : transition === `committed-write` + ? [ + { id: 1, name: `one` }, + { id: 2, name: `two` }, + { id: 3, name: `three` }, + ] + : [], + ) + subscriber([ + change(`insert`, { id: 9, name: `late` }), + { headers: { control: `up-to-date` } }, + ]) + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(Array.from(collection.values())).toEqual([]) + expect(await restartedAdapter.loadSubset(collectionId, {})).toEqual( + durableRowsBeforeLateDelivery, + ) + expect(publications).toBe(0) + } + if (replacesUncertifiedBaseline) { + expect(request.offset).toBeUndefined() + expect(request.handle).toBeUndefined() + } else { + expect(request).toMatchObject({ offset: `10_0`, handle: `shape-old` }) + } + expect(laterSnapshotIncludedRows).toBe( + replacesUncertifiedBaseline || syncMode !== `on-demand`, + ) + } catch (error) { + primaryFailure = error + } finally { + releaseLaterSnapshot.resolve() + try { + unsubscribe?.() + } catch (error) { + cleanupFailures.push(error) + } + try { + if (collection) { + await reachCheckpoint( + collection.cleanup(), + `Electric race collection cleanup`, + ) + } + } catch (error) { + cleanupFailures.push(error) + } + try { + database.close() + } catch (error) { + cleanupFailures.push(error) + } + } + + if (primaryFailure !== undefined) { + const failure = + primaryFailure instanceof Error + ? primaryFailure + : new Error(`Resume snapshot race failed`, { cause: primaryFailure }) + if (cleanupFailures.length > 0) { + Object.defineProperty(failure, `cleanupFailures`, { + value: cleanupFailures, + enumerable: true, + }) + } + throw failure + } + if (cleanupFailures.length > 0) { + throw new AggregateError(cleanupFailures, `Resume snapshot cleanup failed`) + } +} + +type LegacyUnknownResumeObservation = { + checkpoint: `post-restart-up-to-date` + migratedKeySet: { status: `unknown` | `consistent` | `incompatible` } + migratedRows: Array + requestedOffset: string | undefined + requestedHandle: string | undefined + sourceDelivery: Array + publicRows: Array + durableRows: Array + status: string +} + +async function observeLegacyUnknownResume(): Promise { + const database = new DatabaseSync(`:memory:`) + const driver = createDriver(database) + const collectionId = `legacy-loss-fixed-witness` + const rowLostBeforeMigration: Item = { + id: 1, + name: `lost-before-ledger`, + } + const survivingRow: Item = { id: 2, name: `survivor` } + const postOffsetRow: Item = { id: 3, name: `post-offset` } + const canonicalSourceSnapshot = [ + rowLostBeforeMigration, + survivingRow, + postOffsetRow, + ] + let collection: + | Collection> + | undefined + let unsubscribe: (() => void) | undefined + let observation: LegacyUnknownResumeObservation | undefined + let primaryFailure: unknown + const cleanupFailures: Array = [] + + try { + // Produce the persisted row/metadata encodings through the real adapter, + // then reduce only the key-evidence schema to its pre-ledger form. + const legacyAdapter = new SQLiteCorePersistenceAdapter({ driver }) + await legacyAdapter.applyCommittedTx(collectionId, { + txId: `legacy-snapshot-at-10`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [rowLostBeforeMigration, survivingRow].map((row) => ({ + type: `insert` as const, + key: row.id, + value: structuredClone(row), + })), + collectionMetadataMutations: [ + { + type: `set`, + key: `electric:resume`, + value: { + kind: `resume`, + requiresTagState: false, + offset: `10_0`, + handle: `shape-old`, + shapeId: `{"params":{"table":"test_table"},"url":"http://test-url"}`, + updatedAt: 1, + }, + }, + ], + }) + + const collectionTable = createPersistedTableName(collectionId, `c`) + await driver.run( + `DELETE FROM "${collectionTable}" WHERE json_extract(value, '$.id') = ?`, + [rowLostBeforeMigration.id], + ) + await driver.exec( + `DROP TRIGGER "${collectionTable}_key_evidence_insert"; + DROP TRIGGER "${collectionTable}_key_evidence_delete"; + DROP TRIGGER "${collectionTable}_key_evidence_update"`, + ) + await driver.exec(`DROP TABLE collection_expected_keys`) + await driver.exec( + `ALTER TABLE collection_version RENAME TO collection_version_with_ledger`, + ) + await driver.exec( + `CREATE TABLE collection_version ( + collection_id TEXT PRIMARY KEY, + latest_row_version INTEGER NOT NULL + )`, + ) + await driver.exec( + `INSERT INTO collection_version (collection_id, latest_row_version) + SELECT collection_id, latest_row_version + FROM collection_version_with_ledger`, + ) + await driver.exec(`DROP TABLE collection_version_with_ledger`) + + const migratedAdapter = new SQLiteCorePersistenceAdapter({ driver }) + const migratedSnapshot = + await migratedAdapter.loadResumeSnapshot(collectionId) + + collection = createCollection( + persistedCollectionOptions< + Item, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id: collectionId, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `eager`, + getKey: (row) => row.id, + startSync: false, + }), + persistence: { adapter: migratedAdapter }, + }), + ) + collection.startSyncImmediate() + const subscription = collection.subscribeChanges(() => {}, { + includeInitialState: false, + }) + unsubscribe = () => subscription.unsubscribe() + + await vi.waitFor(() => expect(subscribers).toHaveLength(1)) + const request = vi.mocked(ShapeStream).mock.calls[0]![0] as { + offset?: string + handle?: string + } + // The source obeys the request: a resume receives only changes after its + // cursor; a fresh request receives the independently specified snapshot. + const sourceDelivery = + request.offset === undefined + ? canonicalSourceSnapshot.map((row) => structuredClone(row)) + : [structuredClone(postOffsetRow)] + subscribers[0]!([ + ...sourceDelivery.map((row) => change(`insert`, structuredClone(row))), + { headers: { control: `up-to-date` } }, + ]) + + await vi.waitFor(() => expect(collection!.status).toBe(`ready`)) + await new Promise((resolve) => setTimeout(resolve, 0)) + await new Promise((resolve) => setTimeout(resolve, 0)) + observation = { + checkpoint: `post-restart-up-to-date`, + migratedKeySet: migratedSnapshot.keySet, + migratedRows: migratedSnapshot.rows + .map(({ value }) => value as Item) + .sort((left, right) => left.id - right.id), + requestedOffset: request.offset, + requestedHandle: request.handle, + sourceDelivery, + publicRows: Array.from(collection.values(), ({ id, name }) => ({ + id, + name, + })).sort((left, right) => left.id - right.id), + durableRows: (await migratedAdapter.loadSubset(collectionId, {})) + .map(({ value }) => value as Item) + .sort((left, right) => left.id - right.id), + status: collection.status, + } + } catch (error) { + primaryFailure = error + } finally { + try { + unsubscribe?.() + } catch (error) { + cleanupFailures.push(error) + } + try { + await collection?.cleanup() + } catch (error) { + cleanupFailures.push(error) + } + try { + database.close() + } catch (error) { + cleanupFailures.push(error) + } + } + + if (primaryFailure !== undefined) { + const failure = + primaryFailure instanceof Error + ? primaryFailure + : new Error(`Legacy resume witness failed`, { cause: primaryFailure }) + if (cleanupFailures.length > 0) { + Object.defineProperty(failure, `cleanupFailures`, { + value: cleanupFailures, + enumerable: true, + }) + } + throw failure + } + if (cleanupFailures.length > 0) { + throw new AggregateError( + cleanupFailures, + `Legacy resume witness cleanup failed`, + ) + } + if (!observation) { + throw new Error(`Legacy resume witness did not capture an observation`) + } + return observation +} + +/** + * Deterministic companion schedules for the persisted recovery owner in + * electric-recovery-oracle.test.ts. Its scenario matrix proves settled restart + * semantics; these cases hold the adapter's second atomic snapshot so row loss, + * reset, and concurrent commit can be injected between startup metadata and + * resume certification. The separate fixture keeps the held boundary explicit + * and does not claim native SQLite or live Electric service coverage. + */ +describe(`Electric resume snapshot races`, () => { + beforeEach(() => { + subscribers.length = 0 + vi.clearAllMocks() + }) + + it(`rejects row loss between resume metadata and baseline hydration`, async () => { + await runRace(`external-row-loss`) + }) + + it(`rejects a schema reset between resume metadata and baseline hydration`, async () => { + await runRace(`schema-reset`) + }) + + it(`conservatively rejects a committed write between startup snapshots`, async () => { + await runRace(`committed-write`) + }) + + it(`freshly replaces an unknown on-demand resume baseline`, async () => { + await runRace(`none`, `on-demand`, true) + }) + + it(`freshly replaces an unknown eager resume baseline`, async () => { + await runRace(`none`, `eager`, true) + }) + + it(`freshly replaces a resume when snapshot evidence is missing`, async () => { + await runRace(`none`, `eager`, false, true) + }) + + it(`rejects row loss during on-demand resume certification`, async () => { + await runRace(`external-row-loss`, `on-demand`) + }) + + it(`rejects a schema reset for an unknown on-demand resume`, async () => { + await runRace(`schema-reset`, `on-demand`, true) + }) + + it(`freshly replaces an unverifiable pre-ledger resume baseline`, async () => { + const observation = await observeLegacyUnknownResume() + expect(observation).toEqual({ + checkpoint: `post-restart-up-to-date`, + migratedKeySet: { status: `unknown` }, + migratedRows: [{ id: 2, name: `survivor` }], + requestedOffset: undefined, + requestedHandle: undefined, + sourceDelivery: [ + { id: 1, name: `lost-before-ledger` }, + { id: 2, name: `survivor` }, + { id: 3, name: `post-offset` }, + ], + publicRows: [ + { id: 1, name: `lost-before-ledger` }, + { id: 2, name: `survivor` }, + { id: 3, name: `post-offset` }, + ], + durableRows: [ + { id: 1, name: `lost-before-ledger` }, + { id: 2, name: `survivor` }, + { id: 3, name: `post-offset` }, + ], + status: `ready`, + }) + }) +}) From 758532d216a63a0618688b43ee8b366a434ee98d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 17 Sep 2026 08:06:21 -0600 Subject: [PATCH 02/23] chore: add persisted resume integrity changeset --- .changeset/preserve-resume-baseline-integrity.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/preserve-resume-baseline-integrity.md diff --git a/.changeset/preserve-resume-baseline-integrity.md b/.changeset/preserve-resume-baseline-integrity.md new file mode 100644 index 0000000000..2cb5517d15 --- /dev/null +++ b/.changeset/preserve-resume-baseline-integrity.md @@ -0,0 +1,6 @@ +--- +'@tanstack/db-sqlite-persistence-core': patch +'@tanstack/electric-db-collection': patch +--- + +Preserve persisted resume integrity with atomic SQLite baseline evidence and stale-writer rejection, and refresh uncertified Electric baselines before publishing resumed data. From 41c2645247b79097db98848ee80682eb09d32a37 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 17 Sep 2026 08:40:02 -0600 Subject: [PATCH 03/23] test(sqlite): allow CLI oracle for CI load --- .../tests/sqlite-core-adapter.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts index 86809a3eb6..ffb5935c87 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts @@ -1576,7 +1576,7 @@ export function runSQLiteCoreAdapterContractSuite( `cleanup=${reducedFailure.cleanupEvidence}`, { cause: reducedFailure }, ) - }, 30_000) + }, 120_000) it(`leaves externally inconsistent metadata reachable for consumer validation`, async () => { const observation = await observeResetResumeHistory( @@ -1610,7 +1610,7 @@ export function runSQLiteCoreAdapterContractSuite( `oracle:provider`, ]) expect(resumeKindOf(observation.resumeState)).toBe(`resume`) - }) + }, 30_000) it(`requires complete metadata reset while preserving non-reset metadata`, () => { const compatibleEmpty: ResetResumeHistory = { From 9d74e9b770e7eb8b22aa8acfbfc6cc2cd1a3853b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 17 Sep 2026 11:29:37 -0600 Subject: [PATCH 04/23] fix(sqlite): preserve runtime-owned resume resets --- .../src/persisted.ts | 70 +++++++++++++++++-- .../electric-db-collection/src/electric.ts | 15 +++- .../electric-resume-snapshot-races.test.ts | 68 +++++++++++++----- 3 files changed, 130 insertions(+), 23 deletions(-) diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 69bdaaf6de..232b2e4e34 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -223,6 +223,13 @@ export type PersistedKeySetEvidence = { status: `unknown` | `consistent` | `incompatible` } +type PersistedResumeGeneration = { + latestTerm: number + latestSeq: number + latestRowVersion: number + resetEpoch: number +} + export type PersistedTx< T extends object = Record, TKey extends string | number = string | number, @@ -614,6 +621,7 @@ type BufferedSyncTransaction = { > truncate: boolean internal: boolean + expectedResumeGenerationOwner?: symbol signal?: AbortSignal resolveApplied?: () => void rejectApplied?: (error: unknown) => void @@ -832,7 +840,8 @@ class PersistedCollectionRuntime< private resumeBaselinePromise: Promise | null = null private resumeCertificationPromise: Promise | null = null private persistedKeySetEvidence: PersistedKeySetEvidence | undefined - private persistedResumeGeneration: string | undefined + private persistedResumeGeneration: PersistedResumeGeneration | undefined + private resumeGenerationOwner = Symbol(`persisted resume generation owner`) private lifecycleGeneration = 0 private internalApplyDepth = 0 private appliedReceiptSequence = 0 @@ -982,6 +991,10 @@ class PersistedCollectionRuntime< return this.persistence.adapter.loadResumeSnapshot !== undefined } + getResumeGenerationOwner(): symbol { + return this.resumeGenerationOwner + } + private async hydrateBaseline(lifecycleGeneration: number): Promise { if (lifecycleGeneration !== this.lifecycleGeneration) return @@ -1330,6 +1343,7 @@ class PersistedCollectionRuntime< this.resumeCertificationPromise = null this.persistedKeySetEvidence = undefined this.persistedResumeGeneration = undefined + this.resumeGenerationOwner = Symbol(`persisted resume generation owner`) } private withInternalApply(task: () => TResult): TResult { @@ -1463,8 +1477,26 @@ class PersistedCollectionRuntime< latestSeq: number latestRowVersion: number resetEpoch: number - }): string { - return `${snapshot.resetEpoch}:${snapshot.latestTerm}:${snapshot.latestSeq}:${snapshot.latestRowVersion}` + }): PersistedResumeGeneration { + return { + latestTerm: snapshot.latestTerm, + latestSeq: snapshot.latestSeq, + latestRowVersion: snapshot.latestRowVersion, + resetEpoch: snapshot.resetEpoch, + } + } + + private isExpectedResumeGeneration( + generation: PersistedResumeGeneration, + ): boolean { + const expected = this.persistedResumeGeneration + return ( + expected !== undefined && + expected.latestTerm === generation.latestTerm && + expected.latestSeq === generation.latestSeq && + expected.latestRowVersion === generation.latestRowVersion && + expected.resetEpoch === generation.resetEpoch + ) } private bindResumeSnapshotEvidence(snapshot: { @@ -1486,7 +1518,7 @@ class PersistedCollectionRuntime< snapshot.latestRowVersion, ) this.persistedKeySetEvidence = - this.persistedResumeGeneration === generation || remainsUncertified + this.isExpectedResumeGeneration(generation) || remainsUncertified ? snapshot.keySet : { status: `incompatible` } } @@ -1661,6 +1693,18 @@ class PersistedCollectionRuntime< const tx = this.createPersistedTxFromOperations(transaction, streamPosition) await this.persistence.adapter.applyCommittedTx(this.collectionId, tx) + if ( + transaction.expectedResumeGenerationOwner === + this.resumeGenerationOwner && + this.persistedResumeGeneration !== undefined + ) { + this.persistedResumeGeneration = { + ...this.persistedResumeGeneration, + latestTerm: tx.term, + latestSeq: tx.seq, + latestRowVersion: tx.rowVersion, + } + } this.publishTxCommittedEvent( this.createTxCommittedPayload({ term: tx.term, @@ -2599,6 +2643,20 @@ function createWrappedSyncConfig< ? undefined : runtime.getPersistedKeySetEvidence() : undefined, + expectCurrentCommitInResumeSnapshot: + runtime.supportsResumeSnapshot() + ? () => { + if (startupState.cleanedUp) return + const openTransaction = getOpenTransaction() + if (!openTransaction) { + throw new InvalidPersistedCollectionConfigError( + `expectCurrentCommitInResumeSnapshot must be called within an open sync transaction`, + ) + } + openTransaction.expectedResumeGenerationOwner = + runtime.getResumeGenerationOwner() + } + : undefined, set: (key: TKey, value: unknown) => { if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() @@ -2752,6 +2810,8 @@ function createWrappedSyncConfig< openTransaction.collectionMetadataWrites, truncate: openTransaction.truncate, internal: openTransaction.internal, + expectedResumeGenerationOwner: + openTransaction.expectedResumeGenerationOwner, signal, resolveApplied, rejectApplied, @@ -2770,6 +2830,8 @@ function createWrappedSyncConfig< openTransaction.collectionMetadataWrites, truncate: openTransaction.truncate, internal: false, + expectedResumeGenerationOwner: + openTransaction.expectedResumeGenerationOwner, }) } const persisted = persistAfterApplication() diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index b8ccf94fc8..4901f9912c 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -76,6 +76,7 @@ type ElectricSyncMetadataWithHydration = SyncMetadataApi & { getPersistedKeySetEvidence?: () => | { status: `unknown` | `consistent` | `incompatible` } | undefined + expectCurrentCommitInResumeSnapshot?: () => void } } @@ -1692,6 +1693,8 @@ function createElectricSync>( persistedMetadata?.row.certifyPersistedResume const getPersistedKeySetEvidence = persistedMetadata?.row.getPersistedKeySetEvidence + const expectCurrentCommitInResumeSnapshot = + persistedMetadata?.row.expectCurrentCommitInResumeSnapshot const persistedKeySetEvidence = getPersistedKeySetEvidence?.() const persistedResumeState = getNewestElectricResumeState( @@ -1904,7 +1907,9 @@ function createElectricSync>( metadata?.collection.set(`electric:resume`, resumeState) } - const commitResetResumeMetadataImmediately = () => { + const commitResetResumeMetadataImmediately = ( + expectInResumeSnapshot = false, + ) => { const resetState: ElectricResumeState = { kind: `reset`, updatedAt: Date.now(), @@ -1914,6 +1919,9 @@ function createElectricSync>( if (metadata) { begin({ immediate: true }) metadata.collection.set(`electric:resume`, resetState) + if (expectInResumeSnapshot) { + expectCurrentCommitInResumeSnapshot?.() + } commit() } } @@ -1923,7 +1931,10 @@ function createElectricSync>( hasUnverifiablePersistedResume || (needsFullSnapshot && persistedResumeState.kind === `resume`) ) { - commitResetResumeMetadataImmediately() + // This reset is part of the current runtime's startup decision. The + // persisted wrapper may commit it before loading the atomic baseline, + // so carry ownership of exactly this generation into certification. + commitResetResumeMetadataImmediately(true) } /** diff --git a/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts index ff9c848d12..ac81515d07 100644 --- a/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts +++ b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts @@ -20,8 +20,10 @@ type Item = Row & { id: number; name: string } type Subscriber = (messages: Array>) => void const subscribers: Array = [] +let synchronousMessages: Array> | undefined const mockSubscribe = vi.fn((subscriber: Subscriber) => { subscribers.push(subscriber) + if (synchronousMessages) subscriber(synchronousMessages) return vi.fn() }) @@ -123,12 +125,13 @@ async function runRace( syncMode: `eager` | `on-demand` = `eager`, legacyUnknown = false, missingKeySetEvidence = false, + startupReset: `none` | `tag-state` | `shape-identity` = `none`, ): Promise { const database = new DatabaseSync(`:memory:`) const driver = createDriver(database) const collectionId = `resume-snapshot-${syncMode}-${transition}-` + - `${legacyUnknown}-${missingKeySetEvidence}` + `${legacyUnknown}-${missingKeySetEvidence}-${startupReset}` const laterSnapshotEntered = deferred() const releaseLaterSnapshot = deferred() let collection: @@ -157,10 +160,13 @@ async function runRace( key: `electric:resume`, value: { kind: `resume`, - requiresTagState: false, + requiresTagState: startupReset === `tag-state`, offset: `10_0`, handle: `shape-old`, - shapeId: `{"params":{"table":"test_table"},"url":"http://test-url"}`, + shapeId: + startupReset === `shape-identity` + ? `{"params":{"table":"other_table"},"url":"http://test-url"}` + : `{"params":{"table":"test_table"},"url":"http://test-url"}`, updatedAt: 1, }, }, @@ -193,6 +199,7 @@ async function runRace( }) let snapshotCalls = 0 let laterSnapshotIncludedRows: boolean | undefined + let resumeStateAtLaterSnapshot: unknown let reportReceiverFailure!: (error: Error) => void const receiverFailure = new Promise((resolve) => { reportReceiverFailure = resolve @@ -214,6 +221,11 @@ async function runRace( snapshotCalls++ if (snapshotCalls > 1) { laterSnapshotIncludedRows = args[1]?.includeRows + if (startupReset !== `none`) { + resumeStateAtLaterSnapshot = ( + await target.loadCollectionMetadata(collectionId) + ).find(({ key }) => key === `electric:resume`)?.value + } laterSnapshotEntered.resolve() await releaseLaterSnapshot.promise } @@ -250,6 +262,13 @@ async function runRace( ) let publications = 0 + if (startupReset !== `none`) { + synchronousMessages = [ + change(`insert`, { id: 1, name: `one` }), + change(`insert`, { id: 2, name: `two` }), + { headers: { control: `up-to-date` } }, + ] + } collection.startSyncImmediate() const subscription = collection.subscribeChanges( () => { @@ -273,22 +292,28 @@ async function runRace( offset?: string handle?: string } - const replacesUncertifiedBaseline = legacyUnknown || missingKeySetEvidence + const replacesUncertifiedBaseline = + legacyUnknown || missingKeySetEvidence || startupReset !== `none` + if (startupReset !== `none`) { + expect(resumeStateAtLaterSnapshot).toMatchObject({ kind: `reset` }) + } - subscriber( - request.offset === undefined - ? [ - change(`insert`, { id: 1, name: `one` }), - change(`insert`, { id: 2, name: `two` }), - { headers: { control: `up-to-date` } }, - ] - : transition === `none` - ? [{ headers: { control: `up-to-date` } }] - : [ - change(`update`, { id: 2, name: `new-two` }), + if (startupReset === `none`) { + subscriber( + request.offset === undefined + ? [ + change(`insert`, { id: 1, name: `one` }), + change(`insert`, { id: 2, name: `two` }), { headers: { control: `up-to-date` } }, - ], - ) + ] + : transition === `none` + ? [{ headers: { control: `up-to-date` } }] + : [ + change(`update`, { id: 2, name: `new-two` }), + { headers: { control: `up-to-date` } }, + ], + ) + } if (transition === `external-row-loss`) { const tableName = createPersistedTableName(collectionId, `c`) await driver.run( @@ -647,9 +672,18 @@ async function observeLegacyUnknownResume(): Promise { beforeEach(() => { subscribers.length = 0 + synchronousMessages = undefined vi.clearAllMocks() }) + it(`keeps a healthy tagged cache when a fresh reset commits before hydration`, async () => { + await runRace(`none`, `eager`, false, false, `tag-state`) + }) + + it(`keeps a healthy cache when a changed shape commits its reset before hydration`, async () => { + await runRace(`none`, `eager`, false, false, `shape-identity`) + }) + it(`rejects row loss between resume metadata and baseline hydration`, async () => { await runRace(`external-row-loss`) }) From de7412f37eee0b0ea61b167b929585079f10a01f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 18 Sep 2026 17:10:53 -0600 Subject: [PATCH 05/23] ci: trigger PR checks From f6c293e05a502fafc270dff95948f19a3bdfefdf Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 09:57:41 +0100 Subject: [PATCH 06/23] docs(test): explain persisted resume oracles --- .../tests/sqlite-resume-snapshot.test.ts | 33 +++++++++++++++---- .../tests/electric-recovery-oracle.test.ts | 26 +++++++++++++++ .../electric-resume-snapshot-races.test.ts | 33 +++++++++++++++---- 3 files changed, 80 insertions(+), 12 deletions(-) diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts index 0add417097..1213b282cc 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts @@ -196,12 +196,33 @@ async function observeCachedSchemaState( } /** - * Narrow companion to sqlite-core-adapter.test.ts for atomic snapshot and DDL - * interleavings. That owner deliberately drives sqlite3 through a serialized - * copy-on-commit CLI harness, which cannot expose two adapters to the same - * in-flight connection state. This file uses node:sqlite only for that missing - * deterministic seam; expected key membership and reset lineage remain - * independent assertions, not a second production-shaped model. + * # Which generation does a SQLite resume snapshot certify? + * + * `loadResumeSnapshot` must return rows, collection metadata, applied position, + * reset epoch, and key-set evidence from one atomic persisted generation. Raw + * key loss makes that evidence incompatible until a full replacement establishes + * a new baseline; concurrent schema migration may advance but never downgrade or + * repeat the observed generation. These laws refine the shared persistence and + * schema-mismatch contracts exercised by sqlite-core-adapter.test.ts. + * + * `CachedSchemaState` is the independent projection: complete rows and metadata, + * transaction position, schema/reset lineage, and expected keys. The history + * grammar crosses external row loss, full replacement, two legacy-schema + * adapters, stale reads, newer-schema observation, and a cached writer racing a + * reset. Expected membership and lineage come from the declared transition, not + * from the adapter's SQL or internal branch structure. + * + * The production driver runs two real `SQLiteCorePersistenceAdapter` instances + * over one node:sqlite database and holds the exact transaction or snapshot + * boundary needed for each interleaving. At the settled snapshot checkpoint it + * compares the entire projected schema state; the held boundary and reset epoch + * are reach witnesses, while compatible reopen and recertifying truncate cases + * prevent an oracle that merely rejects every resume. + * + * This narrow fixture supplies the same-connection concurrency seam that the + * serialized copy-on-commit CLI harness cannot. It does not claim native host + * execution or judge whether a consumer such as Electric may use the certified + * cursor; those remain separate driver-contract and Electric recovery owners. */ describe(`SQLite resume snapshots`, () => { it(`keeps raw key loss sticky until a full replacement recertifies the baseline`, async () => { diff --git a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts index ea723e6fc5..1f3f77ab9c 100644 --- a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts +++ b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts @@ -20,6 +20,32 @@ import type { } from '../../db-sqlite-persistence-core/src' import type { ElectricCollectionUtils, ElectricSyncMode } from '../src/electric' +/** + * # What remains visible while a persisted Electric replica repairs itself? + * + * Hydrated rows and resume metadata provide the last complete public snapshot. + * A must-refetch starts a private replacement. Until that replacement is fully + * applied, readers may see an earlier permitted snapshot but never a torn mix. + * Failure keeps the old public rows and records repair debt; later success may + * replace them atomically. + * + * A plain persisted row Map and metadata Map form the reference snapshots. The + * driver controls hydration, SDK callbacks, applied receipts, cleanup, restart, + * and eager or progressive mode through the real persistence coordinator and + * Electric adapter. It records every exposed cut, not only the final rows. + * + * Legal histories vary sync mode, hydration and restart timing, reset cause, + * peer publication, stream delta, deletion, and full reload. At each recorded + * publication cut, the complete public rows must refine one permitted snapshot; + * durable rows and resume metadata are compared again after the applied or + * post-restart up-to-date checkpoint. Stale/missing canonical-row controls and + * the repaired-intermediate trace challenge the checker; generated failures + * retain fast-check's seed and shrink path. + * + * The fixture does not establish live HTTP delivery, native SQLite host + * behavior, or callback multiplicity beyond the observations named below. + */ + type Item = Row & { id: number; name: string; stable: string } type Subscriber = (messages: Array>) => void type Exposure = { cut: string; rows: Array } diff --git a/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts index ac81515d07..5dc4f378bc 100644 --- a/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts +++ b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts @@ -662,12 +662,33 @@ async function observeLegacyUnknownResume(): Promise { beforeEach(() => { From f80fa5c188cb375cc9a9a9dadeaa8c03fb5a3797 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 15:16:21 +0100 Subject: [PATCH 07/23] fix: require complete sync persistence capability --- .../preserve-resume-baseline-integrity.md | 4 +- .../tests/browser-coordinator.test.ts | 10 + packages/db-sqlite-persistence-core/README.md | 11 +- .../src/persisted.ts | 163 +++++++-------- .../tests/persisted.test-d.ts | 36 ++-- .../tests/persisted.test.ts | 187 +++++++++++------- .../tests/sqlite-resume-snapshot.test.ts | 28 ++- packages/db/src/errors.ts | 10 + packages/db/src/index.ts | 1 + packages/db/src/sync-persistence.ts | 68 +++++++ packages/db/src/types.ts | 38 ++++ packages/db/tests/sync-persistence.test.ts | 50 +++++ .../electric-db-collection/src/electric.ts | 69 ++----- .../tests/ORACLE_MUTATIONS.md | 15 +- .../electric-descriptor-isolation.test.ts | 41 +++- .../tests/electric-oracle.property.test.ts | 107 +++++----- .../tests/electric-recovery-oracle.test.ts | 33 +++- .../electric-resume-snapshot-races.test.ts | 85 +++++++- .../tests/electric.test.ts | 144 +++++++++++++- packages/query-db-collection/src/query.ts | 26 +-- .../load-subset-lifecycle-oracle.test.ts | 29 +-- .../tests/ownership-lifecycle.oracle.test.ts | 59 +++++- .../query-db-collection/tests/query.test.ts | 66 ++++++- 23 files changed, 910 insertions(+), 370 deletions(-) create mode 100644 packages/db/src/sync-persistence.ts create mode 100644 packages/db/tests/sync-persistence.test.ts diff --git a/.changeset/preserve-resume-baseline-integrity.md b/.changeset/preserve-resume-baseline-integrity.md index 2cb5517d15..daa15b8a2c 100644 --- a/.changeset/preserve-resume-baseline-integrity.md +++ b/.changeset/preserve-resume-baseline-integrity.md @@ -1,6 +1,8 @@ --- '@tanstack/db-sqlite-persistence-core': patch +'@tanstack/db': patch '@tanstack/electric-db-collection': patch +'@tanstack/query-db-collection': patch --- -Preserve persisted resume integrity with atomic SQLite baseline evidence and stale-writer rejection, and refresh uncertified Electric baselines before publishing resumed data. +Preserve persisted resume integrity with atomic SQLite baseline evidence and stale-writer rejection, expose persistence sync metadata as one versioned capability, and refresh uncertified Electric baselines before publishing resumed data. diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 5f554006e2..5ddcb272ac 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -193,6 +193,16 @@ function createStubAdapter(): PersistenceAdapter & { return { appliedTxs, loadSubset: () => Promise.resolve([]), + loadResumeSnapshot: () => + Promise.resolve({ + rows: [], + keySet: { status: `consistent` }, + collectionMetadata: [], + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + resetEpoch: 0, + }), applyCommittedTx: (collectionId, tx) => { appliedTxs.push({ collectionId, txId: tx.txId }) return Promise.resolve() diff --git a/packages/db-sqlite-persistence-core/README.md b/packages/db-sqlite-persistence-core/README.md index 235df60776..6d03b94f9a 100644 --- a/packages/db-sqlite-persistence-core/README.md +++ b/packages/db-sqlite-persistence-core/README.md @@ -67,7 +67,7 @@ while still handling per-collection schema versions correctly. ### Atomic resume snapshots -Persistence adapters may implement +Persistence adapters implement `loadResumeSnapshot(collectionId, options)` to let a sync source certify a persisted resume baseline. One call must read rows, collection metadata, stream position, reset epoch, and key-set evidence from the same atomic database @@ -84,9 +84,12 @@ row-bearing snapshot. - `unknown`: the adapter has no authoritative pre-migration key set and does not claim completeness. -The method is optional so existing adapters remain assignable. Without it, the -wrapper retains the legacy stream-position and metadata path. Adapter methods -are invoked with their receiver and may rely on instance state through `this`. +The method is required because the versioned `metadata.persistence` capability +always carries hydration, durable row scanning, certification, evidence, and +generation ownership as one complete bundle. Sync wrappers must forward the +capability object unchanged rather than copying individual methods. Adapter +methods are invoked with their receiver and may rely on instance state through +`this`. ### SQLite core adapter APIs diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 232b2e4e34..d7683eda02 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -1,4 +1,6 @@ import { + SYNC_PERSISTENCE_PROTOCOL, + SYNC_PERSISTENCE_VERSION, SyncTransactionAbortedError, compileSingleRowExpression, safeRandomUUID, @@ -28,6 +30,9 @@ import type { SyncConfig, SyncConfigRes, SyncMetadataApi, + SyncPersistenceCapabilityV1, + SyncPersistenceKeySetEvidence, + SyncPersistenceScanOptions, UpdateMutationFnParams, UtilsRecord, } from '@tanstack/db' @@ -215,13 +220,9 @@ export type PersistedScannedRow< metadata?: unknown } -export type PersistedRowScanOptions = { - metadataOnly?: boolean -} +export type PersistedRowScanOptions = SyncPersistenceScanOptions -export type PersistedKeySetEvidence = { - status: `unknown` | `consistent` | `incompatible` -} +export type PersistedKeySetEvidence = SyncPersistenceKeySetEvidence type PersistedResumeGeneration = { latestTerm: number @@ -272,7 +273,7 @@ export interface PersistenceAdapter { metadata?: unknown }> > - loadResumeSnapshot?: ( + loadResumeSnapshot: ( collectionId: string, ctx?: { requiredIndexSignatures?: ReadonlyArray @@ -450,9 +451,9 @@ const REQUIRED_COORDINATOR_METHODS: ReadonlyArray< const REQUIRED_ADAPTER_METHODS: ReadonlyArray< keyof Pick< PersistenceAdapter, - `loadSubset` | `applyCommittedTx` | `ensureIndex` + `loadSubset` | `loadResumeSnapshot` | `applyCommittedTx` | `ensureIndex` > -> = [`loadSubset`, `applyCommittedTx`, `ensureIndex`] +> = [`loadSubset`, `loadResumeSnapshot`, `applyCommittedTx`, `ensureIndex`] const TARGETED_INVALIDATION_KEY_LIMIT = 128 const DEFAULT_DB_NAME = `tanstack-db` @@ -971,26 +972,23 @@ class PersistedCollectionRuntime< await this.ensureStarted() if (lifecycleGeneration !== this.lifecycleGeneration) return - const adapter = this.persistence.adapter - if (!adapter.loadResumeSnapshot) return - const snapshot = await adapter.loadResumeSnapshot(this.collectionId, { - requiredIndexSignatures: this.getRequiredIndexSignatures(), - includeRows: false, - }) + const snapshot = await this.persistence.adapter.loadResumeSnapshot( + this.collectionId, + { + requiredIndexSignatures: this.getRequiredIndexSignatures(), + includeRows: false, + }, + ) if (lifecycleGeneration !== this.lifecycleGeneration) return this.bindResumeSnapshotEvidence(snapshot) })() return this.resumeCertificationPromise } - getPersistedKeySetEvidence(): PersistedKeySetEvidence | undefined { + getKeySetEvidence(): PersistedKeySetEvidence | undefined { return this.persistedKeySetEvidence } - supportsResumeSnapshot(): boolean { - return this.persistence.adapter.loadResumeSnapshot !== undefined - } - getResumeGenerationOwner(): symbol { return this.resumeGenerationOwner } @@ -1047,45 +1045,19 @@ class PersistedCollectionRuntime< private async loadStartupMetadataInternal( lifecycleGeneration: number, ): Promise { - if (this.persistence.adapter.loadResumeSnapshot) { - const snapshot = await this.persistence.adapter.loadResumeSnapshot( - this.collectionId, - { includeRows: false }, - ) - if (lifecycleGeneration !== this.lifecycleGeneration) return - this.persistedResumeGeneration = - this.getResumeSnapshotGeneration(snapshot) - this.persistedKeySetEvidence = snapshot.keySet - this.observeStreamPosition( - snapshot.latestTerm, - snapshot.latestSeq, - snapshot.latestRowVersion, - ) - this.replaceCollectionMetadataSnapshot(snapshot.collectionMetadata) - return - } - - // Restore stream position from the database so that new mutations - // don't collide with previously applied transactions. - if (this.persistence.adapter.getStreamPosition) { - const position = await this.persistence.adapter.getStreamPosition( - this.collectionId, - ) - if (lifecycleGeneration !== this.lifecycleGeneration) return - this.persistedKeySetEvidence = - position.keySet?.status === `consistent` - ? { status: `unknown` } - : position.keySet - this.observeStreamPosition( - position.latestTerm, - position.latestSeq, - position.latestRowVersion, - ) - } - - const collectionMetadata = await this.loadCollectionMetadataSnapshot() + const snapshot = await this.persistence.adapter.loadResumeSnapshot( + this.collectionId, + { includeRows: false }, + ) if (lifecycleGeneration !== this.lifecycleGeneration) return - this.replaceCollectionMetadataSnapshot(collectionMetadata) + this.persistedResumeGeneration = this.getResumeSnapshotGeneration(snapshot) + this.persistedKeySetEvidence = snapshot.keySet + this.observeStreamPosition( + snapshot.latestTerm, + snapshot.latestSeq, + snapshot.latestRowVersion, + ) + this.replaceCollectionMetadataSnapshot(snapshot.collectionMetadata) } private async loadCollectionMetadataSnapshot(): Promise< @@ -1403,10 +1375,7 @@ class PersistedCollectionRuntime< this.hydratingGeneration = config.lifecycleGeneration try { let rows: Array<{ key: TKey; value: T; metadata?: unknown }> - if ( - config.bindKeySetEvidence && - this.persistence.adapter.loadResumeSnapshot - ) { + if (config.bindKeySetEvidence) { const snapshot = await this.persistence.adapter.loadResumeSnapshot( this.collectionId, { @@ -2526,6 +2495,38 @@ function createWrappedSyncConfig< params.collection as Collection, ) + const persistenceCapability: SyncPersistenceCapabilityV1 = { + protocol: SYNC_PERSISTENCE_PROTOCOL, + version: SYNC_PERSISTENCE_VERSION, + hydrateBaseline: () => + startupState.cleanedUp + ? Promise.resolve() + : runtime.ensureResumeBaselineHydrated(), + scanPersistedRows: (options) => + startupState.cleanedUp + ? Promise.resolve([]) + : runtime.scanPersistedRows(options), + resumeSnapshot: { + certify: () => + startupState.cleanedUp + ? Promise.resolve() + : runtime.ensureResumeBaselineCertified(), + getKeySetEvidence: () => + startupState.cleanedUp ? undefined : runtime.getKeySetEvidence(), + expectCurrentCommit: () => { + if (startupState.cleanedUp) return + const openTransaction = getOpenTransaction() + if (!openTransaction) { + throw new InvalidPersistedCollectionConfigError( + `resumeSnapshot.expectCurrentCommit must be called within an open sync transaction`, + ) + } + openTransaction.expectedResumeGenerationOwner = + runtime.getResumeGenerationOwner() + }, + }, + } + const wrappedParams = { ...params, markReady: () => { @@ -2607,17 +2608,8 @@ function createWrappedSyncConfig< }, metadata: params.metadata ? { + persistence: persistenceCapability, row: { - whenHydrated: () => - startupState.cleanedUp - ? Promise.resolve() - : runtime.ensureResumeBaselineHydrated(), - certifyPersistedResume: runtime.supportsResumeSnapshot() - ? () => - startupState.cleanedUp - ? Promise.resolve() - : runtime.ensureResumeBaselineCertified() - : undefined, get: (key: TKey) => { if (startupState.cleanedUp) return undefined const openTransaction = getOpenTransaction() @@ -2633,30 +2625,6 @@ function createWrappedSyncConfig< } return params.metadata!.row.get(key) }, - scanPersisted: (options?: PersistedRowScanOptions) => - startupState.cleanedUp - ? Promise.resolve([]) - : runtime.scanPersistedRows(options), - getPersistedKeySetEvidence: runtime.supportsResumeSnapshot() - ? () => - startupState.cleanedUp - ? undefined - : runtime.getPersistedKeySetEvidence() - : undefined, - expectCurrentCommitInResumeSnapshot: - runtime.supportsResumeSnapshot() - ? () => { - if (startupState.cleanedUp) return - const openTransaction = getOpenTransaction() - if (!openTransaction) { - throw new InvalidPersistedCollectionConfigError( - `expectCurrentCommitInResumeSnapshot must be called within an open sync transaction`, - ) - } - openTransaction.expectedResumeGenerationOwner = - runtime.getResumeGenerationOwner() - } - : undefined, set: (key: TKey, value: unknown) => { if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() @@ -2856,6 +2824,9 @@ function createWrappedSyncConfig< ) return sourceResult })() + void sourceResultPromise.catch((error) => { + if (!startupState.cleanedUp) params.markError(error) + }) return { cleanup: () => { diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts b/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts index 336c2ff676..8a1ab5da88 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts @@ -23,35 +23,27 @@ interface SyncExtraUtils extends UtilsRecord { const adapter: PersistenceAdapter = { loadSubset: () => Promise.resolve([]), + loadResumeSnapshot: () => + Promise.resolve({ + rows: [], + keySet: { status: `consistent` }, + collectionMetadata: [], + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + resetEpoch: 0, + }), applyCommittedTx: () => Promise.resolve(), ensureIndex: () => Promise.resolve(), } describe(`persisted collection types`, () => { - it(`keeps the atomic resume snapshot extension optional and exact`, () => { - const legacyAdapter: PersistenceAdapter = adapter - const snapshotAdapter: PersistenceAdapter = { - ...adapter, - loadResumeSnapshot: (_collectionId, _options) => - Promise.resolve({ - rows: [], - keySet: { status: `consistent` }, - collectionMetadata: [], - latestTerm: 1, - latestSeq: 2, - latestRowVersion: 3, - resetEpoch: 4, - }), - } - type LoadResumeSnapshot = NonNullable< - PersistenceAdapter[`loadResumeSnapshot`] - > + it(`requires an exact atomic resume snapshot contract`, () => { + type LoadResumeSnapshot = PersistenceAdapter[`loadResumeSnapshot`] type ResumeSnapshot = Awaited> - expectTypeOf(legacyAdapter).toMatchTypeOf() - expectTypeOf(snapshotAdapter.loadResumeSnapshot).toMatchTypeOf< - LoadResumeSnapshot | undefined - >() + expectTypeOf(adapter).toMatchTypeOf() + expectTypeOf(adapter.loadResumeSnapshot).toMatchTypeOf() expectTypeOf[1]>().toEqualTypeOf< | { requiredIndexSignatures?: ReadonlyArray diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 0895e52375..c28aad9325 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -27,7 +27,11 @@ import type { PullSinceResponse, TxCommitted, } from '../src' -import type { LoadSubsetOptions, SyncConfig } from '@tanstack/db' +import type { + LoadSubsetOptions, + SyncConfig, + SyncMetadataApi, +} from '@tanstack/db' type Todo = { id: string @@ -56,6 +60,11 @@ type RecordingAdapter = PersistenceAdapter & { requiredIndexSignatures: ReadonlyArray }> loadCollectionMetadataCalls: Array + loadResumeSnapshotCalls: Array<{ + collectionId: string + includeRows: boolean | undefined + requiredIndexSignatures: ReadonlyArray + }> rows: Map rowMetadata: Map collectionMetadata: Map @@ -76,6 +85,7 @@ function createRecordingAdapter( markIndexRemovedCalls: [], loadSubsetCalls: [], loadCollectionMetadataCalls: [], + loadResumeSnapshotCalls: [], loadSubset: (collectionId, options, ctx) => { adapter.loadSubsetCalls.push({ collectionId, @@ -90,6 +100,33 @@ function createRecordingAdapter( })), ) }, + loadResumeSnapshot: (collectionId, options) => { + adapter.loadResumeSnapshotCalls.push({ + collectionId, + includeRows: options?.includeRows, + requiredIndexSignatures: options?.requiredIndexSignatures ?? [], + }) + const latest = adapter.applyCommittedTxCalls.at(-1)?.tx + return Promise.resolve({ + rows: + options?.includeRows === false + ? [] + : Array.from(rows.values()).map((value) => ({ + key: value.id, + value, + metadata: rowMetadata.get(value.id), + })), + keySet: { status: `consistent` }, + collectionMetadata: Array.from( + adapter.collectionMetadata, + ([key, value]) => ({ key, value }), + ), + latestTerm: latest?.term ?? 0, + latestSeq: latest?.seq ?? 0, + latestRowVersion: latest?.rowVersion ?? 0, + resetEpoch: 0, + }) + }, loadCollectionMetadata: (collectionId) => { adapter.loadCollectionMetadataCalls.push(collectionId) return Promise.resolve( @@ -178,6 +215,16 @@ function createRecordingAdapter( function createNoopAdapter(): PersistenceAdapter { return { loadSubset: () => Promise.resolve([]), + loadResumeSnapshot: () => + Promise.resolve({ + rows: [], + keySet: { status: `consistent` }, + collectionMetadata: [], + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + resetEpoch: 0, + }), applyCommittedTx: () => Promise.resolve(), ensureIndex: () => Promise.resolve(), } @@ -347,9 +394,10 @@ describe(`persistedCollectionOptions`, () => { await collection.stateWhenReady() - expect(adapter.loadCollectionMetadataCalls).toEqual([ - `persisted-startup-metadata`, - ]) + expect(adapter.loadResumeSnapshotCalls[0]).toMatchObject({ + collectionId: `persisted-startup-metadata`, + includeRows: false, + }) expect( collection._state.syncedCollectionMetadata.get(`electric:resume`), ).toEqual({ @@ -1080,7 +1128,7 @@ describe(`persistedCollectionOptions`, () => { await flushAsyncWork() expect(collection.id).toBe(options.id) - expect(adapter.loadSubsetCalls[0]?.collectionId).toBe(collection.id) + expect(adapter.loadResumeSnapshotCalls[0]?.collectionId).toBe(collection.id) }) it(`keeps hydrated rows ahead of persisted startup rows`, async () => { @@ -1193,19 +1241,14 @@ describe(`persistedCollectionOptions`, () => { }, ]) let resolveLoadSubset: (() => void) | undefined - adapter.loadSubset = async () => { - await new Promise((resolve) => { - resolveLoadSubset = resolve - }) - return [ - { - key: `cached-1`, - value: { - id: `cached-1`, - title: `Cached row`, - }, - }, - ] + const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) + adapter.loadResumeSnapshot = async (...args) => { + if (args[1]?.includeRows === true) { + await new Promise((resolve) => { + resolveLoadSubset = resolve + }) + } + return loadResumeSnapshot(...args) } let remoteBegin: (() => void) | undefined @@ -1271,11 +1314,14 @@ describe(`persistedCollectionOptions`, () => { it(`discards a hydration-buffered transaction aborted before replay`, async () => { const adapter = createRecordingAdapter() let resolveLoadSubset: (() => void) | undefined - adapter.loadSubset = async () => { - await new Promise((resolve) => { - resolveLoadSubset = resolve - }) - return [] + const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) + adapter.loadResumeSnapshot = async (...args) => { + if (args[1]?.includeRows === true) { + await new Promise((resolve) => { + resolveLoadSubset = resolve + }) + } + return loadResumeSnapshot(...args) } let remoteBegin: (() => void) | undefined let remoteWrite: @@ -1334,11 +1380,14 @@ describe(`persistedCollectionOptions`, () => { it(`rejects every hydration-buffered receipt when replay fails`, async () => { const adapter = createRecordingAdapter() let resolveLoadSubset: (() => void) | undefined - adapter.loadSubset = async () => { - await new Promise((resolve) => { - resolveLoadSubset = resolve - }) - return [] + const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) + adapter.loadResumeSnapshot = async (...args) => { + if (args[1]?.includeRows === true) { + await new Promise((resolve) => { + resolveLoadSubset = resolve + }) + } + return loadResumeSnapshot(...args) } const replayError = new Error(`replay key failed`) @@ -1415,8 +1464,12 @@ describe(`persistedCollectionOptions`, () => { it(`marks ready even when persisted startup fails before markReady`, async () => { const adapter = createRecordingAdapter() - adapter.loadSubset = async () => { - throw new Error(`startup failure`) + const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) + adapter.loadResumeSnapshot = (...args) => { + if (args[1]?.includeRows === true) { + return Promise.reject(new Error(`startup failure`)) + } + return loadResumeSnapshot(...args) } const collection = createCollection( @@ -1451,20 +1504,14 @@ describe(`persistedCollectionOptions`, () => { adapter.collectionMetadata.set(`startup:key`, { ready: true }) let resolveLoadSubset: (() => void) | undefined - adapter.loadSubset = async () => { - await new Promise((resolve) => { - resolveLoadSubset = resolve - }) - return [ - { - key: `cached-1`, - value: { - id: `cached-1`, - title: `Cached row`, - }, - metadata: adapter.rowMetadata.get(`cached-1`), - }, - ] + const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) + adapter.loadResumeSnapshot = async (...args) => { + if (args[1]?.includeRows === true) { + await new Promise((resolve) => { + resolveLoadSubset = resolve + }) + } + return loadResumeSnapshot(...args) } let remoteBegin: (() => void) | undefined @@ -1754,16 +1801,12 @@ describe(`persistedCollectionOptions`, () => { const originalLoadSubset = adapter.loadSubset.bind(adapter) let loadCalls = 0 let releaseStaleReload!: () => void - let releaseFreshReload!: () => void const staleReloadGate = new Promise((resolve) => { releaseStaleReload = resolve }) - const freshReloadGate = new Promise((resolve) => { - releaseFreshReload = resolve - }) adapter.loadSubset = async (...args) => { loadCalls++ - if (loadCalls === 2) { + if (loadCalls === 1) { await staleReloadGate return [ { @@ -1772,7 +1815,6 @@ describe(`persistedCollectionOptions`, () => { }, ] } - if (loadCalls === 3) await freshReloadGate return originalLoadSubset(...args) } @@ -1799,22 +1841,17 @@ describe(`persistedCollectionOptions`, () => { latestRowVersion: 1, requiresFullReload: true, }) - for (let attempt = 0; attempt < 20 && loadCalls < 2; attempt++) { + for (let attempt = 0; attempt < 20 && loadCalls < 1; attempt++) { await flushAsyncWork() } - expect(loadCalls).toBe(2) + expect(loadCalls).toBe(1) await collection.cleanup() adapter.rows.set(`1`, { id: `1`, title: `Restarted` }) collection.startSyncImmediate() releaseStaleReload() - for (let attempt = 0; attempt < 20 && loadCalls < 3; attempt++) { - await flushAsyncWork() - } - expect(loadCalls).toBe(3) expect(collection.get(`1`)?.title).not.toBe(`Stale reload`) - releaseFreshReload() for ( let attempt = 0; attempt < 20 && collection.get(`1`)?.title !== `Restarted`; @@ -1866,8 +1903,8 @@ describe(`persistedCollectionOptions`, () => { await collection.preload() await flushAsyncWork() - expect(metadataCalls).toBe(1) - expect(subsetCalls).toBe(1) + expect(metadataCalls).toBe(0) + expect(subsetCalls).toBe(0) coordinator.emit({ type: `tx:committed`, @@ -1877,10 +1914,10 @@ describe(`persistedCollectionOptions`, () => { latestRowVersion: 1, requiresFullReload: true, }) - for (let attempt = 0; attempt < 20 && metadataCalls < 2; attempt++) { + for (let attempt = 0; attempt < 20 && metadataCalls < 1; attempt++) { await flushAsyncWork() } - expect(metadataCalls).toBe(2) + expect(metadataCalls).toBe(1) await collection.cleanup() adapter.rows.set(`1`, { id: `1`, title: `Restarted` }) @@ -1888,14 +1925,14 @@ describe(`persistedCollectionOptions`, () => { releaseStaleMetadata() for ( let attempt = 0; - attempt < 20 && (metadataCalls < 3 || subsetCalls < 2); + attempt < 20 && collection.get(`1`)?.title !== `Restarted`; attempt++ ) { await flushAsyncWork() } - expect(metadataCalls).toBe(3) - expect(subsetCalls).toBe(2) + expect(metadataCalls).toBe(1) + expect(subsetCalls).toBe(1) expect(stripVirtualProps(collection.get(`1`))).toEqual({ id: `1`, title: `Restarted`, @@ -2512,6 +2549,9 @@ describe(`persistedCollectionOptions`, () => { } const coordinator = createCoordinatorHarness() let hydrateBaseline: (() => Promise) | undefined + let persistenceCapability: + | NonNullable[`persistence`]> + | undefined const collection = createCollection( persistedCollectionOptions({ id: `sync-present`, @@ -2519,11 +2559,8 @@ describe(`persistedCollectionOptions`, () => { getKey: (item) => item.id, sync: { sync: ({ markReady, metadata }) => { - hydrateBaseline = ( - metadata?.row as - | { whenHydrated?: () => Promise } - | undefined - )?.whenHydrated + persistenceCapability = metadata?.persistence + hydrateBaseline = metadata?.persistence?.hydrateBaseline markReady() return { loadSubset: () => true } }, @@ -2534,6 +2571,18 @@ describe(`persistedCollectionOptions`, () => { collection.startSyncImmediate() await vi.waitFor(() => expect(hydrateBaseline).toBeTypeOf(`function`)) + expect(persistenceCapability).toMatchObject({ + protocol: `@tanstack/db/sync-persistence`, + version: 1, + }) + expect(persistenceCapability?.scanPersistedRows).toBeTypeOf(`function`) + expect(persistenceCapability?.resumeSnapshot.certify).toBeTypeOf(`function`) + expect(persistenceCapability?.resumeSnapshot.getKeySetEvidence).toBeTypeOf( + `function`, + ) + expect( + persistenceCapability?.resumeSnapshot.expectCurrentCommit, + ).toBeTypeOf(`function`) await hydrateBaseline!() expect(collection.has(`2`)).toBe(true) diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts index 1213b282cc..9f62b50834 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts @@ -1,6 +1,10 @@ import { DatabaseSync } from 'node:sqlite' import { describe, expect, it } from 'vitest' -import { SQLiteCorePersistenceAdapter, createPersistedTableName } from '../src' +import { + SQLiteCorePersistenceAdapter, + createPersistedTableName, + encodePersistedStorageKey, +} from '../src' import type { SQLiteDriver } from '../src' type CachedSchemaState = { @@ -415,6 +419,15 @@ describe(`SQLite resume snapshots`, () => { deleted_at TEXT NOT NULL )`, ) + await driver.run( + `INSERT INTO "${tableName}" (key, value, metadata, row_version) + VALUES (?, ?, ?, 0)`, + [ + encodePersistedStorageKey(`legacy-row`), + JSON.stringify({ id: `legacy-row`, n: 0 }), + JSON.stringify({ source: `legacy` }), + ], + ) await driver.exec( `CREATE TABLE collection_version ( collection_id TEXT PRIMARY KEY, @@ -473,9 +486,16 @@ describe(`SQLite resume snapshots`, () => { ]), ) - expect((await migrated.loadResumeSnapshot(collectionId)).keySet).toEqual({ - status: `unknown`, - }) + const migratedLegacySnapshot = + await migrated.loadResumeSnapshot(collectionId) + expect(migratedLegacySnapshot.keySet).toEqual({ status: `unknown` }) + expect(migratedLegacySnapshot.rows).toEqual([ + { + key: `legacy-row`, + value: { id: `legacy-row`, n: 0 }, + metadata: { source: `legacy` }, + }, + ]) await migrated.applyCommittedTx(collectionId, { txId: `legacy-insert`, term: 1, diff --git a/packages/db/src/errors.ts b/packages/db/src/errors.ts index a97cb117b7..0056dc4b8c 100644 --- a/packages/db/src/errors.ts +++ b/packages/db/src/errors.ts @@ -75,6 +75,16 @@ export class CollectionConfigurationError extends TanStackDBError { } } +export class InvalidSyncPersistenceCapabilityError extends CollectionConfigurationError { + constructor(reason: string) { + super( + `Invalid sync persistence capability at metadata.persistence: ${reason}. ` + + `Custom sync wrappers must forward metadata.persistence unchanged.`, + ) + this.name = `InvalidSyncPersistenceCapabilityError` + } +} + export class CollectionRequiresConfigError extends CollectionConfigurationError { constructor() { super(`Collection requires a config`) diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 4ef64d880c..fc004ab63b 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -20,6 +20,7 @@ export * from './live-query-window-controller' export * from './local-only' export * from './local-storage' export * from './errors' +export * from './sync-persistence' export { deepEquals } from './utils' /** @internal Used by first-party collection adapters. */ export { warnOnce, resetWarnings } from './utils' diff --git a/packages/db/src/sync-persistence.ts b/packages/db/src/sync-persistence.ts new file mode 100644 index 0000000000..6bb90b28a8 --- /dev/null +++ b/packages/db/src/sync-persistence.ts @@ -0,0 +1,68 @@ +import { InvalidSyncPersistenceCapabilityError } from './errors' +import type { SyncPersistenceCapabilityV1 } from './types' + +export const SYNC_PERSISTENCE_PROTOCOL = + `@tanstack/db/sync-persistence` as const +export const SYNC_PERSISTENCE_VERSION = 1 as const + +function isRecord(value: unknown): value is Record { + return typeof value === `object` && value !== null +} + +function requireFunction( + value: Record, + key: string, + path = key, +): void { + if (typeof value[key] !== `function`) { + throw new InvalidSyncPersistenceCapabilityError( + `${path} must be a function`, + ) + } +} + +/** + * Validates the cross-package structural persistence protocol before a sync + * adapter uses it. Undefined means that the collection has no persistence + * capability; any advertised capability must be complete. + */ +export function validateSyncPersistenceCapability< + TKey extends string | number = string | number, +>(value: unknown): SyncPersistenceCapabilityV1 | undefined { + if (value === undefined) return undefined + if (!isRecord(value)) { + throw new InvalidSyncPersistenceCapabilityError(`expected an object`) + } + if (value.protocol !== SYNC_PERSISTENCE_PROTOCOL) { + throw new InvalidSyncPersistenceCapabilityError( + `protocol must be "${SYNC_PERSISTENCE_PROTOCOL}"`, + ) + } + if (value.version !== SYNC_PERSISTENCE_VERSION) { + throw new InvalidSyncPersistenceCapabilityError( + `version must be ${SYNC_PERSISTENCE_VERSION}`, + ) + } + requireFunction(value, `hydrateBaseline`) + requireFunction(value, `scanPersistedRows`) + + const resumeSnapshot = value.resumeSnapshot + if (!isRecord(resumeSnapshot)) { + throw new InvalidSyncPersistenceCapabilityError( + `resumeSnapshot must be an object`, + ) + } + requireFunction(resumeSnapshot, `certify`, `resumeSnapshot.certify`) + requireFunction( + resumeSnapshot, + `getKeySetEvidence`, + `resumeSnapshot.getKeySetEvidence`, + ) + requireFunction( + resumeSnapshot, + `expectCurrentCommit`, + `resumeSnapshot.expectCurrentCommit`, + ) + + return value as SyncPersistenceCapabilityV1 +} diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 44b958aa8a..27fff92020 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -476,6 +476,44 @@ export interface SyncMetadataApi< value: unknown }> } + /** + * Versioned persistence bridge used by sync adapters that can hydrate and + * inspect a durable collection baseline. Custom sync wrappers must forward + * this object unchanged. + */ + persistence?: SyncPersistenceCapabilityV1 +} + +export type SyncPersistenceKeySetEvidence = { + status: `unknown` | `consistent` | `incompatible` +} + +export type SyncPersistenceScanOptions = { + metadataOnly?: boolean +} + +export type SyncPersistenceScannedRow< + TKey extends string | number = string | number, +> = { + key: TKey + value: object + metadata?: unknown +} + +export type SyncPersistenceCapabilityV1< + TKey extends string | number = string | number, +> = { + readonly protocol: `@tanstack/db/sync-persistence` + readonly version: 1 + readonly hydrateBaseline: () => Promise + readonly scanPersistedRows: ( + options?: SyncPersistenceScanOptions, + ) => Promise>> + readonly resumeSnapshot: { + readonly certify: () => Promise + readonly getKeySetEvidence: () => SyncPersistenceKeySetEvidence | undefined + readonly expectCurrentCommit: () => void + } } export interface ChangeMessage< diff --git a/packages/db/tests/sync-persistence.test.ts b/packages/db/tests/sync-persistence.test.ts new file mode 100644 index 0000000000..e7c9a1ef0e --- /dev/null +++ b/packages/db/tests/sync-persistence.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from 'vitest' +import { validateSyncPersistenceCapability } from '../src' + +const baseCapability = { + protocol: `@tanstack/db/sync-persistence`, + version: 1, + hydrateBaseline: () => Promise.resolve(), + scanPersistedRows: () => Promise.resolve([]), +} as const + +describe(`sync persistence capability`, () => { + it(`preserves a complete capability by identity`, () => { + const capability = { + ...baseCapability, + resumeSnapshot: { + certify: () => Promise.resolve(), + getKeySetEvidence: () => ({ status: `consistent` as const }), + expectCurrentCommit: () => {}, + }, + } + + expect(validateSyncPersistenceCapability(capability)).toBe(capability) + }) + + it(`rejects an unsupported protocol version with forwarding guidance`, () => { + expect(() => + validateSyncPersistenceCapability({ + ...baseCapability, + version: 2, + resumeSnapshot: { + certify: () => Promise.resolve(), + getKeySetEvidence: () => ({ status: `consistent` }), + expectCurrentCommit: () => {}, + }, + }), + ).toThrow(/version must be 1.*forward metadata\.persistence unchanged/i) + }) + + it(`rejects an incomplete resume snapshot capability`, () => { + expect(() => + validateSyncPersistenceCapability({ + ...baseCapability, + resumeSnapshot: { + certify: () => Promise.resolve(), + getKeySetEvidence: () => ({ status: `consistent` }), + }, + }), + ).toThrow(/resumeSnapshot\.expectCurrentCommit.*forward/i) + }) +}) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 94c64ee2fe..2947bb79e1 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -9,6 +9,7 @@ import { DeduplicatedLoadSubset, LoadSubsetOperationAbortedError, and, + validateSyncPersistenceCapability, warnOnce, withCollectionConfigFactory, withCollectionSyncConfigCleanup, @@ -52,7 +53,6 @@ import type { LoadSubsetOptions, SyncAppliedReceipt, SyncConfig, - SyncMetadataApi, SyncMode, UpdateMutationFnParams, } from '@tanstack/db' @@ -67,19 +67,6 @@ import type { ShapeStreamOptions, } from '@electric-sql/client' -type ElectricSyncMetadataWithHydration = SyncMetadataApi & { - row: SyncMetadataApi[`row`] & { - whenHydrated?: () => Promise - // Capability marker for wrappers predating the hydration barrier. - scanPersisted?: unknown - certifyPersistedResume?: () => Promise - getPersistedKeySetEvidence?: () => - | { status: `unknown` | `consistent` | `incompatible` } - | undefined - expectCurrentCommitInResumeSnapshot?: () => void - } -} - // Re-export for user convenience in custom match functions export { isChangeMessage, isControlMessage } from '@electric-sql/client' @@ -1391,7 +1378,6 @@ function createElectricSync>( const { getLifecycle, syncMode, collectionId, testHooks } = options let relationSchema: string | undefined - let warnedUnverifiableResume = false const createTagState = () => { const tagCache = new Map() @@ -1762,18 +1748,15 @@ function createElectricSync>( return parseElectricResumeState(persistedResumeState) } - const persistedMetadata = metadata as - | ElectricSyncMetadataWithHydration - | undefined - const scanPersisted = persistedMetadata?.row.scanPersisted - const whenHydrated = persistedMetadata?.row.whenHydrated - const certifyPersistedResume = - persistedMetadata?.row.certifyPersistedResume - const getPersistedKeySetEvidence = - persistedMetadata?.row.getPersistedKeySetEvidence - const expectCurrentCommitInResumeSnapshot = - persistedMetadata?.row.expectCurrentCommitInResumeSnapshot - const persistedKeySetEvidence = getPersistedKeySetEvidence?.() + const persistence = validateSyncPersistenceCapability( + metadata?.persistence, + ) + const hydrateBaseline = persistence?.hydrateBaseline + const resumeSnapshot = persistence?.resumeSnapshot + const certifyResumeSnapshot = resumeSnapshot?.certify + const getKeySetEvidence = resumeSnapshot?.getKeySetEvidence + const expectCurrentCommit = resumeSnapshot?.expectCurrentCommit + const persistedKeySetEvidence = getKeySetEvidence?.() const persistedResumeState = getNewestElectricResumeState( readPersistedResumeState(), @@ -1786,24 +1769,12 @@ function createElectricSync>( const hasIncompatiblePersistedResume = persistedResumeState?.kind === `resume` && persistedResumeState.shapeId !== shapeIdentity - const hasUnverifiablePersistedResume = - shapeOptions.offset === undefined && - shapeOptions.handle === undefined && - persistedResumeState?.kind === `resume` && - scanPersisted !== undefined && - whenHydrated === undefined // A pre-ledger `unknown` baseline cannot justify a non-initial cursor. // One fresh replacement establishes consistent evidence for later resumes. const lacksCompletePersistedKeySet = persistedResumeState?.kind === `resume` && - getPersistedKeySetEvidence !== undefined && + getKeySetEvidence !== undefined && persistedKeySetEvidence?.status !== `consistent` - if (hasUnverifiablePersistedResume && !warnedUnverifiableResume) { - warnedUnverifiableResume = true - console.warn( - `Electric persistence cannot verify hydration for saved resume state. Update the persistence adapter alongside Electric to enable safe resume.`, - ) - } const needsFullSnapshot = shapeOptions.offset === undefined && shapeOptions.handle === undefined && @@ -1816,7 +1787,6 @@ function createElectricSync>( shapeOptions.handle === undefined && persistedResumeState?.kind === `resume` && !hasIncompatiblePersistedResume && - !hasUnverifiablePersistedResume && // Cached rows do not contain authoritative tag/active-condition state. // Only a complete adapter ledger can justify a persisted cursor. !needsFullSnapshot @@ -1827,7 +1797,7 @@ function createElectricSync>( } const receivesCompleteRows = shapeOptions.params?.replica === `full` const requiresKeySetCertification = - canUsePersistedResume && certifyPersistedResume !== undefined + canUsePersistedResume && certifyResumeSnapshot !== undefined // Eager and progressive streams that start after the initial offset can // only apply partial updates when the local materialization is complete. const requiresCompleteResume = @@ -1840,7 +1810,7 @@ function createElectricSync>( (syncMode === `eager` || needsFullSnapshot) && !canUsePersistedResume && !hasExplicitResumeOffset && - whenHydrated !== undefined + hydrateBaseline !== undefined // Wrap markReady to wait for test hook in progressive mode let progressiveReadyGate: Promise | null = null @@ -1998,7 +1968,7 @@ function createElectricSync>( begin({ immediate: true }) metadata.collection.set(`electric:resume`, resetState) if (expectInResumeSnapshot) { - expectCurrentCommitInResumeSnapshot?.() + expectCurrentCommit?.() } commit() } @@ -2006,7 +1976,6 @@ function createElectricSync>( if ( hasIncompatiblePersistedResume || - hasUnverifiablePersistedResume || (needsFullSnapshot && persistedResumeState.kind === `resume`) ) { // This reset is part of the current runtime's startup decision. The @@ -2093,12 +2062,12 @@ function createElectricSync>( const resumeKeysPromise = requiresCompleteResume || freshSnapshotPending - ? whenHydrated + ? hydrateBaseline ? (async () => { - await whenHydrated() + await hydrateBaseline() if ( persistedKeySetEvidence?.status !== `incompatible` && - getPersistedKeySetEvidence?.()?.status === `incompatible` + getKeySetEvidence?.()?.status === `incompatible` ) { throw new Error( `Electric persisted resume baseline became incompatible during hydration`, @@ -2108,8 +2077,8 @@ function createElectricSync>( : undefined : requiresKeySetCertification ? (async () => { - await certifyPersistedResume() - if (getPersistedKeySetEvidence?.()?.status === `incompatible`) { + await certifyResumeSnapshot() + if (getKeySetEvidence?.()?.status === `incompatible`) { throw new Error( `Electric persisted resume baseline became incompatible during certification`, ) diff --git a/packages/electric-db-collection/tests/ORACLE_MUTATIONS.md b/packages/electric-db-collection/tests/ORACLE_MUTATIONS.md index ddb1b16e7a..30ffd810e7 100644 --- a/packages/electric-db-collection/tests/ORACLE_MUTATIONS.md +++ b/packages/electric-db-collection/tests/ORACLE_MUTATIONS.md @@ -76,11 +76,16 @@ counts key iteration for both new and deduplicated acquisitions. ## 7. Resume capability fencing -Accept a persisted offset when `scanPersisted` exists but `whenHydrated` -does not. - -Killed by: `warns once and restarts a persisted resume when hydration completion is unavailable`. -The restart control also verifies that the compatibility warning is not repeated. +Delete `expectCurrentCommit` from an advertised persistence capability, or let +a source wrapper rebuild the capability without forwarding the same complete +`resumeSnapshot` object. + +Killed by: `fails fast when a persistence wrapper drops resume generation +ownership` and `keeps generation ownership when a source wrapper +shallow-forwards the persistence capability`. The direct-source control, `uses +direct resume metadata when no persistence capability is present`, preserves +the intentional no-capability path; completeness is required only after a +persistence capability is advertised. ## 8. Complete-row discrimination diff --git a/packages/electric-db-collection/tests/electric-descriptor-isolation.test.ts b/packages/electric-db-collection/tests/electric-descriptor-isolation.test.ts index e2e63cef03..4704c02deb 100644 --- a/packages/electric-db-collection/tests/electric-descriptor-isolation.test.ts +++ b/packages/electric-db-collection/tests/electric-descriptor-isolation.test.ts @@ -97,11 +97,34 @@ function tagPersistence() { { value: TestRow; metadata?: unknown } >() const metadata = new Map() + let latestTerm = 0 + let latestSeq = 0 + let latestRowVersion = 0 + let resetEpoch = 0 const adapter: PersistenceAdapter = { loadSubset: () => Promise.resolve( Array.from(rows, ([key, row]) => ({ key, ...structuredClone(row) })), ), + loadResumeSnapshot: (_id, ctx) => + Promise.resolve({ + rows: + ctx?.includeRows === false + ? [] + : Array.from(rows, ([key, row]) => ({ + key, + ...structuredClone(row), + })), + keySet: { status: `consistent` }, + collectionMetadata: Array.from(metadata, ([key, value]) => ({ + key, + value: structuredClone(value), + })), + latestTerm, + latestSeq, + latestRowVersion, + resetEpoch, + }), loadCollectionMetadata: () => Promise.resolve( Array.from(metadata, ([key, value]) => ({ @@ -110,7 +133,10 @@ function tagPersistence() { })), ), applyCommittedTx: (_id, transaction) => { - if (transaction.truncate) rows.clear() + if (transaction.truncate) { + rows.clear() + resetEpoch++ + } for (const mutation of transaction.mutations) { if (mutation.type === `delete`) rows.delete(mutation.key) else @@ -136,6 +162,9 @@ function tagPersistence() { if (mutation.type === `delete`) metadata.delete(mutation.key) else metadata.set(mutation.key, structuredClone(mutation.value)) } + latestTerm = transaction.term + latestSeq = transaction.seq + latestRowVersion = transaction.rowVersion return Promise.resolve() }, ensureIndex: () => Promise.resolve(), @@ -556,6 +585,16 @@ fcTest.prop( it(`keeps insert acknowledgements on the owner of a reused persisted descriptor`, async () => { const adapter: PersistenceAdapter = { loadSubset: () => Promise.resolve([]), + loadResumeSnapshot: () => + Promise.resolve({ + rows: [], + keySet: { status: `consistent` }, + collectionMetadata: [], + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + resetEpoch: 0, + }), loadCollectionMetadata: () => Promise.resolve([]), applyCommittedTx: () => Promise.resolve(), ensureIndex: () => Promise.resolve(), diff --git a/packages/electric-db-collection/tests/electric-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-oracle.property.test.ts index 49457d1ea6..e243bd92c9 100644 --- a/packages/electric-db-collection/tests/electric-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-oracle.property.test.ts @@ -143,6 +143,10 @@ function createPersistedAdapter( rows: Map, loadGate: Promise = Promise.resolve(), ): PersistenceAdapter { + let latestTerm = 0 + let latestSeq = 0 + let latestRowVersion = 0 + let resetEpoch = 0 return { loadSubset: () => loadGate.then(() => @@ -151,6 +155,27 @@ function createPersistedAdapter( value: structuredClone(value), })), ), + loadResumeSnapshot: async (_collectionId, ctx) => { + if (ctx?.includeRows !== false) await loadGate + return { + rows: + ctx?.includeRows === false + ? [] + : Array.from(rows, ([key, value]) => ({ + key, + value: structuredClone(value), + })), + keySet: { status: `consistent` }, + collectionMetadata: Array.from(collectionMetadata, ([key, value]) => ({ + key, + value: structuredClone(value), + })), + latestTerm, + latestSeq, + latestRowVersion, + resetEpoch, + } + }, loadCollectionMetadata: () => Promise.resolve( Array.from(collectionMetadata, ([key, value]) => ({ @@ -166,7 +191,10 @@ function createPersistedAdapter( collectionMetadata.set(mutation.key, structuredClone(mutation.value)) } } - if (tx.truncate) rows.clear() + if (tx.truncate) { + rows.clear() + resetEpoch++ + } for (const mutation of tx.mutations) { if (mutation.type === `delete`) { rows.delete(mutation.key) @@ -179,6 +207,9 @@ function createPersistedAdapter( rows.set(mutation.key, structuredClone(mutation.value) as OracleRow) } } + latestTerm = tx.term + latestSeq = tx.seq + latestRowVersion = tx.rowVersion return Promise.resolve() }, ensureIndex: () => Promise.resolve(), @@ -2892,10 +2923,11 @@ describe(`Electric adapter laws`, () => { const metadataStarted = createDeferred() const metadataGate = createDeferred() const adapter = createPersistedAdapter(new Map(), new Map()) - adapter.loadCollectionMetadata = async () => { + const loadResumeSnapshot = adapter.loadResumeSnapshot + adapter.loadResumeSnapshot = async (...args) => { metadataStarted.resolve() await metadataGate.promise - return [] + return loadResumeSnapshot(args[0], args[1]) } const collection = createCollection( persistedCollectionOptions< @@ -2937,9 +2969,10 @@ describe(`Electric adapter laws`, () => { it(`retires pre-start waiters through automatic collection GC`, async () => { const metadataGate = createDeferred() const adapter = createPersistedAdapter(new Map(), new Map()) - adapter.loadCollectionMetadata = vi.fn(async () => { + const loadResumeSnapshot = adapter.loadResumeSnapshot + adapter.loadResumeSnapshot = vi.fn(async (...args) => { await metadataGate.promise - return [] + return loadResumeSnapshot(args[0], args[1]) }) const collection = createCollection( persistedCollectionOptions< @@ -2966,7 +2999,7 @@ describe(`Electric adapter laws`, () => { // A pending preload owns retention; exercise unowned sync for automatic GC. collection.startSyncImmediate() await vi.waitFor( - () => expect(adapter.loadCollectionMetadata).toHaveBeenCalledOnce(), + () => expect(adapter.loadResumeSnapshot).toHaveBeenCalledOnce(), { interval: 1, timeout: 250 }, ) const subscription = collection.subscribeChanges(() => {}) @@ -3639,19 +3672,25 @@ describe(`Electric adapter laws`, () => { [1, { id: 1, name: `current`, stable: `stable-1` }], ]) const adapter = createPersistedAdapter(collectionMetadata, persistedRows) + const loadResumeSnapshot = adapter.loadResumeSnapshot let hydrationCall = 0 - adapter.loadSubset = vi.fn(async () => { + adapter.loadResumeSnapshot = vi.fn(async (collectionId, ctx) => { + const snapshot = await loadResumeSnapshot(collectionId, ctx) + if (ctx?.includeRows === false) return snapshot hydrationCall++ if (hydrationCall === 1) { await firstHydration.promise - return [ - { - key: 1, - value: { id: 1, name: `stale`, stable: `stable-1` }, - }, - ] + return { + ...snapshot, + rows: [ + { + key: 1, + value: { id: 1, name: `stale`, stable: `stable-1` }, + }, + ], + } } - return Array.from(persistedRows, ([key, value]) => ({ key, value })) + return snapshot }) const collection = createCollection( persistedCollectionOptions< @@ -3675,13 +3714,13 @@ describe(`Electric adapter laws`, () => { ) collection.startSyncImmediate() - await vi.waitFor(() => expect(adapter.loadSubset).toHaveBeenCalledTimes(1)) + await vi.waitFor(() => expect(hydrationCall).toBe(1)) await collection.cleanup() collection.startSyncImmediate() await vi.waitFor(() => expect(subscribers).toHaveLength(2)) firstHydration.resolve() - await vi.waitFor(() => expect(adapter.loadSubset).toHaveBeenCalledTimes(2)) + await vi.waitFor(() => expect(hydrationCall).toBe(2)) await vi.waitFor(() => expect(collection.get(1)?.name).toBe(`current`)) subscribers[1]!([upToDate]) await collection.stateWhenReady() @@ -4412,42 +4451,6 @@ describe(`Electric adapter laws`, () => { }, ) - it(`warns once and restarts a persisted resume when hydration completion is unavailable`, async () => { - const warn = vi.spyOn(console, `warn`).mockImplementation(() => {}) - const metadata = createMetadata(resumeState()) - Object.assign(metadata.api.row, { - scanPersisted: () => Promise.resolve([{ key: 1 }]), - }) - const trace = createOracleCollection( - `unverifiable-persisted-resume`, - `eager`, - metadata.api, - ) - - try { - expect(vi.mocked(ShapeStream).mock.calls.at(-1)?.[0]).toMatchObject({ - offset: undefined, - handle: undefined, - }) - trace.subscriber([change(`insert`, 1, `full snapshot`), upToDate]) - - expect(trace.collection.status).toBe(`ready`) - expect(trace.collection.get(1)).toEqual( - expect.objectContaining({ stable: `stable-1` }), - ) - await trace.collection.cleanup() - trace.collection.startSyncImmediate() - mockSubscribe.mock.calls.at(-1)![0]([upToDate]) - expect(warn).toHaveBeenCalledTimes(1) - expect(warn.mock.calls[0]?.[0]).toMatch( - /persistence.*cannot verify hydration.*[Uu]pdate/, - ) - } finally { - await trace.collection.cleanup() - warn.mockRestore() - } - }) - it(`ignores an unseen on-demand update without blocking readiness`, async () => { const metadata = createMetadata(resumeState()) const trace = createOracleCollection( diff --git a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts index 1f3f77ab9c..36a46557dd 100644 --- a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts +++ b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts @@ -176,6 +176,10 @@ function fixture( ]) let hydrationGate = Promise.resolve() const commits: Array = [] + let latestTerm = 0 + let latestSeq = 0 + let latestRowVersion = 0 + let resetEpoch = 0 const adapter: PersistenceAdapter = { loadSubset: () => { const snapshot = Array.from(rows, ([key, value]) => ({ @@ -184,6 +188,27 @@ function fixture( })) return hydrationGate.then(() => snapshot) }, + loadResumeSnapshot: async (_collectionId, ctx) => { + if (ctx?.includeRows !== false) await hydrationGate + return { + rows: + ctx?.includeRows === false + ? [] + : Array.from(rows, ([key, value]) => ({ + key, + value: { ...value }, + })), + keySet: { status: `consistent` }, + collectionMetadata: Array.from(metadata, ([key, value]) => ({ + key, + value: structuredClone(value), + })), + latestTerm, + latestSeq, + latestRowVersion, + resetEpoch, + } + }, loadCollectionMetadata: () => Promise.resolve( Array.from(metadata, ([key, value]) => ({ @@ -196,7 +221,10 @@ function fixture( if (mutation.type === `delete`) metadata.delete(mutation.key) else metadata.set(mutation.key, structuredClone(mutation.value)) } - if (tx.truncate) rows.clear() + if (tx.truncate) { + rows.clear() + resetEpoch++ + } for (const mutation of tx.mutations) { if (mutation.type === `delete`) rows.delete(mutation.key) else { @@ -206,6 +234,9 @@ function fixture( } as Item) } } + latestTerm = tx.term + latestSeq = tx.seq + latestRowVersion = tx.rowVersion commits.push(structuredClone(tx)) return Promise.resolve() }, diff --git a/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts index 5dc4f378bc..40eb2077b0 100644 --- a/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts +++ b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts @@ -126,6 +126,7 @@ async function runRace( legacyUnknown = false, missingKeySetEvidence = false, startupReset: `none` | `tag-state` | `shape-identity` = `none`, + metadataWrapper: `none` | `shallow-persistence` = `none`, ): Promise { const database = new DatabaseSync(`:memory:`) const driver = createDriver(database) @@ -138,6 +139,9 @@ async function runRace( | Collection> | undefined let unsubscribe: (() => void) | undefined + let receivedPersistenceCapability: unknown + let forwardedPersistenceCapability: unknown + let getExpectedCommitCallCount = () => 0 let primaryFailure: unknown const cleanupFailures: Array = [] try { @@ -240,6 +244,49 @@ async function runRace( }, }) as unknown as PersistenceAdapter + const electricOptions = electricCollectionOptions({ + id: collectionId, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode, + getKey: (row) => row.id, + startSync: false, + }) + const electricSync = electricOptions.sync + const wrappedElectricOptions = + metadataWrapper === `shallow-persistence` + ? { + ...electricOptions, + sync: { + ...electricSync, + sync: (params: Parameters[0]) => { + const sourceMetadata = params.metadata + const persistence = sourceMetadata?.persistence + if (!sourceMetadata || !persistence) { + throw new Error(`Expected a persistence resume capability`) + } + + receivedPersistenceCapability = persistence + const expectedCommit = vi.spyOn( + persistence.resumeSnapshot, + `expectCurrentCommit`, + ) + getExpectedCommitCallCount = () => + expectedCommit.mock.calls.length + + const metadata = { + ...sourceMetadata, + persistence, + } + forwardedPersistenceCapability = metadata.persistence + return electricSync.sync({ ...params, metadata }) + }, + }, + } + : electricOptions + collection = createCollection( persistedCollectionOptions< Item, @@ -247,16 +294,7 @@ async function runRace( never, ElectricCollectionUtils >({ - ...electricCollectionOptions({ - id: collectionId, - shapeOptions: { - url: `http://test-url`, - params: { table: `test_table` }, - }, - syncMode, - getKey: (row) => row.id, - startSync: false, - }), + ...wrappedElectricOptions, persistence: { adapter: gatedAdapter }, }), ) @@ -297,6 +335,10 @@ async function runRace( if (startupReset !== `none`) { expect(resumeStateAtLaterSnapshot).toMatchObject({ kind: `reset` }) } + if (metadataWrapper === `shallow-persistence`) { + expect(forwardedPersistenceCapability).toBe(receivedPersistenceCapability) + expect(getExpectedCommitCallCount()).toBe(1) + } if (startupReset === `none`) { subscriber( @@ -359,6 +401,18 @@ async function runRace( { id: 1, name: `one` }, { id: 2, name: `two` }, ]) + if (startupReset !== `none`) { + await vi.waitFor(async () => { + const resumeState = ( + await restartedAdapter.loadCollectionMetadata(collectionId) + ).find(({ key }) => key === `electric:resume`)?.value + expect(resumeState).toMatchObject({ + kind: `resume`, + offset: `20_0`, + handle: `shape-current`, + }) + }) + } } else { await vi.waitFor(() => expect(collection!.status).toBe(`error`)) await vi.waitFor(async () => { @@ -705,6 +759,17 @@ describe(`Electric resume snapshot races`, () => { await runRace(`none`, `eager`, false, false, `shape-identity`) }) + it(`keeps generation ownership when a source wrapper shallow-forwards the persistence capability`, async () => { + await runRace( + `none`, + `eager`, + false, + false, + `shape-identity`, + `shallow-persistence`, + ) + }) + it(`rejects row loss between resume metadata and baseline hydration`, async () => { await runRace(`external-row-loss`) }) diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index bcb7528bd2..d242e7de75 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -117,6 +117,28 @@ describe(`Electric Integration`, () => { Promise.resolve( Array.from(rows.entries()).map(([key, value]) => ({ key, value })), ), + loadResumeSnapshot: ( + _collectionId: string, + options?: { includeRows?: boolean }, + ) => + Promise.resolve({ + rows: + options?.includeRows === false + ? [] + : Array.from(rows.entries()).map(([key, value]) => ({ + key, + value, + })), + keySet: { status: `consistent` as const }, + collectionMetadata: Array.from( + (collectionMetadata ?? new Map()).entries(), + ([key, value]) => ({ key, value }), + ), + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + resetEpoch: 0, + }), loadCollectionMetadata: () => Promise.resolve( Array.from((collectionMetadata ?? new Map()).entries()).map( @@ -3945,7 +3967,7 @@ describe(`Electric Integration`, () => { ) }) - it(`should use persisted resume metadata when no explicit offset or handle is provided`, async () => { + it(`uses direct resume metadata when no persistence capability is present`, async () => { vi.clearAllMocks() const { ShapeStream } = await import(`@electric-sql/client`) @@ -3998,6 +4020,126 @@ describe(`Electric Integration`, () => { ) }) + it(`rejects an incomplete advertised persistence capability before opening ShapeStream`, async () => { + vi.clearAllMocks() + const metadataHarness = createInMemorySyncMetadataApi() + const malformedMetadata = Object.assign(metadataHarness.api, { + persistence: { + protocol: `@tanstack/db/sync-persistence`, + version: 1, + hydrateBaseline: () => Promise.resolve(), + scanPersistedRows: () => Promise.resolve([]), + resumeSnapshot: { + certify: () => Promise.resolve(), + getKeySetEvidence: () => ({ status: `consistent` as const }), + }, + }, + }) + const options = electricCollectionOptions({ + id: `incomplete-persistence-capability-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (item) => item.id as number, + startSync: false, + }) + const originalSync = options.sync + + let cleanup: (() => Promise) | undefined + let configurationError: unknown + try { + const malformedCollection = createCollection({ + ...options, + startSync: true, + sync: { + ...originalSync, + sync: (params: Parameters[0]) => + originalSync.sync({ ...params, metadata: malformedMetadata }), + }, + }) + cleanup = () => malformedCollection.cleanup() + } catch (error) { + configurationError = error + } + + await cleanup?.() + + expect(configurationError).toEqual( + expect.objectContaining({ + message: expect.stringMatching( + /persistence.*capability.*expectCurrentCommit/i, + ), + }), + ) + expect(ShapeStream).not.toHaveBeenCalled() + }) + + it(`fails fast when a persistence wrapper drops resume generation ownership`, async () => { + vi.clearAllMocks() + const { ShapeStream } = await import(`@electric-sql/client`) + const durableResume = { + kind: `resume`, + requiresTagState: false, + offset: `10_0`, + handle: `handle-1`, + shapeId: `{"params":{"table":"test_table"},"url":"http://test-url"}`, + updatedAt: 1, + } + const collectionMetadata = new Map([ + [`electric:resume`, durableResume], + ]) + const electricOptions = electricCollectionOptions({ + id: `malformed-persisted-wrapper-capability-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (item) => item.id as number, + startSync: false, + }) + const electricSync = electricOptions.sync + const persistedCollection = createCollection( + persistedCollectionOptions({ + ...electricOptions, + sync: { + ...electricSync, + sync: (params: Parameters[0]) => { + const persistence = params.metadata?.persistence + if (!persistence) { + throw new Error(`Expected a persistence capability`) + } + const { expectCurrentCommit: _dropped, ...resumeSnapshot } = + persistence.resumeSnapshot + return electricSync.sync({ + ...params, + metadata: { + ...params.metadata, + persistence: { + ...persistence, + resumeSnapshot, + }, + } as unknown as SyncMetadataApi, + }) + }, + }, + persistence: { + adapter: createPersistedAdapter(collectionMetadata), + }, + }) as any, + ) + + const preload = persistedCollection.preload() + await expect(preload).rejects.toThrow( + /persistence.*capability.*expectCurrentCommit/i, + ) + + expect(persistedCollection.status).toBe(`error`) + expect(ShapeStream).not.toHaveBeenCalled() + expect(collectionMetadata.get(`electric:resume`)).toEqual(durableResume) + await persistedCollection.cleanup() + }) + it(`prefers newer persisted resume metadata over hydrated metadata`, () => { vi.clearAllMocks() const metadataHarness = createInMemorySyncMetadataApi( diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index 7867b6cfe6..1b2c146bd8 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -3,6 +3,7 @@ import { LoadSubsetOperationAbortedError, deepEquals, getLoadSubsetDemandKey, + validateSyncPersistenceCapability, warnOnce, withCollectionConfigFactory, withCollectionSyncConfigFactory, @@ -22,7 +23,6 @@ import type { LoadSubsetOptions, SyncAppliedReceipt, SyncConfig, - SyncMetadataApi, } from '@tanstack/db' import type { FetchStatus, @@ -353,22 +353,6 @@ const queryCollectionSuccessfulFetchStarts = new WeakMap() const queryCollectionRequiredFetchStarts = new WeakMap() const queryCollectionCacheOwners = new WeakMap>() -type PersistedScannedRowForQuery = { - key: string | number - value: TItem - metadata?: unknown -} - -type QuerySyncMetadataWithPersistedScan = SyncMetadataApi< - string | number -> & { - row: SyncMetadataApi[`row`] & { - scanPersisted?: (options?: { - metadataOnly?: boolean - }) => Promise>> - } -} - /** * Implementation class for QueryCollectionUtils with explicit dependency injection * for better testability and architectural clarity @@ -994,9 +978,7 @@ export function queryCollectionOptions( ) const { begin, write, commit, markReady, markError, collection, metadata } = params - const persistedMetadata = metadata as - | QuerySyncMetadataWithPersistedScan - | undefined + const persistence = validateSyncPersistenceCapability(metadata?.persistence) // Track whether sync has been started let syncStarted = false @@ -1220,7 +1202,7 @@ export function queryCollectionOptions( return baseline } - const scanPersisted = persistedMetadata?.row.scanPersisted + const scanPersisted = persistence?.scanPersistedRows if (!scanPersisted) { const baseline = new Map< string | number, @@ -2201,7 +2183,7 @@ export function queryCollectionOptions( if ( effectivePersistedGcTime !== undefined && metadata && - persistedMetadata?.row.scanPersisted + persistence?.scanPersistedRows ) { invalidatePendingResultApplication(hashedQueryKey) manualWriteSnapshots.delete(hashedQueryKey) diff --git a/packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts b/packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts index 82b2417608..d97266b2dc 100644 --- a/packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts +++ b/packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts @@ -295,22 +295,15 @@ async function expectDeferredStartupReadyDoesNotOverrideError(): Promise { const maintenanceDeleted = new Promise((resolve) => { resolveMaintenanceDelete = resolve }) - type MetadataWithPersistedScan = SyncMetadataApi & { - row: SyncMetadataApi[`row`] & { - scanPersisted: () => Promise< - Array<{ key: string | number; value: Row; metadata?: unknown }> - > - } - } - const metadata: MetadataWithPersistedScan = { + const scanPersistedRows = vi.fn(async () => { + await scanReleased + return [] + }) + const metadata: SyncMetadataApi = { row: { get: () => undefined, set: () => {}, delete: () => {}, - scanPersisted: async () => { - await scanReleased - return [] - }, }, collection: { get: () => undefined, @@ -325,6 +318,17 @@ async function expectDeferredStartupReadyDoesNotOverrideError(): Promise { }, ], }, + persistence: { + protocol: `@tanstack/db/sync-persistence`, + version: 1, + hydrateBaseline: async () => {}, + scanPersistedRows, + resumeSnapshot: { + certify: async () => {}, + getKeySetEvidence: () => ({ status: `consistent` }), + expectCurrentCommit: () => {}, + }, + }, } collection._lifecycle.setStatus(`cleaned-up`) @@ -333,6 +337,7 @@ async function expectDeferredStartupReadyDoesNotOverrideError(): Promise { try { expect(collection.status).toBe(`error`) + await vi.waitFor(() => expect(scanPersistedRows).toHaveBeenCalledOnce()) releaseScan() await maintenanceDeleted for (let turn = 0; turn < 10; turn++) await Promise.resolve() diff --git a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts index 3a5958f79e..1fdad1ff40 100644 --- a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts +++ b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts @@ -59,7 +59,7 @@ type OwnershipFixtureOptions = { staleTime?: number metadataRecorder?: MetadataRecorder setupMetadata?: (metadata: SyncMetadataApi) => void - scanPersisted?: () => Promise< + scanPersistedRows?: () => Promise< Array<{ key: string | number; value: Item; metadata?: unknown }> > } @@ -106,6 +106,7 @@ function recordMetadata( ): SyncMetadataApi { // These are emitted writes, captured before delegation, not durable commits. return { + ...metadata, row: { get: (key) => metadata.row.get(key), set: (key, value) => { @@ -134,7 +135,7 @@ function createOwnershipFixture({ syncMode = `on-demand`, metadataRecorder, setupMetadata, - scanPersisted, + scanPersistedRows, customHash, staleTime, }: OwnershipFixtureOptions): OwnershipFixture { @@ -156,7 +157,7 @@ function createOwnershipFixture({ const originalSync = baseOptions.sync let pendingSetup = setupMetadata const collection = createCollection( - metadataRecorder || setupMetadata || scanPersisted + metadataRecorder || setupMetadata || scanPersistedRows ? { ...baseOptions, sync: { @@ -167,11 +168,23 @@ function createOwnershipFixture({ const observedMetadata = metadataRecorder ? recordMetadata(params.metadata, metadataRecorder) : params.metadata - const metadataWithPersistedScan = scanPersisted - ? ({ + const metadataWithPersistedScan: SyncMetadataApi< + string | number + > = scanPersistedRows + ? { ...observedMetadata, - row: { ...observedMetadata.row, scanPersisted }, - } as SyncMetadataApi) + persistence: { + protocol: `@tanstack/db/sync-persistence`, + version: 1, + hydrateBaseline: async () => {}, + scanPersistedRows, + resumeSnapshot: { + certify: async () => {}, + getKeySetEvidence: () => ({ status: `consistent` }), + expectCurrentCommit: () => {}, + }, + }, + } : observedMetadata if (pendingSetup) { params.begin() @@ -304,6 +317,9 @@ function createOwnershipStorage(seed?: StoredOwnership, gatedCommit?: number) { const entered = createDeferred() const released = createDeferred() let commitCount = 0 + let latestTerm = 0 + let latestSeq = 0 + let latestRowVersion = 0 const adapter: PersistenceAdapter = { loadSubset: (_id, options) => Promise.resolve( @@ -313,6 +329,26 @@ function createOwnershipStorage(seed?: StoredOwnership, gatedCommit?: number) { metadata: structuredClone(state.rowMetadata.get(value.id)), })), ), + loadResumeSnapshot: (_id, options) => + Promise.resolve({ + rows: + options?.includeRows === false + ? [] + : Array.from(state.rows, ([key, value]) => ({ + key, + value: structuredClone(value), + metadata: structuredClone(state.rowMetadata.get(key)), + })), + keySet: { status: `consistent` }, + collectionMetadata: Array.from( + state.collectionMetadata, + ([key, value]) => ({ key, value: structuredClone(value) }), + ), + latestTerm, + latestSeq, + latestRowVersion, + resetEpoch: 0, + }), loadCollectionMetadata: () => Promise.resolve( Array.from(state.collectionMetadata, ([key, value]) => ({ @@ -376,6 +412,9 @@ function createOwnershipStorage(seed?: StoredOwnership, gatedCommit?: number) { structuredClone(mutation.value), ) } + latestTerm = tx.term + latestSeq = tx.seq + latestRowVersion = tx.rowVersion }, } return { @@ -2215,7 +2254,7 @@ describe(`query collection ownership lifecycle`, () => { createDeferred< Array<{ key: string | number; value: Item; metadata?: unknown }> >() - const scanPersisted = vi + const scanPersistedRows = vi .fn() .mockReturnValueOnce(firstScan.promise) .mockResolvedValue([]) @@ -2223,7 +2262,7 @@ describe(`query collection ownership lifecycle`, () => { id, results: [[stale], [fresh]], syncMode: `eager`, - scanPersisted, + scanPersistedRows, setupMetadata: (metadata) => { metadata.collection.set(`queryCollection:gc:${queryHash}`, { queryHash, @@ -2240,7 +2279,7 @@ describe(`query collection ownership lifecycle`, () => { return Promise.resolve() }) - await vi.waitFor(() => expect(scanPersisted).toHaveBeenCalledOnce()) + await vi.waitFor(() => expect(scanPersistedRows).toHaveBeenCalledOnce()) expect(queryFn).toHaveBeenCalledOnce() const refetch = collection.utils.refetch({ throwOnError: true }) await vi.waitFor(() => expect(queryFn).toHaveBeenCalledTimes(2)) diff --git a/packages/query-db-collection/tests/query.test.ts b/packages/query-db-collection/tests/query.test.ts index a4c14d8d84..4f2fffacb4 100644 --- a/packages/query-db-collection/tests/query.test.ts +++ b/packages/query-db-collection/tests/query.test.ts @@ -95,7 +95,7 @@ function createInMemorySyncMetadataApi< const rowMetadata = new Map(seed?.rowMetadata) const collectionMetadata = new Map(seed?.collectionMetadata) const persistedRows = new Map(seed?.persistedRows) - const api = { + const api: SyncMetadataApi = { row: { get: (key: TKey) => rowMetadata.get(key), set: (key: TKey, value: unknown) => { @@ -104,12 +104,22 @@ function createInMemorySyncMetadataApi< delete: (key: TKey) => { rowMetadata.delete(key) }, - scanPersisted: async () => + }, + persistence: { + protocol: `@tanstack/db/sync-persistence`, + version: 1, + hydrateBaseline: async () => {}, + scanPersistedRows: async () => Array.from(persistedRows.entries()).map(([key, value]) => ({ key, value, metadata: rowMetadata.get(key), })), + resumeSnapshot: { + certify: async () => {}, + getKeySetEvidence: () => ({ status: `consistent` }), + expectCurrentCommit: () => {}, + }, }, collection: { get: (key: string) => collectionMetadata.get(key), @@ -130,7 +140,7 @@ function createInMemorySyncMetadataApi< rowMetadata, collectionMetadata, persistedRows, - api: api as SyncMetadataApi, + api, } } @@ -144,6 +154,9 @@ function createPersistedQueryAdapter( const rows = new Map(seed.rows) const rowMetadata = new Map(seed.rowMetadata) const collectionMetadata = new Map(seed.collectionMetadata) + let latestTerm = 0 + let latestSeq = 0 + let latestRowVersion = 0 return { rows, @@ -155,6 +168,28 @@ function createPersistedQueryAdapter( value, metadata: rowMetadata.get(value.id), })), + loadResumeSnapshot: async ( + _collectionId: string, + options?: { includeRows?: boolean }, + ) => ({ + rows: + options?.includeRows === false + ? [] + : Array.from(rows.values()).map((value) => ({ + key: value.id, + value, + metadata: rowMetadata.get(value.id), + })), + keySet: { status: `consistent` as const }, + collectionMetadata: Array.from( + collectionMetadata.entries(), + ([key, value]) => ({ key, value }), + ), + latestTerm, + latestSeq, + latestRowVersion, + resetEpoch: 0, + }), loadCollectionMetadata: async () => Array.from(collectionMetadata.entries()).map(([key, value]) => ({ key, @@ -193,6 +228,9 @@ function createPersistedQueryAdapter( collectionMetadata.set(mutation.key, mutation.value) } } + latestTerm = tx.term + latestSeq = tx.seq + latestRowVersion = tx.rowVersion }, ensureIndex: async () => {}, } @@ -6693,14 +6731,21 @@ describe(`QueryCollection`, () => { ], ]), }) - const scanPersisted = vi.fn().mockReturnValue(persistedScan.promise) - const metadataApi = { + const scanPersistedRows = vi.fn().mockReturnValue(persistedScan.promise) + const metadataApi: SyncMetadataApi = { ...metadataHarness.api, - row: { - ...metadataHarness.api.row, - scanPersisted, + persistence: { + protocol: `@tanstack/db/sync-persistence`, + version: 1, + hydrateBaseline: async () => {}, + scanPersistedRows, + resumeSnapshot: { + certify: async () => {}, + getKeySetEvidence: () => ({ status: `consistent` }), + expectCurrentCommit: () => {}, + }, }, - } as SyncMetadataApi + } const baseOptions = queryCollectionOptions({ id: `stale-retained-reconciliation`, @@ -6724,7 +6769,8 @@ describe(`QueryCollection`, () => { }) const load = collection._sync.loadSubset({}) await vi.waitFor(() => { - expect(scanPersisted).toHaveBeenCalledOnce() + expect(scanPersistedRows).toHaveBeenCalledOnce() + expect(scanPersistedRows).toHaveBeenCalledWith() }) collection._sync.unloadSubset({}) From 9c85990954090c3bd8fcf5b5159322710ffbf93b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 16:13:09 +0100 Subject: [PATCH 08/23] fix: enforce complete persistence forwarding --- .../preserve-resume-baseline-integrity.md | 7 +- packages/db-sqlite-persistence-core/README.md | 6 + .../tests/persisted.test.ts | 13 ++- packages/db/src/collection/sync.ts | 1 + packages/db/src/sync-persistence.ts | 13 ++- packages/db/src/types.ts | 5 +- packages/db/tests/sync-persistence.test.ts | 10 ++ .../tests/electric-oracle.property.test.ts | 1 + .../tests/electric.test.ts | 62 +++++++++- .../src/main.ts | 14 +++ .../src/protocol.ts | 23 +++- .../src/renderer.ts | 14 +++ .../tests/e2e/fixtures/electron-main.mjs | 6 + .../tests/electron-ipc.test-d.ts | 29 ++++- .../tests/electron-ipc.test.ts | 110 ++++++++++++++++++ .../query-db-collection/tests/query.test.ts | 66 +++++++++++ 16 files changed, 359 insertions(+), 21 deletions(-) diff --git a/.changeset/preserve-resume-baseline-integrity.md b/.changeset/preserve-resume-baseline-integrity.md index daa15b8a2c..7da60be386 100644 --- a/.changeset/preserve-resume-baseline-integrity.md +++ b/.changeset/preserve-resume-baseline-integrity.md @@ -1,8 +1,11 @@ --- -'@tanstack/db-sqlite-persistence-core': patch -'@tanstack/db': patch +'@tanstack/db-sqlite-persistence-core': minor +'@tanstack/db': minor +'@tanstack/electron-db-sqlite-persistence': minor '@tanstack/electric-db-collection': patch '@tanstack/query-db-collection': patch --- Preserve persisted resume integrity with atomic SQLite baseline evidence and stale-writer rejection, expose persistence sync metadata as one versioned capability, and refresh uncertified Electric baselines before publishing resumed data. + +This changes the public persistence contracts: custom `PersistenceAdapter` implementations must now implement `loadResumeSnapshot`, and `SyncMetadataApi.persistence` is required with `null` explicitly representing no persistence. Custom sync wrappers must forward `metadata.persistence` unchanged so consumers receive either that sentinel or the complete versioned capability. The Electron bridge now transports the atomic resume snapshot through IPC protocol v2; Electron main and renderer integrations must upgrade together because mixed v1/v2 peers fail closed. diff --git a/packages/db-sqlite-persistence-core/README.md b/packages/db-sqlite-persistence-core/README.md index 6d03b94f9a..8fa0157568 100644 --- a/packages/db-sqlite-persistence-core/README.md +++ b/packages/db-sqlite-persistence-core/README.md @@ -91,6 +91,12 @@ capability object unchanged rather than copying individual methods. Adapter methods are invoked with their receiver and may rely on instance state through `this`. +`SyncMetadataApi.persistence` is always present. Core sync sources receive +`null`, which explicitly means that no persistence bridge is active. The +persisted wrapper replaces that sentinel with the complete versioned +capability. A wrapper that omits the property is invalid and fails before a +consumer can resume or query against uncertified durable state. + ### SQLite core adapter APIs - `SQLiteCoreAdapterOptions` diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index c28aad9325..f6c4eae871 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -1880,7 +1880,7 @@ describe(`persistedCollectionOptions`, () => { }) adapter.loadCollectionMetadata = async (...args) => { metadataCalls++ - if (metadataCalls === 2) await staleMetadataGate + if (metadataCalls === 1) await staleMetadataGate return originalLoadCollectionMetadata(...args) } adapter.loadSubset = async (...args) => { @@ -1918,6 +1918,7 @@ describe(`persistedCollectionOptions`, () => { await flushAsyncWork() } expect(metadataCalls).toBe(1) + expect(subsetCalls).toBe(0) await collection.cleanup() adapter.rows.set(`1`, { id: `1`, title: `Restarted` }) @@ -1932,7 +1933,7 @@ describe(`persistedCollectionOptions`, () => { } expect(metadataCalls).toBe(1) - expect(subsetCalls).toBe(1) + expect(subsetCalls).toBe(0) expect(stripVirtualProps(collection.get(`1`))).toEqual({ id: `1`, title: `Restarted`, @@ -2559,8 +2560,12 @@ describe(`persistedCollectionOptions`, () => { getKey: (item) => item.id, sync: { sync: ({ markReady, metadata }) => { - persistenceCapability = metadata?.persistence - hydrateBaseline = metadata?.persistence?.hydrateBaseline + const capability = metadata?.persistence + if (!capability) { + throw new Error(`Expected persisted sync capability`) + } + persistenceCapability = capability + hydrateBaseline = capability.hydrateBaseline markReady() return { loadSubset: () => true } }, diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts index a2822f0946..533e7c4025 100644 --- a/packages/db/src/collection/sync.ts +++ b/packages/db/src/collection/sync.ts @@ -452,6 +452,7 @@ export class CollectionSyncManager< isCurrentSync: () => boolean, ): SyncMetadataApi { return { + persistence: null, row: { get: (key) => { if (!isCurrentSync()) return undefined diff --git a/packages/db/src/sync-persistence.ts b/packages/db/src/sync-persistence.ts index 6bb90b28a8..6be13fa4a5 100644 --- a/packages/db/src/sync-persistence.ts +++ b/packages/db/src/sync-persistence.ts @@ -23,15 +23,18 @@ function requireFunction( /** * Validates the cross-package structural persistence protocol before a sync - * adapter uses it. Undefined means that the collection has no persistence - * capability; any advertised capability must be complete. + * adapter uses it. Null explicitly means that the collection has no + * persistence capability; undefined means a wrapper dropped the required + * field. Any advertised capability must be complete. */ export function validateSyncPersistenceCapability< TKey extends string | number = string | number, ->(value: unknown): SyncPersistenceCapabilityV1 | undefined { - if (value === undefined) return undefined +>(value: unknown): SyncPersistenceCapabilityV1 | null { + if (value === null) return null if (!isRecord(value)) { - throw new InvalidSyncPersistenceCapabilityError(`expected an object`) + throw new InvalidSyncPersistenceCapabilityError( + `expected null or a complete capability object`, + ) } if (value.protocol !== SYNC_PERSISTENCE_PROTOCOL) { throw new InvalidSyncPersistenceCapabilityError( diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 27fff92020..62fd1fbe90 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -479,9 +479,10 @@ export interface SyncMetadataApi< /** * Versioned persistence bridge used by sync adapters that can hydrate and * inspect a durable collection baseline. Custom sync wrappers must forward - * this object unchanged. + * this value unchanged. `null` explicitly means that the collection has no + * persistence capability; a missing property is invalid. */ - persistence?: SyncPersistenceCapabilityV1 + persistence: SyncPersistenceCapabilityV1 | null } export type SyncPersistenceKeySetEvidence = { diff --git a/packages/db/tests/sync-persistence.test.ts b/packages/db/tests/sync-persistence.test.ts index e7c9a1ef0e..38a794be70 100644 --- a/packages/db/tests/sync-persistence.test.ts +++ b/packages/db/tests/sync-persistence.test.ts @@ -9,6 +9,16 @@ const baseCapability = { } as const describe(`sync persistence capability`, () => { + it(`accepts null as an explicit no-persistence capability`, () => { + expect(validateSyncPersistenceCapability(null)).toBeNull() + }) + + it(`rejects a missing persistence field with forwarding guidance`, () => { + expect(() => validateSyncPersistenceCapability(undefined)).toThrow( + /expected null or a complete capability object.*forward metadata\.persistence unchanged/i, + ) + }) + it(`preserves a complete capability by identity`, () => { const capability = { ...baseCapability, diff --git a/packages/electric-db-collection/tests/electric-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-oracle.property.test.ts index e243bd92c9..1080a40c90 100644 --- a/packages/electric-db-collection/tests/electric-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-oracle.property.test.ts @@ -98,6 +98,7 @@ function createMetadata(seed: ReadonlyMap): { return { state, api: { + persistence: null, row: { get: () => undefined, set: () => {}, diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index d242e7de75..3fb8c72bc1 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -87,6 +87,7 @@ describe(`Electric Integration`, () => { return { collectionMetadata, api: { + persistence: null, row: { get: () => undefined, set: () => {}, @@ -3967,7 +3968,7 @@ describe(`Electric Integration`, () => { ) }) - it(`uses direct resume metadata when no persistence capability is present`, async () => { + it(`uses direct resume metadata when persistence is explicitly null`, async () => { vi.clearAllMocks() const { ShapeStream } = await import(`@electric-sql/client`) @@ -4001,6 +4002,7 @@ describe(`Electric Integration`, () => { }) const originalSync = baseOptions.sync + expect(metadataHarness.api.persistence).toBeNull() createCollection({ ...baseOptions, sync: { @@ -4020,6 +4022,64 @@ describe(`Electric Integration`, () => { ) }) + it(`rejects a sync wrapper that drops the entire persistence field before opening ShapeStream`, async () => { + vi.clearAllMocks() + const { ShapeStream } = await import(`@electric-sql/client`) + const durableResume = { + kind: `resume`, + requiresTagState: false, + offset: `10_0`, + handle: `handle-1`, + shapeId: `{"params":{"table":"test_table"},"url":"http://test-url"}`, + updatedAt: 1, + } + const collectionMetadata = new Map([ + [`electric:resume`, durableResume], + ]) + const electricOptions = electricCollectionOptions({ + id: `missing-persisted-wrapper-capability-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (item) => item.id as number, + startSync: false, + }) + const electricSync = electricOptions.sync + const persistedCollection = createCollection( + persistedCollectionOptions({ + ...electricOptions, + sync: { + ...electricSync, + sync: (params: Parameters[0]) => { + const { persistence: _dropped, ...metadataWithoutPersistence } = + params.metadata! + return electricSync.sync({ + ...params, + metadata: + metadataWithoutPersistence as unknown as SyncMetadataApi< + string | number + >, + }) + }, + }, + persistence: { + adapter: createPersistedAdapter(collectionMetadata), + }, + }) as any, + ) + + const preload = persistedCollection.preload() + await expect(preload).rejects.toThrow( + /expected null or a complete capability object.*forward metadata\.persistence unchanged/i, + ) + + expect(persistedCollection.status).toBe(`error`) + expect(ShapeStream).not.toHaveBeenCalled() + expect(collectionMetadata.get(`electric:resume`)).toEqual(durableResume) + await persistedCollection.cleanup() + }) + it(`rejects an incomplete advertised persistence capability before opening ShapeStream`, async () => { vi.clearAllMocks() const metadataHarness = createInMemorySyncMetadataApi() diff --git a/packages/electron-db-sqlite-persistence/src/main.ts b/packages/electron-db-sqlite-persistence/src/main.ts index e412ebd4fe..0e765ae70b 100644 --- a/packages/electron-db-sqlite-persistence/src/main.ts +++ b/packages/electron-db-sqlite-persistence/src/main.ts @@ -120,6 +120,20 @@ async function executeRequestAgainstAdapter( } } + case `loadResumeSnapshot`: { + const result = await adapter.loadResumeSnapshot( + request.collectionId, + request.payload.ctx, + ) + return { + v: ELECTRON_PERSISTENCE_PROTOCOL_VERSION, + requestId: request.requestId, + method: request.method, + ok: true, + result, + } + } + case `loadCollectionMetadata`: { if (!adapter.loadCollectionMetadata) { throw new InvalidPersistedCollectionConfigError( diff --git a/packages/electron-db-sqlite-persistence/src/protocol.ts b/packages/electron-db-sqlite-persistence/src/protocol.ts index 7dcdc8dec5..cf64260752 100644 --- a/packages/electron-db-sqlite-persistence/src/protocol.ts +++ b/packages/electron-db-sqlite-persistence/src/protocol.ts @@ -2,11 +2,12 @@ import type { LoadSubsetOptions } from '@tanstack/db' import type { PersistedCollectionMode, PersistedIndexSpec, + PersistedKeySetEvidence, PersistedTx, SQLitePullSinceResult, } from '@tanstack/db-sqlite-persistence-core' -export const ELECTRON_PERSISTENCE_PROTOCOL_VERSION = 1 as const +export const ELECTRON_PERSISTENCE_PROTOCOL_VERSION = 2 as const export const DEFAULT_ELECTRON_PERSISTENCE_CHANNEL = `tanstack-db:sqlite-persistence` export type ElectronPersistedRow = Record @@ -19,6 +20,7 @@ export type ElectronPersistenceResolution = { export type ElectronPersistenceMethod = | `loadSubset` + | `loadResumeSnapshot` | `loadCollectionMetadata` | `scanRows` | `applyCommittedTx` @@ -32,6 +34,12 @@ export type ElectronPersistencePayloadMap = { options: LoadSubsetOptions ctx?: { requiredIndexSignatures?: ReadonlyArray } } + loadResumeSnapshot: { + ctx?: { + requiredIndexSignatures?: ReadonlyArray + includeRows?: boolean + } + } loadCollectionMetadata: {} scanRows: { options?: { @@ -56,6 +64,19 @@ export type ElectronPersistencePayloadMap = { export type ElectronPersistenceResultMap = { loadSubset: Array<{ key: ElectronPersistedKey; value: ElectronPersistedRow }> + loadResumeSnapshot: { + rows: Array<{ + key: ElectronPersistedKey + value: ElectronPersistedRow + metadata?: unknown + }> + keySet?: PersistedKeySetEvidence + collectionMetadata: Array<{ key: string; value: unknown }> + latestTerm: number + latestSeq: number + latestRowVersion: number + resetEpoch: number + } loadCollectionMetadata: Array<{ key: string; value: unknown }> scanRows: Array<{ key: ElectronPersistedKey diff --git a/packages/electron-db-sqlite-persistence/src/renderer.ts b/packages/electron-db-sqlite-persistence/src/renderer.ts index 2ac7203000..b7974344d9 100644 --- a/packages/electron-db-sqlite-persistence/src/renderer.ts +++ b/packages/electron-db-sqlite-persistence/src/renderer.ts @@ -200,6 +200,20 @@ function createResolvedRendererAdapter( value: Record }> }, + loadResumeSnapshot: async ( + collectionId: string, + ctx?: { + requiredIndexSignatures?: ReadonlyArray + includeRows?: boolean + }, + ) => { + return executeRequest( + `loadResumeSnapshot`, + collectionId, + { ctx }, + resolution, + ) + }, applyCommittedTx: async ( collectionId: string, tx: PersistedTx, string | number>, diff --git a/packages/electron-db-sqlite-persistence/tests/e2e/fixtures/electron-main.mjs b/packages/electron-db-sqlite-persistence/tests/e2e/fixtures/electron-main.mjs index 048483e932..dd5127291e 100644 --- a/packages/electron-db-sqlite-persistence/tests/e2e/fixtures/electron-main.mjs +++ b/packages/electron-db-sqlite-persistence/tests/e2e/fixtures/electron-main.mjs @@ -291,6 +291,12 @@ function createMainPersistence(input, driver) { } return adapter.loadSubset(collectionId, options, ctx) }, + loadResumeSnapshot: (collectionId, ctx) => { + if (collectionId !== input.collectionId) { + throw createUnknownCollectionError(collectionId) + } + return adapter.loadResumeSnapshot(collectionId, ctx) + }, applyCommittedTx: (collectionId, tx) => { if (collectionId !== input.collectionId) { throw createUnknownCollectionError(collectionId) diff --git a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test-d.ts b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test-d.ts index f6f1daa3a0..2116bbd4fd 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test-d.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test-d.ts @@ -7,15 +7,31 @@ test(`renderer persistence requires invoke transport`, () => { switch (request.method) { case `loadSubset`: return Promise.resolve({ - v: 1, + v: 2, requestId: request.requestId, method: request.method, ok: true, result: [], }) + case `loadResumeSnapshot`: + return Promise.resolve({ + v: 2, + requestId: request.requestId, + method: request.method, + ok: true, + result: { + rows: [], + keySet: { status: `consistent` }, + collectionMetadata: [], + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + resetEpoch: 0, + }, + }) case `pullSince`: return Promise.resolve({ - v: 1, + v: 2, requestId: request.requestId, method: request.method, ok: true, @@ -26,7 +42,7 @@ test(`renderer persistence requires invoke transport`, () => { }) case `getStreamPosition`: return Promise.resolve({ - v: 1, + v: 2, requestId: request.requestId, method: request.method, ok: true, @@ -38,7 +54,7 @@ test(`renderer persistence requires invoke transport`, () => { }) case `loadCollectionMetadata`: return Promise.resolve({ - v: 1, + v: 2, requestId: request.requestId, method: request.method, ok: true, @@ -46,7 +62,7 @@ test(`renderer persistence requires invoke transport`, () => { }) case `scanRows`: return Promise.resolve({ - v: 1, + v: 2, requestId: request.requestId, method: request.method, ok: true, @@ -54,7 +70,7 @@ test(`renderer persistence requires invoke transport`, () => { }) default: return Promise.resolve({ - v: 1, + v: 2, requestId: request.requestId, method: request.method, ok: true, @@ -68,6 +84,7 @@ test(`renderer persistence requires invoke transport`, () => { }) expectTypeOf(persistence.adapter).toHaveProperty(`loadSubset`) + expectTypeOf(persistence.adapter).toHaveProperty(`loadResumeSnapshot`) createElectronSQLitePersistence({ invoke, diff --git a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts index d1c330fb6f..5afd32a3b4 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts @@ -61,6 +61,10 @@ function createFilteredPersistence( assertKnownCollection(requestedCollectionId) return baseAdapter.loadSubset(requestedCollectionId, options, ctx) }, + loadResumeSnapshot: (requestedCollectionId, ctx) => { + assertKnownCollection(requestedCollectionId) + return baseAdapter.loadResumeSnapshot(requestedCollectionId, ctx) + }, applyCommittedTx: (requestedCollectionId, tx) => { assertKnownCollection(requestedCollectionId) return baseAdapter.applyCommittedTx(requestedCollectionId, tx) @@ -197,6 +201,13 @@ describe(`electron sqlite persistence bridge`, () => { }, }, ], + collectionMetadataMutations: [ + { + type: `set`, + key: `resume:test`, + value: { offset: `1_0` }, + }, + ], }) const rows = await rendererPersistence.adapter.loadSubset(`todos`, {}) @@ -210,6 +221,46 @@ describe(`electron sqlite persistence bridge`, () => { }, }, ]) + + const resumeSnapshot = await rendererPersistence.adapter.loadResumeSnapshot( + `todos`, + { + includeRows: true, + }, + ) + expect(resumeSnapshot).toEqual({ + rows: [ + { + key: `1`, + metadata: undefined, + value: { + id: `1`, + title: `From renderer`, + score: 10, + }, + }, + ], + keySet: { status: `consistent` }, + collectionMetadata: [ + { + key: `resume:test`, + value: { offset: `1_0` }, + }, + ], + latestTerm: 1, + latestSeq: 1, + latestRowVersion: 1, + resetEpoch: 0, + }) + + await expect( + rendererPersistence.adapter.loadResumeSnapshot(`todos`, { + includeRows: false, + }), + ).resolves.toEqual({ + ...resumeSnapshot, + rows: [], + }) }) it(`persists data across main process restarts`, async () => { @@ -297,6 +348,25 @@ describe(`electron sqlite persistence bridge`, () => { ).rejects.toBeInstanceOf(InvalidPersistedCollectionConfigError) }) + it(`rejects a version-1 main response before reading its result`, async () => { + const rendererPersistence = createElectronSQLitePersistence({ + invoke: (_channel, request) => + Promise.resolve({ + v: 1, + requestId: request.requestId, + method: request.method, + ok: true, + result: null, + } as unknown as ElectronPersistenceResponseEnvelope), + }) + + await expect( + rendererPersistence.adapter.loadResumeSnapshot(`todos`), + ).rejects.toThrow( + `Unexpected electron persistence protocol version "1" in response`, + ) + }) + it(`returns remote errors for unknown collections`, async () => { const dbPath = createTempDbPath() const invokeHarness = createInvokeHarness(dbPath, `known`, false) @@ -368,6 +438,46 @@ describe(`electron sqlite persistence bridge`, () => { method: `loadSubset`, }) + const resumeResponse = await registeredHandler?.(undefined, { + v: ELECTRON_PERSISTENCE_PROTOCOL_VERSION, + requestId: `req-2`, + collectionId: `todos`, + method: `loadResumeSnapshot`, + payload: { + ctx: { includeRows: false }, + }, + }) + expect(resumeResponse).toMatchObject({ + ok: true, + requestId: `req-2`, + method: `loadResumeSnapshot`, + result: { + rows: [], + collectionMetadata: [], + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + resetEpoch: 0, + }, + }) + + const legacyVersionResponse = await registeredHandler?.(undefined, { + v: 1, + requestId: `req-v1`, + collectionId: `todos`, + method: `loadResumeSnapshot`, + payload: {}, + }) + expect(legacyVersionResponse).toMatchObject({ + v: ELECTRON_PERSISTENCE_PROTOCOL_VERSION, + requestId: `req-v1`, + method: `loadResumeSnapshot`, + ok: false, + error: { + message: `Unsupported electron persistence protocol version "1"`, + }, + }) + dispose() expect(removedChannels).toEqual([DEFAULT_ELECTRON_PERSISTENCE_CHANNEL]) }) diff --git a/packages/query-db-collection/tests/query.test.ts b/packages/query-db-collection/tests/query.test.ts index 4f2fffacb4..9379b56f1f 100644 --- a/packages/query-db-collection/tests/query.test.ts +++ b/packages/query-db-collection/tests/query.test.ts @@ -318,6 +318,72 @@ describe(`QueryCollection`, () => { queryClient.clear() }) + it(`accepts the explicit null persistence sentinel from core`, async () => { + const queryFn = vi.fn().mockResolvedValue([{ id: `1`, name: `Item 1` }]) + const options = queryCollectionOptions({ + id: `explicit-null-persistence-test`, + queryClient, + queryKey: [`explicit-null-persistence-test`], + queryFn, + getKey, + startSync: false, + }) + const originalSync = options.sync + let observedPersistence: unknown + const collection = createCollection({ + ...options, + sync: { + sync: (params: Parameters[0]) => { + observedPersistence = params.metadata?.persistence + return originalSync.sync(params) + }, + }, + }) + + collection.startSyncImmediate() + await collection.stateWhenReady() + + expect(observedPersistence).toBeNull() + expect(queryFn).toHaveBeenCalledOnce() + await collection.cleanup() + }) + + it(`rejects a sync wrapper that drops the entire persistence field before querying`, async () => { + const queryFn = vi.fn().mockResolvedValue([{ id: `1`, name: `Item 1` }]) + const options = queryCollectionOptions({ + id: `missing-persistence-field-test`, + queryClient, + queryKey: [`missing-persistence-field-test`], + queryFn, + getKey, + startSync: false, + }) + const originalSync = options.sync + const malformedCollection = createCollection({ + ...options, + startSync: false, + sync: { + sync: (params: Parameters[0]) => { + const { persistence: _dropped, ...metadataWithoutPersistence } = + params.metadata! + return originalSync.sync({ + ...params, + metadata: metadataWithoutPersistence as unknown as SyncMetadataApi< + string | number + >, + }) + }, + }, + }) + + expect(() => malformedCollection.startSyncImmediate()).toThrow( + /expected null or a complete capability object.*forward metadata\.persistence unchanged/i, + ) + expect(malformedCollection.status).toBe(`error`) + expect(queryFn).not.toHaveBeenCalled() + await malformedCollection.cleanup() + }) + it(`should pass through additional top-level Query observer options`, async () => { const queryKey = [`query-options-pass-through`] const queryFn = vi.fn().mockResolvedValue([{ id: `1`, name: `Item 1` }]) From a330d43ba7b05a32ebdfbedd1327be8fe07ae566 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 19:01:49 +0100 Subject: [PATCH 09/23] fix(electric-db-collection): require certified resume evidence --- .../electric-db-collection/src/electric.ts | 15 +++-- .../tests/ORACLE_MUTATIONS.md | 2 +- .../electric-resume-snapshot-races.test.ts | 67 +++++++++++++++++-- 3 files changed, 73 insertions(+), 11 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 2947bb79e1..bbe8b3feb9 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -2065,12 +2065,16 @@ function createElectricSync>( ? hydrateBaseline ? (async () => { await hydrateBaseline() + const currentKeySetEvidence = getKeySetEvidence?.() if ( - persistedKeySetEvidence?.status !== `incompatible` && - getKeySetEvidence?.()?.status === `incompatible` + (canUsePersistedResume && + currentKeySetEvidence?.status !== `consistent`) || + (!canUsePersistedResume && + persistedKeySetEvidence?.status !== `incompatible` && + currentKeySetEvidence?.status === `incompatible`) ) { throw new Error( - `Electric persisted resume baseline became incompatible during hydration`, + `Electric persisted resume baseline could not be certified during hydration`, ) } })() @@ -2078,9 +2082,10 @@ function createElectricSync>( : requiresKeySetCertification ? (async () => { await certifyResumeSnapshot() - if (getKeySetEvidence?.()?.status === `incompatible`) { + const currentKeySetEvidence = getKeySetEvidence?.() + if (currentKeySetEvidence?.status !== `consistent`) { throw new Error( - `Electric persisted resume baseline became incompatible during certification`, + `Electric persisted resume baseline could not be certified`, ) } })() diff --git a/packages/electric-db-collection/tests/ORACLE_MUTATIONS.md b/packages/electric-db-collection/tests/ORACLE_MUTATIONS.md index 30ffd810e7..4572673868 100644 --- a/packages/electric-db-collection/tests/ORACLE_MUTATIONS.md +++ b/packages/electric-db-collection/tests/ORACLE_MUTATIONS.md @@ -83,7 +83,7 @@ a source wrapper rebuild the capability without forwarding the same complete Killed by: `fails fast when a persistence wrapper drops resume generation ownership` and `keeps generation ownership when a source wrapper shallow-forwards the persistence capability`. The direct-source control, `uses -direct resume metadata when no persistence capability is present`, preserves +direct resume metadata when persistence is explicitly null`, preserves the intentional no-capability path; completeness is required only after a persistence capability is advertised. diff --git a/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts index 40eb2077b0..5ada96a9d6 100644 --- a/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts +++ b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts @@ -127,6 +127,7 @@ async function runRace( missingKeySetEvidence = false, startupReset: `none` | `tag-state` | `shape-identity` = `none`, metadataWrapper: `none` | `shallow-persistence` = `none`, + laterKeySetEvidence: `unchanged` | `unknown` | `missing` = `unchanged`, ): Promise { const database = new DatabaseSync(`:memory:`) const driver = createDriver(database) @@ -223,7 +224,8 @@ async function runRace( throw error } snapshotCalls++ - if (snapshotCalls > 1) { + const isLaterSnapshot = snapshotCalls > 1 + if (isLaterSnapshot) { laterSnapshotIncludedRows = args[1]?.includeRows if (startupReset !== `none`) { resumeStateAtLaterSnapshot = ( @@ -234,6 +236,15 @@ async function runRace( await releaseLaterSnapshot.promise } const snapshot = await target.loadResumeSnapshot(...args) + if (isLaterSnapshot && laterKeySetEvidence !== `unchanged`) { + return { + ...snapshot, + keySet: + laterKeySetEvidence === `unknown` + ? { status: `unknown` as const } + : undefined, + } + } return missingKeySetEvidence ? { ...snapshot, keySet: undefined } : snapshot @@ -415,6 +426,13 @@ async function runRace( } } else { await vi.waitFor(() => expect(collection!.status).toBe(`error`)) + if (laterKeySetEvidence !== `unchanged`) { + expect(collection._lifecycle.getSyncError()).toEqual( + expect.objectContaining({ + message: `Electric persisted resume baseline could not be certified during hydration`, + }), + ) + } await vi.waitFor(async () => { const metadata = await restartedAdapter.loadCollectionMetadata(collectionId) @@ -429,9 +447,22 @@ async function runRace( expect(resumeState).toMatchObject({ kind: `reset` }) } }) - expect(Array.from(collection.values())).toEqual([]) + // The persisted wrapper can apply the held baseline before Electric + // observes that its evidence was downgraded. Safety here means startup + // fails and never applies or publishes the queued stream batches. + const expectedErroredRows = + transition === `external-row-loss` && + laterKeySetEvidence !== `unchanged` + ? [{ id: 2, name: `two` }] + : [] + expect( + Array.from(collection.values(), ({ id, name }) => ({ id, name })), + ).toEqual(expectedErroredRows) expect(collection.status).not.toBe(`ready`) - expect(publications).toBe(0) + const publicationsBeforeLateDelivery = publications + if (laterKeySetEvidence === `unchanged`) { + expect(publicationsBeforeLateDelivery).toBe(0) + } const durableRowsBeforeLateDelivery = await restartedAdapter.loadSubset( collectionId, {}, @@ -452,11 +483,13 @@ async function runRace( { headers: { control: `up-to-date` } }, ]) await new Promise((resolve) => setTimeout(resolve, 0)) - expect(Array.from(collection.values())).toEqual([]) + expect( + Array.from(collection.values(), ({ id, name }) => ({ id, name })), + ).toEqual(expectedErroredRows) expect(await restartedAdapter.loadSubset(collectionId, {})).toEqual( durableRowsBeforeLateDelivery, ) - expect(publications).toBe(0) + expect(publications).toBe(publicationsBeforeLateDelivery) } if (replacesUncertifiedBaseline) { expect(request.offset).toBeUndefined() @@ -782,6 +815,30 @@ describe(`Electric resume snapshot races`, () => { await runRace(`committed-write`) }) + it(`rejects row loss when later resume evidence becomes unknown`, async () => { + await runRace( + `external-row-loss`, + `eager`, + false, + false, + `none`, + `none`, + `unknown`, + ) + }) + + it(`rejects row loss when later resume evidence becomes missing`, async () => { + await runRace( + `external-row-loss`, + `eager`, + false, + false, + `none`, + `none`, + `missing`, + ) + }) + it(`freshly replaces an unknown on-demand resume baseline`, async () => { await runRace(`none`, `on-demand`, true) }) From 85338021c2bd2dddef6e60a3a33bae0bd475ed03 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 19:11:47 +0100 Subject: [PATCH 10/23] test(electric): cover on-demand resume evidence loss --- .../electric-resume-snapshot-races.test.ts | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts index 5ada96a9d6..d941718909 100644 --- a/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts +++ b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts @@ -429,7 +429,10 @@ async function runRace( if (laterKeySetEvidence !== `unchanged`) { expect(collection._lifecycle.getSyncError()).toEqual( expect.objectContaining({ - message: `Electric persisted resume baseline could not be certified during hydration`, + message: + syncMode === `on-demand` + ? `Electric persisted resume baseline could not be certified` + : `Electric persisted resume baseline could not be certified during hydration`, }), ) } @@ -452,7 +455,8 @@ async function runRace( // fails and never applies or publishes the queued stream batches. const expectedErroredRows = transition === `external-row-loss` && - laterKeySetEvidence !== `unchanged` + laterKeySetEvidence !== `unchanged` && + syncMode !== `on-demand` ? [{ id: 2, name: `two` }] : [] expect( @@ -815,29 +819,25 @@ describe(`Electric resume snapshot races`, () => { await runRace(`committed-write`) }) - it(`rejects row loss when later resume evidence becomes unknown`, async () => { - await runRace( - `external-row-loss`, - `eager`, - false, - false, - `none`, - `none`, - `unknown`, - ) - }) - - it(`rejects row loss when later resume evidence becomes missing`, async () => { - await runRace( - `external-row-loss`, - `eager`, - false, - false, - `none`, - `none`, - `missing`, - ) - }) + it.each([ + [`eager`, `unknown`], + [`eager`, `missing`], + [`on-demand`, `unknown`], + [`on-demand`, `missing`], + ] as const)( + `rejects row loss when %s resume evidence becomes %s`, + async (syncMode, laterKeySetEvidence) => { + await runRace( + `external-row-loss`, + syncMode, + false, + false, + `none`, + `none`, + laterKeySetEvidence, + ) + }, + ) it(`freshly replaces an unknown on-demand resume baseline`, async () => { await runRace(`none`, `on-demand`, true) From 6613b2c48e9faedca8e42c3c59f923b9bf864311 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 13:39:21 +0100 Subject: [PATCH 11/23] fix(sqlite): reduce resume evidence overhead --- .../src/sqlite-core-adapter.ts | 102 +++++++++--------- .../tests/persisted.test.ts | 82 ++++++++++++++ .../tests/sqlite-resume-snapshot.test.ts | 76 +++++++++++++ 3 files changed, 209 insertions(+), 51 deletions(-) diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index de5eef9b53..b82d2de959 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -1206,20 +1206,9 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { WHERE collection_id = ?`, [collectionId], ) - const termRows = await transactionDriver.query<{ latest_term: number }>( - `SELECT latest_term - FROM leader_term - WHERE collection_id = ? - LIMIT 1`, - [collectionId], - ) - const seqRows = await transactionDriver.query<{ max_seq: number }>( - `SELECT MAX(seq) AS max_seq - FROM applied_tx - WHERE collection_id = ? AND term = ( - SELECT latest_term FROM leader_term WHERE collection_id = ? LIMIT 1 - )`, - [collectionId, collectionId], + const { latestTerm, latestSeq } = await this.readStreamPosition( + collectionId, + transactionDriver, ) const resetRows = await transactionDriver.query<{ reset_epoch: number }>( `SELECT reset_epoch @@ -1240,8 +1229,8 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { key: row.key, value: deserializePersistedRowValue(row.value), })), - latestTerm: termRows[0]?.latest_term ?? 0, - latestSeq: seqRows[0]?.max_seq ?? 0, + latestTerm, + latestSeq, latestRowVersion, resetEpoch: resetRows[0]?.reset_epoch ?? 0, } @@ -1662,16 +1651,31 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { keySet?: PersistedKeySetEvidence }> { const tableMapping = await this.ensureCollectionReady(collectionId) - const [termRows, version, seqRows] = await Promise.all([ - this.driver.query<{ latest_term: number }>( + const [position, version] = await Promise.all([ + this.readStreamPosition(collectionId, this.driver), + this.readKeySetEvidence(collectionId, tableMapping, this.driver), + ]) + + return { + ...position, + latestRowVersion: version.latestRowVersion, + keySet: version.keySet, + } + } + + private async readStreamPosition( + collectionId: string, + driver: SQLiteDriver, + ): Promise<{ latestTerm: number; latestSeq: number }> { + const [termRows, seqRows] = await Promise.all([ + driver.query<{ latest_term: number }>( `SELECT latest_term FROM leader_term WHERE collection_id = ? LIMIT 1`, [collectionId], ), - this.readKeySetEvidence(collectionId, tableMapping, this.driver), - this.driver.query<{ max_seq: number }>( + driver.query<{ max_seq: number }>( `SELECT MAX(seq) AS max_seq FROM applied_tx WHERE collection_id = ? AND term = ( @@ -1684,8 +1688,6 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { return { latestTerm: termRows[0]?.latest_term ?? 0, latestSeq: seqRows[0]?.max_seq ?? 0, - latestRowVersion: version.latestRowVersion, - keySet: version.keySet, } } @@ -2173,7 +2175,7 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { ON CONFLICT(collection_id) DO NOTHING`, [collectionId], ) - await this.ensureCollectionKeyEvidenceTriggers(tableName) + await this.ensureCollectionKeyEvidenceTriggers(collectionId, tableName) const mapping = { tableName, tombstoneTableName, @@ -2183,10 +2185,11 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { } private async ensureCollectionKeyEvidenceTriggers( + collectionId: string, tableName: string, ): Promise { const collectionTableSql = quoteIdentifier(tableName) - const tableNameLiteral = toSqliteLiteral(tableName) + const collectionIdLiteral = toSqliteLiteral(collectionId) const insertTriggerSql = quoteIdentifier(`${tableName}_key_evidence_insert`) const deleteTriggerSql = quoteIdentifier(`${tableName}_key_evidence_delete`) const updateTriggerSql = quoteIdentifier(`${tableName}_key_evidence_update`) @@ -2194,59 +2197,56 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { await this.driver.exec( `CREATE TRIGGER IF NOT EXISTS ${insertTriggerSql} AFTER INSERT ON ${collectionTableSql} - WHEN NOT EXISTS ( + WHEN EXISTS ( + SELECT 1 + FROM collection_version + WHERE collection_id = ${collectionIdLiteral} + AND key_set_evidence_available = 1 + ) AND NOT EXISTS ( SELECT 1 FROM collection_expected_keys - WHERE collection_id = ( - SELECT collection_id - FROM collection_registry - WHERE table_name = ${tableNameLiteral} - ) AND key = NEW.key + WHERE collection_id = ${collectionIdLiteral} + AND key = NEW.key ) BEGIN UPDATE collection_version SET key_set_evidence_incompatible = 1 - WHERE collection_id = ( - SELECT collection_id - FROM collection_registry - WHERE table_name = ${tableNameLiteral} - ); + WHERE collection_id = ${collectionIdLiteral}; END`, ) await this.driver.exec( `CREATE TRIGGER IF NOT EXISTS ${deleteTriggerSql} AFTER DELETE ON ${collectionTableSql} WHEN EXISTS ( + SELECT 1 + FROM collection_version + WHERE collection_id = ${collectionIdLiteral} + AND key_set_evidence_available = 1 + ) AND EXISTS ( SELECT 1 FROM collection_expected_keys - WHERE collection_id = ( - SELECT collection_id - FROM collection_registry - WHERE table_name = ${tableNameLiteral} - ) AND key = OLD.key + WHERE collection_id = ${collectionIdLiteral} + AND key = OLD.key ) BEGIN UPDATE collection_version SET key_set_evidence_incompatible = 1 - WHERE collection_id = ( - SELECT collection_id - FROM collection_registry - WHERE table_name = ${tableNameLiteral} - ); + WHERE collection_id = ${collectionIdLiteral}; END`, ) await this.driver.exec( `CREATE TRIGGER IF NOT EXISTS ${updateTriggerSql} AFTER UPDATE OF key ON ${collectionTableSql} - WHEN OLD.key <> NEW.key + WHEN OLD.key <> NEW.key AND EXISTS ( + SELECT 1 + FROM collection_version + WHERE collection_id = ${collectionIdLiteral} + AND key_set_evidence_available = 1 + ) BEGIN UPDATE collection_version SET key_set_evidence_incompatible = 1 - WHERE collection_id = ( - SELECT collection_id - FROM collection_registry - WHERE table_name = ${tableNameLiteral} - ); + WHERE collection_id = ${collectionIdLiteral}; END`, ) } diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index be748264d8..e9d7fff298 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -2630,6 +2630,88 @@ describe(`persistedCollectionOptions`, () => { await collection.cleanup() }) + it(`invalidates resume evidence when storage advances outside the owned commit generation`, async () => { + const adapter = createRecordingAdapter() + let durableGeneration = { + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + resetEpoch: 0, + } + const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) + adapter.loadResumeSnapshot = async (...args) => ({ + ...(await loadResumeSnapshot(...args)), + ...durableGeneration, + }) + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + adapter.applyCommittedTx = async (collectionId, tx) => { + await applyCommittedTx(collectionId, tx) + durableGeneration = { + latestTerm: tx.term, + latestSeq: tx.seq, + latestRowVersion: Math.max( + durableGeneration.latestRowVersion + 1, + tx.rowVersion, + ), + resetEpoch: durableGeneration.resetEpoch, + } + } + + let remoteBegin: (() => void) | undefined + let remoteCommit: (() => true | Promise) | undefined + let remoteMetadata: + | Parameters[`sync`]>[0][`metadata`] + | undefined + let persistenceCapability: + | SyncMetadataApi[`persistence`] + | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `generation-fence`, + syncMode: `on-demand`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, commit, markReady, metadata }) => { + remoteBegin = begin + remoteCommit = commit + remoteMetadata = metadata + persistenceCapability = metadata?.persistence + markReady() + }, + }, + persistence: { adapter }, + }), + ) + + try { + collection.startSyncImmediate() + await vi.waitFor(() => + expect(persistenceCapability).toMatchObject({ + protocol: `@tanstack/db/sync-persistence`, + version: 1, + }), + ) + + // This is not a supported writer path. It models storage advancing + // without the runtime observing the generation that now precedes its + // commit. The exact-generation fence must reject that uncertainty. + durableGeneration.latestRowVersion = 5 + + remoteBegin?.() + persistenceCapability?.resumeSnapshot.expectCurrentCommit() + remoteMetadata?.collection.set(`cursor`, `next`) + const applied = remoteCommit?.() + if (applied !== true) await applied + + await persistenceCapability?.resumeSnapshot.certify() + expect(persistenceCapability?.resumeSnapshot.getKeySetEvidence()).toEqual( + { status: `incompatible` }, + ) + } finally { + await collection.cleanup() + } + }) + it(`ignores late wrapped sync writes after cleanup`, async () => { let lateWrite!: (message: { type: `insert`; value: Todo }) => void const collection = createCollection( diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts index 9f62b50834..a6793f018f 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts @@ -229,6 +229,82 @@ async function observeCachedSchemaState( * cursor; those remain separate driver-contract and Electric recovery owners. */ describe(`SQLite resume snapshots`, () => { + it(`does not amplify legacy writes while key-set evidence is unavailable`, async () => { + const database = new DatabaseSync(`:memory:`) + let primaryFailure: unknown + try { + const collectionId = `legacy-write-'work` + const tableName = createPersistedTableName(collectionId, `c`) + const driver = createDriver(database) + const adapter = new SQLiteCorePersistenceAdapter({ driver }) + + await adapter.loadResumeSnapshot(collectionId, { includeRows: false }) + await driver.run( + `UPDATE collection_version + SET key_set_evidence_available = 0, + key_set_evidence_incompatible = 0 + WHERE collection_id = ?`, + [collectionId], + ) + const before = await driver.query<{ count: number }>( + `SELECT total_changes() AS count`, + ) + + await driver.run( + `INSERT INTO "${tableName}" (key, value, metadata, row_version) + VALUES (?, ?, NULL, 1)`, + [encodePersistedStorageKey(`legacy`), `{}`], + ) + + const after = await driver.query<{ count: number }>( + `SELECT total_changes() AS count`, + ) + const version = await driver.query<{ + key_set_evidence_incompatible: number + }>( + `SELECT key_set_evidence_incompatible + FROM collection_version + WHERE collection_id = ?`, + [collectionId], + ) + expect((after[0]?.count ?? 0) - (before[0]?.count ?? 0)).toBe(1) + expect(version).toEqual([{ key_set_evidence_incompatible: 0 }]) + + const beforeKeyUpdate = await driver.query<{ count: number }>( + `SELECT total_changes() AS count`, + ) + await driver.run(`UPDATE "${tableName}" SET key = ? WHERE key = ?`, [ + encodePersistedStorageKey(`legacy-renamed`), + encodePersistedStorageKey(`legacy`), + ]) + const afterKeyUpdate = await driver.query<{ count: number }>( + `SELECT total_changes() AS count`, + ) + expect( + (afterKeyUpdate[0]?.count ?? 0) - (beforeKeyUpdate[0]?.count ?? 0), + ).toBe(1) + + const triggerDefinitions = await driver.query<{ sql: string }>( + `SELECT sql + FROM sqlite_schema + WHERE type = 'trigger' AND tbl_name = ?`, + [tableName], + ) + expect(triggerDefinitions).toHaveLength(3) + expect(triggerDefinitions.map(({ sql }) => sql).join(`\n`)).not.toContain( + `collection_registry`, + ) + expect( + (await adapter.loadResumeSnapshot(collectionId, { includeRows: false })) + .keySet, + ).toEqual({ status: `unknown` }) + } catch (error) { + primaryFailure = error + } finally { + closeDatabasePreservingPrimary(database, primaryFailure) + } + }) + it(`keeps raw key loss sticky until a full replacement recertifies the baseline`, async () => { const database = new DatabaseSync(`:memory:`) let primaryFailure: unknown From abc74ad2df12313e44dcb99d86a22f7ec3621fb1 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 14:35:15 +0100 Subject: [PATCH 12/23] test(sqlite-persistence): update adapter type oracle --- .../tests/persisted-options-type-oracle.test-d.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/db-sqlite-persistence-core/tests/persisted-options-type-oracle.test-d.ts b/packages/db-sqlite-persistence-core/tests/persisted-options-type-oracle.test-d.ts index 673ebd0c9d..c5db843d49 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted-options-type-oracle.test-d.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted-options-type-oracle.test-d.ts @@ -6,6 +6,15 @@ import type { StandardSchemaV1 } from '@standard-schema/spec' const adapter: PersistenceAdapter = { loadSubset: () => Promise.resolve([]), + loadResumeSnapshot: () => + Promise.resolve({ + rows: [], + collectionMetadata: [], + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + resetEpoch: 0, + }), applyCommittedTx: () => Promise.resolve(), ensureIndex: () => Promise.resolve(), } From 382b38bb41009b68aee8faf86cf5aa0e448a8ffe Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 15:17:04 +0100 Subject: [PATCH 13/23] fix(sqlite): preserve on-demand rows and trim evidence work --- .../src/persisted.ts | 6 +- .../src/sqlite-core-adapter.ts | 51 ++++------ .../tests/persisted.test.ts | 94 +++++++++++++++++++ .../tests/sqlite-resume-snapshot.test.ts | 90 +++++++++++++++++- 4 files changed, 205 insertions(+), 36 deletions(-) diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 556f93a04d..4f5a3f1e55 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -311,7 +311,6 @@ export interface PersistenceAdapter { latestTerm: number latestSeq: number latestRowVersion: number - keySet?: PersistedKeySetEvidence }> } @@ -1396,7 +1395,10 @@ class PersistedCollectionRuntime< } if (config.lifecycleGeneration !== this.lifecycleGeneration) return - if (this.persistedKeySetEvidence?.status !== `incompatible`) { + if ( + !config.bindKeySetEvidence || + this.persistedKeySetEvidence?.status !== `incompatible` + ) { this.applyRowsToCollection(rows) } } finally { diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index b82d2de959..cfbed9d4ac 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -1194,7 +1194,6 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { : [] const { latestRowVersion, keySet } = await this.readKeySetEvidence( collectionId, - tableMapping, transactionDriver, ) const collectionMetadataRows = await transactionDriver.query<{ @@ -1648,21 +1647,33 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { latestTerm: number latestSeq: number latestRowVersion: number - keySet?: PersistedKeySetEvidence }> { - const tableMapping = await this.ensureCollectionReady(collectionId) - const [position, version] = await Promise.all([ + await this.ensureCollectionReady(collectionId) + const [position, latestRowVersion] = await Promise.all([ this.readStreamPosition(collectionId, this.driver), - this.readKeySetEvidence(collectionId, tableMapping, this.driver), + this.readLatestRowVersion(collectionId, this.driver), ]) return { ...position, - latestRowVersion: version.latestRowVersion, - keySet: version.keySet, + latestRowVersion, } } + private async readLatestRowVersion( + collectionId: string, + driver: SQLiteDriver, + ): Promise { + const versionRows = await driver.query<{ latest_row_version: number }>( + `SELECT latest_row_version + FROM collection_version + WHERE collection_id = ? + LIMIT 1`, + [collectionId], + ) + return versionRows[0]?.latest_row_version ?? 0 + } + private async readStreamPosition( collectionId: string, driver: SQLiteDriver, @@ -1693,13 +1704,11 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { private async readKeySetEvidence( collectionId: string, - tableMapping: CollectionTableMapping, driver: SQLiteDriver, ): Promise<{ latestRowVersion: number keySet: PersistedKeySetEvidence }> { - const collectionTableSql = quoteIdentifier(tableMapping.tableName) const versionRows = await driver.query<{ latest_row_version: number key_set_evidence_available: number @@ -1708,31 +1717,11 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { `SELECT latest_row_version, key_set_evidence_available, - CASE - WHEN key_set_evidence_available = 0 THEN 0 - WHEN key_set_evidence_incompatible = 1 THEN 1 - WHEN EXISTS ( - SELECT 1 - FROM ${collectionTableSql} AS actual - LEFT JOIN collection_expected_keys AS expected - ON expected.collection_id = ? - AND expected.key = actual.key - WHERE expected.key IS NULL - UNION ALL - SELECT 1 - FROM collection_expected_keys AS expected - LEFT JOIN ${collectionTableSql} AS actual - ON actual.key = expected.key - WHERE expected.collection_id = ? - AND actual.key IS NULL - LIMIT 1 - ) THEN 1 - ELSE 0 - END AS key_set_incompatible + key_set_evidence_incompatible AS key_set_incompatible FROM collection_version WHERE collection_id = ? LIMIT 1`, - [collectionId, collectionId, collectionId], + [collectionId], ) const version = versionRows[0] diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index e9d7fff298..bf1bdae32f 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -2630,6 +2630,100 @@ describe(`persistedCollectionOptions`, () => { await collection.cleanup() }) + it(`applies on-demand subsets even when the full baseline is incompatible`, async () => { + const hydrateUsing = async ( + route: `loadSubset` | `forceReloadSubset`, + ): Promise => { + const adapter = createRecordingAdapter([ + { id: `1`, title: `Locally cached` }, + ]) + const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) + adapter.loadResumeSnapshot = async (...args) => ({ + ...(await loadResumeSnapshot(...args)), + keySet: { status: `incompatible` }, + }) + const collection = createCollection( + persistedCollectionOptions({ + id: `incompatible-${route}`, + syncMode: `on-demand`, + getKey: (item) => item.id, + persistence: { adapter }, + }), + ) + + try { + collection.startSyncImmediate() + await vi.waitFor(() => + expect(adapter.loadResumeSnapshotCalls.length).toBeGreaterThan(0), + ) + expect(collection.has(`1`)).toBe(false) + if (route === `loadSubset`) { + await collection._sync.loadSubset({}) + } else { + await collection.utils.forceReloadSubset!({}) + } + return collection.has(`1`) + } finally { + await collection.cleanup() + } + } + + expect( + await Promise.all([ + hydrateUsing(`loadSubset`), + hydrateUsing(`forceReloadSubset`), + ]), + ).toEqual([true, true]) + }) + + it(`keeps resume certification consistent after an owned no-op commit`, async () => { + const adapter = createRecordingAdapter() + let remoteBegin: (() => void) | undefined + let remoteCommit: (() => true | Promise) | undefined + let persistenceCapability: + | SyncMetadataApi[`persistence`] + | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `owned-no-op-generation`, + syncMode: `on-demand`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, commit, markReady, metadata }) => { + remoteBegin = begin + remoteCommit = commit + persistenceCapability = metadata?.persistence + markReady() + }, + }, + persistence: { adapter }, + }), + ) + + try { + collection.startSyncImmediate() + await vi.waitFor(() => + expect(persistenceCapability).toMatchObject({ + protocol: `@tanstack/db/sync-persistence`, + version: 1, + }), + ) + + remoteBegin?.() + persistenceCapability?.resumeSnapshot.expectCurrentCommit() + const applied = remoteCommit?.() + if (applied !== true) await applied + + expect(adapter.applyCommittedTxCalls).toHaveLength(0) + await persistenceCapability?.resumeSnapshot.certify() + expect(persistenceCapability?.resumeSnapshot.getKeySetEvidence()).toEqual( + { status: `consistent` }, + ) + } finally { + await collection.cleanup() + } + }) + it(`invalidates resume evidence when storage advances outside the owned commit generation`, async () => { const adapter = createRecordingAdapter() let durableGeneration = { diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts index a6793f018f..631ea3b786 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts @@ -41,19 +41,22 @@ function toBinding(value: unknown): string | number | bigint | null { function createDriver( database: DatabaseSync, failTransactionRun?: (sql: string) => boolean, + observeQuery?: (sql: string) => void, ): SQLiteDriver { const driver: SQLiteDriver = { exec: (sql) => { database.exec(sql) return Promise.resolve() }, - query: (sql, params = []) => - Promise.resolve( + query: (sql, params = []) => { + observeQuery?.(sql) + return Promise.resolve( database .prepare(sql) .all(...params.map(toBinding)) .map((row) => ({ ...row })) as Array, - ), + ) + }, run: (sql, params = []) => { database.prepare(sql).run(...params.map(toBinding)) return Promise.resolve() @@ -229,6 +232,87 @@ async function observeCachedSchemaState( * cursor; those remain separate driver-contract and Electric recovery owners. */ describe(`SQLite resume snapshots`, () => { + it(`reads key-set evidence without rescanning key membership`, async () => { + const database = new DatabaseSync(`:memory:`) + let primaryFailure: unknown + try { + const collectionId = `evidence-work` + const tableName = createPersistedTableName(collectionId, `c`) + let keyEvidenceReads = 0 + let keyMembershipScans = 0 + const driver = createDriver(database, undefined, (sql) => { + if (sql.includes(`key_set_evidence_available`)) keyEvidenceReads += 1 + if (sql.includes(`FROM collection_expected_keys AS expected`)) { + keyMembershipScans += 1 + } + }) + const adapter = new SQLiteCorePersistenceAdapter({ driver }) + await adapter.applyCommittedTx(collectionId, { + txId: `seed`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [{ type: `insert`, key: 1, value: { id: 1, name: `one` } }], + }) + + const observeWork = () => ({ keyEvidenceReads, keyMembershipScans }) + const resetWork = () => { + keyEvidenceReads = 0 + keyMembershipScans = 0 + } + + resetWork() + const position = await adapter.getStreamPosition(collectionId) + const leadershipClaim = observeWork() + + resetWork() + const consistent = await adapter.loadResumeSnapshot(collectionId, { + includeRows: false, + }) + const consistentSnapshot = observeWork() + + await driver.run(`DELETE FROM "${tableName}"`) + resetWork() + const incompatible = await adapter.loadResumeSnapshot(collectionId, { + includeRows: false, + }) + const incompatibleSnapshot = observeWork() + + expect({ + position, + leadershipClaim, + consistentKeySet: consistent.keySet, + consistentSnapshot, + incompatibleKeySet: incompatible.keySet, + incompatibleSnapshot, + }).toEqual({ + position: { + latestTerm: 1, + latestSeq: 1, + latestRowVersion: 1, + }, + leadershipClaim: { + keyEvidenceReads: 0, + keyMembershipScans: 0, + }, + consistentKeySet: { status: `consistent` }, + consistentSnapshot: { + keyEvidenceReads: 1, + keyMembershipScans: 0, + }, + incompatibleKeySet: { status: `incompatible` }, + incompatibleSnapshot: { + keyEvidenceReads: 1, + keyMembershipScans: 0, + }, + }) + } catch (error) { + primaryFailure = error + } finally { + closeDatabasePreservingPrimary(database, primaryFailure) + } + }) + it(`does not amplify legacy writes while key-set evidence is unavailable`, async () => { const database = new DatabaseSync(`:memory:`) let primaryFailure: unknown From 063158ca0a355762357b78a98d95a77e4a3b58c8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 16:10:04 +0100 Subject: [PATCH 14/23] docs(db): mark sync persistence protocol internal --- packages/db/src/index.ts | 1 + packages/db/src/sync-persistence.ts | 8 ++++++++ packages/db/src/types.ts | 14 ++++++++++---- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index fc004ab63b..d58c43974c 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -20,6 +20,7 @@ export * from './live-query-window-controller' export * from './local-only' export * from './local-storage' export * from './errors' +/** @internal Unstable protocol for persistence-aware collection adapters. */ export * from './sync-persistence' export { deepEquals } from './utils' /** @internal Used by first-party collection adapters. */ diff --git a/packages/db/src/sync-persistence.ts b/packages/db/src/sync-persistence.ts index 6be13fa4a5..1291dc403d 100644 --- a/packages/db/src/sync-persistence.ts +++ b/packages/db/src/sync-persistence.ts @@ -1,8 +1,14 @@ import { InvalidSyncPersistenceCapabilityError } from './errors' import type { SyncPersistenceCapabilityV1 } from './types' +/** + * @internal + * Unstable cross-package protocol for persistence-aware collection adapters. + * Application code should not construct or depend on this value directly. + */ export const SYNC_PERSISTENCE_PROTOCOL = `@tanstack/db/sync-persistence` as const +/** @internal See {@link SYNC_PERSISTENCE_PROTOCOL}. */ export const SYNC_PERSISTENCE_VERSION = 1 as const function isRecord(value: unknown): value is Record { @@ -26,6 +32,8 @@ function requireFunction( * adapter uses it. Null explicitly means that the collection has no * persistence capability; undefined means a wrapper dropped the required * field. Any advertised capability must be complete. + * + * @internal This is adapter infrastructure, not an application API. */ export function validateSyncPersistenceCapability< TKey extends string | number = string | number, diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index b19ee4f95f..e9fe2df620 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -477,10 +477,13 @@ export interface SyncMetadataApi< }> } /** - * Versioned persistence bridge used by sync adapters that can hydrate and - * inspect a durable collection baseline. Custom sync wrappers must forward - * this value unchanged. `null` explicitly means that the collection has no - * persistence capability; a missing property is invalid. + * Unstable, versioned bridge between persistence-aware collection adapters + * and sync adapters. Application code should not construct this capability. + * Custom adapter wrappers must forward it unchanged. `null` explicitly means + * that the collection has no persistence capability; a missing property is + * invalid. + * + * @internal Adapter infrastructure; not an application-facing API. */ persistence: SyncPersistenceCapabilityV1 | null } @@ -501,6 +504,9 @@ export type SyncPersistenceScannedRow< metadata?: unknown } +/** + * @internal Unstable cross-package protocol for persistence-aware adapters. + */ export type SyncPersistenceCapabilityV1< TKey extends string | number = string | number, > = { From 4d80452ba402a8c1ce757c42473bba36c4290d40 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 16:12:39 +0100 Subject: [PATCH 15/23] docs(oracles): track reusable boundary laws --- docs/contributing/oracle-coverage.md | 45 ++++++++++++++++++++++++++++ docs/contributing/oracle-tests.md | 35 ++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index eb74148e22..095f0b1749 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -188,6 +188,51 @@ project, worker progress RPC starvation produced passing assertions with a nonzero process exit. Such a run is not green. Raising a test timeout alone does not let the worker process its progress messages. +## Reusable-law backlog + +RFC #1659 reviews found several green oracles whose stated laws remained valid +but whose fixtures, grammars, or observations did not cover a neighboring +boundary. Track the generalized repairs here instead of accumulating isolated +regressions. Completion requires an executable owner, a production-path witness, +a hostile wrong-answer control, and an explicit statement of remaining limits. + +- [ ] **Real-provider conformance fixtures.** Record representative result + envelopes from each supported OP-SQLite runtime and other persistence hosts. + Make the driver-contract suites prove their shims accept those exact shapes. + Owner: SQLite driver contracts and native-host suites. +- [ ] **Minimal ambiguity and name invariance.** Generate one-field and + otherwise minimally distinguishable results. Renaming a selected column to a + structural-looking alias must not turn a data row into a write envelope. + Owner: React Native OP-SQLite decoding. +- [ ] **Carrier coexistence and representation symmetry.** Cross `rows`, + `rawRows`, `columnNames`, and supported result containers, including legal + coexistence. Equivalent array and object forms must agree on rows or on the + documented rejection. Owner: SQLite driver contracts. +- [ ] **Transitions at every relevant await.** Hold each coordination boundary, + then change leadership, remote-subset ownership, abort state, cleanup, or + restart generation before release. Owner: browser/electron coordination and + collection cleanup/restart oracles. +- [ ] **Local-versus-transport refinement.** Compare local-leader and transported + remote-subset behavior for immutable values. Separately prove that local + `signal` and `subscription` references survive delivery and reach matching + unload cleanup. Owner: remote-subset coordination. +- [ ] **Partial-construction cleanup.** Fail database, driver, worker, and + subscription construction after each acquired resource. Preserve the primary + failure while proving all acquired resources are released exactly once. + Owner: native-host harnesses and collection lifecycle owners. +- [ ] **On-demand persistence after evidence changes.** Cross baseline versus + on-demand hydration with consistent, unknown, and incompatible key-set + evidence. A baseline certification failure must not silently erase valid + on-demand rows. Owner: persisted hydration histories. +- [ ] **Deterministic value-and-work laws.** Pair row correctness with stable + statement, scan, trigger, or queue-cardinality observations where the + subsystem promises bounded work. Owner: SQLite persistence and shared-driver + scheduling suites. +- [ ] **Explicit omission records.** Add a short `Known omissions` section to + each primary executable owner touched above and keep this map synchronized as + laws land. An omission record narrows evidence; it does not waive a product + obligation. + ## Deferred contracts and evidence The maintainer assigned offline policy work to diff --git a/docs/contributing/oracle-tests.md b/docs/contributing/oracle-tests.md index bea51eac49..279392fd74 100644 --- a/docs/contributing/oracle-tests.md +++ b/docs/contributing/oracle-tests.md @@ -624,4 +624,39 @@ For a new oracle or a claimed repair, ask: 6. Can capture, cleanup or shrinking turn this into a different failure? 7. Which larger promises remain outside this test, and where are they tracked? +### Reusable boundary-law checklist + +Adapter and lifecycle oracles should consider these laws when the contract has +the corresponding boundary. They are prompts, not universal requirements. State +why an inapplicable law does not belong to the owner instead of adding a vacuous +case. + +- **Real-provider conformance:** freeze representative values from each + supported provider version. Prove the fixture accepts those values before it + stands in for that provider. +- **Minimal ambiguity:** include the smallest valid input for every classifier + branch. Rich values that carry several redundant signals do not cover a + one-field collision. +- **Name invariance:** changing a user-controlled name or SQL alias must not + change envelope classification unless the public contract assigns that name + structural meaning. +- **Representation symmetry:** equivalent array/object forms and coexisting + carriers must produce the same public result or the same documented error. +- **Await-boundary transitions:** hold each relevant `await`, change ownership, + leadership, generation, abort, cleanup, or restart state, then release it. + Compare the result with the contract for that transition. +- **Local/transport refinement:** immutable transported data must have the same + meaning on local and remote paths. Live local references must remain local, + and cleanup must receive the exact lifecycle object delivered locally. +- **Partial-construction cleanup:** fail each construction step after it acquires + a resource. Preserve the primary error and prove every acquired resource is + released exactly once. +- **Value-and-work refinement:** when bounded work is promised, check the exact + result and a deterministic work/cardinality measure. Correct rows alone do + not establish the work law. + +Every owner should also state its known omissions beside the contract. The +coverage map tracks open reusable laws; an unchecked item is not evidence that +the neighboring laws are absent. + The payoff is not a bigger test framework. It is a smaller distance between “this test is green” and a precise account of what that green result protects. From 924c6895bd579401df1af38f87cf9763061fd5f8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 16:21:51 +0100 Subject: [PATCH 16/23] docs(oracles): record reusable law owners --- docs/contributing/oracle-coverage.md | 39 ++++++++++++++++------------ 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 095f0b1749..ce193bc2e6 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -200,35 +200,40 @@ a hostile wrong-answer control, and an explicit statement of remaining limits. envelopes from each supported OP-SQLite runtime and other persistence hosts. Make the driver-contract suites prove their shims accept those exact shapes. Owner: SQLite driver contracts and native-host suites. -- [ ] **Minimal ambiguity and name invariance.** Generate one-field and +- [x] **Minimal ambiguity and name invariance.** Generate one-field and otherwise minimally distinguishable results. Renaming a selected column to a structural-looking alias must not turn a data row into a write envelope. - Owner: React Native OP-SQLite decoding. -- [ ] **Carrier coexistence and representation symmetry.** Cross `rows`, + Owner: `packages/react-native-db-sqlite-persistence/tests/op-sqlite-driver.test.ts`. +- [x] **Carrier coexistence and representation symmetry.** Cross `rows`, `rawRows`, `columnNames`, and supported result containers, including legal coexistence. Equivalent array and object forms must agree on rows or on the - documented rejection. Owner: SQLite driver contracts. -- [ ] **Transitions at every relevant await.** Hold each coordination boundary, + documented rejection. Owner: + `packages/react-native-db-sqlite-persistence/tests/op-sqlite-driver.test.ts` + and the shared SQLite driver contract. +- [x] **Transitions at every relevant await.** Hold each coordination boundary, then change leadership, remote-subset ownership, abort state, cleanup, or - restart generation before release. Owner: browser/electron coordination and - collection cleanup/restart oracles. -- [ ] **Local-versus-transport refinement.** Compare local-leader and transported + restart generation before release. Owners: browser/electron coordinator, + persisted-history, and collection cleanup/restart oracles. +- [x] **Local-versus-transport refinement.** Compare local-leader and transported remote-subset behavior for immutable values. Separately prove that local `signal` and `subscription` references survive delivery and reach matching - unload cleanup. Owner: remote-subset coordination. -- [ ] **Partial-construction cleanup.** Fail database, driver, worker, and + unload cleanup. Owners: browser/electron coordinator and persisted-history + suites. +- [x] **Partial-construction cleanup.** Fail database, driver, worker, and subscription construction after each acquired resource. Preserve the primary failure while proving all acquired resources are released exactly once. - Owner: native-host harnesses and collection lifecycle owners. -- [ ] **On-demand persistence after evidence changes.** Cross baseline versus + Owners: OP-SQLite driver-contract construction and OPFS page/worker lifecycle + suites. +- [x] **On-demand persistence after evidence changes.** Cross baseline versus on-demand hydration with consistent, unknown, and incompatible key-set evidence. A baseline certification failure must not silently erase valid - on-demand rows. Owner: persisted hydration histories. -- [ ] **Deterministic value-and-work laws.** Pair row correctness with stable + on-demand rows. Owner: + `packages/db-sqlite-persistence-core/tests/persisted.test.ts`. +- [x] **Deterministic value-and-work laws.** Pair row correctness with stable statement, scan, trigger, or queue-cardinality observations where the - subsystem promises bounded work. Owner: SQLite persistence and shared-driver - scheduling suites. -- [ ] **Explicit omission records.** Add a short `Known omissions` section to + subsystem promises bounded work. Owners: SQLite resume snapshots, Electric + acquisition work, and shared-driver scheduling suites. +- [x] **Explicit omission records.** Add a short `Known omissions` section to each primary executable owner touched above and keep this map synchronized as laws land. An omission record narrows evidence; it does not waive a product obligation. From 46e7442e34e87290c2deab89b0da957db0266314 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 17:09:15 +0100 Subject: [PATCH 17/23] test(persistence): prove membership scan observer --- .../tests/sqlite-resume-snapshot.test.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts index 631ea3b786..27a6fc287f 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts @@ -225,6 +225,9 @@ async function observeCachedSchemaState( * compares the entire projected schema state; the held boundary and reset epoch * are reach witnesses, while compatible reopen and recertifying truncate cases * prevent an oracle that merely rejects every resume. + * The focused work law first executes one controlled expected-key table read to + * prove its SQL observer can detect the forbidden membership work, then resets + * the counters before measuring the public position and snapshot operations. * * This narrow fixture supplies the same-connection concurrency seam that the * serialized copy-on-commit CLI harness cannot. It does not claim native host @@ -242,7 +245,7 @@ describe(`SQLite resume snapshots`, () => { let keyMembershipScans = 0 const driver = createDriver(database, undefined, (sql) => { if (sql.includes(`key_set_evidence_available`)) keyEvidenceReads += 1 - if (sql.includes(`FROM collection_expected_keys AS expected`)) { + if (sql.includes(`collection_expected_keys`)) { keyMembershipScans += 1 } }) @@ -261,6 +264,14 @@ describe(`SQLite resume snapshots`, () => { keyMembershipScans = 0 } + await driver.query( + `SELECT key FROM collection_expected_keys + WHERE collection_id = ? + LIMIT 0`, + [collectionId], + ) + expect(observeWork().keyMembershipScans).toBe(1) + resetWork() const position = await adapter.getStreamPosition(collectionId) const leadershipClaim = observeWork() From a8c2097b5f928ccb79d1054c500170967109d44c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 17:41:37 +0100 Subject: [PATCH 18/23] test(persistence): execute reusable boundary laws --- docs/contributing/oracle-coverage.md | 17 +- .../tests/persisted.test.ts | 157 +++++++++++++++--- .../tests/sqlite-resume-snapshot.test.ts | 9 +- 3 files changed, 150 insertions(+), 33 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index ce193bc2e6..c1cd537300 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -196,10 +196,14 @@ boundary. Track the generalized repairs here instead of accumulating isolated regressions. Completion requires an executable owner, a production-path witness, a hostile wrong-answer control, and an explicit statement of remaining limits. -- [ ] **Real-provider conformance fixtures.** Record representative result - envelopes from each supported OP-SQLite runtime and other persistence hosts. - Make the driver-contract suites prove their shims accept those exact shapes. - Owner: SQLite driver contracts and native-host suites. +- [x] **Real-provider conformance fixtures.** Frozen 15.2.7 React Native and + Node receipts cover the supported peer version; 18.2.1 React Native, Node, + and browser receipts cover the known forward shapes. Exact-row checks and a + row-dropping hostile control prove the shim accepts those envelopes without + mutating them. Owner: + `packages/react-native-db-sqlite-persistence/tests/fixtures/op-sqlite-provider-results.ts` + and `packages/react-native-db-sqlite-persistence/tests/op-sqlite-driver.test.ts`. + Native device/host execution remains a separate runtime receipt. - [x] **Minimal ambiguity and name invariance.** Generate one-field and otherwise minimally distinguishable results. Renaming a selected column to a structural-looking alias must not turn a data row into a write envelope. @@ -227,7 +231,10 @@ a hostile wrong-answer control, and an explicit statement of remaining limits. - [x] **On-demand persistence after evidence changes.** Cross baseline versus on-demand hydration with consistent, unknown, and incompatible key-set evidence. A baseline certification failure must not silently erase valid - on-demand rows. Owner: + on-demand rows. `loadSubset` checks baseline visibility and on-demand rows; + the sync-absent `forceReloadSubset` route crosses the same startup evidence + states but records baseline visibility as unobserved because that API does + not expose baseline hydration. Owner: `packages/db-sqlite-persistence-core/tests/persisted.test.ts`. - [x] **Deterministic value-and-work laws.** Pair row correctness with stable statement, scan, trigger, or queue-cardinality observations where the diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index bf1bdae32f..2d37ef32bf 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -49,9 +49,9 @@ import type { * cleanup, and restart. They compare durable state, public rows, metadata, * request options, sequence evidence, errors, and late-work fencing. * - * Driver SQL behavior, browser page ownership, native runtimes, and the shared - * conformance portfolio have separate owners. This file models persistence - * protocol state, not a particular SQLite engine. + * Known omissions: driver SQL behavior, browser page ownership, native + * runtimes, and the shared conformance portfolio have separate owners. This + * file models persistence protocol state, not a particular SQLite engine. */ type Todo = { @@ -59,6 +59,40 @@ type Todo = { title: string } +const persistedKeySetEvidenceStatuses = [ + `consistent`, + `unknown`, + `incompatible`, +] as const + +type OnDemandEvidenceObservation = { + status: (typeof persistedKeySetEvidenceStatuses)[number] + route: `loadSubset` | `forceReloadSubset` + baselineVisible: boolean | `not-observed` + onDemandVisible: boolean +} + +function expectOnDemandEvidenceLaw( + observation: OnDemandEvidenceObservation, +): void { + try { + expect(observation).toEqual({ + status: observation.status, + route: observation.route, + baselineVisible: + observation.route === `loadSubset` + ? observation.status !== `incompatible` + : `not-observed`, + onDemandVisible: true, + }) + } catch (cause) { + throw new Error( + `on-demand rows must not inherit baseline evidence rejection`, + { cause }, + ) + } +} + type RecordingAdapter = PersistenceAdapter & { applyCommittedTxCalls: Array<{ collectionId: string @@ -2630,21 +2664,81 @@ describe(`persistedCollectionOptions`, () => { await collection.cleanup() }) - it(`applies on-demand subsets even when the full baseline is incompatible`, async () => { - const hydrateUsing = async ( - route: `loadSubset` | `forceReloadSubset`, - ): Promise => { + it.each(persistedKeySetEvidenceStatuses.map((status) => ({ status })))( + `keeps on-demand rows independent from $status baseline evidence`, + async ({ status }) => { + const adapter = createRecordingAdapter([ + { id: `on-demand`, title: `On-demand row` }, + ]) + const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) + adapter.loadResumeSnapshot = async (...args) => ({ + ...(await loadResumeSnapshot(...args)), + rows: [ + { + key: `baseline`, + value: { id: `baseline`, title: `Baseline row` }, + }, + ], + keySet: { status }, + }) + let hydrateBaseline: (() => Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `${status}-baseline-and-on-demand`, + syncMode: `on-demand`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady, metadata }) => { + const capability = metadata?.persistence + if (!capability) { + throw new Error(`Expected persisted sync capability`) + } + hydrateBaseline = capability.hydrateBaseline + markReady() + return { loadSubset: () => true } + }, + }, + persistence: { adapter }, + }), + ) + + try { + collection.startSyncImmediate() + await vi.waitFor(() => expect(hydrateBaseline).toBeTypeOf(`function`)) + await hydrateBaseline!() + await flushAsyncWork() + await flushAsyncWork() + + const baselineVisible = collection.has(`baseline`) + expect(collection.has(`on-demand`)).toBe(false) + await collection._sync.loadSubset({}) + + expectOnDemandEvidenceLaw({ + status, + route: `loadSubset`, + baselineVisible, + onDemandVisible: collection.has(`on-demand`), + }) + } finally { + await collection.cleanup() + } + }, + ) + + it.each(persistedKeySetEvidenceStatuses.map((status) => ({ status })))( + `keeps force reload rows independent from $status startup evidence`, + async ({ status }) => { const adapter = createRecordingAdapter([ - { id: `1`, title: `Locally cached` }, + { id: `on-demand`, title: `On-demand row` }, ]) const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) adapter.loadResumeSnapshot = async (...args) => ({ ...(await loadResumeSnapshot(...args)), - keySet: { status: `incompatible` }, + keySet: { status }, }) const collection = createCollection( persistedCollectionOptions({ - id: `incompatible-${route}`, + id: `${status}-local-only-force-reload`, syncMode: `on-demand`, getKey: (item) => item.id, persistence: { adapter }, @@ -2656,24 +2750,39 @@ describe(`persistedCollectionOptions`, () => { await vi.waitFor(() => expect(adapter.loadResumeSnapshotCalls.length).toBeGreaterThan(0), ) - expect(collection.has(`1`)).toBe(false) - if (route === `loadSubset`) { - await collection._sync.loadSubset({}) - } else { - await collection.utils.forceReloadSubset!({}) - } - return collection.has(`1`) + await collection.utils.forceReloadSubset!({}) + + expect(adapter.loadResumeSnapshotCalls[0]?.includeRows).toBe(false) + expectOnDemandEvidenceLaw({ + status, + route: `forceReloadSubset`, + // This local-only route reads startup evidence but does not expose + // baseline hydration. Do not claim a baseline observation here. + baselineVisible: `not-observed`, + onDemandVisible: collection.has(`on-demand`), + }) } finally { await collection.cleanup() } - } + }, + ) - expect( - await Promise.all([ - hydrateUsing(`loadSubset`), - hydrateUsing(`forceReloadSubset`), - ]), - ).toEqual([true, true]) + it(`rejects an evidence-coupled on-demand hydration mutant`, () => { + expect(() => + expectOnDemandEvidenceLaw({ + status: `incompatible`, + route: `loadSubset`, + baselineVisible: false, + onDemandVisible: false, + }), + ).toThrow(`on-demand rows must not inherit baseline evidence rejection`) + + expectOnDemandEvidenceLaw({ + status: `incompatible`, + route: `loadSubset`, + baselineVisible: false, + onDemandVisible: true, + }) }) it(`keeps resume certification consistent after an owned no-op commit`, async () => { diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts index 27a6fc287f..1d4e13faec 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts @@ -229,10 +229,11 @@ async function observeCachedSchemaState( * prove its SQL observer can detect the forbidden membership work, then resets * the counters before measuring the public position and snapshot operations. * - * This narrow fixture supplies the same-connection concurrency seam that the - * serialized copy-on-commit CLI harness cannot. It does not claim native host - * execution or judge whether a consumer such as Electric may use the certified - * cursor; those remain separate driver-contract and Electric recovery owners. + * Known omissions: this narrow fixture supplies the same-connection + * concurrency seam that the serialized copy-on-commit CLI harness cannot. It + * does not claim native host execution or judge whether a consumer such as + * Electric may use the certified cursor; those remain separate driver-contract + * and Electric recovery owners. */ describe(`SQLite resume snapshots`, () => { it(`reads key-set evidence without rescanning key membership`, async () => { From ae521cf84f71f4355b19352e1910563937cb1ea7 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 11:03:13 +0100 Subject: [PATCH 19/23] fix(persistence): fence startup and schema generations --- .../preserve-resume-baseline-integrity.md | 1 + docs/contributing/oracle-coverage.md | 16 + .../src/browser-coordinator.ts | 31 +- .../src/browser-persistence.ts | 33 +- .../tests/browser-coordinator.test.ts | 90 ++++ .../src/persisted.ts | 28 +- .../src/sqlite-core-adapter.ts | 486 +++++++++++------- .../tests/persisted.test.ts | 64 +++ .../tests/sqlite-resume-snapshot.test.ts | 369 +++++++++++++ .../electric-db-collection/src/electric.ts | 7 +- .../electric-resume-snapshot-races.test.ts | 82 ++- .../src/electron-coordinator.ts | 31 +- .../src/renderer.ts | 30 +- .../tests/electron-ipc.test.ts | 120 ++++- 14 files changed, 1119 insertions(+), 269 deletions(-) diff --git a/.changeset/preserve-resume-baseline-integrity.md b/.changeset/preserve-resume-baseline-integrity.md index 7da60be386..696e51a1d1 100644 --- a/.changeset/preserve-resume-baseline-integrity.md +++ b/.changeset/preserve-resume-baseline-integrity.md @@ -1,6 +1,7 @@ --- '@tanstack/db-sqlite-persistence-core': minor '@tanstack/db': minor +'@tanstack/browser-db-sqlite-persistence': patch '@tanstack/electron-db-sqlite-persistence': minor '@tanstack/electric-db-collection': patch '@tanstack/query-db-collection': patch diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index c1cd537300..1f9b2c5e69 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -240,10 +240,26 @@ a hostile wrong-answer control, and an explicit statement of remaining limits. statement, scan, trigger, or queue-cardinality observations where the subsystem promises bounded work. Owners: SQLite resume snapshots, Electric acquisition work, and shared-driver scheduling suites. +- [x] **Startup generation interleavings.** Hold persisted startup between its + metadata and hydration snapshots, then cross no write, a managed mutation, + and hostile raw loss. Compare public rows with the atomic durable snapshot, + and require mutation persistence to wait until the prior stream position is + known. Owners: SQLite resume snapshots and Electric resume snapshot races. +- [x] **Schema-generation fences on cached adapters.** Cross a newer-schema + reset with snapshot, row, metadata, delta, position, and index-lifecycle + operations from the cached older adapter. Every stale operation rejects; + the current adapter remains readable and its index registry remains intact. + Owners: SQLite resume snapshots plus browser/electron coordinator routing. - [x] **Explicit omission records.** Add a short `Known omissions` section to each primary executable owner touched above and keep this map synchronized as laws land. An omission record narrows evidence; it does not waive a product obligation. +- [ ] **Atomic active-subset full reload.** Load collection metadata and every + active subset from one adapter generation, including filtered and paginated + on-demand subsets. A sound implementation needs an atomic multi-subset API; + loading all rows or accepting a metadata/row torn pair is not equivalent. + RED evidence and the deferred executable placeholder live in + `packages/db-sqlite-persistence-core/tests/persisted.test.ts` under R5-007. ## Deferred contracts and evidence diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index 1babddc5a7..1b290b186d 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -122,6 +122,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina private readonly nodeId = safeRandomUUID() private readonly dbName: string private adapter: AdapterWithPullSince | null + private readonly collectionAdapters = new Map() private readonly channel: BroadcastChannel private readonly collections = new Map() private readonly pendingRPCs = new Map() @@ -133,13 +134,14 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina return this.disposed } - private requireAdapter(): AdapterWithPullSince { - if (!this.adapter) { + private requireAdapter(collectionId: string): AdapterWithPullSince { + const adapter = this.collectionAdapters.get(collectionId) ?? this.adapter + if (!adapter) { throw new Error( `BrowserCollectionCoordinator: adapter not set. Call setAdapter() before using leader-side operations.`, ) } - return this.adapter + return adapter } constructor(options: BrowserCollectionCoordinatorOptions) { @@ -160,6 +162,14 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina this.adapter = adapter } + /** Register the schema/mode-specific adapter for one collection. */ + setCollectionAdapter( + collectionId: string, + adapter: AdapterWithPullSince, + ): void { + this.collectionAdapters.set(collectionId, adapter) + } + // ----------------------------------------------------------------------- // PersistedCollectionCoordinator interface // ----------------------------------------------------------------------- @@ -223,7 +233,11 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina spec: PersistedIndexSpec, ): Promise { if (this.isLeader(collectionId)) { - await this.requireAdapter().ensureIndex(collectionId, signature, spec) + await this.requireAdapter(collectionId).ensureIndex( + collectionId, + signature, + spec, + ) return } @@ -305,6 +319,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina this.channel.close() this.collections.clear() + this.collectionAdapters.clear() } // ----------------------------------------------------------------------- @@ -348,7 +363,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina try { // Restore stream position from DB before claiming leadership - const adapter = this.requireAdapter() + const adapter = this.requireAdapter(collectionId) if (adapter.getStreamPosition) { const pos = await adapter.getStreamPosition(collectionId) state.latestTerm = pos.latestTerm @@ -610,7 +625,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina }, ): Promise { await this.withWriterLock(() => - this.requireAdapter().ensureIndex( + this.requireAdapter(collectionId).ensureIndex( collectionId, request.signature, request.spec, @@ -676,7 +691,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina } await this.withWriterLock(() => - this.requireAdapter().applyCommittedTx(collectionId, tx), + this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), ) // Track envelope for dedup @@ -736,7 +751,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina ): Promise { const state = this.collections.get(collectionId) - const adapter = this.requireAdapter() + const adapter = this.requireAdapter(collectionId) if (!adapter.pullSince) { return { type: `rpc:pullSince:res`, diff --git a/packages/browser-db-sqlite-persistence/src/browser-persistence.ts b/packages/browser-db-sqlite-persistence/src/browser-persistence.ts index b36e564900..4c6e25ac87 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-persistence.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-persistence.ts @@ -130,34 +130,39 @@ export function createBrowserWASQLitePersistence( ...(schemaVersion === undefined ? {} : { schemaVersion }), }) adapterCache.set(cacheKey, adapter) - - // Wire the adapter into the multi-tab coordinator so it can handle - // leader-side RPCs (applyCommittedTx, pullSince, ensureIndex, etc.) - if (resolvedCoordinator instanceof BrowserCollectionCoordinator) { - resolvedCoordinator.setAdapter(adapter) - } - return adapter } const createCollectionPersistence = ( + collectionId: string | undefined, mode: PersistedCollectionMode, schemaVersion: number | undefined, - ): PersistedCollectionPersistence => ({ - adapter: getAdapterForCollection(mode, schemaVersion), - coordinator: resolvedCoordinator, - }) + ): PersistedCollectionPersistence => { + const adapter = getAdapterForCollection(mode, schemaVersion) + if (resolvedCoordinator instanceof BrowserCollectionCoordinator) { + if (collectionId === undefined) { + resolvedCoordinator.setAdapter(adapter) + } else { + resolvedCoordinator.setCollectionAdapter(collectionId, adapter) + } + } + return { + adapter, + coordinator: resolvedCoordinator, + } + } const defaultPersistence = createCollectionPersistence( + undefined, `sync-absent`, undefined, ) return { ...defaultPersistence, - resolvePersistenceForCollection: ({ mode, schemaVersion }) => - createCollectionPersistence(mode, schemaVersion), + resolvePersistenceForCollection: ({ collectionId, mode, schemaVersion }) => + createCollectionPersistence(collectionId, mode, schemaVersion), resolvePersistenceForMode: (mode) => - createCollectionPersistence(mode, undefined), + createCollectionPersistence(undefined, mode, undefined), } } diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 5ddcb272ac..57b6985875 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -1,5 +1,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { BrowserCollectionCoordinator } from '../src/browser-coordinator' +import { + createBrowserWASQLitePersistence, + persistedCollectionOptions, +} from '../src' +import { createWASQLiteTestDatabase } from './helpers/wa-sqlite-test-db' import type { BrowserCollectionCoordinatorOptions } from '../src/browser-coordinator' import type { PersistenceAdapter } from '@tanstack/db-sqlite-persistence-core' @@ -295,6 +300,91 @@ describe(`BrowserCollectionCoordinator`, () => { coord2.dispose() }) + it.each([ + { schemaV1: 1, schemaV2: 2 }, + { schemaV1: 2, schemaV2: 4 }, + ])( + `routes each collection through its schema-version adapter: $schemaV1/$schemaV2`, + async ({ schemaV1, schemaV2 }) => { + const database = createWASQLiteTestDatabase({ filename: `:memory:` }) + const coordinator = new BrowserCollectionCoordinator({ + dbName: `schema-routed-db`, + }) + const persistence = createBrowserWASQLitePersistence({ + database, + coordinator, + }) + const collectionV1 = `schema-routed-v1` + const collectionV2 = `schema-routed-v2` + + try { + const optionsV1 = persistedCollectionOptions< + { id: string; title: string }, + string + >({ + id: collectionV1, + schemaVersion: schemaV1, + getKey: (row) => row.id, + persistence, + }) + await optionsV1.persistence.adapter.loadResumeSnapshot(collectionV1) + + const optionsV2 = persistedCollectionOptions< + { id: string; title: string }, + string + >({ + id: collectionV2, + schemaVersion: schemaV2, + getKey: (row) => row.id, + persistence, + }) + await optionsV2.persistence.adapter.loadResumeSnapshot(collectionV2) + + coordinator.subscribe(collectionV1, () => {}) + coordinator.subscribe(collectionV2, () => {}) + await vi.waitFor(() => { + expect(coordinator.isLeader(collectionV1)).toBe(true) + expect(coordinator.isLeader(collectionV2)).toBe(true) + }) + + const [resultV1, resultV2] = await Promise.all([ + coordinator.requestApplyLocalMutations(collectionV1, [ + { + mutationId: `mutation-v1`, + type: `insert`, + key: `v1`, + value: { id: `v1`, title: `schema one` }, + }, + ]), + coordinator.requestApplyLocalMutations(collectionV2, [ + { + mutationId: `mutation-v2`, + type: `insert`, + key: `v2`, + value: { id: `v2`, title: `schema two` }, + }, + ]), + ]) + + expect(resultV1.ok).toBe(true) + expect(resultV2.ok).toBe(true) + expect( + await optionsV1.persistence.adapter.loadSubset(collectionV1, {}), + ).toMatchObject([ + { key: `v1`, value: { id: `v1`, title: `schema one` } }, + ]) + expect( + await optionsV2.persistence.adapter.loadSubset(collectionV2, {}), + ).toMatchObject([ + { key: `v2`, value: { id: `v2`, title: `schema two` } }, + ]) + } finally { + coordinator.dispose() + await Promise.resolve(database.close?.()) + } + }, + ) + it(`different collections have independent leaders`, async () => { const coord1 = createCoordinator() const coord2 = createCoordinator() diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 4f5a3f1e55..d1697c75e0 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -936,13 +936,14 @@ class PersistedCollectionRuntime< } } - async ensureStarted(): Promise { + ensureStarted(): Promise { if (this.startPromise) { return this.startPromise } const lifecycleGeneration = this.lifecycleGeneration this.startPromise = this.startInternal(lifecycleGeneration) + void this.startPromise.catch(() => undefined) return this.startPromise } @@ -1247,6 +1248,10 @@ class PersistedCollectionRuntime< } await this.applyMutex.run(async () => { + // Startup metadata establishes the durable term/sequence boundary. A + // mutation admitted before it resolves must wait rather than allocate a + // default position that can collide with an already-applied transaction. + await this.ensureStartupMetadataLoaded() const acceptedMutationIds = await this.persistCollectionMutationsUnsafe(mutations) const acceptedMutationIdSet = new Set(acceptedMutationIds) @@ -1479,18 +1484,27 @@ class PersistedCollectionRuntime< resetEpoch: number }): void { const generation = this.getResumeSnapshotGeneration(snapshot) - // Moving between equally uncertified snapshots cannot make either - // trustworthy; preserve that status so sync performs a fresh replacement. - const remainsUncertified = - this.persistedKeySetEvidence?.status === snapshot.keySet?.status && - snapshot.keySet?.status !== `consistent` + const previousEvidenceStatus = this.persistedKeySetEvidence?.status + // An uncertified sync baseline never authorized a persisted resume cursor, + // so a later atomic snapshot may replace its evidence while the source + // performs the already-required fresh snapshot. Local-only collections + // have no remote cursor to fence; a managed write may advance a still- + // consistent SQLite baseline during startup. Incompatible evidence remains + // fail-closed in both modes. + const mayAcceptUnownedGeneration = + (this.mode === `sync-present` && + previousEvidenceStatus !== `consistent` && + previousEvidenceStatus !== `incompatible`) || + (this.mode === `sync-absent` && + previousEvidenceStatus === `consistent` && + snapshot.keySet?.status === `consistent`) this.observeStreamPosition( snapshot.latestTerm, snapshot.latestSeq, snapshot.latestRowVersion, ) this.persistedKeySetEvidence = - this.isExpectedResumeGeneration(generation) || remainsUncertified + this.isExpectedResumeGeneration(generation) || mayAcceptUnownedGeneration ? snapshot.keySet : { status: `incompatible` } } diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index cfbed9d4ac..03ffc360ec 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -1098,6 +1098,28 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { return this.driver.transaction(fn) } + private async assertCurrentSchemaVersion( + collectionId: string, + driver: SQLiteDriver, + operation: string, + ): Promise { + const schemaRows = await driver.query<{ schema_version: number }>( + `SELECT schema_version + FROM collection_registry + WHERE collection_id = ? + LIMIT 1`, + [collectionId], + ) + const persistedSchemaVersion = schemaRows[0]?.schema_version + if (persistedSchemaVersion !== this.schemaVersion) { + throw new InvalidPersistedCollectionConfigError( + `Schema version mismatch for collection "${collectionId}": ` + + `found ${persistedSchemaVersion ?? `missing`}, expected ${this.schemaVersion}. ` + + `Refusing to ${operation} through a stale cached adapter.`, + ) + } + } + async loadSubset( collectionId: string, options: LoadSubsetOptions, @@ -1110,54 +1132,77 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { }> > { const tableMapping = await this.ensureCollectionReady(collectionId) - await this.touchRequiredIndexes(collectionId, ctx?.requiredIndexSignatures) - - if (options.cursor) { - const whereCurrentOptions: LoadSubsetOptions = { - where: options.where - ? new IR.Func(`and`, [options.where, options.cursor.whereCurrent]) - : options.cursor.whereCurrent, - orderBy: options.orderBy, - } - const whereFromOptions: LoadSubsetOptions = { - where: options.where - ? new IR.Func(`and`, [options.where, options.cursor.whereFrom]) - : options.cursor.whereFrom, - orderBy: options.orderBy, - limit: options.limit, - } + return this.runInTransaction(async (transactionDriver) => { + await this.assertCurrentSchemaVersion( + collectionId, + transactionDriver, + `load persisted rows`, + ) + await this.touchRequiredIndexes( + collectionId, + ctx?.requiredIndexSignatures, + transactionDriver, + ) - const [whereCurrentRows, whereFromRows] = await Promise.all([ - this.loadSubsetInternal(tableMapping, whereCurrentOptions), - this.loadSubsetInternal(tableMapping, whereFromOptions), - ]) + if (options.cursor) { + const whereCurrentOptions: LoadSubsetOptions = { + where: options.where + ? new IR.Func(`and`, [options.where, options.cursor.whereCurrent]) + : options.cursor.whereCurrent, + orderBy: options.orderBy, + } + const whereFromOptions: LoadSubsetOptions = { + where: options.where + ? new IR.Func(`and`, [options.where, options.cursor.whereFrom]) + : options.cursor.whereFrom, + orderBy: options.orderBy, + limit: options.limit, + } + + const [whereCurrentRows, whereFromRows] = await Promise.all([ + this.loadSubsetInternal( + tableMapping, + whereCurrentOptions, + transactionDriver, + ), + this.loadSubsetInternal( + tableMapping, + whereFromOptions, + transactionDriver, + ), + ]) + + const mergedRows = new Map< + string, + InMemoryRow> + >() + for (const row of [...whereCurrentRows, ...whereFromRows]) { + mergedRows.set(encodePersistedStorageKey(row.key), row) + } + + const orderedRows = this.applyInMemoryOrderBy( + Array.from(mergedRows.values()), + options.orderBy, + ) - const mergedRows = new Map< - string, - InMemoryRow> - >() - for (const row of [...whereCurrentRows, ...whereFromRows]) { - mergedRows.set(encodePersistedStorageKey(row.key), row) + return orderedRows.map((row) => ({ + key: row.key, + value: row.value, + metadata: row.metadata, + })) } - const orderedRows = this.applyInMemoryOrderBy( - Array.from(mergedRows.values()), - options.orderBy, + const rows = await this.loadSubsetInternal( + tableMapping, + options, + transactionDriver, ) - - return orderedRows.map((row) => ({ + return rows.map((row) => ({ key: row.key, value: row.value, metadata: row.metadata, })) - } - - const rows = await this.loadSubsetInternal(tableMapping, options) - return rows.map((row) => ({ - key: row.key, - value: row.value, - metadata: row.metadata, - })) + }) } async loadResumeSnapshot( @@ -1181,14 +1226,21 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { }> { const tableMapping = await this.ensureCollectionReady(collectionId) const includeRows = ctx?.includeRows !== false - if (includeRows) { - await this.touchRequiredIndexes( + + return this.runInTransaction(async (transactionDriver) => { + await this.assertCurrentSchemaVersion( collectionId, - ctx?.requiredIndexSignatures, + transactionDriver, + `load a resume snapshot`, ) - } + if (includeRows) { + await this.touchRequiredIndexes( + collectionId, + ctx?.requiredIndexSignatures, + transactionDriver, + ) + } - return this.runInTransaction(async (transactionDriver) => { const rows = includeRows ? await this.loadSubsetInternal(tableMapping, {}, transactionDriver) : [] @@ -1517,17 +1569,28 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { async loadCollectionMetadata( collectionId: string, ): Promise> { - const rows = await this.driver.query<{ key: string; value: string }>( - `SELECT key, value - FROM collection_metadata - WHERE collection_id = ?`, - [collectionId], - ) + await this.ensureCollectionReady(collectionId) + return this.runInTransaction(async (transactionDriver) => { + await this.assertCurrentSchemaVersion( + collectionId, + transactionDriver, + `load collection metadata`, + ) + const rows = await transactionDriver.query<{ + key: string + value: string + }>( + `SELECT key, value + FROM collection_metadata + WHERE collection_id = ?`, + [collectionId], + ) - return rows.map((row) => ({ - key: row.key, - value: deserializePersistedRowValue(row.value), - })) + return rows.map((row) => ({ + key: row.key, + value: deserializePersistedRowValue(row.value), + })) + }) } async scanRows( @@ -1536,21 +1599,27 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { ): Promise> { const tableMapping = await this.ensureCollectionReady(collectionId) const collectionTableSql = quoteIdentifier(tableMapping.tableName) + return this.runInTransaction(async (transactionDriver) => { + await this.assertCurrentSchemaVersion( + collectionId, + transactionDriver, + `scan persisted rows`, + ) + const storedRows = await transactionDriver.query( + options?.metadataOnly + ? `SELECT key, value, metadata, row_version + FROM ${collectionTableSql} + WHERE metadata IS NOT NULL` + : `SELECT key, value, metadata, row_version + FROM ${collectionTableSql}`, + ) - const storedRows = await this.driver.query( - options?.metadataOnly - ? `SELECT key, value, metadata, row_version - FROM ${collectionTableSql} - WHERE metadata IS NOT NULL` - : `SELECT key, value, metadata, row_version - FROM ${collectionTableSql}`, - ) - - return decodeStoredSqliteRows(storedRows).map((row) => ({ - key: row.key, - value: row.value, - metadata: row.metadata, - })) + return decodeStoredSqliteRows(storedRows).map((row) => ({ + key: row.key, + value: row.value, + metadata: row.metadata, + })) + }) } async ensureIndex( @@ -1571,6 +1640,11 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { : undefined await this.runInTransaction(async (transactionDriver) => { + await this.assertCurrentSchemaVersion( + collectionId, + transactionDriver, + `create a persisted index`, + ) await transactionDriver.run( `INSERT INTO persisted_index_registry ( collection_id, @@ -1618,29 +1692,36 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { signature: string, ): Promise { await this.ensureCollectionReady(collectionId) - const rows = await this.driver.query<{ index_name: string }>( - `SELECT index_name - FROM persisted_index_registry - WHERE collection_id = ? AND signature = ? - LIMIT 1`, - [collectionId, signature], - ) - const indexName = rows[0]?.index_name - - await this.driver.run( - `UPDATE persisted_index_registry - SET removed = 1, - updated_at = CAST(strftime('%s', 'now') AS INTEGER), - last_used_at = CAST(strftime('%s', 'now') AS INTEGER) - WHERE collection_id = ? AND signature = ?`, - [collectionId, signature], - ) + await this.runInTransaction(async (transactionDriver) => { + await this.assertCurrentSchemaVersion( + collectionId, + transactionDriver, + `remove a persisted index`, + ) + const rows = await transactionDriver.query<{ index_name: string }>( + `SELECT index_name + FROM persisted_index_registry + WHERE collection_id = ? AND signature = ? + LIMIT 1`, + [collectionId, signature], + ) + const indexName = rows[0]?.index_name - if (indexName) { - await this.driver.exec( - `DROP INDEX IF EXISTS ${quoteIdentifier(indexName)}`, + await transactionDriver.run( + `UPDATE persisted_index_registry + SET removed = 1, + updated_at = CAST(strftime('%s', 'now') AS INTEGER), + last_used_at = CAST(strftime('%s', 'now') AS INTEGER) + WHERE collection_id = ? AND signature = ?`, + [collectionId, signature], ) - } + + if (indexName) { + await transactionDriver.exec( + `DROP INDEX IF EXISTS ${quoteIdentifier(indexName)}`, + ) + } + }) } async getStreamPosition(collectionId: string): Promise<{ @@ -1649,15 +1730,22 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { latestRowVersion: number }> { await this.ensureCollectionReady(collectionId) - const [position, latestRowVersion] = await Promise.all([ - this.readStreamPosition(collectionId, this.driver), - this.readLatestRowVersion(collectionId, this.driver), - ]) + return this.runInTransaction(async (transactionDriver) => { + await this.assertCurrentSchemaVersion( + collectionId, + transactionDriver, + `read the stream position`, + ) + const [position, latestRowVersion] = await Promise.all([ + this.readStreamPosition(collectionId, transactionDriver), + this.readLatestRowVersion(collectionId, transactionDriver), + ]) - return { - ...position, - latestRowVersion, - } + return { + ...position, + latestRowVersion, + } + }) } private async readLatestRowVersion( @@ -1746,131 +1834,138 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { const collectionTableSql = quoteIdentifier(tableMapping.tableName) const tombstoneTableSql = quoteIdentifier(tableMapping.tombstoneTableName) - const [ - changedRows, - deletedRows, - latestVersionRows, - replayRows, - replayAvailabilityRows, - ] = await Promise.all([ - this.driver.query<{ key: string }>( - `SELECT key + return this.runInTransaction(async (transactionDriver) => { + await this.assertCurrentSchemaVersion( + collectionId, + transactionDriver, + `read persisted transaction deltas`, + ) + const [ + changedRows, + deletedRows, + latestVersionRows, + replayRows, + replayAvailabilityRows, + ] = await Promise.all([ + transactionDriver.query<{ key: string }>( + `SELECT key FROM ${collectionTableSql} WHERE row_version > ?`, - [fromRowVersion], - ), - this.driver.query<{ key: string }>( - `SELECT key + [fromRowVersion], + ), + transactionDriver.query<{ key: string }>( + `SELECT key FROM ${tombstoneTableSql} WHERE row_version > ?`, - [fromRowVersion], - ), - this.driver.query<{ latest_row_version: number }>( - `SELECT latest_row_version + [fromRowVersion], + ), + transactionDriver.query<{ latest_row_version: number }>( + `SELECT latest_row_version FROM collection_version WHERE collection_id = ? LIMIT 1`, - [collectionId], - ), - this.driver.query<{ - tx_id: string - row_version: number - replay_json: string | null - replay_requires_full_reload: number - }>( - `SELECT tx_id, row_version, replay_json, replay_requires_full_reload + [collectionId], + ), + transactionDriver.query<{ + tx_id: string + row_version: number + replay_json: string | null + replay_requires_full_reload: number + }>( + `SELECT tx_id, row_version, replay_json, replay_requires_full_reload FROM applied_tx WHERE collection_id = ? AND row_version > ? ORDER BY term ASC, seq ASC`, - [collectionId, fromRowVersion], - ), - this.driver.query<{ min_row_version: number | null }>( - `SELECT MIN(row_version) AS min_row_version + [collectionId, fromRowVersion], + ), + transactionDriver.query<{ min_row_version: number | null }>( + `SELECT MIN(row_version) AS min_row_version FROM applied_tx WHERE collection_id = ?`, - [collectionId], - ), - ]) + [collectionId], + ), + ]) - const latestRowVersion = latestVersionRows[0]?.latest_row_version ?? 0 - const replayFloor = replayAvailabilityRows[0]?.min_row_version - if ( - latestRowVersion > fromRowVersion && - (replayFloor == null || replayFloor > fromRowVersion + 1) - ) { - return { - latestRowVersion, - requiresFullReload: true, + const latestRowVersion = latestVersionRows[0]?.latest_row_version ?? 0 + const replayFloor = replayAvailabilityRows[0]?.min_row_version + if ( + latestRowVersion > fromRowVersion && + (replayFloor == null || replayFloor > fromRowVersion + 1) + ) { + return { + latestRowVersion, + requiresFullReload: true, + } } - } - const changedKeyCount = changedRows.length + deletedRows.length + const changedKeyCount = changedRows.length + deletedRows.length - if (changedKeyCount > this.pullSinceReloadThreshold) { - return { - latestRowVersion, - requiresFullReload: true, + if (changedKeyCount > this.pullSinceReloadThreshold) { + return { + latestRowVersion, + requiresFullReload: true, + } } - } - if ( - replayRows.some( - (row) => - row.replay_requires_full_reload !== 0 || row.replay_json == null, - ) - ) { - return { - latestRowVersion, - requiresFullReload: true, + if ( + replayRows.some( + (row) => + row.replay_requires_full_reload !== 0 || row.replay_json == null, + ) + ) { + return { + latestRowVersion, + requiresFullReload: true, + } } - } - const decodeKey = (encodedKey: string): string | number => { - try { - return decodePersistedStorageKey(encodedKey) - } catch (error) { - throw new InvalidPersistedStorageKeyEncodingError( - `${encodedKey}: ${(error as Error).message}`, - ) + const decodeKey = (encodedKey: string): string | number => { + try { + return decodePersistedStorageKey(encodedKey) + } catch (error) { + throw new InvalidPersistedStorageKeyEncodingError( + `${encodedKey}: ${(error as Error).message}`, + ) + } } - } - const deltas = replayRows.map((row) => { - const parsed = deserializePersistedRowValue( - row.replay_json ?? `null`, - ) - if (!parsed) { - throw new InvalidPersistedCollectionConfigError( - `missing replay payload for applied_tx row`, + const deltas = replayRows.map((row) => { + const parsed = deserializePersistedRowValue( + row.replay_json ?? `null`, ) - } - return parsed - }) + if (!parsed) { + throw new InvalidPersistedCollectionConfigError( + `missing replay payload for applied_tx row`, + ) + } + return parsed + }) + + const replayChangeCount = deltas.reduce( + (count, delta) => + count + + delta.changedRows.length + + delta.deletedKeys.length + + delta.rowMetadataMutations.length + + delta.collectionMetadataMutations.length, + 0, + ) - const replayChangeCount = deltas.reduce( - (count, delta) => - count + - delta.changedRows.length + - delta.deletedKeys.length + - delta.rowMetadataMutations.length + - delta.collectionMetadataMutations.length, - 0, - ) + if (replayChangeCount > this.pullSinceReloadThreshold) { + return { + latestRowVersion, + requiresFullReload: true, + } + } - if (replayChangeCount > this.pullSinceReloadThreshold) { return { latestRowVersion, - requiresFullReload: true, + requiresFullReload: false, + changedKeys: changedRows.map((row) => decodeKey(row.key)), + deletedKeys: deletedRows.map((row) => decodeKey(row.key)), + deltas, } - } - - return { - latestRowVersion, - requiresFullReload: false, - changedKeys: changedRows.map((row) => decodeKey(row.key)), - deletedKeys: deletedRows.map((row) => decodeKey(row.key)), - deltas, - } + }) } private async loadSubsetInternal( @@ -1984,13 +2079,14 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { private async touchRequiredIndexes( collectionId: string, requiredIndexSignatures: ReadonlyArray | undefined, + driver: SQLiteDriver = this.driver, ): Promise { if (!requiredIndexSignatures || requiredIndexSignatures.length === 0) { return } for (const signature of requiredIndexSignatures) { - await this.driver.run( + await driver.run( `UPDATE persisted_index_registry SET last_used_at = CAST(strftime('%s', 'now') AS INTEGER), updated_at = CAST(strftime('%s', 'now') AS INTEGER) diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 2d37ef32bf..7b584c4b6c 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -392,6 +392,66 @@ describe(`persistedCollectionOptions`, () => { expect(collection.utils.getLeadershipState?.().isLeader).toBe(true) }) + it(`hydrates a local-only baseline after a managed write advances startup evidence`, async () => { + const adapter = createRecordingAdapter([ + { id: `baseline`, title: `Persisted before startup` }, + ]) + const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) + let releaseInitialSnapshot!: () => void + const initialSnapshotRelease = new Promise((resolve) => { + releaseInitialSnapshot = resolve + }) + let reportInitialSnapshot!: () => void + const initialSnapshotCaptured = new Promise((resolve) => { + reportInitialSnapshot = resolve + }) + let snapshotCalls = 0 + adapter.loadResumeSnapshot = async (...args) => { + const snapshot = await loadResumeSnapshot(...args) + snapshotCalls++ + if (snapshotCalls === 1) { + reportInitialSnapshot() + await initialSnapshotRelease + } + return snapshot + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `local-only-startup-write`, + startSync: false, + getKey: (item) => item.id, + persistence: { adapter }, + }), + ) + + try { + collection.startSyncImmediate() + await initialSnapshotCaptured + + const insert = collection.insert({ + id: `concurrent`, + title: `Managed during startup`, + }) + releaseInitialSnapshot() + await insert.isPersisted.promise + await collection.stateWhenReady() + await vi.waitFor(() => expect(snapshotCalls).toBeGreaterThanOrEqual(2)) + + expect( + Array.from(collection.values(), (row) => stripVirtualProps(row)).sort( + (left, right) => left.id.localeCompare(right.id), + ), + ).toEqual([ + { id: `baseline`, title: `Persisted before startup` }, + { id: `concurrent`, title: `Managed during startup` }, + ]) + } finally { + releaseInitialSnapshot() + await collection.cleanup() + } + }) + it(`supports acceptMutations for manual transactions`, async () => { const adapter = createRecordingAdapter() const collection = createCollection( @@ -550,6 +610,10 @@ describe(`persistedCollectionOptions`, () => { }) }) + it.todo( + `loads active subsets and collection metadata from one atomic full-reload generation`, + ) + it(`persists metadata-only wrapped sync transactions`, async () => { const adapter = createRecordingAdapter() diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts index 1d4e13faec..207136ab64 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts @@ -1,11 +1,15 @@ import { DatabaseSync } from 'node:sqlite' +import { fc } from '@fast-check/vitest' import { describe, expect, it } from 'vitest' +import { createCollection } from '@tanstack/db' import { SQLiteCorePersistenceAdapter, createPersistedTableName, encodePersistedStorageKey, + persistedCollectionOptions, } from '../src' import type { SQLiteDriver } from '../src' +import type { Collection } from '@tanstack/db' type CachedSchemaState = { schemaVersion: number @@ -225,6 +229,10 @@ async function observeCachedSchemaState( * compares the entire projected schema state; the held boundary and reset epoch * are reach witnesses, while compatible reopen and recertifying truncate cases * prevent an oracle that merely rejects every resume. + * A separate generated startup lane holds the runtime between its metadata and + * hydration snapshots, then crosses no write, a managed public insert, and a + * hostile raw deletion over one-to-three baseline rows. Its independent model + * preserves every managed row but fails closed for unsupported raw loss. * The focused work law first executes one controlled expected-key table read to * prove its SQL observer can detect the forbidden membership work, then resets * the counters before measuring the public position and snapshot operations. @@ -236,6 +244,165 @@ async function observeCachedSchemaState( * and Electric recovery owners. */ describe(`SQLite resume snapshots`, () => { + it(`preserves local-only startup histories across managed generation advancement`, async () => { + const historyArbitrary = fc.record({ + baselineSize: fc.integer({ min: 1, max: 3 }), + transition: fc.constantFrom( + `none` as const, + `managed-insert` as const, + `raw-delete` as const, + ), + }) + + await fc.assert( + fc.asyncProperty(historyArbitrary, async (history) => { + const database = new DatabaseSync(`:memory:`) + let primaryFailure: unknown + let releaseInitialSnapshot = () => {} + let collection: + | Collection<{ id: string; title: string }, string> + | undefined + try { + const driver = createDriver(database) + const collectionId = `generated-local-startup` + const adapter = new SQLiteCorePersistenceAdapter({ driver }) + const baselineRows = Array.from( + { length: history.baselineSize }, + (_, index) => ({ + id: `baseline-${index}`, + title: `baseline-${index}`, + }), + ) + await adapter.applyCommittedTx(collectionId, { + txId: `seed`, + term: 1, + seq: 1, + rowVersion: 1, + truncate: true, + mutations: baselineRows.map((row) => ({ + type: `insert` as const, + key: row.id, + value: row, + })), + }) + + const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) + const reachedInitialSnapshot = deferred() + const reachedHydrationSnapshot = deferred() + const initialSnapshotRelease = deferred() + releaseInitialSnapshot = initialSnapshotRelease.resolve + let snapshotCalls = 0 + adapter.loadResumeSnapshot = async (...args) => { + const snapshot = await loadResumeSnapshot(...args) + snapshotCalls++ + if (snapshotCalls === 1) { + reachedInitialSnapshot.resolve() + await initialSnapshotRelease.promise + } else if (snapshotCalls === 2) { + reachedHydrationSnapshot.resolve() + } + return snapshot + } + + collection = createCollection( + persistedCollectionOptions<{ id: string; title: string }, string>({ + id: collectionId, + startSync: false, + getKey: (row) => row.id, + persistence: { adapter }, + }), + ) + collection.startSyncImmediate() + await reachCheckpoint( + reachedInitialSnapshot.promise, + `generated local startup metadata snapshot`, + ) + + const managedRow = { + id: `managed`, + title: `managed-during-startup`, + } + let managedPersistence: Promise | undefined + let managedPersistenceSettled = false + if (history.transition === `managed-insert`) { + managedPersistence = + collection.insert(managedRow).isPersisted.promise + void managedPersistence.then( + () => { + managedPersistenceSettled = true + }, + () => { + managedPersistenceSettled = true + }, + ) + for (let attempt = 0; attempt < 20; attempt++) { + await Promise.resolve() + } + expect(managedPersistenceSettled).toBe(false) + } else if (history.transition === `raw-delete`) { + const tableName = createPersistedTableName(collectionId, `c`) + await driver.run(`DELETE FROM "${tableName}" WHERE key = ?`, [ + encodePersistedStorageKey(baselineRows[0]!.id), + ]) + } + + releaseInitialSnapshot() + await managedPersistence + await collection.stateWhenReady() + await reachCheckpoint( + reachedHydrationSnapshot.promise, + `generated local startup hydration snapshot`, + ) + const visibleRows = Array.from( + collection.values(), + ({ id, title }) => ({ + id, + title, + }), + ).sort((left, right) => left.id.localeCompare(right.id)) + const expectedRows = + history.transition === `raw-delete` + ? [] + : [ + ...baselineRows, + ...(history.transition === `managed-insert` + ? [managedRow] + : []), + ].sort((left, right) => left.id.localeCompare(right.id)) + + expect(visibleRows).toEqual(expectedRows) + if (history.transition === `managed-insert`) { + const durableRows = (await loadResumeSnapshot(collectionId)).rows + .map(({ value }) => value) + .sort((left, right) => + String(left.id).localeCompare(String(right.id)), + ) + expect(durableRows).toEqual(expectedRows) + } + if (history.transition === `raw-delete`) { + expect((await loadResumeSnapshot(collectionId)).keySet).toEqual({ + status: `incompatible`, + }) + } + } catch (error) { + primaryFailure = error + } finally { + releaseInitialSnapshot() + try { + await collection?.cleanup() + } catch (cleanupError) { + if (primaryFailure === undefined) primaryFailure = cleanupError + } + closeDatabasePreservingPrimary(database, primaryFailure) + } + }), + { + seed: 1659, + numRuns: 12, + }, + ) + }) + it(`reads key-set evidence without rescanning key membership`, async () => { const database = new DatabaseSync(`:memory:`) let primaryFailure: unknown @@ -954,4 +1121,206 @@ describe(`SQLite resume snapshots`, () => { closeDatabasePreservingPrimary(database, primaryFailure) } }) + + it(`rejects a resume snapshot from a cached adapter after another adapter resets the schema`, async () => { + const database = new DatabaseSync(`:memory:`) + let primaryFailure: unknown + try { + const driver = createDriver(database) + const collectionId = `cached-schema-snapshot` + const staleV1 = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 1, + }) + await staleV1.applyCommittedTx(collectionId, { + txId: `seed-v1`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [{ type: `insert`, key: 1, value: { id: 1 } }], + }) + await staleV1.loadResumeSnapshot(collectionId) + + const currentV2 = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 2, + }) + await currentV2.loadSubset(collectionId, {}) + await currentV2.applyCommittedTx(collectionId, { + txId: `seed-v2`, + term: 2, + seq: 1, + rowVersion: 1, + mutations: [{ type: `insert`, key: 2, value: { id: 2 } }], + collectionMetadataMutations: [ + { type: `set`, key: `cursor`, value: `v2` }, + ], + }) + + await expect(staleV1.loadResumeSnapshot(collectionId)).rejects.toThrow( + `Schema version mismatch`, + ) + expect(await currentV2.loadResumeSnapshot(collectionId)).toMatchObject({ + rows: [{ key: 2, value: { id: 2 } }], + collectionMetadata: [{ key: `cursor`, value: `v2` }], + }) + } catch (error) { + primaryFailure = error + } finally { + closeDatabasePreservingPrimary(database, primaryFailure) + } + }) + + it(`rejects cached row readers after another adapter resets the schema`, async () => { + const database = new DatabaseSync(`:memory:`) + let primaryFailure: unknown + try { + const driver = createDriver(database) + const collectionId = `cached-schema-row-readers` + const staleV1 = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 1, + }) + await staleV1.applyCommittedTx(collectionId, { + txId: `seed-v1`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [{ type: `insert`, key: 1, value: { id: 1 } }], + }) + await staleV1.loadSubset(collectionId, {}) + + const currentV2 = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 2, + }) + await currentV2.loadSubset(collectionId, {}) + await currentV2.applyCommittedTx(collectionId, { + txId: `seed-v2`, + term: 2, + seq: 1, + rowVersion: 1, + mutations: [{ type: `insert`, key: 2, value: { id: 2 } }], + }) + + const staleReads = await Promise.allSettled([ + staleV1.loadSubset(collectionId, {}), + staleV1.scanRows(collectionId), + staleV1.pullSince(collectionId, 0), + ]) + expect(staleReads.map(({ status }) => status)).toEqual([ + `rejected`, + `rejected`, + `rejected`, + ]) + expect(await currentV2.loadSubset(collectionId, {})).toMatchObject([ + { key: 2, value: { id: 2 } }, + ]) + } catch (error) { + primaryFailure = error + } finally { + closeDatabasePreservingPrimary(database, primaryFailure) + } + }) + + it(`rejects cached collection-metadata reads after another adapter resets the schema`, async () => { + const database = new DatabaseSync(`:memory:`) + let primaryFailure: unknown + try { + const driver = createDriver(database) + const collectionId = `cached-schema-metadata-reader` + const staleV1 = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 1, + }) + await staleV1.applyCommittedTx(collectionId, { + txId: `seed-v1`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [], + collectionMetadataMutations: [ + { type: `set`, key: `cursor`, value: `v1` }, + ], + }) + await staleV1.loadSubset(collectionId, {}) + + const currentV2 = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 2, + }) + await currentV2.loadSubset(collectionId, {}) + await currentV2.applyCommittedTx(collectionId, { + txId: `seed-v2`, + term: 2, + seq: 1, + rowVersion: 1, + mutations: [], + collectionMetadataMutations: [ + { type: `set`, key: `cursor`, value: `v2` }, + ], + }) + + await expect( + staleV1.loadCollectionMetadata(collectionId), + ).rejects.toThrow(`Schema version mismatch`) + expect(await currentV2.loadCollectionMetadata(collectionId)).toEqual([ + { key: `cursor`, value: `v2` }, + ]) + } catch (error) { + primaryFailure = error + } finally { + closeDatabasePreservingPrimary(database, primaryFailure) + } + }) + + it(`rejects cached index lifecycle writes after another adapter resets the schema`, async () => { + const database = new DatabaseSync(`:memory:`) + let primaryFailure: unknown + try { + const driver = createDriver(database) + const collectionId = `cached-schema-index-writers` + const staleV1 = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 1, + }) + await staleV1.loadSubset(collectionId, {}) + + const currentV2 = new SQLiteCorePersistenceAdapter({ + driver, + schemaVersion: 2, + }) + await currentV2.loadSubset(collectionId, {}) + await currentV2.ensureIndex(collectionId, `v2-index`, { + expressionSql: [`json_extract(value, '$.id')`], + }) + + const staleWrites = await Promise.allSettled([ + staleV1.ensureIndex(collectionId, `stale-v1-index`, { + expressionSql: [`json_extract(value, '$.legacy')`], + }), + staleV1.markIndexRemoved(collectionId, `v2-index`), + ]) + expect(staleWrites.map(({ status }) => status)).toEqual([ + `rejected`, + `rejected`, + ]) + + const currentIndexes = await driver.query<{ + signature: string + removed: number + }>( + `SELECT signature, removed + FROM persisted_index_registry + WHERE collection_id = ? + ORDER BY signature`, + [collectionId], + ) + expect(currentIndexes).toEqual([{ signature: `v2-index`, removed: 0 }]) + } catch (error) { + primaryFailure = error + } finally { + closeDatabasePreservingPrimary(database, primaryFailure) + } + }) }) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index bbe8b3feb9..465b84e483 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -2067,11 +2067,8 @@ function createElectricSync>( await hydrateBaseline() const currentKeySetEvidence = getKeySetEvidence?.() if ( - (canUsePersistedResume && - currentKeySetEvidence?.status !== `consistent`) || - (!canUsePersistedResume && - persistedKeySetEvidence?.status !== `incompatible` && - currentKeySetEvidence?.status === `incompatible`) + canUsePersistedResume && + currentKeySetEvidence?.status !== `consistent` ) { throw new Error( `Electric persisted resume baseline could not be certified during hydration`, diff --git a/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts index d941718909..e39d5a0d63 100644 --- a/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts +++ b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts @@ -121,13 +121,22 @@ function change( } async function runRace( - transition: `none` | `external-row-loss` | `schema-reset` | `committed-write`, + transition: + | `none` + | `external-row-loss` + | `schema-reset` + | `committed-write` + | `committed-replacement`, syncMode: `eager` | `on-demand` = `eager`, legacyUnknown = false, missingKeySetEvidence = false, startupReset: `none` | `tag-state` | `shape-identity` = `none`, metadataWrapper: `none` | `shallow-persistence` = `none`, - laterKeySetEvidence: `unchanged` | `unknown` | `missing` = `unchanged`, + laterKeySetEvidence: + | `unchanged` + | `unknown` + | `missing` + | `incompatible` = `unchanged`, ): Promise { const database = new DatabaseSync(`:memory:`) const driver = createDriver(database) @@ -202,6 +211,7 @@ async function runRace( driver, schemaVersion: 1, }) + let durableObserverAdapter = restartedAdapter let snapshotCalls = 0 let laterSnapshotIncludedRows: boolean | undefined let resumeStateAtLaterSnapshot: unknown @@ -242,7 +252,9 @@ async function runRace( keySet: laterKeySetEvidence === `unknown` ? { status: `unknown` as const } - : undefined, + : laterKeySetEvidence === `incompatible` + ? { status: `incompatible` as const } + : undefined, } } return missingKeySetEvidence @@ -319,6 +331,8 @@ async function runRace( ] } collection.startSyncImmediate() + const readiness = collection.stateWhenReady() + void readiness.catch(() => undefined) const subscription = collection.subscribeChanges( () => { publications++ @@ -379,6 +393,7 @@ async function runRace( schemaVersion: 2, }) await resettingAdapter.loadSubset(collectionId, {}) + durableObserverAdapter = resettingAdapter } else if (transition === `committed-write`) { await seedAdapter.applyCommittedTx(collectionId, { txId: `concurrent-writer`, @@ -389,10 +404,22 @@ async function runRace( { type: `insert`, key: 3, value: { id: 3, name: `three` } }, ], }) + } else if (transition === `committed-replacement`) { + await seedAdapter.applyCommittedTx(collectionId, { + txId: `concurrent-replacement`, + term: 2, + seq: 1, + rowVersion: 2, + truncate: true, + mutations: [ + { type: `insert`, key: 1, value: { id: 1, name: `one` } }, + { type: `insert`, key: 2, value: { id: 2, name: `two` } }, + ], + }) } releaseLaterSnapshot.resolve() - if (transition === `none`) { + if (transition === `none` || replacesUncertifiedBaseline) { await vi.waitFor(() => expect(collection!.status).toBe(`ready`)) expect( Array.from(collection.values(), ({ id, name }) => ({ id, name })), @@ -405,7 +432,7 @@ async function runRace( : [], ) expect( - (await restartedAdapter.loadSubset(collectionId, {})).map( + (await durableObserverAdapter.loadSubset(collectionId, {})).map( ({ value }) => value, ), ).toEqual([ @@ -415,7 +442,7 @@ async function runRace( if (startupReset !== `none`) { await vi.waitFor(async () => { const resumeState = ( - await restartedAdapter.loadCollectionMetadata(collectionId) + await durableObserverAdapter.loadCollectionMetadata(collectionId) ).find(({ key }) => key === `electric:resume`)?.value expect(resumeState).toMatchObject({ kind: `resume`, @@ -426,6 +453,7 @@ async function runRace( } } else { await vi.waitFor(() => expect(collection!.status).toBe(`error`)) + await expect(readiness).rejects.toBe(collection._lifecycle.getSyncError()) if (laterKeySetEvidence !== `unchanged`) { expect(collection._lifecycle.getSyncError()).toEqual( expect.objectContaining({ @@ -438,7 +466,7 @@ async function runRace( } await vi.waitFor(async () => { const metadata = - await restartedAdapter.loadCollectionMetadata(collectionId) + await durableObserverAdapter.loadCollectionMetadata(collectionId) const resumeState = metadata.find( ({ key }) => key === `electric:resume`, )?.value @@ -467,10 +495,8 @@ async function runRace( if (laterKeySetEvidence === `unchanged`) { expect(publicationsBeforeLateDelivery).toBe(0) } - const durableRowsBeforeLateDelivery = await restartedAdapter.loadSubset( - collectionId, - {}, - ) + const durableRowsBeforeLateDelivery = + await durableObserverAdapter.loadSubset(collectionId, {}) expect(durableRowsBeforeLateDelivery.map(({ value }) => value)).toEqual( transition === `external-row-loss` ? [{ id: 2, name: `two` }] @@ -480,7 +506,12 @@ async function runRace( { id: 2, name: `two` }, { id: 3, name: `three` }, ] - : [], + : transition === `committed-replacement` + ? [ + { id: 1, name: `one` }, + { id: 2, name: `two` }, + ] + : [], ) subscriber([ change(`insert`, { id: 9, name: `late` }), @@ -490,7 +521,7 @@ async function runRace( expect( Array.from(collection.values(), ({ id, name }) => ({ id, name })), ).toEqual(expectedErroredRows) - expect(await restartedAdapter.loadSubset(collectionId, {})).toEqual( + expect(await durableObserverAdapter.loadSubset(collectionId, {})).toEqual( durableRowsBeforeLateDelivery, ) expect(publications).toBe(publicationsBeforeLateDelivery) @@ -855,9 +886,28 @@ describe(`Electric resume snapshot races`, () => { await runRace(`external-row-loss`, `on-demand`) }) - it(`rejects a schema reset for an unknown on-demand resume`, async () => { - await runRace(`schema-reset`, `on-demand`, true) - }) + it.each( + ([`eager`, `on-demand`] as const).flatMap((syncMode) => + ([`unknown`, `missing`] as const).flatMap((initialEvidence) => + ([`external-row-loss`, `committed-replacement`] as const).map( + (transition) => ({ syncMode, initialEvidence, transition }), + ), + ), + ), + )( + `freshly replaces a $initialEvidence $syncMode baseline across $transition`, + async ({ syncMode, initialEvidence, transition }) => { + await runRace( + transition, + syncMode, + initialEvidence === `unknown`, + initialEvidence === `missing`, + `none`, + `none`, + transition === `external-row-loss` ? `incompatible` : `unchanged`, + ) + }, + ) it(`freshly replaces an unverifiable pre-ledger resume baseline`, async () => { const observation = await observeLegacyUnknownResume() diff --git a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts index a4c6bb7fe8..ea271d5204 100644 --- a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts +++ b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts @@ -122,6 +122,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin private readonly nodeId = safeRandomUUID() private readonly dbName: string private adapter: AdapterWithPullSince | null + private readonly collectionAdapters = new Map() private readonly channel: BroadcastChannel private readonly collections = new Map() private readonly pendingRPCs = new Map() @@ -133,13 +134,14 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin return this.disposed } - private requireAdapter(): AdapterWithPullSince { - if (!this.adapter) { + private requireAdapter(collectionId: string): AdapterWithPullSince { + const adapter = this.collectionAdapters.get(collectionId) ?? this.adapter + if (!adapter) { throw new Error( `ElectronCollectionCoordinator: adapter not set. Call setAdapter() before using leader-side operations.`, ) } - return this.adapter + return adapter } constructor(options: ElectronCollectionCoordinatorOptions) { @@ -160,6 +162,14 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin this.adapter = adapter } + /** Register the schema/mode-specific adapter for one collection. */ + setCollectionAdapter( + collectionId: string, + adapter: AdapterWithPullSince, + ): void { + this.collectionAdapters.set(collectionId, adapter) + } + // ----------------------------------------------------------------------- // PersistedCollectionCoordinator interface // ----------------------------------------------------------------------- @@ -223,7 +233,11 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin spec: PersistedIndexSpec, ): Promise { if (this.isLeader(collectionId)) { - await this.requireAdapter().ensureIndex(collectionId, signature, spec) + await this.requireAdapter(collectionId).ensureIndex( + collectionId, + signature, + spec, + ) return } @@ -305,6 +319,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin this.channel.close() this.collections.clear() + this.collectionAdapters.clear() } // ----------------------------------------------------------------------- @@ -348,7 +363,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin try { // Restore stream position from DB before claiming leadership - const adapter = this.requireAdapter() + const adapter = this.requireAdapter(collectionId) if (adapter.getStreamPosition) { const pos = await adapter.getStreamPosition(collectionId) state.latestTerm = pos.latestTerm @@ -610,7 +625,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin }, ): Promise { await this.withWriterLock(() => - this.requireAdapter().ensureIndex( + this.requireAdapter(collectionId).ensureIndex( collectionId, request.signature, request.spec, @@ -676,7 +691,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin } await this.withWriterLock(() => - this.requireAdapter().applyCommittedTx(collectionId, tx), + this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), ) // Track envelope for dedup @@ -736,7 +751,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin ): Promise { const state = this.collections.get(collectionId) - const adapter = this.requireAdapter() + const adapter = this.requireAdapter(collectionId) if (!adapter.pullSince) { return { type: `rpc:pullSince:res`, diff --git a/packages/electron-db-sqlite-persistence/src/renderer.ts b/packages/electron-db-sqlite-persistence/src/renderer.ts index b7974344d9..b56e47f2bb 100644 --- a/packages/electron-db-sqlite-persistence/src/renderer.ts +++ b/packages/electron-db-sqlite-persistence/src/renderer.ts @@ -373,35 +373,37 @@ export function createElectronSQLitePersistence( schemaVersion, }) adapterCache.set(cacheKey, adapter) - - // Wire the adapter into the coordinator so it can handle - // leader-side RPCs (applyCommittedTx, pullSince, getStreamPosition, etc.) - if (coordinator instanceof ElectronCollectionCoordinator) { - coordinator.setAdapter(adapter) - } - return adapter } const createCollectionPersistence = ( + collectionId: string | undefined, mode: PersistedCollectionMode, schemaVersion: number | undefined, - ): PersistedCollectionPersistence => ({ - adapter: getAdapterForCollection(mode, schemaVersion), - coordinator, - }) + ): PersistedCollectionPersistence => { + const adapter = getAdapterForCollection(mode, schemaVersion) + if (coordinator instanceof ElectronCollectionCoordinator) { + if (collectionId === undefined) { + coordinator.setAdapter(adapter) + } else { + coordinator.setCollectionAdapter(collectionId, adapter) + } + } + return { adapter, coordinator } + } const defaultPersistence = createCollectionPersistence( + undefined, `sync-absent`, undefined, ) return { ...defaultPersistence, - resolvePersistenceForCollection: ({ mode, schemaVersion }) => - createCollectionPersistence(mode, schemaVersion), + resolvePersistenceForCollection: ({ collectionId, mode, schemaVersion }) => + createCollectionPersistence(collectionId, mode, schemaVersion), // Backward compatible fallback for older callers. resolvePersistenceForMode: (mode) => - createCollectionPersistence(mode, undefined), + createCollectionPersistence(undefined, mode, undefined), } } diff --git a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts index 5afd32a3b4..c0afae9cc0 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts @@ -1,11 +1,15 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterEach, describe, expect, it } from 'vitest' -import { InvalidPersistedCollectionConfigError } from '@tanstack/db-sqlite-persistence-core' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + InvalidPersistedCollectionConfigError, + persistedCollectionOptions, +} from '@tanstack/db-sqlite-persistence-core' import { createNodeSQLitePersistence } from '@tanstack/node-db-sqlite-persistence' import { BetterSqlite3SQLiteDriver } from '../../node-db-sqlite-persistence/src/node-driver' import { + ElectronCollectionCoordinator, createElectronSQLitePersistence, exposeElectronSQLitePersistence, } from '../src' @@ -263,6 +267,118 @@ describe(`electron sqlite persistence bridge`, () => { }) }) + it.each([ + { schemaV1: 1, schemaV2: 2 }, + { schemaV1: 2, schemaV2: 4 }, + ])( + `routes coordinator writes through each collection's renderer adapter: $schemaV1/$schemaV2`, + async ({ schemaV1, schemaV2 }) => { + const dbPath = createTempDbPath() + const invokeHarness = createInvokeHarness(dbPath, `unused`) + activeCleanupFns.push(() => invokeHarness.close()) + const originalNavigator = globalThis.navigator + Object.defineProperty(globalThis, `navigator`, { + configurable: true, + value: { + ...originalNavigator, + locks: { + request: ( + _name: string, + optionsOrCallback: + | { signal?: AbortSignal } + | ((lock: { name: string }) => Promise), + maybeCallback?: (lock: { name: string }) => Promise, + ) => { + const callback = + typeof optionsOrCallback === `function` + ? optionsOrCallback + : maybeCallback! + return callback({ name: _name }) + }, + }, + }, + }) + + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-schema-routing`, + }) + const persistence = createElectronSQLitePersistence({ + invoke: invokeHarness.invoke, + coordinator, + }) + const collectionV1 = `electron-schema-v1` + const collectionV2 = `electron-schema-v2` + + try { + const optionsV1 = persistedCollectionOptions< + { id: string; title: string }, + string + >({ + id: collectionV1, + schemaVersion: schemaV1, + getKey: (row) => row.id, + persistence, + }) + const optionsV2 = persistedCollectionOptions< + { id: string; title: string }, + string + >({ + id: collectionV2, + schemaVersion: schemaV2, + getKey: (row) => row.id, + persistence, + }) + await optionsV1.persistence.adapter.loadResumeSnapshot(collectionV1) + await optionsV2.persistence.adapter.loadResumeSnapshot(collectionV2) + + coordinator.subscribe(collectionV1, () => {}) + coordinator.subscribe(collectionV2, () => {}) + await vi.waitFor(() => { + expect(coordinator.isLeader(collectionV1)).toBe(true) + expect(coordinator.isLeader(collectionV2)).toBe(true) + }) + + const [resultV1, resultV2] = await Promise.all([ + coordinator.requestApplyLocalMutations(collectionV1, [ + { + mutationId: `mutation-v1`, + type: `insert`, + key: `v1`, + value: { id: `v1`, title: `schema one` }, + }, + ]), + coordinator.requestApplyLocalMutations(collectionV2, [ + { + mutationId: `mutation-v2`, + type: `insert`, + key: `v2`, + value: { id: `v2`, title: `schema two` }, + }, + ]), + ]) + + expect(resultV1.ok).toBe(true) + expect(resultV2.ok).toBe(true) + expect( + await optionsV1.persistence.adapter.loadSubset(collectionV1, {}), + ).toMatchObject([ + { key: `v1`, value: { id: `v1`, title: `schema one` } }, + ]) + expect( + await optionsV2.persistence.adapter.loadSubset(collectionV2, {}), + ).toMatchObject([ + { key: `v2`, value: { id: `v2`, title: `schema two` } }, + ]) + } finally { + coordinator.dispose() + Object.defineProperty(globalThis, `navigator`, { + configurable: true, + value: originalNavigator, + }) + } + }, + ) + it(`persists data across main process restarts`, async () => { const dbPath = createTempDbPath() From be803276453d4a07a06f937dcc492b431ec676f4 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 12:01:18 +0100 Subject: [PATCH 20/23] fix(node-sqlite): coordinate shared database handles --- .../preserve-resume-baseline-integrity.md | 3 +- docs/contributing/oracle-coverage.md | 5 +- .../src/node-driver.ts | 51 +++++-- .../tests/node-driver.test.ts | 126 ++++++++++++++++++ 4 files changed, 170 insertions(+), 15 deletions(-) diff --git a/.changeset/preserve-resume-baseline-integrity.md b/.changeset/preserve-resume-baseline-integrity.md index 696e51a1d1..f4692895ad 100644 --- a/.changeset/preserve-resume-baseline-integrity.md +++ b/.changeset/preserve-resume-baseline-integrity.md @@ -5,8 +5,9 @@ '@tanstack/electron-db-sqlite-persistence': minor '@tanstack/electric-db-collection': patch '@tanstack/query-db-collection': patch +'@tanstack/node-db-sqlite-persistence': patch --- Preserve persisted resume integrity with atomic SQLite baseline evidence and stale-writer rejection, expose persistence sync metadata as one versioned capability, and refresh uncertified Electric baselines before publishing resumed data. -This changes the public persistence contracts: custom `PersistenceAdapter` implementations must now implement `loadResumeSnapshot`, and `SyncMetadataApi.persistence` is required with `null` explicitly representing no persistence. Custom sync wrappers must forward `metadata.persistence` unchanged so consumers receive either that sentinel or the complete versioned capability. The Electron bridge now transports the atomic resume snapshot through IPC protocol v2; Electron main and renderer integrations must upgrade together because mixed v1/v2 peers fail closed. +This changes the public persistence contracts: custom `PersistenceAdapter` implementations must now implement `loadResumeSnapshot`, and `SyncMetadataApi.persistence` is required with `null` explicitly representing no persistence. Custom sync wrappers must forward `metadata.persistence` unchanged so consumers receive either that sentinel or the complete versioned capability. The Electron bridge now transports the atomic resume snapshot through IPC protocol v2; Electron main and renderer integrations must upgrade together because mixed v1/v2 peers fail closed. Node persistence instances that wrap one `better-sqlite3` database handle now share transaction admission so concurrent collection startup cannot overlap transactions on that connection. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 1f9b2c5e69..d56e8742dd 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -111,7 +111,7 @@ comment and the current API/architecture contract before extending its model. | Opaque backend pagination | [window oracle](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.oracle.test.ts), [cache histories](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.cache-oracle.test.ts), [cache publication](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.publication-oracle.test.ts), [browser acquisition boundaries](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.boundary-oracle.test.ts), [QueryCollection integration](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.integration.test.ts) | Full filter/sort/slice reference, opaque token transport, actual Query cache expiry/invalidation/GC, forced refresh during growth, protocol failure publication/recovery, bounded slice work, nested cancellation/replacement, reader abort, browser retry defaults, manual-write cache isolation, and production window publications. Stable backend sequences; not snapshot guarantees for changing endpoints. Peek-ahead remains enabled. | | Electric and TrailBase | [Electric histories](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-oracle.property.test.ts), [recovery histories](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts), [held resume snapshots](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts), [PostgreSQL semantics](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/e2e/sql-predicate-semantics.e2e.test.ts), [TrailBase contract](https://github.com/TanStack/db/blob/main/packages/trailbase-db-collection/tests/ORACLE.md) | Installed SDK delivery/framing, independent predicates, exact subscription arguments, restart/reset lineage, held certification races, and late errors. The recovery fixtures use a mocked ShapeStream; they do not establish live Electric-service framing or native persistence-host behavior. | | PowerSync | [tests](https://github.com/TanStack/db/tree/main/packages/powersync-db-collection/tests), `tests/correctness-oracle.test.ts` | Applied receipt positions crossed with held peers, native SQLite/SDK and cleanup evidence. Run the focused owner with the package's `test:oracles` command. A timeout mutant proves a progress failure, not every value assertion. | -| SQLite persistence and native hosts | [persisted histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/persisted.test.ts), [reset/resume histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts), [dual-adapter resume snapshots](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts), [driver contracts](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [browser OPFS lifecycle](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-page-lifecycle-oracle.test.ts), [worker diagnostics](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts), [113-law manifest](https://github.com/TanStack/db/blob/main/packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Cache/remote rejection/peer/reopen histories, atomic reset/resume lineage, key-set evidence, dual-adapter races, exact driver results, controlled page/worker ownership, and diagnostic-cause retention. The reset/resume owners use sqlite3 CLI and in-memory node:sqlite seams; they do not prove multi-process WAL, mobile/Tauri, or other native-device execution. Fake workers and synthetic page events do not prove native handle release or real bfcache admission. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | +| SQLite persistence and native hosts | [persisted histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/persisted.test.ts), [reset/resume histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts), [dual-adapter resume snapshots](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts), [driver contracts](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [Node shared-handle scheduling](https://github.com/TanStack/db/blob/main/packages/node-db-sqlite-persistence/tests/node-driver.test.ts), [browser OPFS lifecycle](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-page-lifecycle-oracle.test.ts), [worker diagnostics](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts), [113-law manifest](https://github.com/TanStack/db/blob/main/packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Cache/remote rejection/peer/reopen histories, atomic reset/resume lineage, key-set evidence, dual-adapter races, exact driver results, same-handle Node transaction admission, controlled page/worker ownership, and diagnostic-cause retention. The reset/resume owners use sqlite3 CLI and in-memory node:sqlite seams; they do not prove multi-process WAL, mobile/Tauri, or other native-device execution. Distinct Node database connections rely on SQLite lock admission rather than one in-process queue. Fake workers and synthetic page events do not prove native handle release or real bfcache admission. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | | Offline execution | [scheduler](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/KeyScheduler.property.test.ts), [leadership](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/leadership-replay.property.test.ts), [settlement](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/transaction-settlement.property.test.ts), [serialization](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/transaction-serializer.property.test.ts) | Declarative FIFO eligibility, per-transaction outcomes, durable state and typed wire trees. Issued work may finish after ownership loss, but new work must not start. Exactly-once network execution is not promised. | | Frameworks | [React conformance](https://github.com/TanStack/db/blob/main/packages/react-db/tests/conformance.test.tsx), [React pagination](https://github.com/TanStack/db/blob/main/packages/react-db/tests/infinite-query-conformance.test.tsx), [shared suites](https://github.com/TanStack/db/tree/main/packages/db-collection-e2e/src/suites) | Exact exposed rows/pages and each framework's own lifecycle cuts. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. | | Structural values and ordered primitives | [hash values](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash.property.test.ts), [hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-graph.property.test.ts), [mixed hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-mixed-graph.property.test.ts), [hash retry](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-failure-retry.property.test.ts), [comparison](https://github.com/TanStack/db/blob/main/packages/db/tests/comparison.property.test.ts), [deep equality](https://github.com/TanStack/db/blob/main/packages/db/tests/utils.property.test.ts), [cursor](https://github.com/TanStack/db/blob/main/packages/db/tests/cursor.property.test.ts), [indexes](https://github.com/TanStack/db/blob/main/packages/db/tests/index-update.property.test.ts), [query identity](https://github.com/TanStack/db/blob/main/packages/db/tests/query/identity-output-shape-oracle.test.ts) | Independent flat values, graph topology, algebraic laws, Map/group/sort recomputation, expression denotation, and compiled output bags. Hash collision freedom is not promised. Unsupported composite cursors reject. | @@ -239,7 +239,8 @@ a hostile wrong-answer control, and an explicit statement of remaining limits. - [x] **Deterministic value-and-work laws.** Pair row correctness with stable statement, scan, trigger, or queue-cardinality observations where the subsystem promises bounded work. Owners: SQLite resume snapshots, Electric - acquisition work, and shared-driver scheduling suites. + acquisition work, shared-driver scheduling suites, and the Node + same-database-handle admission law. - [x] **Startup generation interleavings.** Hold persisted startup between its metadata and hydration snapshots, then cross no write, a managed mutation, and hostile raw loss. Compare public rows with the atomic durable snapshot, diff --git a/packages/node-db-sqlite-persistence/src/node-driver.ts b/packages/node-db-sqlite-persistence/src/node-driver.ts index eb42ee1356..cf4dd34d96 100644 --- a/packages/node-db-sqlite-persistence/src/node-driver.ts +++ b/packages/node-db-sqlite-persistence/src/node-driver.ts @@ -35,6 +35,34 @@ type TransactionContext = { depth: number } +type DatabaseExecutionState = { + transactionContext: AsyncLocalStorage + queue: Promise + nextSavepointId: number +} + +const databaseExecutionStates = new WeakMap< + BetterSqlite3Database, + DatabaseExecutionState +>() + +function getDatabaseExecutionState( + database: BetterSqlite3Database, +): DatabaseExecutionState { + const existing = databaseExecutionStates.get(database) + if (existing) { + return existing + } + + const state: DatabaseExecutionState = { + transactionContext: new AsyncLocalStorage(), + queue: Promise.resolve(), + nextSavepointId: 1, + } + databaseExecutionStates.set(database, state) + return state +} + function assertTransactionCallbackHasDriverArg( fn: (transactionDriver: SQLiteDriver) => Promise, ): void { @@ -56,15 +84,13 @@ function hasExistingDatabase( export class BetterSqlite3SQLiteDriver implements SQLiteDriver { private readonly database: BetterSqlite3Database private readonly ownsDatabase: boolean - private readonly transactionContext = - new AsyncLocalStorage() - private queue: Promise = Promise.resolve() - private nextSavepointId = 1 + private readonly executionState: DatabaseExecutionState constructor(options: BetterSqlite3DriverOptions) { if (hasExistingDatabase(options)) { this.database = options.database this.ownsDatabase = false + this.executionState = getDatabaseExecutionState(this.database) this.applyPragmas(options.pragmas ?? DEFAULT_PRAGMAS) return } @@ -77,6 +103,7 @@ export class BetterSqlite3SQLiteDriver implements SQLiteDriver { this.database = new BetterSqlite3(options.filename, options.options) this.ownsDatabase = true + this.executionState = getDatabaseExecutionState(this.database) this.applyPragmas(options.pragmas ?? DEFAULT_PRAGMAS) } @@ -125,7 +152,7 @@ export class BetterSqlite3SQLiteDriver implements SQLiteDriver { return this.enqueue(async () => { this.database.exec(`BEGIN IMMEDIATE`) try { - const result = await this.transactionContext.run( + const result = await this.executionState.transactionContext.run( { depth: 1 }, async () => fn(this), ) @@ -178,7 +205,7 @@ export class BetterSqlite3SQLiteDriver implements SQLiteDriver { } private isInsideTransaction(): boolean { - return this.transactionContext.getStore() !== undefined + return this.executionState.transactionContext.getStore() !== undefined } private executeQuery( @@ -204,8 +231,8 @@ export class BetterSqlite3SQLiteDriver implements SQLiteDriver { } private enqueue(operation: () => Promise | T): Promise { - const queuedOperation = this.queue.then(operation, operation) - this.queue = queuedOperation.then( + const queuedOperation = this.executionState.queue.then(operation, operation) + this.executionState.queue = queuedOperation.then( () => undefined, () => undefined, ) @@ -215,17 +242,17 @@ export class BetterSqlite3SQLiteDriver implements SQLiteDriver { private async runNestedTransaction( fn: (transactionDriver: SQLiteDriver) => Promise, ): Promise { - const context = this.transactionContext.getStore() + const context = this.executionState.transactionContext.getStore() if (!context) { return fn(this) } - const savepointName = `tsdb_sp_${this.nextSavepointId}` - this.nextSavepointId++ + const savepointName = `tsdb_sp_${this.executionState.nextSavepointId}` + this.executionState.nextSavepointId++ this.database.exec(`SAVEPOINT ${savepointName}`) try { - const result = await this.transactionContext.run( + const result = await this.executionState.transactionContext.run( { depth: context.depth + 1 }, async () => fn(this), ) diff --git a/packages/node-db-sqlite-persistence/tests/node-driver.test.ts b/packages/node-db-sqlite-persistence/tests/node-driver.test.ts index fa356622c2..94527fd497 100644 --- a/packages/node-db-sqlite-persistence/tests/node-driver.test.ts +++ b/packages/node-db-sqlite-persistence/tests/node-driver.test.ts @@ -1,6 +1,8 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' +import BetterSqlite3 from 'better-sqlite3' +import { describe, expect, it } from 'vitest' import { runSQLiteDriverContractSuite } from '../../db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract' import { BetterSqlite3SQLiteDriver } from '../src/node-driver' import type { SQLiteDriverContractHarness } from '../../db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract' @@ -23,3 +25,127 @@ function createDriverHarness(): SQLiteDriverContractHarness { } runSQLiteDriverContractSuite(`better-sqlite3 node driver`, createDriverHarness) + +type SharedDatabaseHarness = { + firstDriver: BetterSqlite3SQLiteDriver + secondDriver: BetterSqlite3SQLiteDriver +} + +async function withSharedDatabaseHandle( + fn: (harness: SharedDatabaseHarness) => Promise, +): Promise { + const tempDirectory = mkdtempSync(join(tmpdir(), `db-node-sqlite-shared-`)) + const dbPath = join(tempDirectory, `state.sqlite`) + const database = new BetterSqlite3(dbPath) + + try { + return await fn({ + firstDriver: new BetterSqlite3SQLiteDriver({ database }), + secondDriver: new BetterSqlite3SQLiteDriver({ database }), + }) + } finally { + database.close() + rmSync(tempDirectory, { recursive: true, force: true }) + } +} + +/** + * The shared contract owns one driver wrapper. This provider-specific law also + * covers the supported case where several persistence instances wrap one + * better-sqlite3 database handle. + * + * Known omission: distinct better-sqlite3 connection objects rely on SQLite's + * own lock admission and are not modeled as one in-process scheduling queue. + */ +describe(`better-sqlite3 shared database handle`, () => { + it(`serializes transactions across driver wrappers`, async () => { + await withSharedDatabaseHandle(async ({ firstDriver, secondDriver }) => { + await firstDriver.exec(`CREATE TABLE events (value INTEGER NOT NULL)`) + + let releaseHold!: () => void + const hold = new Promise((resolve) => { + releaseHold = resolve + }) + let signalEntered!: () => void + const entered = new Promise((resolve) => { + signalEntered = resolve + }) + const firstTransaction = firstDriver.transaction( + async (transactionDriver) => { + await transactionDriver.run( + `INSERT INTO events (value) VALUES (?)`, + [1], + ) + signalEntered() + await hold + await transactionDriver.run( + `INSERT INTO events (value) VALUES (?)`, + [2], + ) + }, + ) + + await entered + let secondSettled = false + const secondTransaction = secondDriver + .transaction(async (transactionDriver) => { + await transactionDriver.run( + `INSERT INTO events (value) VALUES (?)`, + [3], + ) + }) + .then(() => { + secondSettled = true + }) + const observed = Promise.allSettled([firstTransaction, secondTransaction]) + + try { + await Promise.resolve() + expect(secondSettled).toBe(false) + releaseHold() + await Promise.all([firstTransaction, secondTransaction]) + } finally { + releaseHold() + await observed + } + + const rows = await firstDriver.query<{ value: number }>( + `SELECT value FROM events ORDER BY rowid ASC`, + ) + expect(rows.map((row) => row.value)).toEqual([1, 2, 3]) + }) + }) + + it(`uses savepoints for transactions nested through another wrapper`, async () => { + await withSharedDatabaseHandle(async ({ firstDriver, secondDriver }) => { + await firstDriver.exec(`CREATE TABLE events (value INTEGER NOT NULL)`) + + await firstDriver.transaction(async (outerTransactionDriver) => { + await outerTransactionDriver.run( + `INSERT INTO events (value) VALUES (?)`, + [1], + ) + + await expect( + secondDriver.transaction(async (innerTransactionDriver) => { + await innerTransactionDriver.run( + `INSERT INTO events (value) VALUES (?)`, + [2], + ) + throw new Error(`inner failure`) + }), + ).rejects.toThrow(`inner failure`) + + await outerTransactionDriver.run( + `INSERT INTO events (value) VALUES (?)`, + [3], + ) + }) + + const rows = await firstDriver.query<{ value: number }>( + `SELECT value FROM events ORDER BY rowid ASC`, + ) + expect(rows.map((row) => row.value)).toEqual([1, 3]) + }) + }) +}) From 8f31e501c9126612a40c7f81a67ec6f48da168dc Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 12:21:18 +0100 Subject: [PATCH 21/23] fix(react-native-sqlite): coordinate shared database handles --- .../preserve-resume-baseline-integrity.md | 3 +- docs/contributing/oracle-coverage.md | 10 +++- .../src/op-sqlite-driver.ts | 50 ++++++++++++++----- .../tests/op-sqlite-driver.test.ts | 39 +++++++++++++++ 4 files changed, 87 insertions(+), 15 deletions(-) diff --git a/.changeset/preserve-resume-baseline-integrity.md b/.changeset/preserve-resume-baseline-integrity.md index f4692895ad..e8955b4316 100644 --- a/.changeset/preserve-resume-baseline-integrity.md +++ b/.changeset/preserve-resume-baseline-integrity.md @@ -6,8 +6,9 @@ '@tanstack/electric-db-collection': patch '@tanstack/query-db-collection': patch '@tanstack/node-db-sqlite-persistence': patch +'@tanstack/react-native-db-sqlite-persistence': patch --- Preserve persisted resume integrity with atomic SQLite baseline evidence and stale-writer rejection, expose persistence sync metadata as one versioned capability, and refresh uncertified Electric baselines before publishing resumed data. -This changes the public persistence contracts: custom `PersistenceAdapter` implementations must now implement `loadResumeSnapshot`, and `SyncMetadataApi.persistence` is required with `null` explicitly representing no persistence. Custom sync wrappers must forward `metadata.persistence` unchanged so consumers receive either that sentinel or the complete versioned capability. The Electron bridge now transports the atomic resume snapshot through IPC protocol v2; Electron main and renderer integrations must upgrade together because mixed v1/v2 peers fail closed. Node persistence instances that wrap one `better-sqlite3` database handle now share transaction admission so concurrent collection startup cannot overlap transactions on that connection. +This changes the public persistence contracts: custom `PersistenceAdapter` implementations must now implement `loadResumeSnapshot`, and `SyncMetadataApi.persistence` is required with `null` explicitly representing no persistence. Custom sync wrappers must forward `metadata.persistence` unchanged so consumers receive either that sentinel or the complete versioned capability. The Electron bridge now transports the atomic resume snapshot through IPC protocol v2; Electron main and renderer integrations must upgrade together because mixed v1/v2 peers fail closed. Node and React Native persistence instances that wrap one database handle now share transaction admission so concurrent collection startup cannot overlap transactions on that connection. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index d56e8742dd..0c752b98c1 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -62,7 +62,7 @@ that test identifiers must copy production's private data structures. | Opaque backend pagination | Complete | The full-relation value model plus opaque token, cache generation, publication, browser acquisition, and live-window integration owners are literate. | | Electric and TrailBase | Complete | Electric replica and recovery models, installed-SDK HTTP delivery, PostgreSQL serialization, and TrailBase's controlled RecordApi/native-stream lifecycle are literate with their real-provider limits intact. | | PowerSync | Complete | Patch conservation, effective-update receipts, metadata and falsey changes, declared-view keys, transformed schema output, logging, cleanup, and native SQLite reach are literate. | -| SQLite persistence and native hosts | Complete | Persisted hydration/replay and ownership, shared driver transaction laws, OPFS page and diagnostic state machines, and the 113-law native conformance manifest are literate. Native execution remains distinct from registration and shim evidence. | +| SQLite persistence and native hosts | Complete | Persisted hydration/replay and ownership, shared-handle driver transaction laws, OPFS page and diagnostic state machines, and the 113-law native conformance manifest are literate. Native execution remains distinct from registration and shim evidence. | | Offline execution | Complete | FIFO retry, scheduler eligibility, leadership replay, transaction settlement, and typed wire serialization are literate. | | Frameworks | Complete | Shared live-query and infinite-query models are literate. Each framework keeps its own realm, ownership, and scheduling driver. | | Structural values and ordered primitives | Complete | Structural hashing, deep equality, comparison, cursor denotation, index refinement, and query-identity output equivalence are literate. | @@ -111,7 +111,7 @@ comment and the current API/architecture contract before extending its model. | Opaque backend pagination | [window oracle](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.oracle.test.ts), [cache histories](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.cache-oracle.test.ts), [cache publication](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.publication-oracle.test.ts), [browser acquisition boundaries](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.boundary-oracle.test.ts), [QueryCollection integration](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.integration.test.ts) | Full filter/sort/slice reference, opaque token transport, actual Query cache expiry/invalidation/GC, forced refresh during growth, protocol failure publication/recovery, bounded slice work, nested cancellation/replacement, reader abort, browser retry defaults, manual-write cache isolation, and production window publications. Stable backend sequences; not snapshot guarantees for changing endpoints. Peek-ahead remains enabled. | | Electric and TrailBase | [Electric histories](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-oracle.property.test.ts), [recovery histories](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts), [held resume snapshots](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts), [PostgreSQL semantics](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/e2e/sql-predicate-semantics.e2e.test.ts), [TrailBase contract](https://github.com/TanStack/db/blob/main/packages/trailbase-db-collection/tests/ORACLE.md) | Installed SDK delivery/framing, independent predicates, exact subscription arguments, restart/reset lineage, held certification races, and late errors. The recovery fixtures use a mocked ShapeStream; they do not establish live Electric-service framing or native persistence-host behavior. | | PowerSync | [tests](https://github.com/TanStack/db/tree/main/packages/powersync-db-collection/tests), `tests/correctness-oracle.test.ts` | Applied receipt positions crossed with held peers, native SQLite/SDK and cleanup evidence. Run the focused owner with the package's `test:oracles` command. A timeout mutant proves a progress failure, not every value assertion. | -| SQLite persistence and native hosts | [persisted histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/persisted.test.ts), [reset/resume histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts), [dual-adapter resume snapshots](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts), [driver contracts](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [Node shared-handle scheduling](https://github.com/TanStack/db/blob/main/packages/node-db-sqlite-persistence/tests/node-driver.test.ts), [browser OPFS lifecycle](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-page-lifecycle-oracle.test.ts), [worker diagnostics](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts), [113-law manifest](https://github.com/TanStack/db/blob/main/packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Cache/remote rejection/peer/reopen histories, atomic reset/resume lineage, key-set evidence, dual-adapter races, exact driver results, same-handle Node transaction admission, controlled page/worker ownership, and diagnostic-cause retention. The reset/resume owners use sqlite3 CLI and in-memory node:sqlite seams; they do not prove multi-process WAL, mobile/Tauri, or other native-device execution. Distinct Node database connections rely on SQLite lock admission rather than one in-process queue. Fake workers and synthetic page events do not prove native handle release or real bfcache admission. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | +| SQLite persistence and native hosts | [persisted histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/persisted.test.ts), [reset/resume histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts), [dual-adapter resume snapshots](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts), [driver contracts](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [Node shared-handle scheduling](https://github.com/TanStack/db/blob/main/packages/node-db-sqlite-persistence/tests/node-driver.test.ts), [OP-SQLite shared-handle scheduling](https://github.com/TanStack/db/blob/main/packages/react-native-db-sqlite-persistence/tests/op-sqlite-driver.test.ts), [browser OPFS lifecycle](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-page-lifecycle-oracle.test.ts), [worker diagnostics](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts), [113-law manifest](https://github.com/TanStack/db/blob/main/packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Cache/remote rejection/peer/reopen histories, atomic reset/resume lineage, key-set evidence, dual-adapter races, exact driver results, same-handle Node and OP-SQLite transaction admission, controlled page/worker ownership, and diagnostic-cause retention. The reset/resume owners use sqlite3 CLI and in-memory node:sqlite seams; they do not prove multi-process WAL, mobile/Tauri, or other native-device execution. Distinct database handles rely on SQLite lock admission rather than one in-process queue. React Native hosts without async-context propagation must use the transaction driver supplied to the callback for nested work. Fake workers and synthetic page events do not prove native handle release or real bfcache admission. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | | Offline execution | [scheduler](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/KeyScheduler.property.test.ts), [leadership](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/leadership-replay.property.test.ts), [settlement](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/transaction-settlement.property.test.ts), [serialization](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/transaction-serializer.property.test.ts) | Declarative FIFO eligibility, per-transaction outcomes, durable state and typed wire trees. Issued work may finish after ownership loss, but new work must not start. Exactly-once network execution is not promised. | | Frameworks | [React conformance](https://github.com/TanStack/db/blob/main/packages/react-db/tests/conformance.test.tsx), [React pagination](https://github.com/TanStack/db/blob/main/packages/react-db/tests/infinite-query-conformance.test.tsx), [shared suites](https://github.com/TanStack/db/tree/main/packages/db-collection-e2e/src/suites) | Exact exposed rows/pages and each framework's own lifecycle cuts. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. | | Structural values and ordered primitives | [hash values](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash.property.test.ts), [hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-graph.property.test.ts), [mixed hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-mixed-graph.property.test.ts), [hash retry](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-failure-retry.property.test.ts), [comparison](https://github.com/TanStack/db/blob/main/packages/db/tests/comparison.property.test.ts), [deep equality](https://github.com/TanStack/db/blob/main/packages/db/tests/utils.property.test.ts), [cursor](https://github.com/TanStack/db/blob/main/packages/db/tests/cursor.property.test.ts), [indexes](https://github.com/TanStack/db/blob/main/packages/db/tests/index-update.property.test.ts), [query identity](https://github.com/TanStack/db/blob/main/packages/db/tests/query/identity-output-shape-oracle.test.ts) | Independent flat values, graph topology, algebraic laws, Map/group/sort recomputation, expression denotation, and compiled output bags. Hash collision freedom is not promised. Unsupported composite cursors reject. | @@ -214,6 +214,12 @@ a hostile wrong-answer control, and an explicit statement of remaining limits. documented rejection. Owner: `packages/react-native-db-sqlite-persistence/tests/op-sqlite-driver.test.ts` and the shared SQLite driver contract. +- [x] **Shared-handle transaction admission.** Concurrent driver wrappers for + one provider database handle serialize root transactions. Nested work must + use the transaction driver supplied to the callback on hosts without async + context propagation. Owners: + `packages/node-db-sqlite-persistence/tests/node-driver.test.ts` and + `packages/react-native-db-sqlite-persistence/tests/op-sqlite-driver.test.ts`. - [x] **Transitions at every relevant await.** Hold each coordination boundary, then change leadership, remote-subset ownership, abort state, cleanup, or restart generation before release. Owners: browser/electron coordinator, diff --git a/packages/react-native-db-sqlite-persistence/src/op-sqlite-driver.ts b/packages/react-native-db-sqlite-persistence/src/op-sqlite-driver.ts index 3fae4aaf86..8cab5e35de 100644 --- a/packages/react-native-db-sqlite-persistence/src/op-sqlite-driver.ts +++ b/packages/react-native-db-sqlite-persistence/src/op-sqlite-driver.ts @@ -59,6 +59,34 @@ type AsyncLocalStorageLike = { type AsyncLocalStorageCtor = new () => AsyncLocalStorageLike +type DatabaseExecutionState = { + queue: Promise + nextSavepointId: number + transactionContextStoragePromise: Promise | null> | null +} + +const databaseExecutionStates = new WeakMap< + OpSQLiteDatabaseLike, + DatabaseExecutionState +>() + +function getDatabaseExecutionState( + database: OpSQLiteDatabaseLike, +): DatabaseExecutionState { + const existing = databaseExecutionStates.get(database) + if (existing) { + return existing + } + + const state: DatabaseExecutionState = { + queue: Promise.resolve(), + nextSavepointId: 1, + transactionContextStoragePromise: null, + } + databaseExecutionStates.set(database, state) + return state +} + let asyncLocalStorageCtorPromise: Promise | null = null @@ -261,10 +289,7 @@ export class OpSQLiteDriver implements SQLiteDriver { private readonly database: OpSQLiteDatabaseLike private readonly executeMethod: OpSQLiteExecuteFn private readonly ownsDatabase: boolean - private queue: Promise = Promise.resolve() - private nextSavepointId = 1 - private transactionContextStoragePromise: Promise | null> | null = - null + private readonly executionState: DatabaseExecutionState constructor(options: OpSQLiteDriverOptions) { if (hasExistingDatabase(options)) { @@ -276,6 +301,7 @@ export class OpSQLiteDriver implements SQLiteDriver { } this.executeMethod = resolveExecuteMethod(this.database) + this.executionState = getDatabaseExecutionState(this.database) } async exec(sql: string): Promise { @@ -389,8 +415,8 @@ export class OpSQLiteDriver implements SQLiteDriver { } private enqueue(operation: () => Promise): Promise { - const queuedOperation = this.queue.then(operation, operation) - this.queue = queuedOperation.then( + const queuedOperation = this.executionState.queue.then(operation, operation) + this.executionState.queue = queuedOperation.then( () => undefined, () => undefined, ) @@ -398,11 +424,11 @@ export class OpSQLiteDriver implements SQLiteDriver { } private async getTransactionContextStorage(): Promise | null> { - if (this.transactionContextStoragePromise) { - return this.transactionContextStoragePromise + if (this.executionState.transactionContextStoragePromise) { + return this.executionState.transactionContextStoragePromise } - this.transactionContextStoragePromise = (async () => { + this.executionState.transactionContextStoragePromise = (async () => { const asyncLocalStorageCtor = await resolveAsyncLocalStorageCtor() if (!asyncLocalStorageCtor) { return null @@ -411,7 +437,7 @@ export class OpSQLiteDriver implements SQLiteDriver { return new asyncLocalStorageCtor() })() - return this.transactionContextStoragePromise + return this.executionState.transactionContextStoragePromise } private async getActiveTransactionDriver(): Promise { @@ -467,8 +493,8 @@ export class OpSQLiteDriver implements SQLiteDriver { transactionDriver: SQLiteDriver, fn: (transactionDriver: SQLiteDriver) => Promise, ): Promise { - const savepointName = `tsdb_sp_${this.nextSavepointId}` - this.nextSavepointId++ + const savepointName = `tsdb_sp_${this.executionState.nextSavepointId}` + this.executionState.nextSavepointId++ await this.execute(`SAVEPOINT ${savepointName}`) try { diff --git a/packages/react-native-db-sqlite-persistence/tests/op-sqlite-driver.test.ts b/packages/react-native-db-sqlite-persistence/tests/op-sqlite-driver.test.ts index 8c482af1c2..ca92b71470 100644 --- a/packages/react-native-db-sqlite-persistence/tests/op-sqlite-driver.test.ts +++ b/packages/react-native-db-sqlite-persistence/tests/op-sqlite-driver.test.ts @@ -256,6 +256,45 @@ it(`serializes unrelated operations behind an active transaction`, async () => { expect(rows).toEqual([{ id: `outside`, title: `Outside transaction` }]) }) +it(`serializes transactions across drivers for one database handle`, async () => { + const dbPath = createTempSqlitePath() + const database = createOpSQLiteTestDatabase({ filename: dbPath }) + activeCleanupFns.push(() => Promise.resolve(database.close())) + + const firstDriver = new OpSQLiteDriver({ database }) + const secondDriver = new OpSQLiteDriver({ database }) + await firstDriver.exec(`CREATE TABLE shared_tx_test (value INTEGER NOT NULL)`) + + const firstTransaction = firstDriver.transaction( + async (transactionDriver) => { + await transactionDriver.run( + `INSERT INTO shared_tx_test (value) VALUES (?)`, + [1], + ) + await Promise.resolve() + await transactionDriver.run( + `INSERT INTO shared_tx_test (value) VALUES (?)`, + [2], + ) + }, + ) + const secondTransaction = secondDriver.transaction( + async (transactionDriver) => { + await transactionDriver.run( + `INSERT INTO shared_tx_test (value) VALUES (?)`, + [3], + ) + }, + ) + + await Promise.all([firstTransaction, secondTransaction]) + + const rows = await firstDriver.query<{ value: number }>( + `SELECT value FROM shared_tx_test ORDER BY rowid ASC`, + ) + expect(rows.map((row) => row.value)).toEqual([1, 2, 3]) +}) + it(`throws config error when db execute methods are missing`, () => { expect(() => new OpSQLiteDriver({ database: {} as never })).toThrowError( InvalidPersistedCollectionConfigError, From 0bdd139d7a27634a34d771c4c7ba15e520659ca5 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 14:19:15 +0100 Subject: [PATCH 22/23] fix(sync): accept omitted optional metadata --- .../preserve-resume-baseline-integrity.md | 2 +- .../electric-db-collection/src/electric.ts | 7 ++-- .../tests/electric.test.ts | 39 +++++++++++++++++++ packages/query-db-collection/src/query.ts | 5 ++- .../query-db-collection/tests/query.test.ts | 35 +++++++++++++++++ 5 files changed, 83 insertions(+), 5 deletions(-) diff --git a/.changeset/preserve-resume-baseline-integrity.md b/.changeset/preserve-resume-baseline-integrity.md index e8955b4316..38a15669fe 100644 --- a/.changeset/preserve-resume-baseline-integrity.md +++ b/.changeset/preserve-resume-baseline-integrity.md @@ -11,4 +11,4 @@ Preserve persisted resume integrity with atomic SQLite baseline evidence and stale-writer rejection, expose persistence sync metadata as one versioned capability, and refresh uncertified Electric baselines before publishing resumed data. -This changes the public persistence contracts: custom `PersistenceAdapter` implementations must now implement `loadResumeSnapshot`, and `SyncMetadataApi.persistence` is required with `null` explicitly representing no persistence. Custom sync wrappers must forward `metadata.persistence` unchanged so consumers receive either that sentinel or the complete versioned capability. The Electron bridge now transports the atomic resume snapshot through IPC protocol v2; Electron main and renderer integrations must upgrade together because mixed v1/v2 peers fail closed. Node and React Native persistence instances that wrap one database handle now share transaction admission so concurrent collection startup cannot overlap transactions on that connection. +This changes the public persistence contracts: custom `PersistenceAdapter` implementations must now implement `loadResumeSnapshot`, and `SyncMetadataApi.persistence` is required with `null` explicitly representing no persistence. Custom sync wrappers that receive metadata must forward `metadata.persistence` unchanged so consumers receive either that sentinel or the complete versioned capability. A direct sync invocation may still omit the optional metadata object entirely, which consumers treat as no persistence. The Electron bridge now transports the atomic resume snapshot through IPC protocol v2; Electron main and renderer integrations must upgrade together because mixed v1/v2 peers fail closed. Node and React Native persistence instances that wrap one database handle now share transaction admission so concurrent collection startup cannot overlap transactions on that connection. diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 465b84e483..fb0bb614ec 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -1748,9 +1748,10 @@ function createElectricSync>( return parseElectricResumeState(persistedResumeState) } - const persistence = validateSyncPersistenceCapability( - metadata?.persistence, - ) + const persistence = + metadata === undefined + ? null + : validateSyncPersistenceCapability(metadata.persistence) const hydrateBaseline = persistence?.hydrateBaseline const resumeSnapshot = persistence?.resumeSnapshot const certifyResumeSnapshot = resumeSnapshot?.certify diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 3fb8c72bc1..2c77568fba 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -4022,6 +4022,45 @@ describe(`Electric Integration`, () => { ) }) + it(`treats omitted optional sync metadata as no persistence`, async () => { + vi.clearAllMocks() + const options = electricCollectionOptions({ + id: `omitted-sync-metadata-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (item) => item.id as number, + startSync: false, + }) + const electricSync = options.sync + const collectionWithoutMetadata = createCollection({ + ...options, + sync: { + sync: (params: Parameters[0]) => { + const { metadata: _omitted, ...paramsWithoutMetadata } = params + return electricSync.sync(paramsWithoutMetadata) + }, + }, + }) + + let startError: unknown + try { + collectionWithoutMetadata.startSyncImmediate() + } catch (error) { + startError = error + } finally { + await collectionWithoutMetadata.cleanup() + } + + expect(startError).toBeUndefined() + expect(ShapeStream).toHaveBeenCalledWith( + expect.objectContaining({ + url: `http://test-url`, + }), + ) + }) + it(`rejects a sync wrapper that drops the entire persistence field before opening ShapeStream`, async () => { vi.clearAllMocks() const { ShapeStream } = await import(`@electric-sql/client`) diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index 60f3753808..71351b5d02 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -982,7 +982,10 @@ export function queryCollectionOptions( ) const { begin, write, commit, markReady, markError, collection, metadata } = params - const persistence = validateSyncPersistenceCapability(metadata?.persistence) + const persistence = + metadata === undefined + ? null + : validateSyncPersistenceCapability(metadata.persistence) // Track whether sync has been started let syncStarted = false diff --git a/packages/query-db-collection/tests/query.test.ts b/packages/query-db-collection/tests/query.test.ts index 9379b56f1f..2401fc86e9 100644 --- a/packages/query-db-collection/tests/query.test.ts +++ b/packages/query-db-collection/tests/query.test.ts @@ -348,6 +348,41 @@ describe(`QueryCollection`, () => { await collection.cleanup() }) + it(`treats omitted optional sync metadata as no persistence`, async () => { + const queryFn = vi.fn().mockResolvedValue([{ id: `1`, name: `Item 1` }]) + const options = queryCollectionOptions({ + id: `omitted-sync-metadata-test`, + queryClient, + queryKey: [`omitted-sync-metadata-test`], + queryFn, + getKey, + startSync: false, + }) + const querySync = options.sync + const collectionWithoutMetadata = createCollection({ + ...options, + sync: { + sync: (params: Parameters[0]) => { + const { metadata: _omitted, ...paramsWithoutMetadata } = params + return querySync.sync(paramsWithoutMetadata) + }, + }, + }) + + let startError: unknown + try { + collectionWithoutMetadata.startSyncImmediate() + await collectionWithoutMetadata.stateWhenReady() + } catch (error) { + startError = error + } finally { + await collectionWithoutMetadata.cleanup() + } + + expect(startError).toBeUndefined() + expect(queryFn).toHaveBeenCalledOnce() + }) + it(`rejects a sync wrapper that drops the entire persistence field before querying`, async () => { const queryFn = vi.fn().mockResolvedValue([{ id: `1`, name: `Item 1` }]) const options = queryCollectionOptions({ From 06eb56206297d5bc1fda69cbb182fb64f6779077 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 15:16:31 +0100 Subject: [PATCH 23/23] test(oracles): add persistence replay lanes --- .../tests/sqlite-resume-snapshot.test.ts | 335 ++++++++++-------- packages/db/tests/oracle-config.ts | 3 + packages/db/tests/oracle-replay-manifest.ts | 12 +- .../tests/electric-oracle.property.test.ts | 138 +++++--- .../tests/electric-recovery-oracle.test.ts | 243 ++++++++----- 5 files changed, 426 insertions(+), 305 deletions(-) diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts index 207136ab64..3bd4a37862 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts @@ -1,7 +1,8 @@ import { DatabaseSync } from 'node:sqlite' -import { fc } from '@fast-check/vitest' +import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect, it } from 'vitest' import { createCollection } from '@tanstack/db' +import { oraclePropertyOptions, oracleRuns } from '../../db/tests/oracle-config' import { SQLiteCorePersistenceAdapter, createPersistedTableName, @@ -244,165 +245,195 @@ async function observeCachedSchemaState( * and Electric recovery owners. */ describe(`SQLite resume snapshots`, () => { - it(`preserves local-only startup histories across managed generation advancement`, async () => { - const historyArbitrary = fc.record({ - baselineSize: fc.integer({ min: 1, max: 3 }), - transition: fc.constantFrom( - `none` as const, - `managed-insert` as const, - `raw-delete` as const, - ), - }) + const startupHistoryArbitrary = fc.record({ + baselineSize: fc.integer({ min: 1, max: 3 }), + transition: fc.constantFrom( + `none` as const, + `managed-insert` as const, + `raw-delete` as const, + ), + }) - await fc.assert( - fc.asyncProperty(historyArbitrary, async (history) => { - const database = new DatabaseSync(`:memory:`) - let primaryFailure: unknown - let releaseInitialSnapshot = () => {} - let collection: - | Collection<{ id: string; title: string }, string> - | undefined - try { - const driver = createDriver(database) - const collectionId = `generated-local-startup` - const adapter = new SQLiteCorePersistenceAdapter({ driver }) - const baselineRows = Array.from( - { length: history.baselineSize }, - (_, index) => ({ - id: `baseline-${index}`, - title: `baseline-${index}`, - }), - ) - await adapter.applyCommittedTx(collectionId, { - txId: `seed`, - term: 1, - seq: 1, - rowVersion: 1, - truncate: true, - mutations: baselineRows.map((row) => ({ - type: `insert` as const, - key: row.id, - value: row, - })), - }) - - const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) - const reachedInitialSnapshot = deferred() - const reachedHydrationSnapshot = deferred() - const initialSnapshotRelease = deferred() - releaseInitialSnapshot = initialSnapshotRelease.resolve - let snapshotCalls = 0 - adapter.loadResumeSnapshot = async (...args) => { - const snapshot = await loadResumeSnapshot(...args) - snapshotCalls++ - if (snapshotCalls === 1) { - reachedInitialSnapshot.resolve() - await initialSnapshotRelease.promise - } else if (snapshotCalls === 2) { - reachedHydrationSnapshot.resolve() - } - return snapshot - } + type StartupRow = { id: string; title: string } + const expectStartupRows = ( + actual: Array, + expected: Array, + ) => expect(actual).toEqual(expected) - collection = createCollection( - persistedCollectionOptions<{ id: string; title: string }, string>({ - id: collectionId, - startSync: false, - getKey: (row) => row.id, - persistence: { adapter }, - }), - ) - collection.startSyncImmediate() - await reachCheckpoint( - reachedInitialSnapshot.promise, - `generated local startup metadata snapshot`, - ) + const assertStartupHistory = async (history: { + baselineSize: number + transition: `none` | `managed-insert` | `raw-delete` + }) => { + const database = new DatabaseSync(`:memory:`) + let primaryFailure: unknown + let releaseInitialSnapshot = () => {} + let collection: + | Collection<{ id: string; title: string }, string> + | undefined + try { + const driver = createDriver(database) + const collectionId = `generated-local-startup` + const adapter = new SQLiteCorePersistenceAdapter({ driver }) + const baselineRows = Array.from( + { length: history.baselineSize }, + (_, index) => ({ + id: `baseline-${index}`, + title: `baseline-${index}`, + }), + ) + await adapter.applyCommittedTx(collectionId, { + txId: `seed`, + term: 1, + seq: 1, + rowVersion: 1, + truncate: true, + mutations: baselineRows.map((row) => ({ + type: `insert` as const, + key: row.id, + value: row, + })), + }) - const managedRow = { - id: `managed`, - title: `managed-during-startup`, - } - let managedPersistence: Promise | undefined - let managedPersistenceSettled = false - if (history.transition === `managed-insert`) { - managedPersistence = - collection.insert(managedRow).isPersisted.promise - void managedPersistence.then( - () => { - managedPersistenceSettled = true - }, - () => { - managedPersistenceSettled = true - }, - ) - for (let attempt = 0; attempt < 20; attempt++) { - await Promise.resolve() - } - expect(managedPersistenceSettled).toBe(false) - } else if (history.transition === `raw-delete`) { - const tableName = createPersistedTableName(collectionId, `c`) - await driver.run(`DELETE FROM "${tableName}" WHERE key = ?`, [ - encodePersistedStorageKey(baselineRows[0]!.id), - ]) - } + const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) + const reachedInitialSnapshot = deferred() + const reachedHydrationSnapshot = deferred() + const initialSnapshotRelease = deferred() + releaseInitialSnapshot = initialSnapshotRelease.resolve + let snapshotCalls = 0 + adapter.loadResumeSnapshot = async (...args) => { + const snapshot = await loadResumeSnapshot(...args) + snapshotCalls++ + if (snapshotCalls === 1) { + reachedInitialSnapshot.resolve() + await initialSnapshotRelease.promise + } else if (snapshotCalls === 2) { + reachedHydrationSnapshot.resolve() + } + return snapshot + } - releaseInitialSnapshot() - await managedPersistence - await collection.stateWhenReady() - await reachCheckpoint( - reachedHydrationSnapshot.promise, - `generated local startup hydration snapshot`, - ) - const visibleRows = Array.from( - collection.values(), - ({ id, title }) => ({ - id, - title, - }), - ).sort((left, right) => left.id.localeCompare(right.id)) - const expectedRows = - history.transition === `raw-delete` - ? [] - : [ - ...baselineRows, - ...(history.transition === `managed-insert` - ? [managedRow] - : []), - ].sort((left, right) => left.id.localeCompare(right.id)) - - expect(visibleRows).toEqual(expectedRows) - if (history.transition === `managed-insert`) { - const durableRows = (await loadResumeSnapshot(collectionId)).rows - .map(({ value }) => value) - .sort((left, right) => - String(left.id).localeCompare(String(right.id)), - ) - expect(durableRows).toEqual(expectedRows) - } - if (history.transition === `raw-delete`) { - expect((await loadResumeSnapshot(collectionId)).keySet).toEqual({ - status: `incompatible`, - }) - } - } catch (error) { - primaryFailure = error - } finally { - releaseInitialSnapshot() - try { - await collection?.cleanup() - } catch (cleanupError) { - if (primaryFailure === undefined) primaryFailure = cleanupError - } - closeDatabasePreservingPrimary(database, primaryFailure) + collection = createCollection( + persistedCollectionOptions<{ id: string; title: string }, string>({ + id: collectionId, + startSync: false, + getKey: (row) => row.id, + persistence: { adapter }, + }), + ) + collection.startSyncImmediate() + await reachCheckpoint( + reachedInitialSnapshot.promise, + `generated local startup metadata snapshot`, + ) + + const managedRow = { + id: `managed`, + title: `managed-during-startup`, + } + let managedPersistence: Promise | undefined + let managedPersistenceSettled = false + if (history.transition === `managed-insert`) { + managedPersistence = collection.insert(managedRow).isPersisted.promise + void managedPersistence.then( + () => { + managedPersistenceSettled = true + }, + () => { + managedPersistenceSettled = true + }, + ) + for (let attempt = 0; attempt < 20; attempt++) { + await Promise.resolve() } - }), - { - seed: 1659, - numRuns: 12, - }, + expect(managedPersistenceSettled).toBe(false) + } else if (history.transition === `raw-delete`) { + const tableName = createPersistedTableName(collectionId, `c`) + await driver.run(`DELETE FROM "${tableName}" WHERE key = ?`, [ + encodePersistedStorageKey(baselineRows[0]!.id), + ]) + } + + releaseInitialSnapshot() + await managedPersistence + await collection.stateWhenReady() + await reachCheckpoint( + reachedHydrationSnapshot.promise, + `generated local startup hydration snapshot`, + ) + const visibleRows = Array.from(collection.values(), ({ id, title }) => ({ + id, + title, + })).sort((left, right) => left.id.localeCompare(right.id)) + const expectedRows = + history.transition === `raw-delete` + ? [] + : [ + ...baselineRows, + ...(history.transition === `managed-insert` ? [managedRow] : []), + ].sort((left, right) => left.id.localeCompare(right.id)) + + expectStartupRows(visibleRows, expectedRows) + if (history.transition === `managed-insert`) { + const durableRows = (await loadResumeSnapshot(collectionId)).rows + .map(({ value }) => value) + .sort((left, right) => + String(left.id).localeCompare(String(right.id)), + ) + expect(durableRows).toEqual(expectedRows) + } + if (history.transition === `raw-delete`) { + expect((await loadResumeSnapshot(collectionId)).keySet).toEqual({ + status: `incompatible`, + }) + } + } catch (error) { + primaryFailure = error + } finally { + releaseInitialSnapshot() + try { + await collection?.cleanup() + } catch (cleanupError) { + if (primaryFailure === undefined) primaryFailure = cleanupError + } + closeDatabasePreservingPrimary(database, primaryFailure) + } + } + + it(`rejects a startup answer that suppresses the older baseline after a managed generation advance`, () => { + const baseline = { id: `baseline-0`, title: `baseline-0` } + const managed = { id: `managed`, title: `managed-during-startup` } + expect(() => expectStartupRows([managed], [baseline, managed])).toThrow() + }) + + it(`fixed startup corpus reaches every baseline size and transition`, () => { + const histories = fc.sample(startupHistoryArbitrary, { + seed: 1659, + numRuns: 12, + }) + expect(new Set(histories.map(({ baselineSize }) => baselineSize))).toEqual( + new Set([1, 2, 3]), + ) + expect(new Set(histories.map(({ transition }) => transition))).toEqual( + new Set([`none`, `managed-insert`, `raw-delete`]), ) }) + fcTest.prop([startupHistoryArbitrary], { + seed: 1659, + numRuns: oracleRuns(12), + })( + `preserves local-only startup histories across managed generation advancement (fixed)`, + assertStartupHistory, + ) + + fcTest.prop( + [startupHistoryArbitrary], + oraclePropertyOptions(12, `sqlite-resume.startup-generation`), + )( + `preserves local-only startup histories across managed generation advancement (random or replayed)`, + assertStartupHistory, + ) + it(`reads key-set evidence without rescanning key membership`, async () => { const database = new DatabaseSync(`:memory:`) let primaryFailure: unknown diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 00b8eab32d..767ed24b73 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -23,8 +23,11 @@ const staticOracleProperties = [ `electric.bound-descriptor-history`, `electric.persisted-tag-history`, `electric.match-reentry`, + `electric.publication-epoch-convergence`, `electric.sdk-snapshot-delivery`, `electric.sdk-dnf-membership`, + `electric-recovery.publication-stream-convergence`, + `sqlite-resume.startup-generation`, `collection-sync.reentrant-drain`, `collection-state.retention`, `collection-state.optimistic-history`, diff --git a/packages/db/tests/oracle-replay-manifest.ts b/packages/db/tests/oracle-replay-manifest.ts index 7b1eeae867..b1b25ddd6c 100644 --- a/packages/db/tests/oracle-replay-manifest.ts +++ b/packages/db/tests/oracle-replay-manifest.ts @@ -39,7 +39,12 @@ const ownerGroups: ReadonlyArray = [ [ `electric-db-collection/tests/electric-oracle.property.test.ts`, `electric`, - `match-reentry`, + `match-reentry publication-epoch-convergence`, + ], + [ + `electric-db-collection/tests/electric-recovery-oracle.test.ts`, + `electric-recovery`, + `publication-stream-convergence`, ], [ `electric-db-collection/tests/electric-sdk-delivery.property.test.ts`, @@ -91,6 +96,11 @@ const ownerGroups: ReadonlyArray = [ `live-query-observer`, `granular-history wholesale-history`, ], + [ + `db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts`, + `sqlite-resume`, + `startup-generation`, + ], [ `db/tests/query/derived-delete-reconciliation.test.ts`, `derived-publication`, diff --git a/packages/electric-db-collection/tests/electric-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-oracle.property.test.ts index f69a15973a..8d20a12554 100644 --- a/packages/electric-db-collection/tests/electric-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-oracle.property.test.ts @@ -2224,69 +2224,91 @@ describe(`Electric adapter laws`, () => { }, ) + const publicationEpochTokensArbitrary = fc.array(designTokenArb, { + minLength: 1, + maxLength: 7, + }) + const publicationEpochPartitionArbitrary = fc.nat() + const assertPublicationEpochHistory = async ( + tokens: Array, + partitionSeed: number, + ) => { + const messages = buildDifferentialHistory(tokens) + const partitions = everyContiguousPartition(messages).filter( + isSdkResetFramedPartition, + ) + const partition = partitions[partitionSeed % partitions.length]! + const referenceSnapshots = recomputedSnapshots([], partition) + + for (const syncMode of [`eager`, `on-demand`, `progressive`] as const) { + const direct = await runTrace( + `direct-differential-${syncMode}`, + syncMode, + [], + partition, + ) + const persisted = await runPersistedTrace( + `persisted-differential-${syncMode}`, + syncMode, + partition, + ) + const query = await runQueryTrace( + `query-differential-${syncMode}`, + partition, + ) + + expect(direct.snapshots).toEqual(referenceSnapshots) + expect(persisted.snapshots).toEqual(referenceSnapshots) + expect(query.snapshots).toEqual(referenceSnapshots) + expect(persisted.rows).toEqual(direct.rows) + expect(query.rows).toEqual(direct.rows) + expect(persisted.status).toBe(direct.status) + expect(query.status).toBe(`ready`) + expect(persisted.resume).toEqual(direct.resume) + expect(persisted.durableRows).toEqual(direct.rows) + expect(persisted.durableResume).toEqual(direct.resume) + expect(persisted.persistenceCommits).toBeGreaterThan(0) + } + } + const publicationEpochExamples: Array<[Array, number]> = [ + [ + [ + { operation: `insert`, id: 2, name: `` }, + { operation: `commit` }, + { operation: `delete`, id: 2, name: `` }, + ], + 2032071466, + ], + [ + [{ operation: `reset` }, { operation: `subset` }, { operation: `reset` }], + 30, + ], + ] + + it.each(publicationEpochExamples)( + `reconstructs the authored publication-epoch history`, + assertPublicationEpochHistory, + 30_000, + ) + fcTest.prop( - [fc.array(designTokenArb, { minLength: 1, maxLength: 7 }), fc.nat()], + [publicationEpochTokensArbitrary, publicationEpochPartitionArbitrary], { - numRuns: 20, - examples: [ - [ - [ - { operation: `insert`, id: 2, name: `` }, - { operation: `commit` }, - { operation: `delete`, id: 2, name: `` }, - ], - 2032071466, - ], - [ - [ - { operation: `reset` }, - { operation: `subset` }, - { operation: `reset` }, - ], - 30, - ], - ], + seed: 42714, + numRuns: oracleRuns(20), }, )( - `Electric drivers converge with the denotational reference and model-fed Query projection across publication epochs`, - async (tokens, partitionSeed) => { - const messages = buildDifferentialHistory(tokens) - const partitions = everyContiguousPartition(messages).filter( - isSdkResetFramedPartition, - ) - const partition = partitions[partitionSeed % partitions.length]! - const referenceSnapshots = recomputedSnapshots([], partition) - - for (const syncMode of [`eager`, `on-demand`, `progressive`] as const) { - const direct = await runTrace( - `direct-differential-${syncMode}`, - syncMode, - [], - partition, - ) - const persisted = await runPersistedTrace( - `persisted-differential-${syncMode}`, - syncMode, - partition, - ) - const query = await runQueryTrace( - `query-differential-${syncMode}`, - partition, - ) + `Electric drivers converge with the denotational reference and model-fed Query projection across publication epochs (fixed)`, + assertPublicationEpochHistory, + 30_000, + ) - expect(direct.snapshots).toEqual(referenceSnapshots) - expect(persisted.snapshots).toEqual(referenceSnapshots) - expect(query.snapshots).toEqual(referenceSnapshots) - expect(persisted.rows).toEqual(direct.rows) - expect(query.rows).toEqual(direct.rows) - expect(persisted.status).toBe(direct.status) - expect(query.status).toBe(`ready`) - expect(persisted.resume).toEqual(direct.resume) - expect(persisted.durableRows).toEqual(direct.rows) - expect(persisted.durableResume).toEqual(direct.resume) - expect(persisted.persistenceCommits).toBeGreaterThan(0) - } - }, + fcTest.prop( + [publicationEpochTokensArbitrary, publicationEpochPartitionArbitrary], + oraclePropertyOptions(20, `electric.publication-epoch-convergence`), + )( + `Electric drivers converge with the denotational reference and model-fed Query projection across publication epochs (random or replayed)`, + assertPublicationEpochHistory, 30_000, ) diff --git a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts index 36a46557dd..e6607105e5 100644 --- a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts +++ b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts @@ -4,6 +4,7 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { beforeEach, describe, expect, it, vi } from 'vitest' import { createCollection } from '@tanstack/db' import { ShapeStream } from '@electric-sql/client' +import { oraclePropertyOptions, oracleRuns } from '../../db/tests/oracle-config' import { SQLiteCorePersistenceAdapter, createPersistedTableName, @@ -180,8 +181,10 @@ function fixture( let latestSeq = 0 let latestRowVersion = 0 let resetEpoch = 0 + let subsetLoads = 0 const adapter: PersistenceAdapter = { loadSubset: () => { + subsetLoads++ const snapshot = Array.from(rows, ([key, value]) => ({ key, value: { ...value }, @@ -293,6 +296,7 @@ function fixture( durableRows, exposures, record, + subsetLoadCount: () => subsetLoads, start, stopObserving: () => stopObserving(), pauseHydration: (gate: Promise) => { @@ -957,106 +961,157 @@ describe(`persisted Electric recovery laws`, () => { }, ) - fcTest.prop( - [ - fc.array( - fc.record({ - id: fc.integer({ min: 2, max: 4 }), - name: fc.string({ maxLength: 8 }), - deleted: fc.boolean(), - fullReload: fc.boolean(), - }), - { minLength: 1, maxLength: 8 }, - ), - ], - { - numRuns: 20, - examples: [ - [ - [ - { id: 2, name: `external`, deleted: false, fullReload: false }, - { id: 2, name: `removed`, deleted: true, fullReload: true }, - ], - ], - ], - }, - )( - `independent persistence publications and stream deltas agree with complete-row state`, - async (commands) => { - subscribers.length = 0 - const peer = externalPublisher() - const f = fixture(`on-demand`, peer.coordinator) - const expected = new Map([[oldRow.id, structuredClone(oldRow)]]) - const expectedRows = () => - structuredClone([...expected.values()].sort((a, b) => a.id - b.id)) - try { - f.start() - await vi.waitFor(() => expect(subscribers).toHaveLength(1), { - interval: 1, - }) - await f.collection._sync.loadSubset({}) - subscribers[0]!([upToDate]) - for (const command of commands) { - const before = expectedRows() - const cut = f.exposures.length - f.record(`before peer ${JSON.stringify(command)}`) - const row = { - id: command.id, - name: command.name, - stable: `peer-${command.id}`, - } - if (command.deleted) { - f.rows.delete(row.id) - expected.delete(row.id) - } else { - f.rows.set(row.id, structuredClone(row)) - expected.set(row.id, structuredClone(row)) - } - const revision = peer.publish( - row, - command.deleted, - command.fullReload, - f.metadata, - ) - f.record(`after peer revision ${revision}`) - // An unchanged row set is not proof that the peer publication ran. - // Its metadata marker commits with the rows, including empty deletes. + type PublicationHistoryCommand = { + id: number + name: string + deleted: boolean + fullReload: boolean + } + + const publicationHistoryArbitrary = fc.array( + fc.record({ + id: fc.integer({ min: 2, max: 4 }), + name: fc.string({ maxLength: 8 }), + deleted: fc.boolean(), + fullReload: fc.boolean(), + }), + { minLength: 1, maxLength: 8 }, + ) + + const assertPublicationHistory = async ( + commands: Array, + ) => { + subscribers.length = 0 + const peer = externalPublisher() + const f = fixture(`on-demand`, peer.coordinator) + const expected = new Map([[oldRow.id, structuredClone(oldRow)]]) + const expectedRows = () => + structuredClone([...expected.values()].sort((a, b) => a.id - b.id)) + try { + f.start() + await vi.waitFor(() => expect(subscribers).toHaveLength(1), { + interval: 1, + }) + await f.collection._sync.loadSubset({}) + subscribers[0]!([upToDate]) + for (const command of commands) { + const before = expectedRows() + const cut = f.exposures.length + f.record(`before peer ${JSON.stringify(command)}`) + const row = { + id: command.id, + name: command.name, + stable: `peer-${command.id}`, + } + if (command.deleted) { + f.rows.delete(row.id) + expected.delete(row.id) + } else { + f.rows.set(row.id, structuredClone(row)) + expected.set(row.id, structuredClone(row)) + } + const subsetLoadsBeforeSettlement = f.subsetLoadCount() + const revision = peer.publish( + row, + command.deleted, + command.fullReload, + f.metadata, + ) + f.record(`after peer revision ${revision}`) + // An unchanged row set is not proof that the peer publication ran. + // Its metadata marker commits with the rows, including empty deletes. + await vi.waitFor( + () => + expect( + f.collection._state.syncedCollectionMetadata.get( + `oracle:publication`, + ), + ).toBe(revision), + { interval: 1 }, + ) + if (command.fullReload) { await vi.waitFor( () => - expect( - f.collection._state.syncedCollectionMetadata.get( - `oracle:publication`, - ), - ).toBe(revision), - { interval: 1 }, - ) - expect(f.publicRows()).toEqual(expectedRows()) - f.record(`peer revision ${revision} settled`) - const afterPeer = expectedRows() - expectWholeRecoveryTrace(f.exposures.slice(cut), [before, afterPeer]) - const streamCut = f.exposures.length - f.record(`before stream revision ${revision}`) - subscribers[0]!([ - change(`update`, { id: row.id, name: `stream` }), - upToDate, - ]) - if (!command.deleted) expected.set(row.id, { ...row, name: `stream` }) - f.record(`after stream revision ${revision}`) - expect(f.publicRows()).toEqual(expectedRows()) - await vi.waitFor( - () => expect(f.durableRows()).toEqual(expectedRows()), + expect(f.subsetLoadCount()).toBeGreaterThan( + subsetLoadsBeforeSettlement, + ), { interval: 1 }, ) - expectWholeRecoveryTrace(f.exposures.slice(streamCut), [ - afterPeer, - expectedRows(), - ]) + } else { + expect(f.subsetLoadCount()).toBe(subsetLoadsBeforeSettlement) } - } finally { - f.stopObserving() - await f.collection.cleanup() + expect(f.publicRows()).toEqual(expectedRows()) + f.record(`peer revision ${revision} settled`) + const afterPeer = expectedRows() + expectWholeRecoveryTrace(f.exposures.slice(cut), [before, afterPeer]) + const streamCut = f.exposures.length + f.record(`before stream revision ${revision}`) + subscribers[0]!([ + change(`update`, { id: row.id, name: `stream` }), + upToDate, + ]) + if (!command.deleted) expected.set(row.id, { ...row, name: `stream` }) + f.record(`after stream revision ${revision}`) + expect(f.publicRows()).toEqual(expectedRows()) + await vi.waitFor( + () => expect(f.durableRows()).toEqual(expectedRows()), + { interval: 1 }, + ) + expectWholeRecoveryTrace(f.exposures.slice(streamCut), [ + afterPeer, + expectedRows(), + ]) } - }, + } finally { + f.stopObserving() + await f.collection.cleanup() + } + } + + const publicationHistoryExamples: Array<[Array]> = + [ + [ + [ + { id: 2, name: `external`, deleted: false, fullReload: false }, + { id: 2, name: `removed`, deleted: true, fullReload: true }, + ], + ], + ] + + it.each(publicationHistoryExamples)( + `reconstructs the authored persistence publication history`, + assertPublicationHistory, + ) + + it(`fixed recovery corpus reaches every deletion and reload combination`, () => { + const combinations = new Set( + fc + .sample(publicationHistoryArbitrary, { seed: 1659, numRuns: 20 }) + .flat() + .map(({ deleted, fullReload }) => `${deleted}:${fullReload}`), + ) + expect(combinations).toEqual( + new Set([`false:false`, `false:true`, `true:false`, `true:true`]), + ) + }) + + fcTest.prop([publicationHistoryArbitrary], { + seed: 1659, + numRuns: oracleRuns(20), + })( + `independent persistence publications and stream deltas agree with complete-row state (fixed)`, + assertPublicationHistory, + ) + + fcTest.prop( + [publicationHistoryArbitrary], + oraclePropertyOptions( + 20, + `electric-recovery.publication-stream-convergence`, + ), + )( + `independent persistence publications and stream deltas agree with complete-row state (random or replayed)`, + assertPublicationHistory, ) it.each(scenarios)(